diff --git a/.github/workflows/modelkit-ci.yml b/.github/workflows/modelkit-ci.yml index b61b2a5cb..aa013dc6c 100644 --- a/.github/workflows/modelkit-ci.yml +++ b/.github/workflows/modelkit-ci.yml @@ -26,7 +26,8 @@ jobs: tests/unit/models tests/unit/loader tests/unit/datasets tests/unit/export - group: optim - paths: tests/unit/optim + paths: >- + tests/unit/optim tests/unit/test_quant_passes.py tests/unit/test_quantizer.py - group: commands paths: >- tests/unit/commands tests/unit/config tests/unit/build diff --git a/examples/recipes/BAAI_bge-base-en-v1.5/feature-extraction_fp16_config.json b/examples/recipes/BAAI_bge-base-en-v1.5/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/BAAI_bge-base-en-v1.5/feature-extraction_fp16_config.json rename to examples/recipes/BAAI_bge-base-en-v1.5/feature-extraction_fp32_config.json diff --git a/examples/recipes/BAAI_bge-base-en-v1.5/sentence-similarity_fp16_config.json b/examples/recipes/BAAI_bge-base-en-v1.5/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/BAAI_bge-base-en-v1.5/sentence-similarity_fp16_config.json rename to examples/recipes/BAAI_bge-base-en-v1.5/sentence-similarity_fp32_config.json diff --git a/examples/recipes/BAAI_bge-large-en-v1.5/sentence-similarity_fp16_config.json b/examples/recipes/BAAI_bge-large-en-v1.5/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/BAAI_bge-large-en-v1.5/sentence-similarity_fp16_config.json rename to examples/recipes/BAAI_bge-large-en-v1.5/sentence-similarity_fp32_config.json diff --git a/examples/recipes/BAAI_bge-m3/feature-extraction_fp16_config.json b/examples/recipes/BAAI_bge-m3/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/BAAI_bge-m3/feature-extraction_fp16_config.json rename to examples/recipes/BAAI_bge-m3/feature-extraction_fp32_config.json diff --git a/examples/recipes/BAAI_bge-m3/sentence-similarity_fp16_config.json b/examples/recipes/BAAI_bge-m3/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/BAAI_bge-m3/sentence-similarity_fp16_config.json rename to examples/recipes/BAAI_bge-m3/sentence-similarity_fp32_config.json diff --git a/examples/recipes/BAAI_bge-small-en-v1.5/feature-extraction_fp16_config.json b/examples/recipes/BAAI_bge-small-en-v1.5/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/BAAI_bge-small-en-v1.5/feature-extraction_fp16_config.json rename to examples/recipes/BAAI_bge-small-en-v1.5/feature-extraction_fp32_config.json diff --git a/examples/recipes/BAAI_bge-small-en-v1.5/sentence-similarity_fp16_config.json b/examples/recipes/BAAI_bge-small-en-v1.5/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/BAAI_bge-small-en-v1.5/sentence-similarity_fp16_config.json rename to examples/recipes/BAAI_bge-small-en-v1.5/sentence-similarity_fp32_config.json diff --git a/examples/recipes/Babelscape_wikineural-multilingual-ner/token-classification_fp16_config.json b/examples/recipes/Babelscape_wikineural-multilingual-ner/token-classification_fp32_config.json similarity index 100% rename from examples/recipes/Babelscape_wikineural-multilingual-ner/token-classification_fp16_config.json rename to examples/recipes/Babelscape_wikineural-multilingual-ner/token-classification_fp32_config.json diff --git a/examples/recipes/FacebookAI_roberta-base/fill-mask_fp16_config.json b/examples/recipes/FacebookAI_roberta-base/fill-mask_fp32_config.json similarity index 100% rename from examples/recipes/FacebookAI_roberta-base/fill-mask_fp16_config.json rename to examples/recipes/FacebookAI_roberta-base/fill-mask_fp32_config.json diff --git a/examples/recipes/FacebookAI_roberta-large/fill-mask_fp16_config.json b/examples/recipes/FacebookAI_roberta-large/fill-mask_fp32_config.json similarity index 100% rename from examples/recipes/FacebookAI_roberta-large/fill-mask_fp16_config.json rename to examples/recipes/FacebookAI_roberta-large/fill-mask_fp32_config.json diff --git a/examples/recipes/FacebookAI_xlm-roberta-base/fill-mask_fp16_config.json b/examples/recipes/FacebookAI_xlm-roberta-base/fill-mask_fp32_config.json similarity index 100% rename from examples/recipes/FacebookAI_xlm-roberta-base/fill-mask_fp16_config.json rename to examples/recipes/FacebookAI_xlm-roberta-base/fill-mask_fp32_config.json diff --git a/examples/recipes/Intel_bert-base-uncased-mrpc/feature-extraction_fp16_config.json b/examples/recipes/Intel_bert-base-uncased-mrpc/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/Intel_bert-base-uncased-mrpc/feature-extraction_fp16_config.json rename to examples/recipes/Intel_bert-base-uncased-mrpc/feature-extraction_fp32_config.json diff --git a/examples/recipes/Intel_bert-base-uncased-mrpc/text-classification_fp16_config.json b/examples/recipes/Intel_bert-base-uncased-mrpc/text-classification_fp32_config.json similarity index 100% rename from examples/recipes/Intel_bert-base-uncased-mrpc/text-classification_fp16_config.json rename to examples/recipes/Intel_bert-base-uncased-mrpc/text-classification_fp32_config.json diff --git a/examples/recipes/Intel_dpt-hybrid-midas/depth-estimation_fp16_config.json b/examples/recipes/Intel_dpt-hybrid-midas/depth-estimation_fp32_config.json similarity index 100% rename from examples/recipes/Intel_dpt-hybrid-midas/depth-estimation_fp16_config.json rename to examples/recipes/Intel_dpt-hybrid-midas/depth-estimation_fp32_config.json diff --git a/examples/recipes/Isotonic_distilbert_finetuned_ai4privacy_v2/token-classification_fp16_config.json b/examples/recipes/Isotonic_distilbert_finetuned_ai4privacy_v2/token-classification_fp32_config.json similarity index 100% rename from examples/recipes/Isotonic_distilbert_finetuned_ai4privacy_v2/token-classification_fp16_config.json rename to examples/recipes/Isotonic_distilbert_finetuned_ai4privacy_v2/token-classification_fp32_config.json diff --git a/examples/recipes/LiheYoung_depth-anything-small-hf/depth-estimation_fp16_config.json b/examples/recipes/LiheYoung_depth-anything-small-hf/depth-estimation_fp32_config.json similarity index 100% rename from examples/recipes/LiheYoung_depth-anything-small-hf/depth-estimation_fp16_config.json rename to examples/recipes/LiheYoung_depth-anything-small-hf/depth-estimation_fp32_config.json diff --git a/examples/recipes/ProsusAI_finbert/text-classification_fp16_config.json b/examples/recipes/ProsusAI_finbert/text-classification_fp32_config.json similarity index 100% rename from examples/recipes/ProsusAI_finbert/text-classification_fp16_config.json rename to examples/recipes/ProsusAI_finbert/text-classification_fp32_config.json diff --git a/examples/recipes/Salesforce_blip-image-captioning-base/image-to-text_fp16_config_decoder.json b/examples/recipes/Salesforce_blip-image-captioning-base/image-to-text_fp32_config_decoder.json similarity index 100% rename from examples/recipes/Salesforce_blip-image-captioning-base/image-to-text_fp16_config_decoder.json rename to examples/recipes/Salesforce_blip-image-captioning-base/image-to-text_fp32_config_decoder.json diff --git a/examples/recipes/Salesforce_blip-image-captioning-base/image-to-text_fp16_config_encoder.json b/examples/recipes/Salesforce_blip-image-captioning-base/image-to-text_fp32_config_encoder.json similarity index 100% rename from examples/recipes/Salesforce_blip-image-captioning-base/image-to-text_fp16_config_encoder.json rename to examples/recipes/Salesforce_blip-image-captioning-base/image-to-text_fp32_config_encoder.json diff --git a/examples/recipes/StanfordAIMI_dinov2-base-xray-224/image-feature-extraction_fp16_config.json b/examples/recipes/StanfordAIMI_dinov2-base-xray-224/image-feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/StanfordAIMI_dinov2-base-xray-224/image-feature-extraction_fp16_config.json rename to examples/recipes/StanfordAIMI_dinov2-base-xray-224/image-feature-extraction_fp32_config.json diff --git a/examples/recipes/ahotrod_electra_large_discriminator_squad2_512/question-answering_fp16_config.json b/examples/recipes/ahotrod_electra_large_discriminator_squad2_512/question-answering_fp32_config.json similarity index 100% rename from examples/recipes/ahotrod_electra_large_discriminator_squad2_512/question-answering_fp16_config.json rename to examples/recipes/ahotrod_electra_large_discriminator_squad2_512/question-answering_fp32_config.json diff --git a/examples/recipes/apple_mobilevit-small/image-classification_fp16_config.json b/examples/recipes/apple_mobilevit-small/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/apple_mobilevit-small/image-classification_fp16_config.json rename to examples/recipes/apple_mobilevit-small/image-classification_fp32_config.json diff --git a/examples/recipes/baidu_Unlimited-OCR/cpu/cpu/feature-extraction_fp16_config.json b/examples/recipes/baidu_Unlimited-OCR/cpu/cpu/feature-extraction_fp16_config.json deleted file mode 100644 index 7ec8e3ba4..000000000 --- a/examples/recipes/baidu_Unlimited-OCR/cpu/cpu/feature-extraction_fp16_config.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "pixel_values", - "dtype": "float32", - "shape": [ - 1, - 3, - 1024, - 1024 - ], - "value_range": [ - 0, - 1 - ] - } - ], - "output_tensors": [ - { - "name": "image_embeds" - } - ] - }, - "optim": {}, - "quant": null, - "loader": { - "task": "feature-extraction", - "model_type": "unlimited-ocr", - "trust_remote_code": true - } -} diff --git a/examples/recipes/baidu_Unlimited-OCR/dml/gpu/feature-extraction_fp16_config.json b/examples/recipes/baidu_Unlimited-OCR/dml/gpu/feature-extraction_fp16_config.json deleted file mode 100644 index 7ec8e3ba4..000000000 --- a/examples/recipes/baidu_Unlimited-OCR/dml/gpu/feature-extraction_fp16_config.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "pixel_values", - "dtype": "float32", - "shape": [ - 1, - 3, - 1024, - 1024 - ], - "value_range": [ - 0, - 1 - ] - } - ], - "output_tensors": [ - { - "name": "image_embeds" - } - ] - }, - "optim": {}, - "quant": null, - "loader": { - "task": "feature-extraction", - "model_type": "unlimited-ocr", - "trust_remote_code": true - } -} diff --git a/examples/recipes/baidu_Unlimited-OCR/openvino/cpu/feature-extraction_fp16_config.json b/examples/recipes/baidu_Unlimited-OCR/openvino/cpu/feature-extraction_fp16_config.json deleted file mode 100644 index 7ec8e3ba4..000000000 --- a/examples/recipes/baidu_Unlimited-OCR/openvino/cpu/feature-extraction_fp16_config.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "pixel_values", - "dtype": "float32", - "shape": [ - 1, - 3, - 1024, - 1024 - ], - "value_range": [ - 0, - 1 - ] - } - ], - "output_tensors": [ - { - "name": "image_embeds" - } - ] - }, - "optim": {}, - "quant": null, - "loader": { - "task": "feature-extraction", - "model_type": "unlimited-ocr", - "trust_remote_code": true - } -} diff --git a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp32_config.json similarity index 100% rename from examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json rename to examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp32_config.json diff --git a/examples/recipes/dbmdz_bert-large-cased-finetuned-conll03-english/token-classification_fp16_config.json b/examples/recipes/dbmdz_bert-large-cased-finetuned-conll03-english/token-classification_fp32_config.json similarity index 100% rename from examples/recipes/dbmdz_bert-large-cased-finetuned-conll03-english/token-classification_fp16_config.json rename to examples/recipes/dbmdz_bert-large-cased-finetuned-conll03-english/token-classification_fp32_config.json diff --git a/examples/recipes/deepset_bert-large-uncased-whole-word-masking-squad2/question-answering_fp16_config.json b/examples/recipes/deepset_bert-large-uncased-whole-word-masking-squad2/question-answering_fp32_config.json similarity index 100% rename from examples/recipes/deepset_bert-large-uncased-whole-word-masking-squad2/question-answering_fp16_config.json rename to examples/recipes/deepset_bert-large-uncased-whole-word-masking-squad2/question-answering_fp32_config.json diff --git a/examples/recipes/deepset_roberta-base-squad2/question-answering_fp16_config.json b/examples/recipes/deepset_roberta-base-squad2/question-answering_fp32_config.json similarity index 100% rename from examples/recipes/deepset_roberta-base-squad2/question-answering_fp16_config.json rename to examples/recipes/deepset_roberta-base-squad2/question-answering_fp32_config.json diff --git a/examples/recipes/deepset_tinyroberta-squad2/question-answering_fp16_config.json b/examples/recipes/deepset_tinyroberta-squad2/question-answering_fp32_config.json similarity index 100% rename from examples/recipes/deepset_tinyroberta-squad2/question-answering_fp16_config.json rename to examples/recipes/deepset_tinyroberta-squad2/question-answering_fp32_config.json diff --git a/examples/recipes/dell-research-harvard_lt-un-data-fine-fine-es/cpu/cpu/feature-extraction_fp16_config.json b/examples/recipes/dell-research-harvard_lt-un-data-fine-fine-es/cpu/cpu/feature-extraction_fp16_config.json deleted file mode 100644 index 515917997..000000000 --- a/examples/recipes/dell-research-harvard_lt-un-data-fine-fine-es/cpu/cpu/feature-extraction_fp16_config.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "input_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 31002 - ] - }, - { - "name": "attention_mask", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - }, - { - "name": "token_type_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - } - ], - "output_tensors": [ - { - "name": "last_hidden_state" - } - ], - "compatibility": { - "transformers_attention": "eager" - } - }, - "optim": { - "clamp_constant_values": true - }, - "quant": null, - "compile": null, - "loader": { - "task": "feature-extraction", - "model_class": "AutoModel", - "model_type": "bert" - } -} diff --git a/examples/recipes/dell-research-harvard_lt-un-data-fine-fine-es/cpu/cpu/sentence-similarity_fp16_config.json b/examples/recipes/dell-research-harvard_lt-un-data-fine-fine-es/cpu/cpu/sentence-similarity_fp16_config.json deleted file mode 100644 index 77ccfd279..000000000 --- a/examples/recipes/dell-research-harvard_lt-un-data-fine-fine-es/cpu/cpu/sentence-similarity_fp16_config.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "input_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 31002 - ] - }, - { - "name": "attention_mask", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - }, - { - "name": "token_type_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - } - ], - "output_tensors": [ - { - "name": "last_hidden_state" - } - ], - "compatibility": { - "transformers_attention": "eager" - } - }, - "optim": { - "clamp_constant_values": true - }, - "quant": null, - "compile": null, - "loader": { - "task": "sentence-similarity", - "model_class": "AutoModel", - "model_type": "bert" - } -} diff --git a/examples/recipes/depth-anything_Depth-Anything-V2-Small-hf/depth-estimation_fp16_config.json b/examples/recipes/depth-anything_Depth-Anything-V2-Small-hf/depth-estimation_fp32_config.json similarity index 100% rename from examples/recipes/depth-anything_Depth-Anything-V2-Small-hf/depth-estimation_fp16_config.json rename to examples/recipes/depth-anything_Depth-Anything-V2-Small-hf/depth-estimation_fp32_config.json diff --git a/examples/recipes/dima806_fairface_age_image_detection/image-classification_fp16_config.json b/examples/recipes/dima806_fairface_age_image_detection/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/dima806_fairface_age_image_detection/image-classification_fp16_config.json rename to examples/recipes/dima806_fairface_age_image_detection/image-classification_fp32_config.json diff --git a/examples/recipes/distilbert_distilbert-base-cased-distilled-squad/question-answering_fp16_config.json b/examples/recipes/distilbert_distilbert-base-cased-distilled-squad/question-answering_fp32_config.json similarity index 100% rename from examples/recipes/distilbert_distilbert-base-cased-distilled-squad/question-answering_fp16_config.json rename to examples/recipes/distilbert_distilbert-base-cased-distilled-squad/question-answering_fp32_config.json diff --git a/examples/recipes/distilbert_distilbert-base-uncased-distilled-squad/question-answering_fp16_config.json b/examples/recipes/distilbert_distilbert-base-uncased-distilled-squad/question-answering_fp32_config.json similarity index 100% rename from examples/recipes/distilbert_distilbert-base-uncased-distilled-squad/question-answering_fp16_config.json rename to examples/recipes/distilbert_distilbert-base-uncased-distilled-squad/question-answering_fp32_config.json diff --git a/examples/recipes/distilbert_distilbert-base-uncased-finetuned-sst-2-english/text-classification_fp16_config.json b/examples/recipes/distilbert_distilbert-base-uncased-finetuned-sst-2-english/text-classification_fp32_config.json similarity index 100% rename from examples/recipes/distilbert_distilbert-base-uncased-finetuned-sst-2-english/text-classification_fp16_config.json rename to examples/recipes/distilbert_distilbert-base-uncased-finetuned-sst-2-english/text-classification_fp32_config.json diff --git a/examples/recipes/distilbert_distilbert-base-uncased/fill-mask_fp16_config.json b/examples/recipes/distilbert_distilbert-base-uncased/fill-mask_fp32_config.json similarity index 100% rename from examples/recipes/distilbert_distilbert-base-uncased/fill-mask_fp16_config.json rename to examples/recipes/distilbert_distilbert-base-uncased/fill-mask_fp32_config.json diff --git a/examples/recipes/dslim_bert-base-NER/token-classification_fp16_config.json b/examples/recipes/dslim_bert-base-NER/token-classification_fp32_config.json similarity index 100% rename from examples/recipes/dslim_bert-base-NER/token-classification_fp16_config.json rename to examples/recipes/dslim_bert-base-NER/token-classification_fp32_config.json diff --git a/examples/recipes/facebook_convnext-tiny-224/image-classification_fp16_config.json b/examples/recipes/facebook_convnext-tiny-224/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/facebook_convnext-tiny-224/image-classification_fp16_config.json rename to examples/recipes/facebook_convnext-tiny-224/image-classification_fp32_config.json diff --git a/examples/recipes/facebook_dino-vitb16/image-feature-extraction_fp16_config.json b/examples/recipes/facebook_dino-vitb16/image-feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/facebook_dino-vitb16/image-feature-extraction_fp16_config.json rename to examples/recipes/facebook_dino-vitb16/image-feature-extraction_fp32_config.json diff --git a/examples/recipes/facebook_dino-vits16/image-feature-extraction_fp16_config.json b/examples/recipes/facebook_dino-vits16/image-feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/facebook_dino-vits16/image-feature-extraction_fp16_config.json rename to examples/recipes/facebook_dino-vits16/image-feature-extraction_fp32_config.json diff --git a/examples/recipes/facebook_dinov2-base/image-feature-extraction_fp16_config.json b/examples/recipes/facebook_dinov2-base/image-feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/facebook_dinov2-base/image-feature-extraction_fp16_config.json rename to examples/recipes/facebook_dinov2-base/image-feature-extraction_fp32_config.json diff --git a/examples/recipes/facebook_dinov2-large/image-feature-extraction_fp16_config.json b/examples/recipes/facebook_dinov2-large/image-feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/facebook_dinov2-large/image-feature-extraction_fp16_config.json rename to examples/recipes/facebook_dinov2-large/image-feature-extraction_fp32_config.json diff --git a/examples/recipes/facebook_dinov2-small/image-feature-extraction_fp16_config.json b/examples/recipes/facebook_dinov2-small/image-feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/facebook_dinov2-small/image-feature-extraction_fp16_config.json rename to examples/recipes/facebook_dinov2-small/image-feature-extraction_fp32_config.json diff --git a/examples/recipes/google-bert_bert-base-multilingual-cased/feature-extraction_fp16_config.json b/examples/recipes/google-bert_bert-base-multilingual-cased/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/google-bert_bert-base-multilingual-cased/feature-extraction_fp16_config.json rename to examples/recipes/google-bert_bert-base-multilingual-cased/feature-extraction_fp32_config.json diff --git a/examples/recipes/google-bert_bert-base-multilingual-cased/fill-mask_fp16_config.json b/examples/recipes/google-bert_bert-base-multilingual-cased/fill-mask_fp32_config.json similarity index 100% rename from examples/recipes/google-bert_bert-base-multilingual-cased/fill-mask_fp16_config.json rename to examples/recipes/google-bert_bert-base-multilingual-cased/fill-mask_fp32_config.json diff --git a/examples/recipes/google-bert_bert-base-multilingual-uncased/fill-mask_fp16_config.json b/examples/recipes/google-bert_bert-base-multilingual-uncased/fill-mask_fp32_config.json similarity index 100% rename from examples/recipes/google-bert_bert-base-multilingual-uncased/fill-mask_fp16_config.json rename to examples/recipes/google-bert_bert-base-multilingual-uncased/fill-mask_fp32_config.json diff --git a/examples/recipes/google-bert_bert-base-uncased/fill-mask_fp16_config.json b/examples/recipes/google-bert_bert-base-uncased/fill-mask_fp32_config.json similarity index 100% rename from examples/recipes/google-bert_bert-base-uncased/fill-mask_fp16_config.json rename to examples/recipes/google-bert_bert-base-uncased/fill-mask_fp32_config.json diff --git a/examples/recipes/google-bert_bert-large-uncased-whole-word-masking-finetuned-squad/question-answering_fp16_config.json b/examples/recipes/google-bert_bert-large-uncased-whole-word-masking-finetuned-squad/question-answering_fp32_config.json similarity index 100% rename from examples/recipes/google-bert_bert-large-uncased-whole-word-masking-finetuned-squad/question-answering_fp16_config.json rename to examples/recipes/google-bert_bert-large-uncased-whole-word-masking-finetuned-squad/question-answering_fp32_config.json diff --git a/examples/recipes/google_vit-base-patch16-224-in21k/image-feature-extraction_fp16_config.json b/examples/recipes/google_vit-base-patch16-224-in21k/image-feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/google_vit-base-patch16-224-in21k/image-feature-extraction_fp16_config.json rename to examples/recipes/google_vit-base-patch16-224-in21k/image-feature-extraction_fp32_config.json diff --git a/examples/recipes/google_vit-base-patch16-224/image-classification_fp16_config.json b/examples/recipes/google_vit-base-patch16-224/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/google_vit-base-patch16-224/image-classification_fp16_config.json rename to examples/recipes/google_vit-base-patch16-224/image-classification_fp32_config.json diff --git a/examples/recipes/impira_layoutlm-document-qa/cpu/cpu/question-answering_fp16_config.json b/examples/recipes/impira_layoutlm-document-qa/cpu/cpu/question-answering_fp16_config.json deleted file mode 100644 index 0fe7d1a21..000000000 --- a/examples/recipes/impira_layoutlm-document-qa/cpu/cpu/question-answering_fp16_config.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "input_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 50265 - ] - }, - { - "name": "bbox", - "dtype": "int32", - "shape": [ - 1, - 512, - 4 - ], - "value_range": [ - 0, - 1000 - ] - }, - { - "name": "attention_mask", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - }, - { - "name": "token_type_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 1 - ] - } - ], - "output_tensors": [ - { - "name": "start_logits" - }, - { - "name": "end_logits" - } - ] - }, - "optim": { - "clamp_constant_values": true - }, - "quant": null, - "loader": { - "task": "question-answering", - "model_class": "LayoutLMForQuestionAnswering", - "model_type": "layoutlm" - } -} diff --git a/examples/recipes/joeddav_xlm-roberta-large-xnli/zero-shot-classification_fp16_config.json b/examples/recipes/joeddav_xlm-roberta-large-xnli/zero-shot-classification_fp32_config.json similarity index 100% rename from examples/recipes/joeddav_xlm-roberta-large-xnli/zero-shot-classification_fp16_config.json rename to examples/recipes/joeddav_xlm-roberta-large-xnli/zero-shot-classification_fp32_config.json diff --git a/examples/recipes/laion_CLIP-ViT-B-32-laion2B-s34B-b79K/feature-extraction_fp16_config.json b/examples/recipes/laion_CLIP-ViT-B-32-laion2B-s34B-b79K/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/laion_CLIP-ViT-B-32-laion2B-s34B-b79K/feature-extraction_fp16_config.json rename to examples/recipes/laion_CLIP-ViT-B-32-laion2B-s34B-b79K/feature-extraction_fp32_config.json diff --git a/examples/recipes/mattmdjaga_segformer_b2_clothes/image-segmentation_fp16_config.json b/examples/recipes/mattmdjaga_segformer_b2_clothes/image-segmentation_fp32_config.json similarity index 100% rename from examples/recipes/mattmdjaga_segformer_b2_clothes/image-segmentation_fp16_config.json rename to examples/recipes/mattmdjaga_segformer_b2_clothes/image-segmentation_fp32_config.json diff --git a/examples/recipes/microsoft_beit-base-patch16-224/cpu/cpu/image-classification_fp16_config.json b/examples/recipes/microsoft_beit-base-patch16-224/cpu/cpu/image-classification_fp16_config.json deleted file mode 100644 index 2c9c0acdb..000000000 --- a/examples/recipes/microsoft_beit-base-patch16-224/cpu/cpu/image-classification_fp16_config.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "pixel_values", - "dtype": "float32", - "shape": [ - 1, - 3, - 224, - 224 - ], - "value_range": [ - 0, - 1 - ] - } - ], - "output_tensors": [ - { - "name": "logits" - } - ] - }, - "optim": {}, - "quant": null, - "compile": null, - "loader": { - "task": "image-classification", - "model_class": "AutoModelForImageClassification", - "model_type": "beit" - } -} diff --git a/examples/recipes/microsoft_rad-dino/image-feature-extraction_fp16_config.json b/examples/recipes/microsoft_rad-dino/image-feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/microsoft_rad-dino/image-feature-extraction_fp16_config.json rename to examples/recipes/microsoft_rad-dino/image-feature-extraction_fp32_config.json diff --git a/examples/recipes/microsoft_resnet-18/image-classification_fp16_config.json b/examples/recipes/microsoft_resnet-18/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/microsoft_resnet-18/image-classification_fp16_config.json rename to examples/recipes/microsoft_resnet-18/image-classification_fp32_config.json diff --git a/examples/recipes/microsoft_resnet-50/image-classification_fp16_config.json b/examples/recipes/microsoft_resnet-50/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/microsoft_resnet-50/image-classification_fp16_config.json rename to examples/recipes/microsoft_resnet-50/image-classification_fp32_config.json diff --git a/examples/recipes/microsoft_swin-large-patch4-window7-224/image-classification_fp16_config.json b/examples/recipes/microsoft_swin-large-patch4-window7-224/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/microsoft_swin-large-patch4-window7-224/image-classification_fp16_config.json rename to examples/recipes/microsoft_swin-large-patch4-window7-224/image-classification_fp32_config.json diff --git a/examples/recipes/microsoft_swinv2-tiny-patch4-window16-256/image-classification_fp16_config.json b/examples/recipes/microsoft_swinv2-tiny-patch4-window16-256/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/microsoft_swinv2-tiny-patch4-window16-256/image-classification_fp16_config.json rename to examples/recipes/microsoft_swinv2-tiny-patch4-window16-256/image-classification_fp32_config.json diff --git a/examples/recipes/microsoft_swinv2-tiny-patch4-window16-256/qnn/npu/image-classification_fp16_config.json b/examples/recipes/microsoft_swinv2-tiny-patch4-window16-256/qnn/npu/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/microsoft_swinv2-tiny-patch4-window16-256/qnn/npu/image-classification_fp16_config.json rename to examples/recipes/microsoft_swinv2-tiny-patch4-window16-256/qnn/npu/image-classification_fp32_config.json diff --git a/examples/recipes/microsoft_trocr-base-handwritten/image-to-text_fp16_config_decoder.json b/examples/recipes/microsoft_trocr-base-handwritten/image-to-text_fp32_config_decoder.json similarity index 100% rename from examples/recipes/microsoft_trocr-base-handwritten/image-to-text_fp16_config_decoder.json rename to examples/recipes/microsoft_trocr-base-handwritten/image-to-text_fp32_config_decoder.json diff --git a/examples/recipes/microsoft_trocr-base-handwritten/image-to-text_fp16_config_encoder.json b/examples/recipes/microsoft_trocr-base-handwritten/image-to-text_fp32_config_encoder.json similarity index 100% rename from examples/recipes/microsoft_trocr-base-handwritten/image-to-text_fp16_config_encoder.json rename to examples/recipes/microsoft_trocr-base-handwritten/image-to-text_fp32_config_encoder.json diff --git a/examples/recipes/microsoft_trocr-base-printed/image-to-text_fp16_config_decoder.json b/examples/recipes/microsoft_trocr-base-printed/image-to-text_fp32_config_decoder.json similarity index 100% rename from examples/recipes/microsoft_trocr-base-printed/image-to-text_fp16_config_decoder.json rename to examples/recipes/microsoft_trocr-base-printed/image-to-text_fp32_config_decoder.json diff --git a/examples/recipes/microsoft_trocr-base-printed/image-to-text_fp16_config_encoder.json b/examples/recipes/microsoft_trocr-base-printed/image-to-text_fp32_config_encoder.json similarity index 100% rename from examples/recipes/microsoft_trocr-base-printed/image-to-text_fp16_config_encoder.json rename to examples/recipes/microsoft_trocr-base-printed/image-to-text_fp32_config_encoder.json diff --git a/examples/recipes/microsoft_trocr-large-handwritten/image-to-text_fp16_config_decoder.json b/examples/recipes/microsoft_trocr-large-handwritten/image-to-text_fp32_config_decoder.json similarity index 100% rename from examples/recipes/microsoft_trocr-large-handwritten/image-to-text_fp16_config_decoder.json rename to examples/recipes/microsoft_trocr-large-handwritten/image-to-text_fp32_config_decoder.json diff --git a/examples/recipes/microsoft_trocr-large-handwritten/image-to-text_fp16_config_encoder.json b/examples/recipes/microsoft_trocr-large-handwritten/image-to-text_fp32_config_encoder.json similarity index 100% rename from examples/recipes/microsoft_trocr-large-handwritten/image-to-text_fp16_config_encoder.json rename to examples/recipes/microsoft_trocr-large-handwritten/image-to-text_fp32_config_encoder.json diff --git a/examples/recipes/microsoft_trocr-large-printed/image-to-text_fp16_config_decoder.json b/examples/recipes/microsoft_trocr-large-printed/image-to-text_fp32_config_decoder.json similarity index 100% rename from examples/recipes/microsoft_trocr-large-printed/image-to-text_fp16_config_decoder.json rename to examples/recipes/microsoft_trocr-large-printed/image-to-text_fp32_config_decoder.json diff --git a/examples/recipes/microsoft_trocr-large-printed/image-to-text_fp16_config_encoder.json b/examples/recipes/microsoft_trocr-large-printed/image-to-text_fp32_config_encoder.json similarity index 100% rename from examples/recipes/microsoft_trocr-large-printed/image-to-text_fp16_config_encoder.json rename to examples/recipes/microsoft_trocr-large-printed/image-to-text_fp32_config_encoder.json diff --git a/examples/recipes/mixedbread-ai_mxbai-rerank-base-v1/cpu/cpu/text-classification_fp16_config.json b/examples/recipes/mixedbread-ai_mxbai-rerank-base-v1/cpu/cpu/text-classification_fp16_config.json deleted file mode 100644 index 7b36e6952..000000000 --- a/examples/recipes/mixedbread-ai_mxbai-rerank-base-v1/cpu/cpu/text-classification_fp16_config.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "input_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 128100 - ] - }, - { - "name": "attention_mask", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - } - ], - "output_tensors": [ - { - "name": "logits" - } - ] - }, - "optim": { - "clamp_constant_values": true - }, - "quant": null, - "compile": null, - "loader": { - "task": "text-classification", - "model_class": "AutoModelForSequenceClassification", - "model_type": "deberta-v2" - } -} diff --git a/examples/recipes/mixedbread-ai_mxbai-rerank-xsmall-v1/cpu/cpu/text-classification_fp16_config.json b/examples/recipes/mixedbread-ai_mxbai-rerank-xsmall-v1/cpu/cpu/text-classification_fp16_config.json deleted file mode 100644 index 7b36e6952..000000000 --- a/examples/recipes/mixedbread-ai_mxbai-rerank-xsmall-v1/cpu/cpu/text-classification_fp16_config.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "input_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 128100 - ] - }, - { - "name": "attention_mask", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - } - ], - "output_tensors": [ - { - "name": "logits" - } - ] - }, - "optim": { - "clamp_constant_values": true - }, - "quant": null, - "compile": null, - "loader": { - "task": "text-classification", - "model_class": "AutoModelForSequenceClassification", - "model_type": "deberta-v2" - } -} diff --git a/examples/recipes/monologg_koelectra-small-v2-distilled-korquad-384/question-answering_fp16_config.json b/examples/recipes/monologg_koelectra-small-v2-distilled-korquad-384/question-answering_fp32_config.json similarity index 100% rename from examples/recipes/monologg_koelectra-small-v2-distilled-korquad-384/question-answering_fp16_config.json rename to examples/recipes/monologg_koelectra-small-v2-distilled-korquad-384/question-answering_fp32_config.json diff --git a/examples/recipes/nvidia_segformer-b1-finetuned-ade-512-512/image-segmentation_fp16_config.json b/examples/recipes/nvidia_segformer-b1-finetuned-ade-512-512/image-segmentation_fp32_config.json similarity index 100% rename from examples/recipes/nvidia_segformer-b1-finetuned-ade-512-512/image-segmentation_fp16_config.json rename to examples/recipes/nvidia_segformer-b1-finetuned-ade-512-512/image-segmentation_fp32_config.json diff --git a/examples/recipes/nvidia_segformer-b2-finetuned-ade-512-512/image-segmentation_fp16_config.json b/examples/recipes/nvidia_segformer-b2-finetuned-ade-512-512/image-segmentation_fp32_config.json similarity index 100% rename from examples/recipes/nvidia_segformer-b2-finetuned-ade-512-512/image-segmentation_fp16_config.json rename to examples/recipes/nvidia_segformer-b2-finetuned-ade-512-512/image-segmentation_fp32_config.json diff --git a/examples/recipes/nvidia_segformer-b5-finetuned-ade-640-640/image-segmentation_fp16_config.json b/examples/recipes/nvidia_segformer-b5-finetuned-ade-640-640/image-segmentation_fp32_config.json similarity index 100% rename from examples/recipes/nvidia_segformer-b5-finetuned-ade-640-640/image-segmentation_fp16_config.json rename to examples/recipes/nvidia_segformer-b5-finetuned-ade-640-640/image-segmentation_fp32_config.json diff --git a/examples/recipes/openai_clip-vit-base-patch16/feature-extraction_fp16_config.json b/examples/recipes/openai_clip-vit-base-patch16/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/openai_clip-vit-base-patch16/feature-extraction_fp16_config.json rename to examples/recipes/openai_clip-vit-base-patch16/feature-extraction_fp32_config.json diff --git a/examples/recipes/openai_clip-vit-base-patch16/zero-shot-image-classification_fp16_config_image-encoder.json b/examples/recipes/openai_clip-vit-base-patch16/zero-shot-image-classification_fp32_config_image-encoder.json similarity index 100% rename from examples/recipes/openai_clip-vit-base-patch16/zero-shot-image-classification_fp16_config_image-encoder.json rename to examples/recipes/openai_clip-vit-base-patch16/zero-shot-image-classification_fp32_config_image-encoder.json diff --git a/examples/recipes/openai_clip-vit-base-patch16/zero-shot-image-classification_fp16_config_text-encoder.json b/examples/recipes/openai_clip-vit-base-patch16/zero-shot-image-classification_fp32_config_text-encoder.json similarity index 100% rename from examples/recipes/openai_clip-vit-base-patch16/zero-shot-image-classification_fp16_config_text-encoder.json rename to examples/recipes/openai_clip-vit-base-patch16/zero-shot-image-classification_fp32_config_text-encoder.json diff --git a/examples/recipes/openai_clip-vit-base-patch32/feature-extraction_fp16_config.json b/examples/recipes/openai_clip-vit-base-patch32/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/openai_clip-vit-base-patch32/feature-extraction_fp16_config.json rename to examples/recipes/openai_clip-vit-base-patch32/feature-extraction_fp32_config.json diff --git a/examples/recipes/openai_clip-vit-large-patch14-336/zero-shot-image-classification_fp16_config_image-encoder.json b/examples/recipes/openai_clip-vit-large-patch14-336/zero-shot-image-classification_fp32_config_image-encoder.json similarity index 100% rename from examples/recipes/openai_clip-vit-large-patch14-336/zero-shot-image-classification_fp16_config_image-encoder.json rename to examples/recipes/openai_clip-vit-large-patch14-336/zero-shot-image-classification_fp32_config_image-encoder.json diff --git a/examples/recipes/openai_clip-vit-large-patch14-336/zero-shot-image-classification_fp16_config_text-encoder.json b/examples/recipes/openai_clip-vit-large-patch14-336/zero-shot-image-classification_fp32_config_text-encoder.json similarity index 100% rename from examples/recipes/openai_clip-vit-large-patch14-336/zero-shot-image-classification_fp16_config_text-encoder.json rename to examples/recipes/openai_clip-vit-large-patch14-336/zero-shot-image-classification_fp32_config_text-encoder.json diff --git a/examples/recipes/openai_clip-vit-large-patch14/zero-shot-image-classification_fp16_config_image-encoder.json b/examples/recipes/openai_clip-vit-large-patch14/zero-shot-image-classification_fp32_config_image-encoder.json similarity index 100% rename from examples/recipes/openai_clip-vit-large-patch14/zero-shot-image-classification_fp16_config_image-encoder.json rename to examples/recipes/openai_clip-vit-large-patch14/zero-shot-image-classification_fp32_config_image-encoder.json diff --git a/examples/recipes/openai_clip-vit-large-patch14/zero-shot-image-classification_fp16_config_text-encoder.json b/examples/recipes/openai_clip-vit-large-patch14/zero-shot-image-classification_fp32_config_text-encoder.json similarity index 100% rename from examples/recipes/openai_clip-vit-large-patch14/zero-shot-image-classification_fp16_config_text-encoder.json rename to examples/recipes/openai_clip-vit-large-patch14/zero-shot-image-classification_fp32_config_text-encoder.json diff --git a/examples/recipes/rizvandwiki_gender-classification/image-classification_fp16_config.json b/examples/recipes/rizvandwiki_gender-classification/image-classification_fp32_config.json similarity index 100% rename from examples/recipes/rizvandwiki_gender-classification/image-classification_fp16_config.json rename to examples/recipes/rizvandwiki_gender-classification/image-classification_fp32_config.json diff --git a/examples/recipes/sentence-transformers_all-MiniLM-L6-v2/feature-extraction_fp16_config.json b/examples/recipes/sentence-transformers_all-MiniLM-L6-v2/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/sentence-transformers_all-MiniLM-L6-v2/feature-extraction_fp16_config.json rename to examples/recipes/sentence-transformers_all-MiniLM-L6-v2/feature-extraction_fp32_config.json diff --git a/examples/recipes/sentence-transformers_all-MiniLM-L6-v2/sentence-similarity_fp16_config.json b/examples/recipes/sentence-transformers_all-MiniLM-L6-v2/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/sentence-transformers_all-MiniLM-L6-v2/sentence-similarity_fp16_config.json rename to examples/recipes/sentence-transformers_all-MiniLM-L6-v2/sentence-similarity_fp32_config.json diff --git a/examples/recipes/sentence-transformers_all-mpnet-base-v2/feature-extraction_fp16_config.json b/examples/recipes/sentence-transformers_all-mpnet-base-v2/feature-extraction_fp16_config.json deleted file mode 100644 index e731b2a2a..000000000 --- a/examples/recipes/sentence-transformers_all-mpnet-base-v2/feature-extraction_fp16_config.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "input_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 30527 - ] - }, - { - "name": "attention_mask", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - } - ], - "output_tensors": [ - { - "name": "last_hidden_state" - } - ] - }, - "optim": {}, - "quant": null, - "loader": { - "task": "feature-extraction", - "model_class": "AutoModel", - "model_type": "mpnet" - }, - "eval": { - "task": "feature-extraction", - "dataset": { - "path": "mteb/stsbenchmark-sts", - "split": "test", - "samples": 100, - "columns_mapping": { - "input_column_1": "sentence1", - "input_column_2": "sentence2", - "score_column": "score" - } - } - } -} diff --git a/examples/recipes/dell-research-harvard_lt-wikidata-comp-en/cpu/cpu/feature-extraction_fp16_config.json b/examples/recipes/sentence-transformers_all-mpnet-base-v2/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/dell-research-harvard_lt-wikidata-comp-en/cpu/cpu/feature-extraction_fp16_config.json rename to examples/recipes/sentence-transformers_all-mpnet-base-v2/feature-extraction_fp32_config.json diff --git a/examples/recipes/dell-research-harvard_lt-wikidata-comp-en/cpu/cpu/sentence-similarity_fp16_config.json b/examples/recipes/sentence-transformers_all-mpnet-base-v2/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/dell-research-harvard_lt-wikidata-comp-en/cpu/cpu/sentence-similarity_fp16_config.json rename to examples/recipes/sentence-transformers_all-mpnet-base-v2/sentence-similarity_fp32_config.json diff --git a/examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/feature-extraction_fp16_config.json b/examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/feature-extraction_fp16_config.json deleted file mode 100644 index e731b2a2a..000000000 --- a/examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/feature-extraction_fp16_config.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "input_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 30527 - ] - }, - { - "name": "attention_mask", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - } - ], - "output_tensors": [ - { - "name": "last_hidden_state" - } - ] - }, - "optim": {}, - "quant": null, - "loader": { - "task": "feature-extraction", - "model_class": "AutoModel", - "model_type": "mpnet" - }, - "eval": { - "task": "feature-extraction", - "dataset": { - "path": "mteb/stsbenchmark-sts", - "split": "test", - "samples": 100, - "columns_mapping": { - "input_column_1": "sentence1", - "input_column_2": "sentence2", - "score_column": "score" - } - } - } -} diff --git a/examples/recipes/dell-research-harvard_names-linking-model/cpu/cpu/feature-extraction_fp16_config.json b/examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/dell-research-harvard_names-linking-model/cpu/cpu/feature-extraction_fp16_config.json rename to examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/feature-extraction_fp32_config.json diff --git a/examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/sentence-similarity_fp16_config.json b/examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/sentence-similarity_fp16_config.json deleted file mode 100644 index 25b22809c..000000000 --- a/examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/sentence-similarity_fp16_config.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "export": { - "opset_version": 17, - "batch_size": 1, - "export_params": true, - "do_constant_folding": true, - "verbose": false, - "dynamo": false, - "enable_hierarchy_tags": true, - "clean_onnx": false, - "hierarchy_tag_format": "full", - "input_tensors": [ - { - "name": "input_ids", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 30527 - ] - }, - { - "name": "attention_mask", - "dtype": "int32", - "shape": [ - 1, - 512 - ], - "value_range": [ - 0, - 2 - ] - } - ], - "output_tensors": [ - { - "name": "last_hidden_state" - } - ] - }, - "optim": {}, - "quant": null, - "loader": { - "task": "sentence-similarity", - "model_class": "AutoModel", - "model_type": "mpnet" - }, - "eval": { - "task": "sentence-similarity", - "dataset": { - "path": "mteb/stsbenchmark-sts", - "split": "test", - "samples": 100, - "columns_mapping": { - "input_column_1": "sentence1", - "input_column_2": "sentence2", - "score_column": "score" - } - } - } -} diff --git a/examples/recipes/sentence-transformers_all-mpnet-base-v2/sentence-similarity_fp16_config.json b/examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/sentence-transformers_all-mpnet-base-v2/sentence-similarity_fp16_config.json rename to examples/recipes/sentence-transformers_multi-qa-mpnet-base-dot-v1/sentence-similarity_fp32_config.json diff --git a/examples/recipes/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/feature-extraction_fp16_config.json b/examples/recipes/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/feature-extraction_fp32_config.json similarity index 100% rename from examples/recipes/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/feature-extraction_fp16_config.json rename to examples/recipes/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/feature-extraction_fp32_config.json diff --git a/examples/recipes/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/sentence-similarity_fp16_config.json b/examples/recipes/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/sentence-similarity_fp16_config.json rename to examples/recipes/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/sentence-similarity_fp32_config.json diff --git a/examples/recipes/sentence-transformers_paraphrase-multilingual-mpnet-base-v2/sentence-similarity_fp16_config.json b/examples/recipes/sentence-transformers_paraphrase-multilingual-mpnet-base-v2/sentence-similarity_fp32_config.json similarity index 100% rename from examples/recipes/sentence-transformers_paraphrase-multilingual-mpnet-base-v2/sentence-similarity_fp16_config.json rename to examples/recipes/sentence-transformers_paraphrase-multilingual-mpnet-base-v2/sentence-similarity_fp32_config.json diff --git a/examples/recipes/usyd-community_vitpose-base-simple/keypoint-detection_fp16_config.json b/examples/recipes/usyd-community_vitpose-base-simple/keypoint-detection_fp32_config.json similarity index 100% rename from examples/recipes/usyd-community_vitpose-base-simple/keypoint-detection_fp16_config.json rename to examples/recipes/usyd-community_vitpose-base-simple/keypoint-detection_fp32_config.json diff --git a/examples/recipes/usyd-community_vitpose-plus-base/keypoint-detection_fp16_config.json b/examples/recipes/usyd-community_vitpose-plus-base/keypoint-detection_fp32_config.json similarity index 100% rename from examples/recipes/usyd-community_vitpose-plus-base/keypoint-detection_fp16_config.json rename to examples/recipes/usyd-community_vitpose-plus-base/keypoint-detection_fp32_config.json diff --git a/examples/recipes/usyd-community_vitpose-plus-large/keypoint-detection_fp16_config.json b/examples/recipes/usyd-community_vitpose-plus-large/keypoint-detection_fp32_config.json similarity index 100% rename from examples/recipes/usyd-community_vitpose-plus-large/keypoint-detection_fp16_config.json rename to examples/recipes/usyd-community_vitpose-plus-large/keypoint-detection_fp32_config.json diff --git a/examples/recipes/usyd-community_vitpose-plus-small/keypoint-detection_fp16_config.json b/examples/recipes/usyd-community_vitpose-plus-small/keypoint-detection_fp32_config.json similarity index 100% rename from examples/recipes/usyd-community_vitpose-plus-small/keypoint-detection_fp16_config.json rename to examples/recipes/usyd-community_vitpose-plus-small/keypoint-detection_fp32_config.json diff --git a/examples/recipes/w11wo_indonesian-roberta-base-posp-tagger/token-classification_fp16_config.json b/examples/recipes/w11wo_indonesian-roberta-base-posp-tagger/token-classification_fp32_config.json similarity index 100% rename from examples/recipes/w11wo_indonesian-roberta-base-posp-tagger/token-classification_fp16_config.json rename to examples/recipes/w11wo_indonesian-roberta-base-posp-tagger/token-classification_fp32_config.json diff --git a/scripts/e2e_eval/README.md b/scripts/e2e_eval/README.md index d918f8c64..85ef7c1e5 100644 --- a/scripts/e2e_eval/README.md +++ b/scripts/e2e_eval/README.md @@ -51,6 +51,169 @@ uv run python scripts/e2e_eval/build_registry.py --dry-run ### `run_eval.py` — Run Evaluation (recipe-driven perf + accuracy) +**Release evaluation is opt-in with `--release`.** Without it, the runner keeps +the existing registry, single-model, build-only and baseline behavior, with +`--priority` defaulting to `P0 P1 P2 P3`. In release mode it reads +[testsets/models_release_validation.json](testsets/models_release_validation.json), the machine-readable +projection of [testsets/models_release_validation.md](testsets/models_release_validation.md). +It runs one job per selected model/task using the selected config precision for the +resolved `--ep` and `--device`, not the Markdown reference precision. It uses a +recipe with that exact label when available, otherwise a single config/build job. +This also applies to `--no-recipes`; release never expands extra variants. + +`--release` rejects explicitly supplied `--priority`, `--hf-model`, `--registry`, +`--task`, `--group` and `--model-type`: the manifest defines the complete case set. +It also rejects `--build-only` and `--update-baseline`; those modes keep their +existing workflows without `--release`. Default argument values are not conflicts, +and ordinary-mode combinations remain unchanged. `release` is not a priority +value; model priority metadata remains P0-P3. + +Execution controls such as `--ep`, `--device`, `--eval-type`, `--output-dir`, +`--continue`, `--retry-failed`, timeouts and cache cleanup remain available, as +do `--list` and `--list-json`. Recipes and `--no-recipes` still choose the build +source, but cannot expand the manifest's selected precisions. + +Each manifest entry contains `hf_id`, `task`, `priority` and a `targets` map keyed +by `machine/EP_device`. A target stores only `precision`. Historical evidence +links remain in the sibling Markdown report for audit and are not opened by the +runner, so the sibling artifacts repo is not required to load the release list. +Registry metadata still supplies dataset and perf/eval overrides. +`default` means omit the precision flag and leave resolution to WinML, not FP32. +The target's `precision` selects a recipe/config, not a measured artifact dtype. +Checked-in recipe filenames now follow their `quant` configuration: + +| `quant` configuration | Filename precision | +|---|---| +| `null` (no conversion) | `fp32` | +| `mode: fp16` | `fp16` | +| QDQ/static quantization | `w{weight_bits}a{activation_bits}` | + +This naming convention describes the configured build stage. It is not an +independent guarantee of every tensor dtype or the EP's internal computation. +A matching recipe is still built unchanged. No FP16 conversion was introduced +by renaming an old `fp16` file whose config has `quant: null`. + +The filename correction renamed 91 files and removed 12 byte-identical copies +whose `fp32` destination already existed. All configuration contents were +preserved. The release manifest and target-column labels were updated for the +456 selections that used a renamed recipe; unmatched config fallbacks were not +changed. Historical evidence URLs, original JSON and the Markdown reference +column retain their old labels. New jobs use `__fp32` result directories for +these renamed cases, so `--continue` does not reuse old `__fp16` results. + +Precision locking constrains case selection. With a recipe, the consistency +check compares recipe labels, not the loaded quantization config or tensor +dtypes. Without a recipe (including `--no-recipes`), a non-default label is +instead passed as an explicit precision flag. That fallback is a new build +configuration and is not proof of reproducing the historical case. +Keep the case label separate from `perf.result.benchmark_info.precision` (the +perf command's requested policy) and `perf.result.model_info.precision` (the +artifact precision reported by perf). `auto` for a prebuilt ONNX input does not +mean that the model was converted to the device's default precision. +Historical PASS evidence does not guarantee a new run with changed software or +configuration will pass. + +Targets such as OpenVINO GPU have one matching manifest key. For a shared EP +such as DML GPU or MLAS CPU, use a consecutive `/_` pair +from the manifest in `--output-dir`, for example +`nvidia/DmlExecutionProvider_GPU/2026-09-10`. The path is normalized before +matching, and unrelated ancestor machine names do not select a target. +Missing or multiple matching pairs cause an error instead of guessing. +The output folder does not override `--ep/--device`; name it consistently to +avoid filing OpenVINO results under DML. + +```bash +# Preview the release plan without downloading/building models +uv run --project . python scripts/e2e_eval/run_eval.py --release --ep openvino --device gpu --list + +# Resume/retry release evaluation +uv run --project . python scripts/e2e_eval/run_eval.py --release --ep openvino --device gpu --eval-type both --output-dir ../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-09-10 --continue --timeout 1000 --retry-failed HF_FETCH_FAIL --clean-cache winml + +# Disambiguate a shared EP through its output path +uv run python scripts/e2e_eval/run_eval.py --release --ep dml --device gpu --output-dir eval_results/intel/DmlExecutionProvider_GPU --list +``` + +#### Historical Precision Evidence + +The release set is a regression selection, not a requirement to convert every +model to the precision named in its historical directory. The 2026-09-14 audit +of the table's 800 linked results found: + +| Historical case label | Cases | Artifact precision explicitly reported | Not recovered from precision fields/logs | +|---|---:|---|---:| +| `fp16` | 464 | 237 report `fp32` | 227 | +| `default` | 282 | 133 report `fp32`; 1 reports `w8a16` | 148 | +| `w8a16` | 52 | 42 report `w8a16` | 10 | +| `fp32` | 1 | None | 1 | +| `w8a8` | 1 | None | 1 | + +The audit reads `perf.result.model_info.precision` first, then the older +`Model Precision:` perf log entry. A missing observation remains unknown; +neither a directory suffix, an input tensor dtype nor file size proves the +precision of all weights, components or accelerator computations. Of the +historical eval commands that reference a recipe, 457 `fp16` cases referred to +configs with `quant: null`; 16 `w8a16` cases referred to QDQ configs. These are +historical labels, before the filename correction described above. + +For ResNet-50, compare the +[historical OpenVINO GPU result](../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) +with the +[September OpenVINO GPU result](../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-09-10/models/microsoft__resnet-50__image-classification__fp16/eval_result.json): + +| Property | Historical result | September result | +|---|---|---| +| Case label | `fp16` | `fp16` | +| Artifact precision reported by perf | `fp32` in logs | `fp32` in `model_info` | +| Explicit perf/eval precision flag | Not supplied | Not supplied (`auto` policy) | +| ONNX size | 102,188,898 bytes | 102,188,898 bytes | +| Accuracy data | `timm/mini-imagenet`, test, 100 samples | Same | +| Accuracy metric | 0.78 | 0.78 | +| Perf / accuracy execution status | PASS / PASS | PASS / PASS | + +Both use the historically named `image-classification_fp16_config.json` recipe +for evaluation, now named +[image-classification_fp32_config.json](../../examples/recipes/microsoft_resnet-50/image-classification_fp32_config.json). +Its `quant: null` content has not changed since its June 23 introduction. +The current release recipe build also supplies no `--precision` +override. Across all 16 ResNet-50 references, 9 explicitly report `fp32` and 7 +do not independently report artifact precision. All 12 September ResNet-50 +results available in the initial audit report `fp32`. Equal sizes and metrics +are supporting evidence, +not proof of byte-identical artifacts or a fresh release run: inspect the +result timestamp because `--continue` can reuse earlier results. + +Conversely, the historical +[QNN NPU splinter result](../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-06-30/models/tau__splinter-base__question-answering/eval_result.json) +has the `default` case label but reports `w8a16`. Do not turn every `default` +case into an explicit `fp32` build. + +Use these rules when maintaining release cases: + +1. Identify a case by model, task, target and selected config precision. When + correcting a filename, update the manifest to keep selecting the same recipe + contents and preserve links to the old evidence. Do not rewrite historical JSON + or copy old results into the newly named result directories. +2. Reuse the referenced recipe/configuration, including component configs, + export shapes, quantization settings, dataset, sample count and perf/eval + overrides. Do not inject a conversion solely to satisfy the case label. +3. Treat a missing recipe or `--no-recipes` as a different build source. The + current runner still permits config fallback; historical equivalence is not + established by that fallback. Verify its configuration and run it before + replacing the case's accepted evidence. +4. Separate the case label, requested build/perf policy and observed artifact + precision. Existing top-level `eval_result.precision` is not an independent + measurement: recipe builds return their label, whereas config builds can + return their resolved precision. For stronger replay guarantees, retain the + build command and configuration snapshot/hash alongside the evidence; + historical eval JSON alone does not contain a complete build snapshot. +5. Compare perf and accuracy on the same case and artifact, with the recorded + dataset settings and the existing metric-grading policy. These 800 references + all have perf PASS, but only 609 also have accuracy PASS with positive dataset + sample counts; 191 have failed, missing or skipped accuracy. A perf-only + reference is not a known-good accuracy baseline. + +Without `--release`, the existing recipe expansion applies: + For each filtered model the runner builds the model, runs `winml perf`, and — when perf passes — runs `winml eval`. **Recipes drive the build on every device (they carry the accuracy eval/dataset config), but quantized precision variants are @@ -110,27 +273,58 @@ uv run python scripts/e2e_eval/run_eval.py --update-baseline --eval-type accurac | `--registry` | `testsets/models_all.json` | Model registry file | | `--hf-model` | — | Single model (overrides registry) | | `--output-dir` | `eval_results/{date}` | Output directory | -| `--recipes-dir` | `examples/recipes` | Authored recipe configs, selected from `//` with legacy `` fallback. NPU builds every precision variant (`winml config` `w8a8`+`w8a16` fallback when a model has none); CPU/GPU (and unquantized-track EPs like VitisAI) build only non-quantized variants, else a `winml config` fallback | -| `--no-recipes` | off | Ignore recipes; build every model via `winml config` (on NPU still expands the `w8a8`+`w8a16` fallback) | +| `--recipes-dir` | `examples/recipes` | Release: use only the matching target precision recipe. P0-P3: target-directory recipe expansion as described above | +| `--no-recipes` | off | Release: one config/build per selected precision. P0-P3: legacy config fallback (including NPU expansion) | | `--copy-recipes-from EP DEVICE` | — | Copy missing configs from each model's source target into `--ep/--device` before building; existing target configs are never overwritten | | `--eval-type` | `perf` | `perf`, `accuracy`, or `both` (perf-gated accuracy) | | `--task` | — | Filter by HF task | -| `--priority` | `P0 P1 P2` | Filter: one or more of `P0`, `P1`, `P2`, `P3` (e.g. `--priority P0 P1`). Pass `P3` explicitly to include P3 models. | +| `--priority` | `P0 P1 P2 P3` | Filter: one or more of `P0`, `P1`, `P2`, `P3` (e.g. `--priority P0 P1`) | +| `--release` | off | Use the fixed release manifest and target precisions; rejects explicit model selectors, `--build-only` and `--update-baseline` | | `--model-type` | — | Filter by model_type (e.g. `bert`) | | `--group` | — | Filter by group (e.g. `Foundry Toolkit`) | | `--device` | `auto` | Target device | | `--ep` | — | Execution provider (e.g. `qnn`, `dml`, `openvino`); applied at perf/eval time | | `--pin-single-dml-gpu` | off | CI RDP workaround: resolve the sole DXCore GPU after build and pass its current LUID to winml perf. Requires --ep dml --device gpu --eval-type perf; errors on ambiguous/missing hardware. Does not change pytest suites, build selection, or product defaults. | | `--timeout` | 600 | Per-subprocess execution timeout (seconds). Observable Hugging Face download time is excluded; the full timeout restarts when the download completes. | -| `--hf-download-stall-timeout` | 600 | Fail as `HF_FETCH_FAIL` when a Hugging Face partial download has no size/mtime progress for this many seconds. | +| `--hf-download-stall-timeout` | 120 | Fail as `HF_FETCH_FAIL` when a Hugging Face partial download has no size/mtime progress for this many seconds. Also sets Hugging Face Hub download and metadata request timeouts. | | `--clean-cache [TARGET ...]` | off | Clean caches after each job. `TARGET`: `winml`, `huggingface`, `others` (others = VitisAI cache + temp/cwd leaked scratch files). Use `--clean-cache` without TARGET to clear all (legacy behavior). | | `--update-baseline` | off | Offline mode: refresh `cache/baseline_cache.json` via the PyTorch baseline, then exit (no build/perf/eval) | | `--list` | off | List models and exit | +| `--list-json PATH` | — | Write filtered models and exit; release includes `precision` (null for default) and `release_target`. Continue/retry checks use the selected precision's result directory | | `--verbose` | off | Print stderr for failed models | | `--continue` | off | Skip jobs with existing results (but backfill accuracy onto perf-only results when `--eval-type` wants it) | | `--retry-failed [TYPE ...]` | — | Re-run failed jobs (implies `--continue`); unknown types are rejected as argument errors. Retry criteria are not mutually exclusive: `HF_FETCH_FAIL` also checks failed perf and accuracy logs for `WinError 10060`, `we couldn't connect to 'https://huggingface.co'`, or `thrown while requesting HEAD https://huggingface.co`, even when the primary perf classification is another type or accuracy is `FAIL`. | | `--build-only` | off | Build with `--no-compile`, writing each stage's ONNX (no EP needed). Loops the EP matrix when `--ep`/`--device` omitted | +Subprocess diagnostics are retained under `temp/e2e-eval-logs/` in the CLI project +root, with a unique timestamped directory for each command. The directory is +printed before launch. `stdout.log` and `stderr.log` receive raw output directly; +`events.jsonl` records the command, runner/child PIDs, stage, timestamps, timeout +decisions, and cleanup lifecycle. `monitor.log` captures download-monitor errors, +and `download_progress.json` contains its latest observation. Logs remain after +success, failure, or interruption and are separate from the generated reports. +Remove old diagnostic directories when no longer needed; raw commands/output +may contain private paths or data and should be reviewed before sharing. + +Every 30 seconds a flushed console heartbeat reports elapsed time, remaining +execution budget, download-monitor state, stdout/stderr byte counts, and time +since output growth was last observed. The same fields are written to +`events.jsonl`. Output inactivity is diagnostic only, not a kill condition: +inference can be silent while consuming its execution budget. Python children +use unbuffered output and enable fault-handler output for fatal errors. + +Capture does not need reader threads or wait for pipe EOF, so full pipes and +inherited output handles cannot stall the runner. Raw output is not streamed +live to the console; the named files can be inspected while the command runs. +Download ownership and cache progress are sampled in a separate helper process. +If it stops reporting for five seconds or exits, it is stopped and execution +timeout enforcement continues without trusting stale download observations. +Timeouts print the reason and subprocess PID before process-tree cleanup. + +The Hugging Face HTTP settings are per-request limits, not a two-minute wall-clock +limit across retries. Requests that never create an observable partial download +remain subject to the subprocess execution timeout. + ### `run_llm_eval.py` — Run GenAI Context Sweep Runs an existing ONNX Runtime GenAI bundle through `winml perf --runtime diff --git a/scripts/e2e_eval/run_eval.py b/scripts/e2e_eval/run_eval.py index 729f0be39..8d3a59eb7 100644 --- a/scripts/e2e_eval/run_eval.py +++ b/scripts/e2e_eval/run_eval.py @@ -5,6 +5,20 @@ """E2E evaluation runner — unified, recipe-driven perf + accuracy. +``--release`` reads ``testsets/models_release_validation.json`` and builds one +job per selected model/task using the target EP's selected config precision. +Recipe filenames follow their quant config; a matching recipe is built unchanged. +The config label is not a measurement of the artifact's numerical precision. +Without a matching recipe the build uses ``winml config``; +this fallback does not establish equivalence to the historical configuration. +``default`` omits the precision flag without expanding NPU variants. Without +``--release``, the existing P0-P3 selection and recipe expansion below apply. + +Explicit ``--ep`` / ``--device`` pairs are checked against locally discovered +runtime devices before loading models. Unavailable targets print ``[SKIP]`` +and exit successfully without installing EPs. Listing, build-only, and baseline +update modes do not require the target hardware. + Batch-builds models and runs winml perf, then (when perf passes) winml eval, writing one unified eval_result.json per (model, task, precision). Recipes drive the build on every device (they carry the accuracy eval/dataset config), but @@ -24,6 +38,9 @@ the offline baseline cache the site grades against. Usage: + # Explicit release set at the target EP's chosen precision + python scripts/e2e_eval/run_eval.py --release --ep openvino --device gpu --eval-type both + # Perf only (default), recipe-driven across precision variants python scripts/e2e_eval/run_eval.py --priority P0 @@ -56,7 +73,6 @@ import subprocess import sys import tempfile -import threading import time from dataclasses import dataclass from datetime import date, datetime, timezone @@ -73,6 +89,7 @@ ModelEntry, filter_registry, load_registry, + load_release_registry, make_adhoc_entry, op_tracing_target_key, ) @@ -95,13 +112,17 @@ WINML_CLI = [sys.executable, "-m", "winml.modelkit.cli"] BASELINE_SCRIPT = Path(__file__).parent / "run_pytorch_baseline.py" BASELINE_CACHE_PATH = Path(__file__).parent / "cache" / "baseline_cache.json" +_RELEASE_MANIFEST = Path(__file__).parent / "testsets" / "models_release_validation.json" EVAL_DATASETS_CACHE = Path.home() / ".cache" / "winml" / "eval_datasets" TIMEOUT_SKIP_LIST_PATH = Path(__file__).parent / "cache" / "timeout_skip_list.json" _DEFAULT_SAMPLES = 1000 _DEFAULT_PRECISION_NPU = "w8a16" -_DEFAULT_HF_DOWNLOAD_STALL_TIMEOUT = 600 +_DEFAULT_HF_DOWNLOAD_STALL_TIMEOUT = 120 _HF_DOWNLOAD_STALL_TIMEOUT = float(_DEFAULT_HF_DOWNLOAD_STALL_TIMEOUT) _SUBPROCESS_POLL_INTERVAL = 0.25 +_HF_DOWNLOAD_MONITOR_TIMEOUT = 5.0 +_SUBPROCESS_HEARTBEAT_INTERVAL = 30.0 +_SUBPROCESS_LOG_ROOT = Path(__file__).resolve().parents[2] / "temp" / "e2e-eval-logs" _PRIORITY_RANK = {f"P{index}": index for index in range(4)} _RETRY_FAILED_TYPES = ( *(failure_type.value for failure_type in FailureType), @@ -162,6 +183,34 @@ def _resolve_eval_target(ep: str | None, device: str | None) -> tuple[str, str]: return target.ep, target.device +def _is_eval_target_available(ep: str | None, device: str | None) -> bool: + """Check an explicit EP/device pair against the host's runtime inventory.""" + if not ep or ep.lower() == "auto" or not device or device.lower() == "auto": + return True + + from winml.modelkit.session import ( + DeviceNotFound, + EPDeviceTarget, + WinMLEPNotDiscovered, + WinMLEPRegistrationFailed, + WinMLEPRegistry, + expand_ep_name, + ) + + target = EPDeviceTarget(ep=ep, device=device) + try: + registry = WinMLEPRegistry.instance() + if expand_ep_name(target.ep) not in registry.available_eps(): + raise WinMLEPNotDiscovered(f"No locally installed EP found for {target.ep}.") + registry.auto_device(target) + except (DeviceNotFound, WinMLEPNotDiscovered, WinMLEPRegistrationFailed) as exc: + safe_print( + f"[SKIP] {target.ep}/{target.device} is not available on this machine: {exc}" + ) + return False + return True + + def _validate_recipe_copy_target(ep: str | None, device: str | None) -> None: """Require a concrete destination so copied recipes are discoverable.""" if not ep or ep.lower() == "auto" or not device or device.lower() == "auto": @@ -490,9 +539,14 @@ def _clean_stray_cwd_artifacts(directory: Path) -> int: def safe_print(text: str) -> None: """Cross-platform safe print (handles Windows Unicode issues).""" try: - print(text) + print(text, flush=True) except UnicodeEncodeError: - print(text.encode("ascii", errors="replace").decode("ascii")) + print(text.encode("ascii", errors="replace").decode("ascii"), flush=True) + + +def _progress_prefix(index: int, total: int, now: datetime | None = None) -> str: + timestamp = (now or datetime.now()).strftime("%Y-%m-%d %H:%M:%S") + return f"[{timestamp}] [{index}/{total}]" def _utc_now() -> str: @@ -610,10 +664,13 @@ def _kill_process_tree(pid: int) -> None: # Fallback: taskkill on Windows, killpg on Unix if platform.system() == "Windows": - subprocess.run( # noqa: S603 - ["taskkill", "/F", "/T", "/PID", str(pid)], # noqa: S607 - capture_output=True, - ) + with contextlib.suppress(OSError, subprocess.TimeoutExpired): + subprocess.run( # noqa: S603 + ["taskkill", "/F", "/T", "/PID", str(pid)], # noqa: S607 + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=5, + ) else: import signal @@ -705,7 +762,7 @@ class _HfDownloadTracker: def __init__(self, env: dict[str, str], now: float) -> None: self._env = env - self._previous = _snapshot_hf_downloads(env) + self._previous: dict[Path, tuple[int, int]] = {} self._active_paths: set[Path] = set() self._pid: int | None = None self.last_progress = now @@ -723,7 +780,7 @@ def poll(self, now: float) -> bool: } if progressed: self._active_paths.update(progressed) - self.last_progress = now + self.last_progress = max(now, time.perf_counter()) self._active_paths.intersection_update( path for path in current if _normalized_path(path) in open_paths ) @@ -731,6 +788,65 @@ def poll(self, now: float) -> bool: return bool(self._active_paths) +_HF_DOWNLOAD_MONITOR_CODE = "\n".join( + ( + "import sys", + "from pathlib import Path", + "sys.path.insert(0, sys.argv[1])", + "from run_eval import _monitor_hf_downloads", + "_monitor_hf_downloads(int(sys.argv[2]), Path(sys.argv[3]))", + ) +) + + +def _monitor_hf_downloads(pid: int, progress_path: Path) -> None: + """Publish download observations outside the timeout-enforcing interpreter.""" + import psutil + + tracker = _HfDownloadTracker(dict(os.environ), time.perf_counter()) + tracker.bind(pid) + pending_path = progress_path.with_suffix(".tmp") + while psutil.pid_exists(pid): + active = tracker.poll(time.perf_counter()) + snapshot = { + "observed_at": time.perf_counter(), + "active": active, + "last_progress": tracker.last_progress, + } + with contextlib.suppress(OSError): + pending_path.write_text(json.dumps(snapshot), encoding="utf-8") + pending_path.replace(progress_path) + time.sleep(_SUBPROCESS_POLL_INTERVAL) + + +def _start_hf_download_monitor( + pid: int, env: dict[str, str], progress_path: Path +) -> subprocess.Popen: + kwargs: dict = { + "env": env, + "stdin": subprocess.DEVNULL, + "stdout": subprocess.DEVNULL, + "stderr": subprocess.DEVNULL, + } + if platform.system() == "Windows": + kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW + else: + kwargs["start_new_session"] = True + with (progress_path.parent / "monitor.log").open("wb") as monitor_log: + kwargs["stderr"] = monitor_log + return subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-c", + _HF_DOWNLOAD_MONITOR_CODE, + str(Path(__file__).parent), + str(pid), + str(progress_path), + ], + **kwargs, + ) + + def _run_subprocess(args: list[str], timeout: int) -> dict: """Run a subprocess with execution and HF-download-stall timeouts. @@ -741,125 +857,240 @@ def _run_subprocess(args: list[str], timeout: int) -> dict: ``_HF_DOWNLOAD_STALL_TIMEOUT`` seconds, the process is terminated as an HF fetch failure. - Windows fix: On Windows, child processes can inherit pipe handles, causing - pipe reads to block indefinitely even after ``taskkill`` kills the process - tree. We work around this by: - 1. Using ``CREATE_NO_WINDOW`` to prevent console inheritance issues. - 2. Reading stdout/stderr in background threads. - 3. Polling process state independently of pipe EOF. + Output goes directly to retained log files so thread startup, full pipes and + inherited pipe handles cannot block the timeout loop or output cleanup. + Download scans run in a separate process; missing or stale observations + disable download suspension after ``_HF_DOWNLOAD_MONITOR_TIMEOUT`` seconds. """ env = { **os.environ, "PYTHONIOENCODING": "utf-8", + "PYTHONUNBUFFERED": "1", + "PYTHONFAULTHANDLER": "1", "HF_HUB_DOWNLOAD_TIMEOUT": str(int(_HF_DOWNLOAD_STALL_TIMEOUT)), + "HF_HUB_ETAG_TIMEOUT": str(int(_HF_DOWNLOAD_STALL_TIMEOUT)), } start = time.perf_counter() timed_out = False hf_download_stalled = False execution_elapsed = 0.0 last_poll = start - download_tracker = _HfDownloadTracker(env, start) download_was_active = False + last_download_progress = start + last_download_observation = start + download_state_known = False + download_monitor_stalled = False + last_heartbeat = start + last_output_progress = start + last_output_sizes = (0, 0) + stage = Path(args[0]).name + if len(args) > 3 and args[1:3] == ["-m", "winml.modelkit.cli"]: + stage = args[3] + elif len(args) > 1 and str(args[1]).endswith(".py"): + stage = Path(args[1]).name + _SUBPROCESS_LOG_ROOT.mkdir(parents=True, exist_ok=True) + log_dir = Path( + tempfile.mkdtemp( + prefix=f"{datetime.now():%Y%m%d-%H%M%S}-{stage}-", dir=_SUBPROCESS_LOG_ROOT + ) + ) + pid: int | None = None - popen_kwargs: dict = { - "stdout": subprocess.PIPE, - "stderr": subprocess.PIPE, - "env": env, - } - if platform.system() == "Windows": - popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW - else: - popen_kwargs["start_new_session"] = True - proc = subprocess.Popen(args, **popen_kwargs) # noqa: S603 - download_tracker.bind(proc.pid) - - # Read pipes in background threads so communicate() timeout works even - # when grandchild processes keep pipe handles alive (Windows issue). - stdout_chunks: list[bytes] = [] - stderr_chunks: list[bytes] = [] + with ( + (log_dir / "stdout.log").open("w+b") as stdout_file, + (log_dir / "stderr.log").open("w+b") as stderr_file, + (log_dir / "events.jsonl").open("a", encoding="utf-8") as events_file, + ): + def report(event: str, message: str, **details) -> None: + record = { + "timestamp": _utc_now(), + "event": event, + "stage": stage, + "runner_pid": os.getpid(), + "pid": pid, + "elapsed": round(time.perf_counter() - start, 3), + **details, + } + with contextlib.suppress(OSError): + events_file.write(json.dumps(record) + "\n") + events_file.flush() + with contextlib.suppress(OSError): + safe_print( + f" [{datetime.now():%Y-%m-%d %H:%M:%S}] [{event}] " + f"stage={stage} pid={pid} {message}" + ) - def _reader(pipe, dest: list[bytes]) -> None: + report( + "starting", + f"timeout={timeout:g}s logs={log_dir}", + command=[str(arg) for arg in args], + cwd=str(Path.cwd()), + timeout_seconds=timeout, + hf_stall_seconds=_HF_DOWNLOAD_STALL_TIMEOUT, + ) + progress_path = log_dir / "download_progress.json" + popen_kwargs: dict = { + "stdout": stdout_file, + "stderr": stderr_file, + "env": env, + } + if platform.system() == "Windows": + popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW + else: + popen_kwargs["start_new_session"] = True try: + proc = subprocess.Popen(args, **popen_kwargs) # noqa: S603 + except OSError as exc: + report("spawn_failed", str(exc), error=str(exc)) + raise + pid = proc.pid + report("spawned", f"timeout={timeout:g}s") + monitor: subprocess.Popen | None = None + try: + report("monitor_starting", "starting isolated download scan") + try: + monitor = _start_hf_download_monitor(proc.pid, env, progress_path) + report("monitor_started", f"monitor_pid={monitor.pid}", monitor_pid=monitor.pid) + except OSError as exc: + download_monitor_stalled = True + download_state_known = True + report("monitor_disabled", "using execution timeout", error=str(exc)) + logger.warning("Download monitor unavailable; enforcing execution timeout.") + while True: - chunk = pipe.read(8192) - if not chunk: + remaining = max(0.01, timeout - execution_elapsed) + try: + proc.wait(timeout=min(_SUBPROCESS_POLL_INTERVAL, remaining)) + exit_code = proc.returncode break - dest.append(chunk) - except (OSError, ValueError): - pass # Pipe closed or broken; stop reading - - stdout_thread = threading.Thread(target=_reader, args=(proc.stdout, stdout_chunks), daemon=True) - stderr_thread = threading.Thread(target=_reader, args=(proc.stderr, stderr_chunks), daemon=True) - stdout_thread.start() - stderr_thread.start() - - try: - while True: - remaining = max(0.01, timeout - execution_elapsed) - try: - proc.wait(timeout=min(_SUBPROCESS_POLL_INTERVAL, remaining)) - now = time.perf_counter() - download_active = download_tracker.poll(now) - if download_was_active and not download_active: - execution_elapsed = 0.0 - elif not download_active: - execution_elapsed += now - last_poll - exit_code = proc.returncode - break - except subprocess.TimeoutExpired: - now = time.perf_counter() - download_active = download_tracker.poll(now) - if download_active: - download_was_active = True - if now - download_tracker.last_progress >= _HF_DOWNLOAD_STALL_TIMEOUT: - hf_download_stalled = True - else: - if download_was_active: - execution_elapsed = 0.0 - download_was_active = False + except subprocess.TimeoutExpired: + now = time.perf_counter() + download_active = False + if not download_monitor_stalled: + try: + snapshot = json.loads(progress_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + snapshot = None + if snapshot is not None: + last_download_observation = snapshot["observed_at"] + last_download_progress = snapshot["last_progress"] + download_active = snapshot["active"] + download_state_known = True + if ( + now - last_download_observation >= _HF_DOWNLOAD_MONITOR_TIMEOUT + or (monitor is not None and monitor.poll() is not None) + ): + download_monitor_stalled = True + download_active = False + download_state_known = True + if monitor is not None: + with contextlib.suppress(OSError): + monitor.kill() + logger.warning( + "Download monitor unavailable or stale; enforcing execution timeout." + ) + report( + "monitor_disabled", + "stale or exited; using execution timeout", + monitor_pid=monitor.pid if monitor is not None else None, + observation_age_seconds=round(now - last_download_observation, 3), + ) + + if download_active: + if not download_was_active: + report("download_active", "execution timeout suspended") + download_was_active = True + if now - last_download_progress >= _HF_DOWNLOAD_STALL_TIMEOUT: + hf_download_stalled = True else: - execution_elapsed += now - last_poll - if execution_elapsed >= timeout: - timed_out = True - last_poll = now + if download_was_active: + execution_elapsed = 0.0 + download_was_active = False + report("execution_resumed", f"execution budget reset to {timeout:g}s") + else: + execution_elapsed += now - last_poll + if execution_elapsed >= timeout and download_state_known: + timed_out = True + last_poll = now + + if now - last_heartbeat >= _SUBPROCESS_HEARTBEAT_INTERVAL: + output_sizes = ( + os.fstat(stdout_file.fileno()).st_size, + os.fstat(stderr_file.fileno()).st_size, + ) + if output_sizes != last_output_sizes: + last_output_progress = now + last_output_sizes = output_sizes + monitor_state = ( + "disabled" if download_monitor_stalled + else "starting" if not download_state_known + else "downloading" if download_active + else "idle" + ) + execution_remaining = max(0.0, timeout - execution_elapsed) + output_idle = now - last_output_progress + report( + "heartbeat", + f"elapsed={now - start:.1f}s exec_left={execution_remaining:.1f}s " + f"monitor={monitor_state} stdout={output_sizes[0]}B " + f"stderr={output_sizes[1]}B output_idle={output_idle:.1f}s", + execution_remaining=round(execution_remaining, 3), + monitor_state=monitor_state, + monitor_age_seconds=round(now - last_download_observation, 3), + hf_download_idle_seconds=round(now - last_download_progress, 3), + stdout_bytes=output_sizes[0], + stderr_bytes=output_sizes[1], + output_idle_seconds=round(output_idle, 3), + ) + last_heartbeat = now - if not timed_out and not hf_download_stalled: - continue + if not timed_out and not hf_download_stalled: + continue + reason = ( + "Hugging Face download stalled" + if hf_download_stalled + else f"execution timeout ({timeout:g}s)" + ) + report( + "timeout", reason, + timeout=timed_out, hf_download_stalled=hf_download_stalled, + ) + exit_code = -1 + break + except KeyboardInterrupt: + report("interrupted", "stopping subprocess tree; logs retained") + safe_print("\n [Ctrl+C] Killing subprocess...") + raise + finally: + report("cleanup_starting", "reaping subprocess and download monitor") + if proc.poll() is None: _kill_process_tree(proc.pid) with contextlib.suppress(OSError): proc.kill() - exit_code = -1 - break - - # Give reader threads a moment to finish draining - stdout_thread.join(timeout=10) - stderr_thread.join(timeout=10) - except KeyboardInterrupt: - safe_print("\n [Ctrl+C] Killing subprocess...") - _kill_process_tree(proc.pid) - with contextlib.suppress(OSError): - proc.kill() - stdout_thread.join(timeout=5) - stderr_thread.join(timeout=5) - raise - finally: - # Force-close pipes to unblock any stuck reader threads - for pipe in (proc.stdout, proc.stderr): - if pipe: - try: - pipe.close() - except OSError: - pass # Pipe already closed - # Final attempt: if reader threads are still alive after pipe close, - # don't block forever — just proceed with whatever was collected. - if stdout_thread.is_alive(): - stdout_thread.join(timeout=2) - if stderr_thread.is_alive(): - stderr_thread.join(timeout=2) - - stdout = b"".join(stdout_chunks).decode("utf-8", errors="replace") - stderr = b"".join(stderr_chunks).decode("utf-8", errors="replace") + with contextlib.suppress(OSError, subprocess.TimeoutExpired): + proc.wait(timeout=5) + if monitor is not None: + with contextlib.suppress(OSError): + monitor.kill() + with contextlib.suppress(OSError, subprocess.TimeoutExpired): + monitor.wait(timeout=1) + report("cleanup_complete", f"process_returncode={proc.poll()}") + + stdout_size = os.fstat(stdout_file.fileno()).st_size + stderr_size = os.fstat(stderr_file.fileno()).st_size + report("collecting_output", f"stdout={stdout_size}B stderr={stderr_size}B") + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read(stdout_size).decode("utf-8", errors="replace") + stderr = stderr_file.read(stderr_size).decode("utf-8", errors="replace") + report( + "exited", + f"exit_code={exit_code} elapsed={time.perf_counter() - start:.1f}s logs={log_dir}", + exit_code=exit_code, + timeout=timed_out, + hf_download_stalled=hf_download_stalled, + ) if hf_download_stalled: stderr += ( "\nError while downloading from https://huggingface.co: " @@ -876,6 +1107,7 @@ def _reader(pipe, dest: list[bytes]) -> None: "timeout": timed_out, "hf_download_stalled": hf_download_stalled, "command": " ".join(str(a) for a in args), + "log_dir": str(log_dir), } # Retry once after clearing caches if the failure was due to disk full. @@ -901,6 +1133,7 @@ def _run_build( model_dir: Path, ep: str | None = None, build_only: bool = False, + honor_precision: bool = False, ) -> dict: """Run winml config + winml build for one model. Returns build result dict. @@ -922,6 +1155,9 @@ def _run_build( dict reports that effective precision back under ``precision`` (read from the generated config when the caller passed none), so the recorded eval_result never claims "no precision" for a quantized build. + + ``honor_precision`` keeps an explicit release precision even on EPs whose + legacy evaluation track normally passes ``--no-quant``. """ composite_onnx = getattr(entry, "composite_onnx", None) if isinstance(composite_onnx, dict) and composite_onnx: @@ -978,7 +1214,8 @@ def _run_build( # written with quant=None up-front; otherwise on NPU the config command # would still apply its default precision (w8a16) and we'd be relying on # --no-quant at build time alone to override it. - if _should_skip_winml_quant(ep, device): + skip_quant = _should_skip_winml_quant(ep, device) and not (honor_precision and precision) + if skip_quant: config_args += ["--no-quant"] config_proc = _run_subprocess(config_args, timeout) @@ -1037,7 +1274,7 @@ def _run_build( # Mirror the --no-quant passed to winml config above so the build # stage also skips QDQ regardless of what the config carries (defence # in depth; see _should_skip_winml_quant for the rationale). - if _should_skip_winml_quant(ep, device): + if skip_quant: build_args += ["--no-quant"] build_proc = _run_subprocess(build_args, timeout) @@ -1216,23 +1453,33 @@ def _run_recipe_build( def _extract_onnx_path(build_proc: dict, hf_id: str, task: str | None) -> str | None: """Extract ONNX path from build subprocess output.""" - # Patterns used by winml build to report the artifact path + # Rich may wrap a long artifact path across physical output lines. Rejoin + # those fragments before falling back to cache discovery. markers = ("Final artifact:", "Existing artifact found:", "Artifact:") - onnx_path = None - for line in (build_proc["stderr"] + build_proc["stdout"]).splitlines(): + output = re.sub( + r"\x1b\[[0-?]*[ -/]*[@-~]", + "", + build_proc["stderr"] + build_proc["stdout"], + ) + lines = output.splitlines() + for index, line in enumerate(lines): for marker in markers: - if marker in line: - candidate = line.split(marker)[-1].strip() - if candidate and Path(candidate).exists(): - onnx_path = candidate + if marker not in line: + continue + fragments = [line.split(marker, 1)[1].strip()] + for continuation in lines[index + 1 : index + 11]: + candidate = "".join(fragments) + if candidate and Path(candidate).is_file(): + return candidate + if candidate.lower().endswith(".onnx"): break - if onnx_path: - break + fragments.append(continuation.strip()) - if not onnx_path or not Path(onnx_path).exists(): - onnx_path = _find_cached_model(hf_id, build_proc, task) + candidate = "".join(fragments) + if candidate and Path(candidate).is_file(): + return candidate - return onnx_path + return _find_cached_model(hf_id, build_proc, task) def _extract_task_from_config(config_path: Path) -> str | None: @@ -1249,8 +1496,9 @@ def _find_cached_model(hf_id: str, build_proc: dict, task: str | None = None) -> """Try to find the built ONNX model in the WinML cache. Requires task to safely identify the correct artifact when a model has - multiple cached tasks (e.g. feat_* and txtcls_*). Returns None if task is - not provided to avoid picking the wrong model. + multiple cached tasks (e.g. feat_* and txtcls_*). A task can also have + multiple precision/config variants, so only an unambiguous single match is + safe. Returns None when task is absent or multiple candidates exist. """ if not task: return None @@ -1264,12 +1512,8 @@ def _find_cached_model(hf_id: str, build_proc: dict, task: str | None = None) -> prefix = get_task_abbrev(task) + "_" - model_files = sorted( - (p for p in cache_dir.glob("*_model.onnx") if p.name.startswith(prefix)), - key=lambda p: p.stat().st_mtime, - reverse=True, - ) - return str(model_files[0]) if model_files else None + model_files = [p for p in cache_dir.glob("*_model.onnx") if p.name.startswith(prefix)] + return str(model_files[0]) if len(model_files) == 1 else None # --------------------------------------------------------------------------- @@ -1799,7 +2043,9 @@ def _record( shared_dir = model_dir / "_shared" canonical_hash: str | None = None - safe_print(f"\n[{i}/{len(entries)}] {label} ({entry.priority}, {entry.group})") + safe_print( + f"\n{_progress_prefix(i, len(entries))} {label} ({entry.priority}, {entry.group})" + ) for combo_label, ep, device in combos: build_dir = model_dir / combo_label if combo_label else model_dir @@ -2610,10 +2856,12 @@ def _run_update_baseline(entries: list[ModelEntry], args: argparse.Namespace) -> ds_config = get_dataset_config(entry.hf_id, entry.task) or {} cached = _lookup_baseline_cache(entry.hf_id, entry.task, ds_config) if cached is not None and not args.retry_failed: - safe_print(f"[{i}/{len(entries)}] {label} (cached {cached['metric']})") + safe_print( + f"{_progress_prefix(i, len(entries))} {label} (cached {cached['metric']})" + ) continue - safe_print(f"[{i}/{len(entries)}] {label} running baseline ...") + safe_print(f"{_progress_prefix(i, len(entries))} {label} running baseline ...") _build_dataset(ds_config, args.timeout) baseline = _run_pytorch_baseline(entry, args.device, args.timeout) if baseline["status"] == "PASS": @@ -2866,11 +3114,17 @@ class EvalJob: one job per NPU quantization scheme (see :data:`_NPU_FALLBACK_PRECISIONS`); it stays ``None`` for the single default-precision fallback. Each job produces one ``eval_result.json``. + + ``precision_locked`` fixes the selected config precision for release jobs. + Recipes keep their authored configuration; fallback builds pass a non-default + label as a precision flag without applying legacy expansion/skip-quant policy. + The label is not a measurement of the produced ONNX tensor dtypes. """ entry: ModelEntry variant: RecipeVariant | None fallback_precision: str | None = None + precision_locked: bool = False @property def precision(self) -> str | None: @@ -2907,10 +3161,19 @@ def _is_quantized_precision(precision: str) -> bool: def _build_jobs( - entries: list[ModelEntry], recipes_dir: Path | None, device: str, ep: str | None = None + entries: list[ModelEntry], + recipes_dir: Path | None, + device: str, + ep: str | None = None, + *, + release: bool = False, ) -> list[EvalJob]: """Expand entries into jobs. Recipes apply on every device; quant is NPU-only. + With ``release=True``, create exactly one selected-config job per entry, + including a single fallback for ``default``. This mode can use a quantized + GPU recipe when the manifest selected it; legacy expansion is bypassed. + Automatic EP/device axes are resolved through the runtime target policy before recipe lookup so target-specific directories always use concrete ``/`` names. @@ -2955,6 +3218,12 @@ def _build_jobs( if recipes_dir is not None else [] ) + if release: + matching = next( + (variant for variant in variants if variant.precision == entry.precision), None + ) + jobs.append(EvalJob(entry, matching, precision_locked=True)) + continue if not npu or skip_quant: # Off-NPU and skip-quant EPs still use recipes for their eval # config, but drop quantized variants -- quantization here is only @@ -2988,6 +3257,9 @@ def _build_for_job( otherwise the ``winml config`` fallback is used. ``recipe_meta`` is the eval/dataset config for ``winml eval -c`` (None for the fallback) and ``trust`` is whether the recipe's dataset needs ``--trust-remote-code``. + Release checks compare declared case/recipe labels, not the loaded recipe's + quantization config or the ONNX artifact's dtypes. Recipes are not converted + to match a label: preserving their configuration preserves the historical case. """ if job.variant is not None: build_result = _run_recipe_build( @@ -3002,13 +3274,26 @@ def _build_for_job( build_result = _run_build( job.entry, args.device, - _resolve_precision(args.device, explicit_precision, ep=args.ep), + explicit_precision + if job.precision_locked + else _resolve_precision(args.device, explicit_precision, ep=args.ep), args.timeout, model_dir, ep=args.ep, + **({"honor_precision": True} if job.precision_locked else {}), ) recipe_meta = None trust = False + if ( + job.precision_locked + and job.precision is not None + and build_result["success"] + and build_result.get("precision") != job.precision + ): + raise ValueError( + f"Release precision mismatch for {job.entry.hf_id}/{job.entry.task}: " + f"requested {job.precision}, built {build_result.get('precision')}" + ) return build_result, recipe_meta, trust @@ -3101,6 +3386,11 @@ def parse_args() -> argparse.Namespace: "Default: P0 P1 P2 P3." ), ) + parser.add_argument( + "--release", + action="store_true", + help="Evaluate the fixed release manifest at each model's selected target precision.", + ) parser.add_argument("--model-type", help="Filter by model_type") parser.add_argument("--group", help="Filter by group") parser.add_argument("--device", default="auto", help="Target device (default: auto)") @@ -3229,7 +3519,7 @@ def parse_args() -> argparse.Namespace: help=( "Hugging Face download inactivity timeout in seconds; active download " "time is excluded and --timeout restarts after download completion " - "(default: 600)" + f"(default: {_DEFAULT_HF_DOWNLOAD_STALL_TIMEOUT})" ), ) parser.add_argument( @@ -3298,6 +3588,27 @@ def parse_args() -> argparse.Namespace: ), ) args = parser.parse_args() + if args.release: + conflict_names = ( + "priority", + "hf_model", + "registry", + "task", + "group", + "model_type", + "build_only", + "update_baseline", + ) + explicit_args = parser.parse_args( + namespace=argparse.Namespace(**dict.fromkeys(conflict_names)) + ) + conflicts = [ + f"--{name.replace('_', '-')}" + for name in conflict_names + if getattr(explicit_args, name) is not None + ] + if conflicts: + parser.error(f"--release cannot be combined with {', '.join(conflicts)}") if args.pin_single_dml_gpu and ( args.device != "gpu" or args.eval_type != "perf" @@ -3328,8 +3639,25 @@ def main() -> None: clean_cache_targets = _resolve_clean_cache_targets(args.clean_cache) args.clean_cache_targets = clean_cache_targets + if ( + not (args.list or args.list_json or args.update_baseline or args.build_only) + and not _is_eval_target_available(args.ep, args.device) + ): + return + # 1. Load registry - if args.hf_model: + release_mode = args.release + if release_mode: + args.ep, args.device = _resolve_eval_target(args.ep, args.device) + entries, args.release_target = load_release_registry( + _RELEASE_MANIFEST, + load_registry(args.registry), + ep=args.ep, + device=args.device, + output_dir=args.output_dir, + ) + safe_print(f"Release: {_RELEASE_MANIFEST} | Target: {args.release_target}") + elif args.hf_model: # Try to find the model in the registry (preserves dataset_config, etc.) matched_entry: ModelEntry | None = None try: @@ -3384,8 +3712,9 @@ def main() -> None: for e in entries: ds = get_dataset_config(e.hf_id, e.task) skip_acc = "" if args.eval_type == "perf" else " [task_default]" if ds is None else "" + precision_label = f" [precision={e.precision or 'default'}]" if release_mode else "" safe_print( - f" [{e.priority}] {e.hf_id} / {e.task} ({e.model_type}, {e.group}){skip_acc}" + f" [{e.priority}] {e.hf_id} / {e.task} ({e.model_type}, {e.group}){precision_label}{skip_acc}" ) sys.exit(0) @@ -3402,7 +3731,12 @@ def main() -> None: filtered: list[ModelEntry] = [] skipped_count = 0 for e in entries: - result_path = model_result_dir(output_dir, e.hf_id, e.task) / "eval_result.json" + result_path = ( + model_result_dir( + output_dir, e.hf_id, e.task, e.precision if release_mode else None + ) + / "eval_result.json" + ) if args.continue_run and result_path.exists(): try: existing = load_result_json(result_path) @@ -3428,6 +3762,11 @@ def main() -> None: "model_type": e.model_type, "group": e.group, "priority": e.priority, + **( + {"precision": e.precision, "release_target": args.release_target} + if release_mode + else {} + ), } for e in entries ] @@ -3500,7 +3839,7 @@ def main() -> None: f"Recipe copy: {copied_count} configs from {source_ep}/{source_device} " f"to {args.ep}/{args.device}" ) - jobs = _build_jobs(entries, recipes_dir, args.device, ep=args.ep) + jobs = _build_jobs(entries, recipes_dir, args.device, ep=args.ep, release=release_mode) total_jobs = len(jobs) safe_print(f"E2E Evaluation: {len(entries)} models -> {total_jobs} jobs -> {output_dir}") @@ -3509,7 +3848,11 @@ def main() -> None: f"Device: {args.device} | EP: {ep_label} | Timeout: {args.timeout}s | Eval: {args.eval_type}" ) safe_print(f"Disk free: {_get_disk_free_gb():.1f} GB") - if recipes_dir is not None and args.device == "npu": + if release_mode: + safe_print( + "Release jobs: one precision per model from the manifest; matching recipe or winml config fallback" + ) + elif recipes_dir is not None and args.device == "npu": safe_print( f"Recipes: {recipes_dir} " f"(NPU; winml config {'+'.join(_NPU_FALLBACK_PRECISIONS)} fallback when a model has none)" @@ -3579,7 +3922,9 @@ def main() -> None: ) if timeout_rule is not None: reason = timeout_rule.get("reason") or "timeout" - safe_print(f"\n[{i}/{total_jobs}] {label} (SKIP - TIMEOUT: {reason})") + safe_print( + f"\n{_progress_prefix(i, total_jobs)} {label} (SKIP - TIMEOUT: {reason})" + ) model_dir.mkdir(parents=True, exist_ok=True) timeout_result = build_eval_result( entry=entry, @@ -3617,7 +3962,8 @@ def main() -> None: perf_tag = "PASS" if perf.get("passed") else f"FAIL/{perf_cls}" acc_tag = f" acc={accuracy_status(acc)}" if acc is not None else "" safe_print( - f"\n[{i}/{total_jobs}] {label} (SKIP - {perf_tag}{acc_tag}, cached)" + f"\n{_progress_prefix(i, total_jobs)} {label} " + f"(SKIP - {perf_tag}{acc_tag}, cached)" ) continue @@ -3625,19 +3971,28 @@ def main() -> None: # Perf already recorded (and passed); only (re)build + run # accuracy, then merge it into the existing result. backfill_existing = existing - safe_print(f"\n[{i}/{total_jobs}] {label} (BACKFILL accuracy - perf cached)") + safe_print( + f"\n{_progress_prefix(i, total_jobs)} {label} " + "(BACKFILL accuracy - perf cached)" + ) else: retry_label = classify_result(existing) or ( accuracy_status(existing.get("accuracy")) if existing.get("accuracy") else "?" ) - safe_print(f"\n[{i}/{total_jobs}] {label} (RETRY - was {retry_label})") + safe_print( + f"\n{_progress_prefix(i, total_jobs)} {label} " + f"(RETRY - was {retry_label})" + ) except (json.JSONDecodeError, KeyError): pass # Corrupted result file — re-run if backfill_existing is None: - safe_print(f"\n[{i}/{total_jobs}] {label} ({entry.priority}, {entry.group})") + safe_print( + f"\n{_progress_prefix(i, total_jobs)} {label} " + f"({entry.priority}, {entry.group})" + ) try: perf_proc: dict | None = None diff --git a/scripts/e2e_eval/testsets/models_release_validation.json b/scripts/e2e_eval/testsets/models_release_validation.json new file mode 100644 index 000000000..d8a43fb0f --- /dev/null +++ b/scripts/e2e_eval/testsets/models_release_validation.json @@ -0,0 +1,2755 @@ +{ + "schema_version": 1, + "models": [ + { + "hf_id": "depth-anything/Depth-Anything-V2-Small-hf", + "task": "depth-estimation", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "Intel/dpt-hybrid-midas", + "task": "depth-estimation", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "BAAI/bge-small-en-v1.5", + "task": "feature-extraction", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "task": "feature-extraction", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "openai/clip-vit-base-patch32", + "task": "feature-extraction", + "priority": "P0", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "task": "feature-extraction", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "sentence-transformers/multi-qa-mpnet-base-dot-v1", + "task": "feature-extraction", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "google-bert/bert-base-uncased", + "task": "fill-mask", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "FacebookAI/xlm-roberta-base", + "task": "fill-mask", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "w8a16" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "FacebookAI/roberta-base", + "task": "fill-mask", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "distilbert/distilbert-base-uncased", + "task": "fill-mask", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "timm/mobilenetv3_small_100.lamb_in1k", + "task": "image-classification", + "priority": "P1", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "google/vit-base-patch16-224", + "task": "image-classification", + "priority": "P0", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "microsoft/resnet-50", + "task": "image-classification", + "priority": "P0", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "microsoft/swinv2-tiny-patch4-window16-256", + "task": "image-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "facebook/convnext-tiny-224", + "task": "image-classification", + "priority": "P0", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "microsoft/beit-base-patch16-224-pt22k-ft22k", + "task": "image-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "facebook/dinov2-base", + "task": "image-feature-extraction", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "google/vit-base-patch16-224-in21k", + "task": "image-feature-extraction", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "facebook/dino-vitb16", + "task": "image-feature-extraction", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "nvidia/segformer-b2-finetuned-ade-512-512", + "task": "image-segmentation", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "mattmdjaga/segformer_b2_clothes", + "task": "image-segmentation", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "Salesforce/blip-image-captioning-base", + "task": "image-to-text", + "priority": "P0", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "microsoft/trocr-base-printed", + "task": "image-to-text", + "priority": "P1", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "facebook/nougat-base", + "task": "image-to-text", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a8" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "google-bert/bert-base-multilingual-cased", + "task": "masked-lm", + "priority": "P0", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "microsoft/table-transformer-structure-recognition", + "task": "object-detection", + "priority": "P1", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "hustvl/yolos-small", + "task": "object-detection", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "facebook/detr-resnet-50", + "task": "object-detection", + "priority": "P0", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "ahotrod/electra_large_discriminator_squad2_512", + "task": "question-answering", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "tau/splinter-base", + "task": "question-answering", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "default" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "timpal0l/mdeberta-v3-base-squad2", + "task": "question-answering", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "sentence-transformers/all-MiniLM-L6-v2", + "task": "sentence-similarity", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "BAAI/bge-m3", + "task": "sentence-similarity", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "w8a16" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "w8a16" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "w8a16" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "w8a16" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "w8a16" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "w8a16" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "w8a16" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "w8a16" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "w8a16" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "w8a16" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "w8a16" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "w8a16" + } + } + }, + { + "hf_id": "sentence-transformers/all-mpnet-base-v2", + "task": "sentence-similarity", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "google/flan-t5-base", + "task": "summarization", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "google/pegasus-xsum", + "task": "summarization", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "cross-encoder/ms-marco-MiniLM-L6-v2", + "task": "text-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp16" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp16" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp16" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp16" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp16" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "BAAI/bge-reranker-v2-m3", + "task": "text-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "distilbert/distilbert-base-uncased-finetuned-sst-2-english", + "task": "text-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "cardiffnlp/twitter-roberta-base-sentiment-latest", + "task": "text-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "dbmdz/bert-large-cased-finetuned-conll03-english", + "task": "token-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "w11wo/indonesian-roberta-base-posp-tagger", + "task": "token-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "kredor/punctuate-all", + "task": "token-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "google-t5/t5-small", + "task": "translation", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "lxyuan/distilbert-base-multilingual-cased-sentiments-student", + "task": "zero-shot-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "cross-encoder/nli-deberta-v3-small", + "task": "zero-shot-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "openai/clip-vit-large-patch14", + "task": "zero-shot-image-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "fp32" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "fp32" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "fp32" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "fp32" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "w8a16" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "fp32" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "fp32" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "fp32" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "fp32" + } + } + }, + { + "hf_id": "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + "task": "zero-shot-image-classification", + "priority": "P0", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + }, + { + "hf_id": "google/siglip-base-patch16-224", + "task": "zero-shot-image-classification", + "priority": "P2", + "targets": { + "amd/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MIGraphExecutionProvider_GPU": { + "precision": "default" + }, + "amd/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "amd/VitisAIExecutionProvider_NPU": { + "precision": "default" + }, + "intel/DmlExecutionProvider_GPU": { + "precision": "default" + }, + "intel/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_CPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_GPU": { + "precision": "default" + }, + "intel/OpenVINOExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "nvidia/TRTRTXARMExecutionProvider_GPU": { + "precision": "default" + }, + "nvidia/TRTRTXExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/MLASExecutionProvider_CPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_GPU": { + "precision": "default" + }, + "qnn/QNNExecutionProvider_NPU": { + "precision": "w8a16" + }, + "nvidia/DmlExecutionProvider_GPU": { + "precision": "default" + } + } + } + ] +} diff --git a/scripts/e2e_eval/testsets/models_release_validation.md b/scripts/e2e_eval/testsets/models_release_validation.md new file mode 100644 index 000000000..a448e453e --- /dev/null +++ b/scripts/e2e_eval/testsets/models_release_validation.md @@ -0,0 +1,65 @@ +# Release Validation Cases + +Target-column labels select the current recipe/config precision. Recipes with +`quant: null` were renamed from `fp16` to `fp32`, with their configuration contents +unchanged, and the corresponding target labels were updated here and in the JSON +manifest. These names describe configuration, not independently measured tensor +dtypes or EP execution precision. + +Historical evidence paths and JSON files are unchanged. The reference precision +column retains its historical label; a link labeled `fp32` can therefore point to +an old `__fp16` result directory. New runs use the corrected names, and +`--continue` will not reuse results from the old directories for renamed cases. + +| # | Model | Task | Reference precision | Priority | Downloads / 30d | Size (MB) | Perf PASS | Perf elapsed (s) | Acc PASS | Acc elapsed (s) | amd/DmlExecutionProvider_GPU | amd/MIGraphExecutionProvider_GPU | amd/MLASExecutionProvider_CPU | amd/VitisAIExecutionProvider_NPU | intel/DmlExecutionProvider_GPU | intel/MLASExecutionProvider_CPU | intel/OpenVINOExecutionProvider_CPU | intel/OpenVINOExecutionProvider_GPU | intel/OpenVINOExecutionProvider_NPU | nvidia/DmlExecutionProvider_GPU | nvidia/MLASExecutionProvider_CPU | nvidia/TRTRTXARMExecutionProvider_GPU | nvidia/TRTRTXExecutionProvider_GPU | qnn/MLASExecutionProvider_CPU | qnn/QNNExecutionProvider_GPU | qnn/QNNExecutionProvider_NPU | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 1 | [depth-anything/Depth-Anything-V2-Small-hf](https://huggingface.co/depth-anything/Depth-Anything-V2-Small-hf) | depth-estimation | fp16 | P2 | 1,752,152 | 99.01 | True | 16.2 | True | 26.6 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-04/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-06/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-07-12/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/depth-anything__Depth-Anything-V2-Small-hf__depth-estimation__fp16/eval_result.json) | +| 2 | [Intel/dpt-hybrid-midas](https://huggingface.co/Intel/dpt-hybrid-midas) | depth-estimation | fp16 | P2 | 782,772 | 485.17 | True | 20.0 | True | 30.3 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/Intel__dpt-hybrid-midas__depth-estimation/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-06/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/Intel__dpt-hybrid-midas__depth-estimation__fp16/eval_result.json) | +| 3 | [BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5) | feature-extraction | fp16 | P2 | 64,933,788 | 132.99 | True | 13.8 | True | 21.0 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/BAAI__bge-small-en-v1.5__feature-extraction__fp16/eval_result.json) | +| 4 | [sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2) | feature-extraction | fp16 | P2 | 46,316,902 | 470.16 | True | 12.7 | True | 43.1 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-06-30/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-07-12/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-07-12/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-06-30/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2__feature-extraction__fp16/eval_result.json) | +| 5 | [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) | feature-extraction | fp16 | P0 | 20,778,503 | 253.89 | True | 13.1 | True | 25.9 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/openai__clip-vit-base-patch32__feature-extraction__fp16/eval_result.json) | +| 6 | [sentence-transformers/paraphrase-multilingual-mpnet-base-v2](https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2) | feature-extraction | w8a16 | P2 | 9,246,884 | 471.52 | True | 19.2 | True | 160.2 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/sentence-transformers__paraphrase-multilingual-mpnet-base-v2__feature-extraction__w8a16/eval_result.json) | +| 7 | [sentence-transformers/multi-qa-mpnet-base-dot-v1](https://huggingface.co/sentence-transformers/multi-qa-mpnet-base-dot-v1) | feature-extraction | fp16 | P2 | 1,694,110 | 448.31 | True | 13.0 | True | 44.1 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/sentence-transformers__multi-qa-mpnet-base-dot-v1__feature-extraction__fp16/eval_result.json) | +| 8 | [google-bert/bert-base-uncased](https://huggingface.co/google-bert/bert-base-uncased) | fill-mask | fp16 | P2 | 48,848,285 | 531.97 | True | 15.5 | True | 86.0 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-07-12/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-07-12/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/google-bert__bert-base-uncased__fill-mask__fp16/eval_result.json) | +| 9 | [FacebookAI/xlm-roberta-base](https://huggingface.co/FacebookAI/xlm-roberta-base) | fill-mask | fp16 | P2 | 21,379,308 | 1,882.91 | True | 16.9 | True | 419.2 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-06/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-07-12/models/FacebookAI__xlm-roberta-base__fill-mask__w8a16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/FacebookAI__xlm-roberta-base__fill-mask__fp16/eval_result.json) | +| 10 | [FacebookAI/roberta-base](https://huggingface.co/FacebookAI/roberta-base) | fill-mask | fp16 | P2 | 7,990,786 | 654.93 | True | 14.9 | True | 109.7 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/FacebookAI__roberta-base__fill-mask__fp16/eval_result.json) | +| 11 | [distilbert/distilbert-base-uncased](https://huggingface.co/distilbert/distilbert-base-uncased) | fill-mask | fp16 | P2 | 7,178,154 | 361.79 | True | 13.5 | True | 73.1 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-04/models/distilbert__distilbert-base-uncased__fill-mask/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/distilbert__distilbert-base-uncased__fill-mask__fp16/eval_result.json) | +| 12 | [timm/mobilenetv3_small_100.lamb_in1k](https://huggingface.co/timm/mobilenetv3_small_100.lamb_in1k) | image-classification | w8a16 | P1 | 16,672,221 | 2.81 | True | 13.5 | False | 30.1 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/timm__mobilenetv3_small_100.lamb_in1k__image-classification__w8a16/eval_result.json) | +| 13 | [google/vit-base-patch16-224](https://huggingface.co/google/vit-base-patch16-224) | image-classification | fp16 | P0 | 5,510,787 | 346.41 | True | 15.1 | True | 1,180.6 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/google__vit-base-patch16-224__image-classification__fp16/eval_result.json) | +| 14 | [microsoft/resnet-50](https://huggingface.co/microsoft/resnet-50) | image-classification | fp16 | P0 | 1,105,417 | 102.19 | True | 12.3 | True | 22.5 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-06-30/models/microsoft__resnet-50__image-classification__fp16/eval_result.json) | +| 15 | [microsoft/swinv2-tiny-patch4-window16-256](https://huggingface.co/microsoft/swinv2-tiny-patch4-window16-256) | image-classification | fp16 | P2 | 414,654 | 142.92 | True | 16.0 | True | 29.0 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/microsoft__swinv2-tiny-patch4-window16-256__image-classification__fp16/eval_result.json) | +| 16 | [facebook/convnext-tiny-224](https://huggingface.co/facebook/convnext-tiny-224) | image-classification | fp16 | P0 | 38,563 | 114.46 | True | 13.5 | True | 24.4 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-06-30/models/facebook__convnext-tiny-224__image-classification__fp16/eval_result.json) | +| 17 | [microsoft/beit-base-patch16-224-pt22k-ft22k](https://huggingface.co/microsoft/beit-base-patch16-224-pt22k-ft22k) | image-classification | w8a16 | P2 | 12,628 | 114.65 | True | 17.5 | False | 25.2 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/microsoft__beit-base-patch16-224-pt22k-ft22k__image-classification__w8a16/eval_result.json) | +| 18 | [facebook/dinov2-base](https://huggingface.co/facebook/dinov2-base) | image-feature-extraction | fp16 | P2 | 3,081,152 | 343.04 | True | 14.7 | True | 21.4 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | [fp16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/facebook__dinov2-base__image-feature-extraction__fp16/eval_result.json) | +| 19 | [google/vit-base-patch16-224-in21k](https://huggingface.co/google/vit-base-patch16-224-in21k) | image-feature-extraction | fp16 | P2 | 1,384,812 | 343.33 | True | 13.5 | True | 25.7 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/google__vit-base-patch16-224-in21k__image-feature-extraction__fp16/eval_result.json) | +| 20 | [facebook/dino-vitb16](https://huggingface.co/facebook/dino-vitb16) | image-feature-extraction | fp16 | P2 | 492,125 | 343.33 | True | 14.3 | True | 21.0 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/facebook__dino-vitb16__image-feature-extraction__fp16/eval_result.json) | +| 21 | [nvidia/segformer-b2-finetuned-ade-512-512](https://huggingface.co/nvidia/segformer-b2-finetuned-ade-512-512) | image-segmentation | fp16 | P2 | 293,787 | 110.13 | True | 18.9 | True | 33.6 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-06/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/nvidia__segformer-b2-finetuned-ade-512-512__image-segmentation__fp16/eval_result.json) | +| 22 | [mattmdjaga/segformer_b2_clothes](https://huggingface.co/mattmdjaga/segformer_b2_clothes) | image-segmentation | fp16 | P2 | 219,901 | 109.72 | True | 19.0 | True | 148.0 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/mattmdjaga__segformer_b2_clothes__image-segmentation__fp16/eval_result.json) | +| 23 | [Salesforce/blip-image-captioning-base](https://huggingface.co/Salesforce/blip-image-captioning-base) | image-to-text | fp16 | P0 | 1,832,246 | 990.09 | True | 24.2 | True | 573.7 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-04/models/Salesforce__blip-image-captioning-base__image-to-text/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-06/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-06/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/Salesforce__blip-image-captioning-base__image-to-text__fp16/eval_result.json) | +| 24 | [microsoft/trocr-base-printed](https://huggingface.co/microsoft/trocr-base-printed) | image-to-text | fp16 | P1 | 213,912 | 1,539.66 | True | 23.3 | True | 263.5 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-06-30/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-07-12/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-07-12/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-07-12/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-06-30/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-06-30/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-06-30/models/microsoft__trocr-base-printed__image-to-text__fp16/eval_result.json) | +| 25 | [facebook/nougat-base](https://huggingface.co/facebook/nougat-base) | image-to-text | w8a16 | P2 | 95,979 | 415.29 | True | 389.6 | False | 0.3 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/facebook__nougat-base__image-to-text/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/facebook__nougat-base__image-to-text__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/facebook__nougat-base__image-to-text/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/facebook__nougat-base__image-to-text/eval_result.json) | [w8a8](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/facebook__nougat-base__image-to-text__w8a8/eval_result.json) | +| 26 | [google-bert/bert-base-multilingual-cased](https://huggingface.co/google-bert/bert-base-multilingual-cased) | masked-lm | w8a16 | P0 | 1,858,502 | 363.28 | True | 27.0 | False | 0.3 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/google-bert__bert-base-multilingual-cased__masked-lm__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/google-bert__bert-base-multilingual-cased__masked-lm/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/google-bert__bert-base-multilingual-cased__masked-lm__w8a16/eval_result.json) | +| 27 | [microsoft/table-transformer-structure-recognition](https://huggingface.co/microsoft/table-transformer-structure-recognition) | object-detection | default | P1 | 1,164,999 | 117.07 | True | 8.5 | True | 208.5 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-07-12/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/microsoft__table-transformer-structure-recognition__object-detection__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/microsoft__table-transformer-structure-recognition__object-detection/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/microsoft__table-transformer-structure-recognition__object-detection__w8a16/eval_result.json) | +| 28 | [hustvl/yolos-small](https://huggingface.co/hustvl/yolos-small) | object-detection | default | P2 | 756,251 | 122.89 | True | 14.7 | True | 66.0 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/hustvl__yolos-small__object-detection/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/hustvl__yolos-small__object-detection__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/hustvl__yolos-small__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/hustvl__yolos-small__object-detection/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/hustvl__yolos-small__object-detection__w8a16/eval_result.json) | +| 29 | [facebook/detr-resnet-50](https://huggingface.co/facebook/detr-resnet-50) | object-detection | default | P0 | 659,491 | 167.84 | True | 16.5 | True | 62.9 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/facebook__detr-resnet-50__object-detection__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/facebook__detr-resnet-50__object-detection/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/facebook__detr-resnet-50__object-detection__w8a16/eval_result.json) | +| 30 | [ahotrod/electra_large_discriminator_squad2_512](https://huggingface.co/ahotrod/electra_large_discriminator_squad2_512) | question-answering | fp16 | P2 | 760,978 | 1,336.68 | True | 17.3 | True | 42.5 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-04/models/ahotrod__electra_large_discriminator_squad2_512__question-answering/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/ahotrod__electra_large_discriminator_squad2_512__question-answering__fp16/eval_result.json) | +| 31 | [tau/splinter-base](https://huggingface.co/tau/splinter-base) | question-answering | w8a16 | P2 | 87,213 | 134.93 | True | 16.9 | True | 89.2 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-06-30/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-07-12/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-07-12/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/tau__splinter-base__question-answering/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/tau__splinter-base__question-answering__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-06-30/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/tau__splinter-base__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-06-30/models/tau__splinter-base__question-answering/eval_result.json) | +| 32 | [timpal0l/mdeberta-v3-base-squad2](https://huggingface.co/timpal0l/mdeberta-v3-base-squad2) | question-answering | w8a16 | P2 | 73,957 | 531.59 | True | 34.2 | True | 310.5 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/timpal0l__mdeberta-v3-base-squad2__question-answering__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/timpal0l__mdeberta-v3-base-squad2__question-answering/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/timpal0l__mdeberta-v3-base-squad2__question-answering__w8a16/eval_result.json) | +| 33 | [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | sentence-similarity | fp16 | P2 | 253,331,994 | 90.32 | True | 13.3 | True | 24.8 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/sentence-transformers__all-MiniLM-L6-v2__sentence-similarity__fp16/eval_result.json) | +| 34 | [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3) | sentence-similarity | w8a16 | P2 | 37,985,566 | 835.17 | True | 52.3 | True | 165.8 | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-06-30/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-07-12/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-07-12/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-06-30/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/BAAI__bge-m3__sentence-similarity__fp16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-06-30/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/BAAI__bge-m3__sentence-similarity__fp16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-06-30/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/BAAI__bge-m3__sentence-similarity__w8a16/eval_result.json) | +| 35 | [sentence-transformers/all-mpnet-base-v2](https://huggingface.co/sentence-transformers/all-mpnet-base-v2) | sentence-similarity | fp16 | P2 | 23,908,257 | 448.31 | True | 13.0 | True | 36.1 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/sentence-transformers__all-mpnet-base-v2__sentence-similarity__fp16/eval_result.json) | +| 36 | [google/flan-t5-base](https://huggingface.co/google/flan-t5-base) | summarization | w8a16 | P2 | 1,535,321 | 323.49 | True | 44.5 | False | 0.3 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/google__flan-t5-base__summarization/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/google__flan-t5-base__summarization__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/google__flan-t5-base__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/google__flan-t5-base__summarization/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/google__flan-t5-base__summarization__w8a16/eval_result.json) | +| 37 | [google/pegasus-xsum](https://huggingface.co/google/pegasus-xsum) | summarization | w8a16 | P2 | 195,197 | 770.67 | True | 46.2 | False | 0.3 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/google__pegasus-xsum__summarization/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/google__pegasus-xsum__summarization__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-05/models/google__pegasus-xsum__summarization/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/google__pegasus-xsum__summarization/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/google__pegasus-xsum__summarization__w8a16/eval_result.json) | +| 38 | [cross-encoder/ms-marco-MiniLM-L6-v2](https://huggingface.co/cross-encoder/ms-marco-MiniLM-L6-v2) | text-classification | fp16 | P2 | 86,755,068 | 45.50 | True | 12.8 | False | 15.7 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-04/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification/eval_result.json) | [fp16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification__fp16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification/eval_result.json) | [fp16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification__fp16/eval_result.json) | [fp16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification__fp16/eval_result.json) | [fp16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-06/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification__fp16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification__fp32/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification/eval_result.json) | [fp16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-06/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification__fp16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification/eval_result.json) | [fp16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification__fp16/eval_result.json) | [fp16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/cross-encoder__ms-marco-MiniLM-L6-v2__text-classification__fp16/eval_result.json) | +| 39 | [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) | text-classification | w8a16 | P2 | 18,159,906 | 836.31 | True | 56.7 | False | 52.7 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/BAAI__bge-reranker-v2-m3__text-classification__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/BAAI__bge-reranker-v2-m3__text-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/BAAI__bge-reranker-v2-m3__text-classification__w8a16/eval_result.json) | +| 40 | [distilbert/distilbert-base-uncased-finetuned-sst-2-english](https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english) | text-classification | fp16 | P2 | 3,852,454 | 267.90 | True | 13.3 | True | 49.8 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/distilbert__distilbert-base-uncased-finetuned-sst-2-english__text-classification__fp16/eval_result.json) | +| 41 | [cardiffnlp/twitter-roberta-base-sentiment-latest](https://huggingface.co/cardiffnlp/twitter-roberta-base-sentiment-latest) | text-classification | fp16 | P2 | 3,245,087 | 500.32 | True | 14.8 | True | 34.1 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-04/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/cardiffnlp__twitter-roberta-base-sentiment-latest__text-classification__fp16/eval_result.json) | +| 42 | [dbmdz/bert-large-cased-finetuned-conll03-english](https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english) | token-classification | fp16 | P2 | 4,484,839 | 1,330.45 | True | 16.9 | True | 29.4 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-04/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/dbmdz__bert-large-cased-finetuned-conll03-english__token-classification__fp16/eval_result.json) | +| 43 | [w11wo/indonesian-roberta-base-posp-tagger](https://huggingface.co/w11wo/indonesian-roberta-base-posp-tagger) | token-classification | fp16 | P2 | 2,468,370 | 498.03 | True | 15.9 | True | 35.5 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-06/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/w11wo__indonesian-roberta-base-posp-tagger__token-classification__fp16/eval_result.json) | +| 44 | [kredor/punctuate-all](https://huggingface.co/kredor/punctuate-all) | token-classification | w8a16 | P2 | 781,094 | 471.58 | True | 19.1 | True | 82.5 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/kredor__punctuate-all__token-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/kredor__punctuate-all__token-classification__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/kredor__punctuate-all__token-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/kredor__punctuate-all__token-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/kredor__punctuate-all__token-classification__w8a16/eval_result.json) | +| 45 | [google-t5/t5-small](https://huggingface.co/google-t5/t5-small) | translation | w8a16 | P2 | 23,855,459 | 127.07 | True | 31.5 | False | 0.3 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/google-t5__t5-small__translation/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/google-t5__t5-small__translation__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/google-t5__t5-small__translation/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/google-t5__t5-small__translation/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/google-t5__t5-small__translation__w8a16/eval_result.json) | +| 46 | [lxyuan/distilbert-base-multilingual-cased-sentiments-student](https://huggingface.co/lxyuan/distilbert-base-multilingual-cased-sentiments-student) | zero-shot-classification | w8a16 | P2 | 990,630 | 228.01 | True | 15.6 | True | 169.3 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/lxyuan__distilbert-base-multilingual-cased-sentiments-student__zero-shot-classification__w8a16/eval_result.json) | +| 47 | [cross-encoder/nli-deberta-v3-small](https://huggingface.co/cross-encoder/nli-deberta-v3-small) | zero-shot-classification | w8a16 | P2 | 558,230 | 295.56 | True | 37.8 | True | 684.4 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-04/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-06/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-04/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/cross-encoder__nli-deberta-v3-small__zero-shot-classification__w8a16/eval_result.json) | +| 48 | [openai/clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) | zero-shot-image-classification | fp16 | P2 | 7,407,606 | 1,710.94 | True | 36.4 | True | 48.5 | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-08-06/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-28/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-07-12/models/openai__clip-vit-large-patch14__zero-shot-image-classification__w8a16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-08-04/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | [fp32](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-04/models/openai__clip-vit-large-patch14__zero-shot-image-classification__fp16/eval_result.json) | +| 49 | [laion/CLIP-ViT-B-32-laion2B-s34B-b79K](https://huggingface.co/laion/CLIP-ViT-B-32-laion2B-s34B-b79K) | zero-shot-image-classification | w8a16 | P0 | 3,860,781 | 178.35 | True | 35.0 | True | 551.1 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-08-06/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/laion__CLIP-ViT-B-32-laion2B-s34B-b79K__zero-shot-image-classification__w8a16/eval_result.json) | +| 50 | [google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) | zero-shot-image-classification | w8a16 | P2 | 1,854,284 | 229.50 | True | 37.3 | True | 961.6 | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/DmlExecutionProvider_GPU/2026-06-30/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MIGraphExecutionProvider_GPU/2026-06-30/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/MLASExecutionProvider_CPU/2026-08-06/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/amd/VitisAIExecutionProvider_NPU/2026-06-30/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/DmlExecutionProvider_GPU/2026-07-12/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/MLASExecutionProvider_CPU/2026-08-06/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_CPU/2026-08-06/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_GPU/2026-08-06/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/intel/OpenVINOExecutionProvider_NPU/2026-08-28/models/google__siglip-base-patch16-224__zero-shot-image-classification__w8a16/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/DmlExecutionProvider_GPU/2026-09-10/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/MLASExecutionProvider_CPU/2026-08-06/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXARMExecutionProvider_GPU/2026-08-15/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/nvidia/TRTRTXExecutionProvider_GPU/2026-08-28/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/MLASExecutionProvider_CPU/2026-06-30/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [default](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_GPU/2026-08-28/models/google__siglip-base-patch16-224__zero-shot-image-classification/eval_result.json) | [w8a16](../../../../ModelKitArtifacts/site/e2e_model_coverage_result/qnn/QNNExecutionProvider_NPU/2026-08-28/models/google__siglip-base-patch16-224__zero-shot-image-classification__w8a16/eval_result.json) | diff --git a/scripts/e2e_eval/utils/registry.py b/scripts/e2e_eval/utils/registry.py index f8007dd21..eb5cd3ba5 100644 --- a/scripts/e2e_eval/utils/registry.py +++ b/scripts/e2e_eval/utils/registry.py @@ -9,7 +9,8 @@ import json from collections.abc import Sequence # noqa: TC003 -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from itertools import pairwise from pathlib import Path # noqa: TC003 @@ -35,6 +36,11 @@ class ModelEntry: _REQUIRED_FIELDS = {"hf_id", "task", "model_type", "group", "priority"} _VALID_PRIORITIES = {"P0", "P1", "P2", "P3"} +_RELEASE_EP_ALIASES = { + "trtrtxexecutionprovider": "nv_tensorrt_rtx", + "mlasexecutionprovider": "cpu", + "migraphexecutionprovider": "migraphx", +} def _canonical_ep_device_key(ep: str, device: str) -> str: @@ -125,6 +131,127 @@ def load_registry(path: Path) -> list[ModelEntry]: return entries +def _release_manifest_rows(path: Path) -> tuple[list[str], list[dict]]: + from winml.modelkit.config.precision import is_quantized_precision + + data = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(data, dict) or data.get("schema_version") != 1: + raise ValueError(f"Release manifest requires schema_version=1: {path}") + rows = data.get("models") + if not isinstance(rows, list) or not rows: + raise ValueError(f"Release manifest requires a nonempty models array: {path}") + target_names: list[str] = [] + seen = set() + for row in rows: + if not isinstance(row, dict) or any( + not isinstance(row.get(field), str) or not row[field].strip() + for field in ("hf_id", "task") + ): + raise ValueError("Release manifest requires nonempty hf_id and task strings") + key = (row["hf_id"], row["task"]) + if key in seen: + raise ValueError(f"Release manifest contains a duplicate model/task: {key}") + seen.add(key) + if "priority" in row and row["priority"] not in _VALID_PRIORITIES: + raise ValueError(f"Invalid release model priority for {key}: {row['priority']!r}") + targets = row.get("targets") + if not isinstance(targets, dict) or not targets: + raise ValueError(f"Release model requires a nonempty targets object: {key}") + if not target_names: + target_names = list(targets) + if set(targets) != set(target_names): + raise ValueError(f"Release models must have the same EP/device targets: {key}") + for target_name, target in targets.items(): + machine, slash, ep_device = target_name.rpartition("/") + ep, underscore, device = ep_device.rpartition("_") + if not slash or not underscore or not machine or not ep or not device: + raise ValueError(f"Invalid release EP/device target: {target_name!r}") + precision = target.get("precision") if isinstance(target, dict) else None + if not isinstance(precision, str) or ( + precision not in {"default", "fp16", "fp32"} + and not is_quantized_precision(precision) + ): + raise ValueError( + f"Invalid release precision {precision!r} for {key} in {target_name}" + ) + return target_names, rows + + +def _release_ep_device_key(ep: str, device: str) -> str: + """Normalize historical result-folder names without changing runtime EP aliases.""" + ep = ep.strip() + return _canonical_ep_device_key(_RELEASE_EP_ALIASES.get(ep.casefold(), ep), device) + + +def _release_target_column( + headers: list[str], + ep: str, + device: str, + machine: str | None, + output_dir: Path | None, +) -> str: + target_key = _release_ep_device_key(ep, device) + matches = [] + for header in headers: + column_machine, slash, target = header.rpartition("/") + column_ep, underscore, column_device = target.rpartition("_") + if slash and underscore and _release_ep_device_key(column_ep, column_device) == target_key: + matches.append((column_machine, header)) + if machine: + matches = [match for match in matches if match[0].casefold() == machine.casefold()] + elif len(matches) > 1 and output_dir is not None: + path_parts = output_dir.resolve().parts + target_folders = { + f"{parent.casefold()}/{child.casefold()}" + for parent, child in pairwise(path_parts) + } + inferred = [match for match in matches if match[1].casefold() in target_folders] + if len(inferred) == 1: + matches = inferred + if not matches: + raise ValueError( + f"Release manifest has no target for {target_key}" + + (f" on machine {machine!r}" if machine else "") + ) + if len(matches) > 1: + machines = ", ".join(sorted({match[0] for match in matches})) + raise ValueError( + f"Release target {target_key} is ambiguous; use /_ " + f"in --output-dir (machines: {machines})" + ) + return matches[0][1] + + +def load_release_registry( + path: Path, + registry: list[ModelEntry], + *, + ep: str, + device: str, + machine: str | None = None, + output_dir: Path | None = None, +) -> tuple[list[ModelEntry], str]: + """Load release JSON in manifest order without opening historical evidence files.""" + headers, rows = _release_manifest_rows(path) + column = _release_target_column(headers, ep, device, machine, output_dir) + entries = [] + for row in rows: + hf_id = row["hf_id"] + task = row["task"] + key = (hf_id, task) + label = row["targets"][column]["precision"] + precision = None if label == "default" else label + matching = [entry for entry in registry if (entry.hf_id, entry.task) == key] + base = next( + (entry for entry in matching if entry.precision == precision), + matching[0] if matching else make_adhoc_entry(hf_id, task), + ) + entries.append( + replace(base, precision=precision, priority=row.get("priority") or base.priority) + ) + return entries, column + + def filter_registry( entries: list[ModelEntry], *, diff --git a/src/winml/modelkit/data/hub_models.json b/src/winml/modelkit/data/hub_models.json index 4be17ce63..ba6b7062b 100644 --- a/src/winml/modelkit/data/hub_models.json +++ b/src/winml/modelkit/data/hub_models.json @@ -2,649 +2,9 @@ "version": "1.0", "models": [ { - "model_id": "BAAI/bge-base-en-v1.5", - "task": "feature-extraction", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 127.4 - }, - { - "model_id": "BAAI/bge-base-en-v1.5", - "task": "sentence-similarity", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 128.0 - }, - { - "model_id": "BAAI/bge-large-en-v1.5", - "task": "sentence-similarity", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 351.8 - }, - { - "model_id": "BAAI/bge-m3", - "task": "feature-extraction", - "model_type": "xlm-roberta", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 796.5 - }, - { - "model_id": "BAAI/bge-m3", - "task": "sentence-similarity", - "model_type": "xlm-roberta", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 797.5 - }, - { - "model_id": "BAAI/bge-small-en-v1.5", - "task": "feature-extraction", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 43.7 - }, - { - "model_id": "BAAI/bge-small-en-v1.5", - "task": "sentence-similarity", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 43.9 - }, - { - "model_id": "Babelscape/wikineural-multilingual-ner", - "task": "token-classification", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 257.9 - }, - { - "model_id": "FacebookAI/roberta-base", - "task": "fill-mask", - "model_type": "roberta", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 194.7 - }, - { - "model_id": "FacebookAI/roberta-large", - "task": "fill-mask", - "model_type": "roberta", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 440.8 - }, - { - "model_id": "FacebookAI/xlm-roberta-base", - "task": "fill-mask", - "model_type": "xlm-roberta", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 634.4 - }, - { - "model_id": "Intel/bert-base-uncased-mrpc", - "task": "feature-extraction", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 127.4 - }, - { - "model_id": "Intel/bert-base-uncased-mrpc", - "task": "text-classification", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 128.0 - }, - { - "model_id": "Isotonic/distilbert_finetuned_ai4privacy_v2", - "task": "token-classification", - "model_type": "distilbert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 86.6 - }, - { - "model_id": "ProsusAI/finbert", - "task": "text-classification", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 128.0 - }, - { - "model_id": "apple/mobilevit-small", - "task": "image-classification", - "model_type": "mobilevit", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 6.1 - }, - { - "model_id": "cardiffnlp/twitter-roberta-base-sentiment-latest", - "task": "text-classification", - "model_type": "roberta", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 157.7 - }, - { - "model_id": "dbmdz/bert-large-cased-finetuned-conll03-english", - "task": "token-classification", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 347.9 - }, - { - "model_id": "deepset/bert-large-uncased-whole-word-masking-squad2", - "task": "question-answering", - "model_type": "bert", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 350.9 - }, - { - "model_id": "deepset/roberta-base-squad2", - "task": "question-answering", - "model_type": "roberta", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 157.2 - }, - { - "model_id": "deepset/tinyroberta-squad2", - "task": "question-answering", - "model_type": "roberta", + "model_id": "BAAI/bge-reranker-v2-m3", + "task": "text-classification", + "model_type": "xlm-roberta", "supported_eps": { "cpu": [ "CPU" @@ -671,12 +31,12 @@ "GPU" ] }, - "size_mb": 116.2 + "size_mb": 797.5 }, { - "model_id": "dima806/fairface_age_image_detection", - "task": "image-classification", - "model_type": "vit", + "model_id": "BAAI/bge-small-en-v1.5", + "task": "feature-extraction", + "model_type": "bert", "supported_eps": { "cpu": [ "CPU" @@ -703,12 +63,12 @@ "GPU" ] }, - "size_mb": 82.8 + "size_mb": 43.7 }, { - "model_id": "distilbert/distilbert-base-cased-distilled-squad", - "task": "question-answering", - "model_type": "distilbert", + "model_id": "FacebookAI/roberta-base", + "task": "fill-mask", + "model_type": "roberta", "supported_eps": { "cpu": [ "CPU" @@ -735,12 +95,12 @@ "GPU" ] }, - "size_mb": 84.2 + "size_mb": 194.7 }, { - "model_id": "distilbert/distilbert-base-uncased", + "model_id": "FacebookAI/xlm-roberta-base", "task": "fill-mask", - "model_type": "distilbert", + "model_type": "xlm-roberta", "supported_eps": { "cpu": [ "CPU" @@ -767,12 +127,12 @@ "GPU" ] }, - "size_mb": 109.5 + "size_mb": 634.4 }, { - "model_id": "distilbert/distilbert-base-uncased-distilled-squad", - "task": "question-answering", - "model_type": "distilbert", + "model_id": "Salesforce/blip-image-captioning-base", + "task": "image-to-text", + "model_type": "blip", "supported_eps": { "cpu": [ "CPU" @@ -799,12 +159,12 @@ "GPU" ] }, - "size_mb": 86.5 + "size_mb": 944.2 }, { - "model_id": "distilbert/distilbert-base-uncased-finetuned-sst-2-english", - "task": "text-classification", - "model_type": "distilbert", + "model_id": "ahotrod/electra_large_discriminator_squad2_512", + "task": "question-answering", + "model_type": "electra", "supported_eps": { "cpu": [ "CPU" @@ -831,12 +191,12 @@ "GPU" ] }, - "size_mb": 87.0 + "size_mb": 1274.8 }, { - "model_id": "dslim/bert-base-NER", - "task": "token-classification", - "model_type": "bert", + "model_id": "cardiffnlp/twitter-roberta-base-sentiment-latest", + "task": "text-classification", + "model_type": "roberta", "supported_eps": { "cpu": [ "CPU" @@ -863,12 +223,12 @@ "GPU" ] }, - "size_mb": 125.2 + "size_mb": 157.7 }, { - "model_id": "facebook/convnext-tiny-224", - "task": "image-classification", - "model_type": "convnext", + "model_id": "cross-encoder/ms-marco-MiniLM-L6-v2", + "task": "text-classification", + "model_type": "bert", "supported_eps": { "cpu": [ "CPU" @@ -895,12 +255,12 @@ "GPU" ] }, - "size_mb": 27.8 + "size_mb": 43.4 }, { - "model_id": "facebook/dino-vitb16", - "task": "image-feature-extraction", - "model_type": "vit", + "model_id": "cross-encoder/nli-deberta-v3-small", + "task": "zero-shot-classification", + "model_type": "deberta-v2", "supported_eps": { "cpu": [ "CPU" @@ -927,12 +287,12 @@ "GPU" ] }, - "size_mb": 83.4 + "size_mb": 281.9 }, { - "model_id": "facebook/dino-vits16", - "task": "image-feature-extraction", - "model_type": "vit", + "model_id": "dbmdz/bert-large-cased-finetuned-conll03-english", + "task": "token-classification", + "model_type": "bert", "supported_eps": { "cpu": [ "CPU" @@ -959,12 +319,12 @@ "GPU" ] }, - "size_mb": 21.6 + "size_mb": 347.9 }, { - "model_id": "facebook/dinov2-small", - "task": "image-feature-extraction", - "model_type": "dinov2", + "model_id": "depth-anything/Depth-Anything-V2-Small-hf", + "task": "depth-estimation", + "model_type": "depth_anything", "supported_eps": { "cpu": [ "CPU" @@ -991,12 +351,12 @@ "GPU" ] }, - "size_mb": 21.4 + "size_mb": 94.4 }, { - "model_id": "google-bert/bert-base-multilingual-cased", - "task": "feature-extraction", - "model_type": "bert", + "model_id": "distilbert/distilbert-base-uncased-finetuned-sst-2-english", + "task": "text-classification", + "model_type": "distilbert", "supported_eps": { "cpu": [ "CPU" @@ -1023,12 +383,12 @@ "GPU" ] }, - "size_mb": 257.8 + "size_mb": 87.0 }, { - "model_id": "google-bert/bert-base-multilingual-cased", + "model_id": "distilbert/distilbert-base-uncased", "task": "fill-mask", - "model_type": "bert", + "model_type": "distilbert", "supported_eps": { "cpu": [ "CPU" @@ -1055,12 +415,12 @@ "GPU" ] }, - "size_mb": 346.5 + "size_mb": 109.5 }, { - "model_id": "google-bert/bert-base-multilingual-uncased", - "task": "fill-mask", - "model_type": "bert", + "model_id": "facebook/convnext-tiny-224", + "task": "image-classification", + "model_type": "convnext", "supported_eps": { "cpu": [ "CPU" @@ -1087,12 +447,12 @@ "GPU" ] }, - "size_mb": 316.4 + "size_mb": 27.8 }, { - "model_id": "google-bert/bert-base-uncased", - "task": "fill-mask", - "model_type": "bert", + "model_id": "facebook/detr-resnet-50", + "task": "object-detection", + "model_type": "detr", "supported_eps": { "cpu": [ "CPU" @@ -1119,12 +479,12 @@ "GPU" ] }, - "size_mb": 150.5 + "size_mb": 41.8 }, { - "model_id": "google-bert/bert-large-uncased-whole-word-masking-finetuned-squad", - "task": "question-answering", - "model_type": "bert", + "model_id": "facebook/dino-vitb16", + "task": "image-feature-extraction", + "model_type": "vit", "supported_eps": { "cpu": [ "CPU" @@ -1151,12 +511,12 @@ "GPU" ] }, - "size_mb": 350.9 + "size_mb": 83.4 }, { - "model_id": "google/vit-base-patch16-224", - "task": "image-classification", - "model_type": "vit", + "model_id": "facebook/dinov2-base", + "task": "image-feature-extraction", + "model_type": "dinov2", "supported_eps": { "cpu": [ "CPU" @@ -1183,12 +543,12 @@ "GPU" ] }, - "size_mb": 83.6 + "size_mb": 163.6 }, { - "model_id": "google/vit-base-patch16-224-in21k", - "task": "image-feature-extraction", - "model_type": "vit", + "model_id": "google-bert/bert-base-multilingual-cased", + "task": "masked-lm", + "model_type": "bert", "supported_eps": { "cpu": [ "CPU" @@ -1215,12 +575,12 @@ "GPU" ] }, - "size_mb": 83.4 + "size_mb": 346.4 }, { - "model_id": "joeddav/xlm-roberta-large-xnli", - "task": "zero-shot-classification", - "model_type": "xlm-roberta", + "model_id": "google-bert/bert-base-uncased", + "task": "fill-mask", + "model_type": "bert", "supported_eps": { "cpu": [ "CPU" @@ -1247,12 +607,12 @@ "GPU" ] }, - "size_mb": 781.6 + "size_mb": 150.5 }, { - "model_id": "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", - "task": "feature-extraction", - "model_type": "clip", + "model_id": "google-t5/t5-small", + "task": "translation", + "model_type": "t5", "supported_eps": { "cpu": [ "CPU" @@ -1279,12 +639,12 @@ "GPU" ] }, - "size_mb": 85.4 + "size_mb": 121.2 }, { - "model_id": "mattmdjaga/segformer_b2_clothes", - "task": "image-segmentation", - "model_type": "segformer", + "model_id": "google/flan-t5-base", + "task": "summarization", + "model_type": "t5", "supported_eps": { "cpu": [ "CPU" @@ -1311,12 +671,12 @@ "GPU" ] }, - "size_mb": 27.4 + "size_mb": 308.5 }, { - "model_id": "microsoft/resnet-18", - "task": "image-classification", - "model_type": "resnet", + "model_id": "google/siglip-base-patch16-224", + "task": "zero-shot-image-classification", + "model_type": "siglip", "supported_eps": { "cpu": [ "CPU" @@ -1343,12 +703,12 @@ "GPU" ] }, - "size_mb": 11.2 + "size_mb": 218.8 }, { - "model_id": "microsoft/resnet-50", - "task": "image-classification", - "model_type": "resnet", + "model_id": "google/vit-base-patch16-224-in21k", + "task": "image-feature-extraction", + "model_type": "vit", "supported_eps": { "cpu": [ "CPU" @@ -1375,12 +735,12 @@ "GPU" ] }, - "size_mb": 24.6 + "size_mb": 83.4 }, { - "model_id": "microsoft/swin-large-patch4-window7-224", + "model_id": "google/vit-base-patch16-224", "task": "image-classification", - "model_type": "swin", + "model_type": "vit", "supported_eps": { "cpu": [ "CPU" @@ -1407,12 +767,12 @@ "GPU" ] }, - "size_mb": 192.8 + "size_mb": 83.6 }, { - "model_id": "microsoft/trocr-base-handwritten", - "task": "image-to-text", - "model_type": "vision-encoder-decoder", + "model_id": "hustvl/yolos-small", + "task": "object-detection", + "model_type": "yolos", "supported_eps": { "cpu": [ "CPU" @@ -1439,12 +799,12 @@ "GPU" ] }, - "size_mb": 419.5 + "size_mb": 38.1 }, { - "model_id": "microsoft/trocr-base-printed", - "task": "image-to-text", - "model_type": "vision-encoder-decoder", + "model_id": "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + "task": "zero-shot-image-classification", + "model_type": "clip", "supported_eps": { "cpu": [ "CPU" @@ -1471,12 +831,12 @@ "GPU" ] }, - "size_mb": 419.5 + "size_mb": 170.1 }, { - "model_id": "microsoft/trocr-large-handwritten", - "task": "image-to-text", - "model_type": "vision-encoder-decoder", + "model_id": "lxyuan/distilbert-base-multilingual-cased-sentiments-student", + "task": "zero-shot-classification", + "model_type": "distilbert", "supported_eps": { "cpu": [ "CPU" @@ -1503,10 +863,10 @@ "GPU" ] }, - "size_mb": 634.1 + "size_mb": 217.4 }, { - "model_id": "nvidia/segformer-b1-finetuned-ade-512-512", + "model_id": "mattmdjaga/segformer_b2_clothes", "task": "image-segmentation", "model_type": "segformer", "supported_eps": { @@ -1535,12 +895,12 @@ "GPU" ] }, - "size_mb": 13.8 + "size_mb": 27.4 }, { - "model_id": "nvidia/segformer-b2-finetuned-ade-512-512", - "task": "image-segmentation", - "model_type": "segformer", + "model_id": "microsoft/beit-base-patch16-224-pt22k-ft22k", + "task": "image-classification", + "model_type": "beit", "supported_eps": { "cpu": [ "CPU" @@ -1567,12 +927,12 @@ "GPU" ] }, - "size_mb": 27.5 + "size_mb": 109.3 }, { - "model_id": "nvidia/segformer-b5-finetuned-ade-640-640", - "task": "image-segmentation", - "model_type": "segformer", + "model_id": "microsoft/resnet-50", + "task": "image-classification", + "model_type": "resnet", "supported_eps": { "cpu": [ "CPU" @@ -1599,12 +959,12 @@ "GPU" ] }, - "size_mb": 85.1 + "size_mb": 24.6 }, { - "model_id": "openai/clip-vit-base-patch16", - "task": "feature-extraction", - "model_type": "clip", + "model_id": "microsoft/table-transformer-structure-recognition", + "task": "object-detection", + "model_type": "table-transformer", "supported_eps": { "cpu": [ "CPU" @@ -1631,12 +991,12 @@ "GPU" ] }, - "size_mb": 85.5 + "size_mb": 29.7 }, { - "model_id": "openai/clip-vit-base-patch16", - "task": "zero-shot-image-classification", - "model_type": "clip", + "model_id": "microsoft/trocr-base-printed", + "task": "image-to-text", + "model_type": "vision-encoder-decoder", "supported_eps": { "cpu": [ "CPU" @@ -1663,12 +1023,12 @@ "GPU" ] }, - "size_mb": 168.7 + "size_mb": 419.5 }, { - "model_id": "openai/clip-vit-base-patch32", - "task": "feature-extraction", - "model_type": "clip", + "model_id": "nvidia/segformer-b2-finetuned-ade-512-512", + "task": "image-segmentation", + "model_type": "segformer", "supported_eps": { "cpu": [ "CPU" @@ -1695,12 +1055,12 @@ "GPU" ] }, - "size_mb": 85.5 + "size_mb": 27.5 }, { - "model_id": "rizvandwiki/gender-classification", - "task": "image-classification", - "model_type": "vit", + "model_id": "openai/clip-vit-base-patch32", + "task": "feature-extraction", + "model_type": "clip", "supported_eps": { "cpu": [ "CPU" @@ -1727,12 +1087,12 @@ "GPU" ] }, - "size_mb": 82.8 + "size_mb": 85.5 }, { - "model_id": "sentence-transformers/all-MiniLM-L6-v2", - "task": "feature-extraction", - "model_type": "bert", + "model_id": "openai/clip-vit-large-patch14", + "task": "zero-shot-image-classification", + "model_type": "clip", "supported_eps": { "cpu": [ "CPU" @@ -1759,7 +1119,7 @@ "GPU" ] }, - "size_mb": 33.2 + "size_mb": 447.2 }, { "model_id": "sentence-transformers/all-MiniLM-L6-v2", @@ -1793,38 +1153,6 @@ }, "size_mb": 33.4 }, - { - "model_id": "sentence-transformers/all-mpnet-base-v2", - "task": "feature-extraction", - "model_type": "mpnet", - "supported_eps": { - "cpu": [ - "CPU" - ], - "dml": [ - "GPU" - ], - "qnn": [ - "GPU", - "NPU" - ], - "openvino": [ - "CPU", - "GPU", - "NPU" - ], - "vitisai": [ - "NPU" - ], - "trtrtx": [ - "GPU" - ], - "migraphx": [ - "GPU" - ] - }, - "size_mb": 133.4 - }, { "model_id": "sentence-transformers/all-mpnet-base-v2", "task": "sentence-similarity", @@ -1890,9 +1218,9 @@ "size_mb": 133.4 }, { - "model_id": "sentence-transformers/multi-qa-mpnet-base-dot-v1", - "task": "sentence-similarity", - "model_type": "mpnet", + "model_id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "task": "feature-extraction", + "model_type": "bert", "supported_eps": { "cpu": [ "CPU" @@ -1919,12 +1247,12 @@ "GPU" ] }, - "size_mb": 134.0 + "size_mb": 204.5 }, { - "model_id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", - "task": "feature-extraction", - "model_type": "bert", + "model_id": "tau/splinter-base", + "task": "question-answering", + "model_type": "splinter", "supported_eps": { "cpu": [ "CPU" @@ -1951,12 +1279,12 @@ "GPU" ] }, - "size_mb": 204.5 + "size_mb": 128.7 }, { - "model_id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", - "task": "sentence-similarity", - "model_type": "bert", + "model_id": "timm/mobilenetv3_small_100.lamb_in1k", + "task": "image-classification", + "model_type": "timm_wrapper", "supported_eps": { "cpu": [ "CPU" @@ -1983,12 +1311,12 @@ "GPU" ] }, - "size_mb": 204.7 + "size_mb": 2.7 }, { - "model_id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", - "task": "sentence-similarity", - "model_type": "xlm-roberta", + "model_id": "timpal0l/mdeberta-v3-base-squad2", + "task": "question-answering", + "model_type": "deberta-v2", "supported_eps": { "cpu": [ "CPU" @@ -2015,7 +1343,7 @@ "GPU" ] }, - "size_mb": 450.3 + "size_mb": 506.9 }, { "model_id": "w11wo/indonesian-roberta-base-posp-tagger", @@ -2048,30 +1376,6 @@ ] }, "size_mb": 157.2 - }, - { - "model_id": "onnx-community/sam3-tracker-ONNX", - "onnx_path": "onnx/vision_encoder_int8.onnx", - "task": "image-feature-extraction", - "model_type": "sam3_tracker", - "supported_eps": { - "cpu": [ - "CPU" - ] - }, - "size_mb": 504.2 - }, - { - "model_id": "onnx-community/sam3-tracker-ONNX", - "onnx_path": "onnx/prompt_encoder_mask_decoder_int8.onnx", - "task": "mask-generation", - "model_type": "sam3_tracker", - "supported_eps": { - "cpu": [ - "CPU" - ] - }, - "size_mb": 9.6 } ] } diff --git a/src/winml/modelkit/datasets/random_dataset.py b/src/winml/modelkit/datasets/random_dataset.py index a8da3d029..105ca8c40 100644 --- a/src/winml/modelkit/datasets/random_dataset.py +++ b/src/winml/modelkit/datasets/random_dataset.py @@ -11,10 +11,8 @@ from __future__ import annotations import logging -import random from typing import Any, ClassVar, cast -import numpy as np import torch from datasets import Dataset @@ -27,6 +25,7 @@ class RandomDataset: Generates synthetic data by reading ONNX model input specs (shapes, dtypes) and winml.io.inputs metadata (value ranges) via get_io_config(). + Samples are generated on demand and replayed deterministically by index. Model-agnostic and requires no real data or downloads. Args: @@ -50,11 +49,6 @@ def __init__( self.max_samples = max_samples self.seed = seed - # Set random seeds for reproducibility - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - # Cache io_config (loads ONNX once) from ..onnx import get_io_config @@ -98,25 +92,31 @@ def label_col(self) -> str: """Label column name (readonly). RandomDataset has no labels.""" return "sample_id" - def _generate_random_sample(self) -> dict[str, Any]: + def _generate_random_sample(self, index: int) -> dict[str, Any]: """Generate a single random sample as torch tensors. Uses cached InputTensorSpec list built from ONNX model I/O config. Each spec's to_tensor() handles value_range, dtype, and shape correctly. """ - return {spec.name: spec.to_tensor() for spec in self._input_specs if spec.name} + generator = torch.Generator().manual_seed(self.seed + index) + sample = { + spec.name: spec.to_tensor(generator=generator) + for spec in self._input_specs + if spec.name + } + sample[self.label_col] = torch.tensor(index, dtype=torch.long) + return sample def _load_dataset(self) -> Dataset: - """Generate synthetic dataset with random samples as tensors.""" - samples = [] - for i in range(self.max_samples): - sample = self._generate_random_sample() - sample[self.label_col] = torch.tensor(i, dtype=torch.long) - samples.append(sample) - - dataset = Dataset.from_list(samples) - dataset.set_format("torch") - return dataset + """Store sample indices and generate tensors only for requested rows.""" + columns = [spec.name for spec in self._input_specs if spec.name] + [self.label_col] + + def generate_batch(batch: dict[str, list[int]]) -> dict[str, list[Any]]: + samples = [self._generate_random_sample(index) for index in batch[self.label_col]] + return {name: [sample[name] for sample in samples] for name in columns} + + dataset = Dataset.from_dict({self.label_col: range(self.max_samples)}) + return dataset.with_transform(generate_batch) def get_data_config(self) -> dict[str, Any]: """Get Olive data configuration for random dataset.""" diff --git a/src/winml/modelkit/export/io.py b/src/winml/modelkit/export/io.py index ad4b02881..d11237a32 100644 --- a/src/winml/modelkit/export/io.py +++ b/src/winml/modelkit/export/io.py @@ -360,52 +360,6 @@ def _populate_sequence_length_from_config( ) -def _coerce_binary_attention_masks_to_bool( - dummy_inputs: dict[str, torch.Tensor], -) -> dict[str, torch.Tensor]: - """Normalize binary attention masks to bool for export tracing. - - Some transformer attention paths reject integral masks under newer torch - SDPA checks, while accepting bool masks. To keep behavior architecture- - agnostic, coerce only tensors whose names indicate an attention mask and - whose runtime values are binary (0/1). - """ - import torch - - integer_dtypes = { - torch.int8, - torch.int16, - torch.int32, - torch.int64, - torch.uint8, - } - normalized = dict(dummy_inputs) - - for name, tensor in dummy_inputs.items(): - if "attention_mask" not in name: - continue - if tensor.dtype not in integer_dtypes: - continue - if tensor.numel() == 0: - continue - - min_value = int(tensor.min().item()) - max_value = int(tensor.max().item()) - if min_value < 0 or max_value > 1: - logger.debug( - "Skipping bool coercion for %s: non-binary range=(%d, %d)", - name, - min_value, - max_value, - ) - continue - - normalized[name] = tensor.to(dtype=torch.bool) - logger.debug("Coerced %s from %s to bool for export tracing", name, tensor.dtype) - - return normalized - - def generate_dummy_inputs( model_type: str, task: str, @@ -465,11 +419,10 @@ def generate_dummy_inputs( ) # Optimum's OnnxConfig is untyped; the dummy-inputs dict matches our return type. - dummy_inputs = cast( + return cast( "dict[str, torch.Tensor]", onnx_config.generate_dummy_inputs(framework="pt", **shape_kwargs), ) - return _coerce_binary_attention_masks_to_bool(dummy_inputs) def resolve_io_specs( @@ -534,8 +487,6 @@ def resolve_io_specs( onnx_config.generate_dummy_inputs(framework="pt", **shape_kwargs), ) - dummy_inputs = _coerce_binary_attention_masks_to_bool(dummy_inputs) - input_shapes = [tuple(t.shape) for t in dummy_inputs.values()] input_dtypes = [str(t.dtype).replace("torch.", "") for t in dummy_inputs.values()] diff --git a/src/winml/modelkit/onnx/io.py b/src/winml/modelkit/onnx/io.py index d2ad85878..80693bdf7 100644 --- a/src/winml/modelkit/onnx/io.py +++ b/src/winml/modelkit/onnx/io.py @@ -23,11 +23,15 @@ import logging from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np +if TYPE_CHECKING: + import torch + + logger = logging.getLogger(__name__) ShapeDim = int | str @@ -70,7 +74,7 @@ class InputTensorSpec: shape: tuple[ShapeDim, ...] | None = None value_range: tuple[float, float] | None = None # (min, max_exclusive) - def to_tensor(self) -> Any: + def to_tensor(self, *, generator: torch.Generator | None = None) -> Any: """Generate a dummy tensor from this spec. When value_range is set, generates values in the correct range: @@ -79,6 +83,9 @@ def to_tensor(self) -> Any: Falls back to ones (int) or rand [0,1) (float) when no range set. + Args: + generator: Optional independent random generator for reproducible sampling. + Returns: torch.Tensor with the correct shape, dtype, and value range. @@ -91,6 +98,7 @@ def to_tensor(self) -> Any: raise ValueError(f"Cannot create tensor: shape is None for '{self.name}'") dtype_map = { + "bool": torch.bool, "float32": torch.float32, "float16": torch.float16, "bfloat16": torch.bfloat16, @@ -105,7 +113,11 @@ def to_tensor(self) -> Any: if self.value_range is not None: lo, hi = self.value_range if torch_dtype.is_floating_point: - return torch.rand(concrete_shape, dtype=torch_dtype) * (hi - lo) + lo + return ( + torch.rand(concrete_shape, dtype=torch_dtype, generator=generator) + * (hi - lo) + + lo + ) # bbox uses the HF convention: a [..., 4] tensor of box corners # (x0, y0, x1, y1). We key on the input name because "bbox" is the # canonical HF/Optimum name for 2D box coordinates; a model using a @@ -113,17 +125,21 @@ def to_tensor(self) -> Any: # randint path below. Reorder so x0<=x1 and y0<=y1 to keep each box # well-formed for layout models. if self.name == "bbox" and len(concrete_shape) >= 1 and concrete_shape[-1] == 4: - coords = torch.randint(int(lo), int(hi), concrete_shape, dtype=torch_dtype) + coords = torch.randint( + int(lo), int(hi), concrete_shape, dtype=torch_dtype, generator=generator + ) x0 = torch.minimum(coords[..., 0], coords[..., 2]) y0 = torch.minimum(coords[..., 1], coords[..., 3]) x1 = torch.maximum(coords[..., 0], coords[..., 2]) y1 = torch.maximum(coords[..., 1], coords[..., 3]) return torch.stack((x0, y0, x1, y1), dim=-1) - return torch.randint(int(lo), int(hi), concrete_shape, dtype=torch_dtype) + return torch.randint( + int(lo), int(hi), concrete_shape, dtype=torch_dtype, generator=generator + ) # Fallback: no range info (backward compatible) if torch_dtype.is_floating_point: - return torch.rand(concrete_shape, dtype=torch_dtype) + return torch.rand(concrete_shape, dtype=torch_dtype, generator=generator) return torch.ones(concrete_shape, dtype=torch_dtype) def concrete_shape(self) -> tuple[int, ...]: @@ -211,6 +227,7 @@ def from_dict(cls, data: dict[str, Any]) -> OutputTensorSpec: 5: np.dtype("int16"), # INT16 6: np.dtype("int32"), # INT32 7: np.dtype("int64"), # INT64 + 9: np.dtype("bool"), 10: np.dtype("float16"), # FLOAT16 11: np.dtype("float64"), # DOUBLE 12: np.dtype("uint32"), # UINT32 diff --git a/src/winml/modelkit/quant/passes/static.py b/src/winml/modelkit/quant/passes/static.py index 709c245c7..57e172d44 100644 --- a/src/winml/modelkit/quant/passes/static.py +++ b/src/winml/modelkit/quant/passes/static.py @@ -19,12 +19,49 @@ if TYPE_CHECKING: + from onnx import GraphProto + from ..config import QuantizeResult, WinMLQuantizationConfig logger = logging.getLogger(__name__) +def _restore_external_initializers(original: GraphProto, converted: GraphProto) -> None: + """Restore external weights within their graph and nested attribute scopes.""" + from onnx import external_data_helper + + external_initializers = { + tensor.name: tensor + for tensor in original.initializer + if external_data_helper.uses_external_data(tensor) + } + for tensor in converted.initializer: + source = external_initializers.get(tensor.name) + if source is not None: + if tensor.data_type != source.data_type or tensor.dims != source.dims: + raise ValueError(f"Opset conversion changed external initializer {tensor.name!r}") + tensor.CopyFrom(source) + + subgraphs = { + (tuple(node.output), attribute.name): attribute + for node in original.node + for attribute in node.attribute + if attribute.HasField("g") or attribute.graphs + } + for node in converted.node: + for attribute in node.attribute: + source_attribute = subgraphs.get((tuple(node.output), attribute.name)) + if source_attribute is None: + continue + if attribute.HasField("g"): + _restore_external_initializers(source_attribute.g, attribute.g) + for source_graph, converted_graph in zip( + source_attribute.graphs, attribute.graphs, strict=True + ): + _restore_external_initializers(source_graph, converted_graph) + + def _publish_staged_model(staged_path: Path, output_path: Path) -> None: """Publish a validated staged ONNX model while preserving prior output on failure.""" staged_sidecar = staged_path.parent / f"{staged_path.name}.data" @@ -87,6 +124,7 @@ def run( use_external_data: bool = True, ) -> QuantizeResult: """Apply QDQ calibrated quantization to *model_path*.""" + from onnx import external_data_helper, load_model, version_converter from onnxruntime.quantization import ( CalibrationMethod, QuantType, @@ -161,9 +199,29 @@ def run( ), } + from onnxruntime.quantization.quant_utils import add_pre_process_metadata + + from ...onnx import capture_metadata, load_onnx, restore_metadata, save_onnx + from ..qdq_fix import fix_qdq_dtype_info + + input_model = load_model(model_path, load_external_data=False) + metadata_snapshot = capture_metadata(input_model) + onnx_opset = next( + opset.version for opset in input_model.opset_import if opset.domain in ("", "ai.onnx") + ) + if onnx_opset < 21 and ( + self._config.weight_type in ("uint16", "int16") + or self._config.activation_type in ("uint16", "int16") + ): + logger.info("Converting QDQ input to opset 21 before loading external weights...") + converted_model = version_converter.convert_version(input_model, 21) + _restore_external_initializers(input_model.graph, converted_model.graph) + input_model = converted_model + add_pre_process_metadata(input_model) + logger.info("Generating QDQ config...") qdq_config = get_qdq_config( - model_input=str(model_path), + model_input=input_model, calibration_data_reader=data_reader, weight_type=weight_type, activation_type=activation_type, @@ -174,14 +232,7 @@ def run( extra_options=extra_options, ) - from onnxruntime.quantization.quant_utils import add_pre_process_metadata - - from ...onnx import capture_metadata, load_onnx, restore_metadata, save_onnx - from ..qdq_fix import fix_qdq_dtype_info - - input_model = load_onnx(model_path, validate=False) - metadata_snapshot = capture_metadata(input_model) - add_pre_process_metadata(input_model) + external_data_helper.load_external_data_for_model(input_model, str(model_path.parent)) if use_external_data: qdq_config.use_external_data_format = True diff --git a/tests/unit/datasets/test_random_dataset.py b/tests/unit/datasets/test_random_dataset.py index 696a3d912..fec9e8413 100644 --- a/tests/unit/datasets/test_random_dataset.py +++ b/tests/unit/datasets/test_random_dataset.py @@ -7,6 +7,7 @@ from __future__ import annotations from typing import TYPE_CHECKING +from unittest.mock import patch import numpy as np @@ -60,6 +61,63 @@ def simple_onnx_model(tmp_path: Path) -> Path: class TestRandomDataset: """Tests for RandomDataset class.""" + def test_random_inputs_are_generated_only_when_requested( + self, simple_onnx_model: Path + ) -> None: + from winml.modelkit.datasets import RandomDataset + from winml.modelkit.onnx import InputTensorSpec + + with patch.object( + InputTensorSpec, "to_tensor", autospec=True, side_effect=InputTensorSpec.to_tensor + ) as generate: + dataset = RandomDataset(str(simple_onnx_model), max_samples=10) + assert len(dataset) == 10 + generate.assert_not_called() + + sample = dataset[3] + assert sample["A"].shape == (1, 4) + assert sample["sample_id"].item() == 3 + assert generate.call_count == 1 + + def test_indexed_random_samples_replay_without_changing_global_rng( + self, simple_onnx_model: Path + ) -> None: + import torch + + from winml.modelkit.datasets import RandomDataset + + rng_state = torch.random.get_rng_state().clone() + dataset = RandomDataset(str(simple_onnx_model), max_samples=3, seed=123) + last = dataset[2]["A"] + first = dataset[0]["A"] + replay = dataset[2]["A"] + + torch.testing.assert_close(last, replay) + assert not torch.equal(first, last) + assert torch.equal(torch.random.get_rng_state(), rng_state) + + def test_bool_inputs_remain_boolean_during_calibration(self, tmp_path: Path) -> None: + from winml.modelkit.datasets import DatasetCalibrationReader + from winml.modelkit.onnx import get_io_config + + graph = helper.make_graph( + [helper.make_node("Identity", ["selector"], ["selected"])], + "boolean_input", + [helper.make_tensor_value_info("selector", TensorProto.BOOL, [1, 64])], + [helper.make_tensor_value_info("selected", TensorProto.BOOL, [1, 64])], + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + model_path = tmp_path / "boolean_input.onnx" + onnx.save(model, str(model_path)) + + assert get_io_config(str(model_path))["input_types"] == [np.dtype("bool")] + reader = DatasetCalibrationReader( + model_name="test-model", task="random", max_samples=1, model_path=str(model_path) + ) + sample = reader.get_next() + assert sample is not None + assert sample["selector"].dtype == np.bool_ + def test_random_dataset_with_model_path(self, simple_onnx_model: Path) -> None: """RandomDataset should work when model_path is provided.""" from winml.modelkit.datasets import RandomDataset diff --git a/tests/unit/eval/test_recipes.py b/tests/unit/eval/test_recipes.py index 60c5dec3f..4d22d40b3 100644 --- a/tests/unit/eval/test_recipes.py +++ b/tests/unit/eval/test_recipes.py @@ -245,7 +245,7 @@ def test_resnet50_has_two_single_variants(self, recipes, recipes_dir): recipes_dir, "microsoft/resnet-50", "image-classification" ) precisions = {v.precision for v in variants} - assert {"fp16", "w8a16"} <= precisions + assert precisions == {"fp32", "w8a16"} for v in variants: assert v.is_composite is False @@ -260,7 +260,7 @@ def test_trocr_is_composite(self, recipes, recipes_dir): assert v.is_composite is True assert v.roles == ["encoder", "decoder"] - def test_unlimited_ocr_feature_extraction_has_six_precision_tuples( + def test_unlimited_ocr_feature_extraction_deduplicates_unquantized_recipes( self, recipes, recipes_dir ): if not recipes_dir.is_dir(): @@ -268,15 +268,12 @@ def test_unlimited_ocr_feature_extraction_has_six_precision_tuples( expected = { ("cpu", "cpu"): { - "feature-extraction_fp16_config.json", "feature-extraction_fp32_config.json", }, ("dml", "gpu"): { - "feature-extraction_fp16_config.json", "feature-extraction_fp32_config.json", }, ("openvino", "cpu"): { - "feature-extraction_fp16_config.json", "feature-extraction_fp32_config.json", }, } @@ -290,6 +287,6 @@ def test_unlimited_ocr_feature_extraction_has_six_precision_tuples( device=device, ) - assert [variant.precision for variant in variants] == ["fp16", "fp32"] + assert [variant.precision for variant in variants] == ["fp32"] found_names = {variant.components[0].path.name for variant in variants} assert found_names == expected_names diff --git a/tests/unit/eval/test_run_eval_script.py b/tests/unit/eval/test_run_eval_script.py index d67393f86..96a61c90a 100644 --- a/tests/unit/eval/test_run_eval_script.py +++ b/tests/unit/eval/test_run_eval_script.py @@ -17,8 +17,9 @@ import importlib.util import json import sys +import threading import time -from io import BytesIO +from datetime import datetime from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -26,6 +27,12 @@ import pytest +def test_progress_prefix_includes_local_timestamp(run_eval): + now = datetime(2026, 9, 13, 2, 25, 10) + + assert run_eval._progress_prefix(27, 50, now) == "[2026-09-13 02:25:10] [27/50]" + + def _load_run_eval(): """Load scripts/e2e_eval/run_eval.py as a module.""" repo_root = Path(__file__).resolve().parents[3] @@ -56,8 +63,10 @@ def _load_run_eval(): @pytest.fixture(scope="module") -def run_eval(): - return _load_run_eval() +def run_eval(tmp_path_factory): + module = _load_run_eval() + module._SUBPROCESS_LOG_ROOT = tmp_path_factory.mktemp("e2e-eval-logs") + return module @pytest.fixture(autouse=True) @@ -107,6 +116,145 @@ def test_preserves_cli_and_legacy_boolean_semantics(self, run_eval, raw_targets, assert run_eval._resolve_clean_cache_targets(raw_targets) == expected +class TestEvalTargetAvailability: + @pytest.mark.parametrize( + ("ep", "device"), + [ + ("cpu", "cpu"), + ("openvino", "gpu"), + ("QNNExecutionProvider", "npu"), + ("nv_tensorrt_rtx", "gpu"), + ("migraphx", "gpu"), + ], + ) + def test_available_target_uses_runtime_binding(self, run_eval, ep, device): + from winml.modelkit.session import EPDeviceTarget, WinMLEPRegistry, expand_ep_name + + with patch.object(WinMLEPRegistry, "instance") as registry: + registry.return_value.available_eps.return_value = {expand_ep_name(ep)} + assert run_eval._is_eval_target_available(ep, device) + + registry.return_value.auto_device.assert_called_once_with( + EPDeviceTarget(ep=ep, device=device) + ) + + def test_undiscovered_ep_skips_without_acquiring_packages(self, run_eval, capsys): + from winml.modelkit.session import WinMLEPRegistry, expand_ep_name + + with patch.object(WinMLEPRegistry, "instance") as registry: + registry.return_value.available_eps.return_value = {expand_ep_name("cpu")} + assert not run_eval._is_eval_target_available("openvino", "npu") + + registry.return_value.auto_device.assert_not_called() + output = capsys.readouterr().out + assert "[SKIP]" in output + assert "No locally installed EP" in output + + @pytest.mark.parametrize( + "error_name", ["DeviceNotFound", "WinMLEPNotDiscovered", "WinMLEPRegistrationFailed"] + ) + def test_unavailable_target_reports_skip(self, run_eval, capsys, error_name): + from winml.modelkit import session + + error = getattr(session, error_name)("target unavailable") + with patch.object(session.WinMLEPRegistry, "instance") as registry: + registry.return_value.available_eps.return_value = {session.expand_ep_name("openvino")} + registry.return_value.auto_device.side_effect = error + assert not run_eval._is_eval_target_available("openvino", "npu") + + output = capsys.readouterr().out + assert "[SKIP]" in output + assert "openvino/npu" in output + assert "target unavailable" in output + + def test_unexpected_probe_error_is_not_silently_skipped(self, run_eval): + from winml.modelkit.session import WinMLEPRegistry + + with ( + patch.object(WinMLEPRegistry, "instance", side_effect=RuntimeError("probe failed")), + pytest.raises(RuntimeError, match="probe failed"), + ): + run_eval._is_eval_target_available("openvino", "gpu") + + @pytest.mark.parametrize(("ep", "device"), [("unknown_ep", "gpu"), ("openvino", "tpu")]) + def test_invalid_target_is_not_silently_skipped(self, run_eval, ep, device): + from winml.modelkit.session import WinMLEPRegistry + + with ( + patch.object(WinMLEPRegistry, "instance") as registry, + pytest.raises(ValueError), + ): + run_eval._is_eval_target_available(ep, device) + + registry.assert_not_called() + + @pytest.mark.parametrize( + ("ep", "device"), + [(None, "gpu"), ("auto", "npu"), ("qnn", "auto"), ("qnn", None)], + ) + def test_automatic_axes_keep_existing_policy(self, run_eval, ep, device): + from winml.modelkit.session import WinMLEPRegistry + + with patch.object(WinMLEPRegistry, "instance") as registry: + assert run_eval._is_eval_target_available(ep, device) + + registry.assert_not_called() + + @pytest.mark.parametrize("release", [False, True]) + def test_main_skips_before_loading_models_or_creating_output(self, run_eval, tmp_path, release): + output_dir = tmp_path / "results" + argv = [ + "run_eval.py", "--ep", "openvino", "--device", "npu", + "--output-dir", str(output_dir), + ] + if release: + argv.append("--release") + with ( + patch.object(sys, "argv", argv), + patch.object(run_eval, "_is_eval_target_available", return_value=False) as available, + patch.object(run_eval, "load_registry") as load_registry, + patch.object(run_eval, "load_release_registry") as load_release_registry, + patch.object(run_eval, "save_environment_info") as save_environment, + ): + assert run_eval.main() is None + + available.assert_called_once_with("openvino", "npu") + load_registry.assert_not_called() + load_release_registry.assert_not_called() + save_environment.assert_not_called() + assert not output_dir.exists() + + def test_available_target_continues_to_model_selection(self, run_eval): + with ( + patch.object(sys, "argv", ["run_eval.py", "--ep", "cpu", "--device", "cpu"]), + patch.object(run_eval, "_is_eval_target_available", return_value=True) as available, + patch.object(run_eval, "load_registry", return_value=[]) as load_registry, + pytest.raises(SystemExit) as exc_info, + ): + run_eval.main() + + available.assert_called_once_with("cpu", "cpu") + load_registry.assert_called_once() + assert exc_info.value.code == 1 + + @pytest.mark.parametrize("mode", ["--list", "--list-json", "--build-only", "--update-baseline"]) + def test_auxiliary_modes_do_not_probe_hardware(self, run_eval, tmp_path, mode): + argv = ["run_eval.py", "--ep", "openvino", "--device", "npu", mode] + if mode == "--list-json": + argv.append(str(tmp_path / "list.json")) + with ( + patch.object(sys, "argv", argv), + patch.object(run_eval, "_is_eval_target_available", return_value=False) as available, + patch.object(run_eval, "load_registry", return_value=[]) as load_registry, + pytest.raises(SystemExit) as exc_info, + ): + run_eval.main() + + available.assert_not_called() + load_registry.assert_called_once() + assert exc_info.value.code == 1 + + class TestFailureClassifier: @pytest.fixture(scope="class") def classifier(self, run_eval): @@ -224,9 +372,28 @@ def test_children_race_falls_back_to_platform_kill(self, run_eval): subprocess_run.assert_called_once_with( ["taskkill", "/F", "/T", "/PID", "123"], - capture_output=True, + stdout=run_eval.subprocess.DEVNULL, + stderr=run_eval.subprocess.DEVNULL, + timeout=5, ) + def test_platform_kill_timeout_does_not_block_cleanup(self, run_eval): + import psutil + + parent = MagicMock() + parent.children.side_effect = psutil.NoSuchProcess(pid=123) + + with ( + patch.object(psutil, "Process", return_value=parent), + patch.object(run_eval.platform, "system", return_value="Windows"), + patch.object( + run_eval.subprocess, + "run", + side_effect=run_eval.subprocess.TimeoutExpired("taskkill", 5), + ), + ): + run_eval._kill_process_tree(123) + class TestRunSubprocessTimeouts: _HF_CACHE_ENV_VARS = ( @@ -248,6 +415,19 @@ def _cache_env(cls, run_eval, **overrides): env.update({name: str(value) for name, value in overrides.items()}) return patch.dict(run_eval.os.environ, env, clear=True) + @staticmethod + def _monitor_script(setup: str) -> str: + return "\n".join( + [ + "import sys, time", + "from pathlib import Path", + "sys.path.insert(0, sys.argv[1])", + "import run_eval", + setup, + "run_eval._monitor_hf_downloads(int(sys.argv[2]), Path(sys.argv[3]))", + ] + ) + @staticmethod def _download_script( incomplete: Path, @@ -285,7 +465,7 @@ def _run_download_timeline( The old test allowed only 150 ms for Python startup before a 500-ms deadline, and depended on Windows open_files observing a brief handle. - This clock controls only process wait/handle discovery; cache-root + This clock controls process wait, handle discovery and monitor publication; cache-root resolution, snapshots, ownership matching and budget accounting are real. """ cache_home = tmp_path / "huggingface" if cache_variable == "XDG_CACHE_HOME" else tmp_path @@ -294,7 +474,10 @@ def _run_download_timeline( download_start, download_end = 0.35, 1.35 finish = download_end + after_download observed = [] - proc = MagicMock(pid=123, returncode=0, stdout=BytesIO(), stderr=BytesIO()) + proc = MagicMock(pid=123, returncode=0) + proc.poll.side_effect = lambda: 0 if now >= finish else None + monitor = MagicMock(pid=456) + monitor.poll.return_value = None def open_paths(pid): nonlocal now @@ -309,18 +492,32 @@ def open_paths(pid): incomplete.unlink(missing_ok=True) return set() - def wait(timeout): - nonlocal now - now = min(now + timeout, max(now, finish)) - if now >= finish: - return 0 - raise run_eval.subprocess.TimeoutExpired("controlled-child", timeout) + def start_monitor(pid, env, progress_path): + tracker = run_eval._HfDownloadTracker(env, now) + tracker.bind(pid) + + def wait(timeout): + nonlocal now + now = min(now + timeout, max(now, finish)) + if now >= finish: + return 0 + active = tracker.poll(now) + snapshot = { + "observed_at": now, + "active": active, + "last_progress": tracker.last_progress, + } + progress_path.write_text(json.dumps(snapshot), encoding="utf-8") + raise run_eval.subprocess.TimeoutExpired("controlled-child", timeout) + + proc.wait.side_effect = wait + return monitor - proc.wait.side_effect = wait with ( self._cache_env(run_eval, **{cache_variable: tmp_path}), patch.object(run_eval, "time", SimpleNamespace(perf_counter=lambda: now)), patch.object(run_eval.subprocess, "Popen", return_value=proc), + patch.object(run_eval, "_start_hf_download_monitor", side_effect=start_monitor), patch.object(run_eval, "_process_tree_open_paths", side_effect=open_paths), patch.object(run_eval, "_kill_process_tree") as kill_tree, patch.object(run_eval, "_HF_DOWNLOAD_STALL_TIMEOUT", 2.0), @@ -439,6 +636,250 @@ def test_execution_without_hf_download_uses_original_timeout(self, run_eval): assert result["timeout"] is True assert result["hf_download_stalled"] is False + def test_hf_http_timeouts_match_stall_timeout(self, run_eval): + script = ( + "import os; " + "print(os.environ['HF_HUB_DOWNLOAD_TIMEOUT']); " + "print(os.environ['HF_HUB_ETAG_TIMEOUT'])" + ) + + with patch.object(run_eval, "_HF_DOWNLOAD_STALL_TIMEOUT", 120.0): + result = run_eval._run_subprocess([sys.executable, "-c", script], timeout=5) + + assert result["exit_code"] == 0 + assert result["stdout"].splitlines() == ["120", "120"] + + def test_subprocess_diagnostics_survive_timeout(self, run_eval, tmp_path, capsys): + script = ( + "import sys, time; print('started', flush=True); " + "print('waiting for response', file=sys.stderr, flush=True); time.sleep(5)" + ) + with ( + self._cache_env(run_eval, HF_HOME=tmp_path / "hf"), + patch.object(run_eval, "_SUBPROCESS_LOG_ROOT", tmp_path / "logs", create=True), + patch.object(run_eval, "_SUBPROCESS_HEARTBEAT_INTERVAL", 0.1, create=True), + patch.object(run_eval, "_HF_DOWNLOAD_MONITOR_TIMEOUT", 0.1), + ): + result = run_eval._run_subprocess([sys.executable, "-c", script], timeout=0.8) + + event_paths = list((tmp_path / "logs").glob("*/events.jsonl")) + assert len(event_paths) == 1 + log_dir = event_paths[0].parent + events = [json.loads(line) for line in event_paths[0].read_text().splitlines()] + assert events[0]["event"] == "starting" + assert events[0]["command"] == [sys.executable, "-c", script] + spawned = next(event for event in events if event["event"] == "spawned") + assert spawned["pid"] > 0 + heartbeats = [event for event in events if event["event"] == "heartbeat"] + assert heartbeats + assert heartbeats[-1]["stdout_bytes"] > 0 + assert heartbeats[-1]["stderr_bytes"] > 0 + assert heartbeats[-1]["execution_remaining"] <= 0.8 + assert "output_idle_seconds" in heartbeats[-1] + assert "monitor_state" in heartbeats[-1] + assert any(event["event"] == "timeout" for event in events) + assert any(event["event"] == "cleanup_complete" for event in events) + assert events[-1]["event"] == "exited" + assert events[-1]["timeout"] is True + assert result["timeout"] is True + assert (log_dir / "stdout.log").read_text().strip() == "started" + assert (log_dir / "stderr.log").read_text().strip() == "waiting for response" + console = capsys.readouterr().out + assert str(log_dir) in console + assert "[heartbeat]" in console + assert f"pid={spawned['pid']}" in console + + def test_download_progress_timestamp_includes_scan_time(self, run_eval, tmp_path): + incomplete = tmp_path / "model.incomplete" + tracker = run_eval._HfDownloadTracker({}, now=1.0) + tracker.bind(123) + + with ( + patch.object( + run_eval, "_process_tree_open_paths", + return_value={run_eval._normalized_path(incomplete)}, + ), + patch.object(run_eval, "_snapshot_hf_downloads", return_value={incomplete: (1, 1)}), + patch.object(run_eval.time, "perf_counter", return_value=7.0), + ): + assert tracker.poll(now=2.0) is True + + assert tracker.last_progress == 7.0 + + @pytest.mark.parametrize( + "blocked_function", ["_process_tree_open_paths", "_snapshot_hf_downloads"] + ) + def test_blocked_download_monitor_cannot_disable_execution_timeout( + self, run_eval, tmp_path, blocked_function + ): + monitor_script = self._monitor_script( + "def blocked_scan(*args):\n" + " time.sleep(30)\n" + " return set()\n" + f"run_eval.{blocked_function} = blocked_scan" + ) + monitors = [] + original_start = run_eval._start_hf_download_monitor + + def start_monitor(*args): + monitor = original_start(*args) + monitors.append(monitor) + return monitor + + with ( + self._cache_env(run_eval, HF_HOME=tmp_path), + patch.object(run_eval, "_HF_DOWNLOAD_MONITOR_TIMEOUT", 0.2), + patch.object(run_eval, "_HF_DOWNLOAD_MONITOR_CODE", monitor_script), + patch.object(run_eval, "_start_hf_download_monitor", side_effect=start_monitor), + ): + result = run_eval._run_subprocess( + [sys.executable, "-c", "import time; time.sleep(5)"], timeout=0.2 + ) + + assert result["exit_code"] == -1 + assert result["elapsed"] < 0.8 + assert result["timeout"] is True + assert result["hf_download_stalled"] is False + assert len(monitors) == 1 + assert monitors[0].poll() is not None + + def test_stale_active_download_monitor_cannot_suspend_timeout_forever(self, run_eval, tmp_path): + monitor_script = "\n".join( + [ + "import json, sys, time", + "from pathlib import Path", + "now = time.perf_counter()", + "state = {'observed_at': now, 'active': True, 'last_progress': now}", + "Path(sys.argv[3]).write_text(json.dumps(state), encoding='utf-8')", + "time.sleep(30)", + ] + ) + + with ( + self._cache_env(run_eval, HF_HOME=tmp_path), + patch.object(run_eval, "_HF_DOWNLOAD_MONITOR_TIMEOUT", 0.5), + patch.object(run_eval, "_HF_DOWNLOAD_STALL_TIMEOUT", 5.0), + patch.object(run_eval, "_HF_DOWNLOAD_MONITOR_CODE", monitor_script), + ): + result = run_eval._run_subprocess( + [sys.executable, "-c", "import time; time.sleep(5)"], timeout=0.5 + ) + + assert result["timeout"] is True + assert result["hf_download_stalled"] is False + assert result["elapsed"] < 2.0 + + def test_monitor_start_failure_keeps_execution_timeout(self, run_eval, tmp_path): + with ( + self._cache_env(run_eval, HF_HOME=tmp_path), + patch.object( + run_eval, "_start_hf_download_monitor", side_effect=OSError("unavailable") + ), + ): + result = run_eval._run_subprocess( + [sys.executable, "-c", "import time; time.sleep(5)"], timeout=0.2 + ) + + assert result["timeout"] is True + assert result["elapsed"] < 1.0 + + def test_monitor_retries_temporarily_locked_state_file(self, run_eval, tmp_path): + import psutil + + progress_path = tmp_path / "progress.json" + tracker = MagicMock() + tracker.poll.return_value = True + tracker.last_progress = time.perf_counter() + original_replace = Path.replace + attempts = [] + + def replace_when_unlocked(source, target): + attempts.append(source) + if len(attempts) == 1: + raise PermissionError("state file is being read") + return original_replace(source, target) + + with ( + patch.object(run_eval, "_HfDownloadTracker", return_value=tracker), + patch.object(psutil, "pid_exists", side_effect=[True, True, False]), + patch.object(Path, "replace", replace_when_unlocked), + patch.object(run_eval.time, "sleep"), + ): + run_eval._monitor_hf_downloads(123, progress_path) + + snapshot = json.loads(progress_path.read_text(encoding="utf-8")) + assert snapshot["active"] is True + assert snapshot["last_progress"] == tracker.last_progress + assert tracker.poll.call_count == 2 + + def test_thread_start_cannot_block_subprocess_timeout(self, run_eval, tmp_path): + original_start = threading.Thread.start + script = ( + "import sys, time; sys.stderr.write('x' * 262144); " + "sys.stderr.flush(); time.sleep(5)" + ) + + def delayed_start(thread): + threading.Event().wait(1.0) + return original_start(thread) + + with ( + self._cache_env(run_eval, HF_HOME=tmp_path), + patch.object(threading.Thread, "start", delayed_start), + patch.object(run_eval, "_HF_DOWNLOAD_MONITOR_TIMEOUT", 0.1), + ): + result = run_eval._run_subprocess( + [sys.executable, "-c", script], + timeout=0.2, + ) + + assert result["exit_code"] == -1 + assert result["timeout"] is True + assert result["elapsed"] < 1.0 + + def test_large_stdout_and_stderr_are_captured_without_reader_threads(self, run_eval, tmp_path): + output_size = 262144 + script = ( + "import sys; " + f"sys.stdout.buffer.write(b'o' * {output_size} + b'\\xff'); " + f"sys.stderr.buffer.write(b'e' * {output_size} + b'\\xfe')" + ) + + with ( + self._cache_env(run_eval, HF_HOME=tmp_path), + patch.object(threading.Thread, "start", side_effect=AssertionError("thread startup")), + ): + result = run_eval._run_subprocess([sys.executable, "-c", script], timeout=5) + + assert result["exit_code"] == 0 + assert result["stdout"] == (b"o" * output_size + b"\xff").decode("utf-8", errors="replace") + assert result["stderr"] == (b"e" * output_size + b"\xfe").decode("utf-8", errors="replace") + + def test_inherited_output_handles_do_not_delay_collection(self, run_eval, tmp_path): + descendant_pid_path = tmp_path / "descendant.pid" + script = "\n".join( + [ + "import subprocess, sys", + "from pathlib import Path", + "child = subprocess.Popen(", + " [sys.executable, '-c', 'import time; time.sleep(30)'],", + " stdout=sys.stdout, stderr=sys.stderr,", + ")", + f"Path({str(descendant_pid_path)!r}).write_text(str(child.pid))", + "print('parent completed', flush=True)", + ] + ) + try: + with self._cache_env(run_eval, HF_HOME=tmp_path): + result = run_eval._run_subprocess([sys.executable, "-c", script], timeout=5) + + assert result["exit_code"] == 0 + assert result["stdout"].strip() == "parent completed" + assert result["elapsed"] < 2.0 + finally: + if descendant_pid_path.exists(): + run_eval._kill_process_tree(int(descendant_pid_path.read_text())) + def test_curated_target_models_preserve_existing_priorities(run_eval): testsets_dir = ( @@ -783,6 +1224,48 @@ def test_run_build_uses_composite_onnx_without_subprocess(self, run_eval, tmp_pa mock_subprocess.assert_not_called() +class TestExtractOnnxPath: + def test_rejoins_rich_wrapped_artifact_path(self, run_eval, tmp_path): + artifact = tmp_path / "model-with-a-long-name_model.onnx" + artifact.touch() + path = str(artifact) + split_at = len(path) - 12 + build_proc = { + "stderr": ( + "Existing artifact found:\n" + f"{path[:split_at]}\n" + f"{path[split_at:]}\n" + "Use --rebuild to force rebuild.\n" + ), + "stdout": "", + } + + assert ( + run_eval._extract_onnx_path( + build_proc, + "microsoft/beit-base-patch16-224-pt22k-ft22k", + "image-classification", + ) + == path + ) + + def test_cache_fallback_rejects_multiple_task_candidates( + self, run_eval, tmp_path, monkeypatch + ): + cache_dir = tmp_path / ".cache" / "winml" / "artifacts" / "microsoft_beit" + cache_dir.mkdir(parents=True) + (cache_dir / "imgcls_fp32_model.onnx").touch() + (cache_dir / "imgcls_w8a16_model.onnx").touch() + monkeypatch.setattr(run_eval.Path, "home", lambda: tmp_path) + + assert ( + run_eval._find_cached_model( + "microsoft/beit", {"stdout": "", "stderr": ""}, "image-classification" + ) + is None + ) + + class TestRunBuildNoQuantInjection: """``_run_build`` must append ``--no-quant`` to both winml config and winml build invocations when the EP quantizes internally (``EPS_WITH_INTERNAL_QUANT``). @@ -1130,6 +1613,92 @@ def fake_subprocess(cmd, _timeout): assert all("--precision" not in cmd for cmd in captured) assert build_result["precision"] is None + @pytest.mark.parametrize("precision", [None, "fp16", "fp32", "w8a16", "w8a8"]) + @pytest.mark.parametrize("ep", ["qnn", "vitisai"]) + def test_release_fallback_keeps_requested_precision(self, run_eval, tmp_path, precision, ep): + entry = _entry("release-test/model", "text-classification") + entry.precision = precision + job = run_eval.EvalJob(entry, None, precision_locked=True) + args = argparse.Namespace(ep=ep, device="npu", timeout=300) + captured = [] + + def fake_subprocess(command, timeout): + captured.append(command) + if "config" in command: + (tmp_path / "build_config.json").write_text( + json.dumps({"quant": None}), encoding="utf-8" + ) + return { + "exit_code": 0, + "stdout": "", + "stderr": "", + "elapsed": 0.1, + "command": " ".join(command), + } + + with ( + patch.object(run_eval, "_run_subprocess", side_effect=fake_subprocess), + patch.object(run_eval, "_extract_onnx_path", return_value=str(tmp_path / "model.onnx")), + ): + result, _, _ = run_eval._build_for_job(job, args, tmp_path) + + assert len(captured) == 2 + for command in captured: + if precision is None: + assert "--precision" not in command + else: + assert command[command.index("--precision") + 1] == precision + assert "--no-quant" not in command + assert result["precision"] == precision + + def test_release_default_preserves_case_when_config_resolves_precision( + self, run_eval, tmp_path + ): + from winml.modelkit.config import resolve_quant_compile_config + + entry = _entry("release-test/default", "question-answering") + job = run_eval.EvalJob(entry, None, precision_locked=True) + args = argparse.Namespace(ep="qnn", device="npu", timeout=300) + quant_config, _ = resolve_quant_compile_config( + device=args.device, precision="auto", ep=args.ep + ) + assert quant_config is not None + config_path = tmp_path / "build_config.json" + commands = [] + + def fake_subprocess(command, timeout): + commands.append(command) + if "config" in command: + config_path.write_text( + json.dumps({"quant": quant_config.to_dict()}), encoding="utf-8" + ) + return {"exit_code": 0, "stdout": "", "stderr": ""} + + with ( + patch.object(run_eval, "_run_subprocess", side_effect=fake_subprocess), + patch.object(run_eval, "_extract_onnx_path", return_value=str(tmp_path / "model.onnx")), + ): + result, _, _ = run_eval._build_for_job(job, args, tmp_path) + + assert result["success"] is True + assert result["precision"] is not None + assert result["precision"] == run_eval._precision_from_build_config(config_path) + assert job.precision is None + assert all("--precision" not in command for command in commands) + + def test_release_rejects_inconsistent_precision_metadata(self, run_eval, tmp_path): + entry = _entry("release-test/model", "text-classification") + entry.precision = "fp16" + job = run_eval.EvalJob(entry, None, precision_locked=True) + args = argparse.Namespace(ep="openvino", device="gpu", timeout=300) + with ( + patch.object( + run_eval, "_run_build", return_value={"success": True, "precision": "fp32"} + ), + pytest.raises(ValueError, match="precision mismatch"), + ): + run_eval._build_for_job(job, args, tmp_path) + class TestMergeBackfillResult: """``_merge_backfill_result`` splices accuracy into an existing result. @@ -1823,6 +2392,45 @@ def _make_single_recipe(self, recipes_dir: Path, slug: str, task: str, precision json.dumps({"eval": {"task": task, "dataset": {"path": "x"}}}), encoding="utf-8" ) + @pytest.mark.parametrize("precision", [None, "fp16", "fp32", "w8a16", "w8a8"]) + @pytest.mark.parametrize( + "device,ep", [("gpu", "qnn"), ("cpu", "mlas"), ("npu", "vitisai"), ("npu", "qnn")] + ) + def test_release_runs_only_one_selected_recipe(self, run_eval, tmp_path, precision, device, ep): + entry = _entry("release-test/model", "text-classification") + entry.precision = precision + self._make_single_recipe( + tmp_path, "release-test_model", entry.task, ["fp16", "fp32", "w8a16", "w8a8"] + ) + jobs = run_eval._build_jobs([entry], tmp_path, device, ep=ep, release=True) + assert len(jobs) == 1 + assert jobs[0].precision == precision + assert jobs[0].precision_locked is True + if precision is None: + assert jobs[0].variant is None + else: + assert jobs[0].variant.precision == precision + + @pytest.mark.parametrize("precision", [None, "fp32", "w8a8"]) + def test_release_missing_recipe_does_not_expand_or_substitute( + self, run_eval, tmp_path, precision + ): + entry = _entry("release-test/model", "text-classification") + entry.precision = precision + self._make_single_recipe(tmp_path, "release-test_model", entry.task, ["fp16", "w8a16"]) + jobs = run_eval._build_jobs([entry], tmp_path, "npu", ep="qnn", release=True) + assert len(jobs) == 1 + assert jobs[0].precision == precision + assert jobs[0].variant is None + + def test_release_without_recipes_still_honors_precision(self, run_eval): + entry = _entry("release-test/model", "text-classification") + entry.precision = "w8a8" + jobs = run_eval._build_jobs([entry], None, "gpu", ep="qnn", release=True) + assert len(jobs) == 1 + assert jobs[0].precision == entry.precision + assert jobs[0].precision_locked is True + def test_npu_recipe_expands_to_one_job_per_precision(self, run_eval, tmp_path): self._make_single_recipe( tmp_path, "microsoft_resnet-50", "image-classification", ["fp16", "w8a16"] @@ -2287,6 +2895,608 @@ def test_schema_has_metrics_and_no_baseline(self, run_eval, tmp_path): assert kwargs["trust_remote_code"] is True +class TestReleaseRegistry: + @pytest.fixture + def release_table(self, run_eval, tmp_path): + columns = [ + "intel/OpenVINOExecutionProvider_GPU", + "intel/DmlExecutionProvider_GPU", + "amd/DmlExecutionProvider_GPU", + "nvidia/DmlExecutionProvider_GPU", + "nvidia/TRTRTXARMExecutionProvider_GPU", + "nvidia/TRTRTXExecutionProvider_GPU", + "intel/MLASExecutionProvider_CPU", + "amd/MLASExecutionProvider_CPU", + "nvidia/MLASExecutionProvider_CPU", + "amd/MIGraphExecutionProvider_GPU", + ] + precisions = ["default", "fp16", "fp32", "w8a16", "w8a8"] + entries = [ + run_eval.ModelEntry( + hf_id=f"release-test/model-{index}", + task=f"task-{index}", + model_type=f"type-{index}", + group="test-release", + priority=list(run_eval._PRIORITY_RANK)[index % len(run_eval._PRIORITY_RANK)], + precision="fp32", + dataset_config={"dataset_path": f"dataset-{index}"}, + perf_args=["--warmup", str(index)], + ) + for index in range(len(precisions)) + ] + by_column = { + column: [ + precisions[(index + offset) % len(precisions)] for index in range(len(entries)) + ] + for offset, column in enumerate(columns) + } + rows = [] + for index, entry in enumerate(entries): + rows.append( + { + "hf_id": entry.hf_id, + "task": entry.task, + "priority": entry.priority, + "targets": { + column: {"precision": by_column[column][index]} for column in columns + }, + } + ) + path = tmp_path / "models_release_validation.json" + path.write_text(json.dumps({"schema_version": 1, "models": rows}), encoding="utf-8") + return path, entries, by_column + + @pytest.mark.parametrize("ep", ["OPENVINO", "openvino", "OpenVINOExecutionProvider"]) + def test_target_precision_and_registry_metadata(self, run_eval, release_table, ep): + path, registry, by_column = release_table + loader = sys.modules["utils.registry"].load_release_registry + entries, column = loader(path, registry, ep=ep, device="gpu") + expected = [None if value == "default" else value for value in by_column[column]] + assert [entry.precision for entry in entries] == expected + assert [(entry.hf_id, entry.task) for entry in entries] == [ + (entry.hf_id, entry.task) for entry in registry + ] + for actual, original in zip(entries, registry, strict=True): + assert actual is not original + assert actual.dataset_config == original.dataset_config + assert actual.perf_args == original.perf_args + assert original.precision == "fp32" + + @pytest.mark.parametrize( + ("ep", "device", "machine", "expected_column"), + [ + ("nv_tensorrt_rtx", "gpu", None, "nvidia/TRTRTXExecutionProvider_GPU"), + ("nvtensorrtrtx", "gpu", None, "nvidia/TRTRTXExecutionProvider_GPU"), + ( + "NvTensorRTRTXExecutionProvider", + "GPU", + None, + "nvidia/TRTRTXExecutionProvider_GPU", + ), + ( + "NVTENSORRTRTXEXECUTIONPROVIDER", + "gpu", + None, + "nvidia/TRTRTXExecutionProvider_GPU", + ), + ("TRTRTXExecutionProvider", "gpu", None, "nvidia/TRTRTXExecutionProvider_GPU"), + ("cpu", "cpu", "intel", "intel/MLASExecutionProvider_CPU"), + ("CPUExecutionProvider", "CPU", "amd", "amd/MLASExecutionProvider_CPU"), + ("migraphx", "gpu", None, "amd/MIGraphExecutionProvider_GPU"), + ("MIGraphXExecutionProvider", "gpu", None, "amd/MIGraphExecutionProvider_GPU"), + ], + ) + def test_runtime_names_match_historical_targets( + self, run_eval, release_table, ep, device, machine, expected_column + ): + path, registry, by_column = release_table + entries, column = sys.modules["utils.registry"].load_release_registry( + path, registry, ep=ep, device=device, machine=machine + ) + assert column == expected_column + assert [entry.precision or "default" for entry in entries] == by_column[column] + + def test_tensorrt_does_not_fall_back_to_arm_target(self, run_eval, release_table): + path, registry, _ = release_table + manifest = json.loads(path.read_text(encoding="utf-8")) + for model in manifest["models"]: + del model["targets"]["nvidia/TRTRTXExecutionProvider_GPU"] + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ValueError, match="no target"): + run_eval.load_release_registry(path, registry, ep="nv_tensorrt_rtx", device="gpu") + + def test_legacy_alias_matching_still_requires_correct_device(self, run_eval, release_table): + path, registry, _ = release_table + with pytest.raises(ValueError, match="no target"): + run_eval.load_release_registry(path, registry, ep="nv_tensorrt_rtx", device="npu") + + @pytest.mark.parametrize(("ep", "device"), [("dml", "gpu"), ("cpu", "cpu")]) + def test_ambiguous_machine_requires_selection(self, run_eval, release_table, ep, device): + path, registry, _ = release_table + with pytest.raises(ValueError, match="output-dir"): + sys.modules["utils.registry"].load_release_registry( + path, registry, ep=ep, device=device + ) + + @pytest.mark.parametrize("explicit", [True, False]) + def test_machine_selection(self, run_eval, release_table, tmp_path, explicit): + path, registry, by_column = release_table + entries, column = sys.modules["utils.registry"].load_release_registry( + path, + registry, + ep="dml", + device="gpu", + machine="AMD" if explicit else None, + output_dir=tmp_path / "amd" / "DmlExecutionProvider_GPU" / "results", + ) + assert column == "amd/DmlExecutionProvider_GPU" + assert [entry.precision or "default" for entry in entries] == by_column[column] + + @pytest.mark.parametrize( + ("ep", "device", "folder"), + [("dml", "gpu", "DmlExecutionProvider_GPU"), ("cpu", "cpu", "MLASExecutionProvider_CPU")], + ) + def test_machine_path_is_normalized_and_target_scoped( + self, run_eval, release_table, tmp_path, ep, device, folder + ): + path, registry, by_column = release_table + output = tmp_path / "intel" / "archive" / "amd" / ".." / "NVIDIA" / folder / "results" + entries, column = run_eval.load_release_registry( + path, registry, ep=ep, device=device, output_dir=output + ) + assert column == f"nvidia/{folder}" + assert [entry.precision or "default" for entry in entries] == by_column[column] + + @pytest.mark.parametrize( + "parts", + [ + ("intel", "archive", "results"), + ("intel", "archive", "DmlExecutionProvider_GPU"), + ("intel", "MLASExecutionProvider_CPU", "results"), + ("intel", "DmlExecutionProvider_GPU", "amd", "DmlExecutionProvider_GPU"), + ], + ) + def test_machine_path_rejects_unrelated_or_ambiguous_components( + self, run_eval, release_table, tmp_path, parts + ): + path, registry, _ = release_table + with pytest.raises(ValueError, match="output-dir"): + run_eval.load_release_registry( + path, registry, ep="dml", device="gpu", output_dir=tmp_path.joinpath(*parts) + ) + + def test_cli_ep_wins_over_output_directory_name(self, run_eval, release_table, tmp_path): + path, registry, _ = release_table + _, column = sys.modules["utils.registry"].load_release_registry( + path, + registry, + ep="openvino", + device="gpu", + output_dir=tmp_path / "intel" / "DmlExecutionProvider_GPU" / "results", + ) + assert column == "intel/OpenVINOExecutionProvider_GPU" + + def test_missing_target_is_an_error(self, run_eval, release_table): + path, registry, _ = release_table + with pytest.raises(ValueError, match="no target"): + sys.modules["utils.registry"].load_release_registry( + path, registry, ep="qnn", device="npu" + ) + + def test_unknown_ep_alias_is_not_silently_accepted(self, run_eval, release_table): + path, registry, _ = release_table + with pytest.raises(ValueError, match="no target"): + sys.modules["utils.registry"].load_release_registry( + path, registry, ep="ov", device="gpu" + ) + + def test_missing_registry_entry_keeps_table_model_task(self, run_eval, release_table): + path, registry, _ = release_table + entries, _ = sys.modules["utils.registry"].load_release_registry( + path, [], ep="openvino", device="gpu" + ) + assert [(entry.hf_id, entry.task) for entry in entries] == [ + (entry.hf_id, entry.task) for entry in registry + ] + + @pytest.mark.parametrize("problem", ["duplicate", "precision", "targets", "version", "empty"]) + def test_invalid_manifest_is_rejected(self, run_eval, release_table, problem): + path, registry, _ = release_table + data = json.loads(path.read_text(encoding="utf-8")) + if problem == "duplicate": + data["models"].append(data["models"][-1]) + elif problem == "precision": + next(iter(data["models"][-1]["targets"].values()))["precision"] = "not-a-precision" + elif problem == "targets": + data["models"][-1]["targets"].popitem() + elif problem == "version": + data["schema_version"] += 1 + else: + data["models"] = [] + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError): + sys.modules["utils.registry"].load_release_registry( + path, registry, ep="openvino", device="gpu" + ) + + @pytest.mark.parametrize("json_output", [True, False]) + @pytest.mark.parametrize( + ("ep", "column"), + [ + ("openvino", "intel/OpenVINOExecutionProvider_GPU"), + ("nv_tensorrt_rtx", "nvidia/TRTRTXExecutionProvider_GPU"), + ], + ) + def test_explicit_release_list_uses_manifest( + self, run_eval, release_table, tmp_path, capsys, json_output, ep, column + ): + path, registry, by_column = release_table + listing = tmp_path / "list.json" + argv = ["run_eval.py", "--release", "--ep", ep, "--device", "gpu"] + argv += ["--list-json", str(listing)] if json_output else ["--list"] + with ( + patch.object(sys, "argv", argv), + patch.object(run_eval, "_RELEASE_MANIFEST", path), + patch.object(run_eval, "load_registry", return_value=registry), + patch.object(run_eval, "register_from_registry"), + patch.object(run_eval, "_run_build") as build, + pytest.raises(SystemExit) as exc_info, + ): + run_eval.main() + assert exc_info.value.code == 0 + build.assert_not_called() + if json_output: + values = json.loads(listing.read_text(encoding="utf-8")) + assert {row["hf_id"] for row in values} == {entry.hf_id for entry in registry} + expected = dict( + zip([entry.hf_id for entry in registry], by_column[column], strict=True) + ) + for row in values: + assert (row["precision"] or "default") == expected[row["hf_id"]] + assert row["release_target"] == column + else: + output = capsys.readouterr().out + assert column in output + for entry, precision in zip(registry, by_column[column], strict=True): + assert entry.hf_id in output + assert f"[precision={precision}]" in output + + @pytest.mark.parametrize("registered", [True, False]) + def test_single_model_preserves_legacy_selection( + self, run_eval, release_table, tmp_path, registered + ): + _, registry, _ = release_table + listing = tmp_path / "filtered.json" + custom_registry = tmp_path / "custom.json" + hf_id = registry[-1].hf_id if registered else "release-test/not-selected" + task = registry[-1].task if registered else "text-classification" + argv = [ + "run_eval.py", + "--registry", + str(custom_registry), + "--hf-model", + hf_id, + "--task", + task, + "--list-json", + str(listing), + ] + with ( + patch.object(sys, "argv", argv), + patch.object(run_eval, "load_registry", return_value=registry) as loader, + patch.object(run_eval, "load_release_registry") as release_loader, + patch.object(run_eval, "register_from_registry"), + pytest.raises(SystemExit) as exc_info, + ): + run_eval.main() + assert exc_info.value.code == 0 + loader.assert_called_once_with(custom_registry) + release_loader.assert_not_called() + values = json.loads(listing.read_text(encoding="utf-8")) + assert [(row["hf_id"], row["task"]) for row in values] == [(hf_id, task)] + assert all("release_target" not in row and "precision" not in row for row in values) + + @pytest.mark.parametrize( + ("mode", "handler_name"), + [("--build-only", "_run_build_only"), ("--update-baseline", "_run_update_baseline")], + ) + def test_auxiliary_modes_preserve_automatic_target( + self, run_eval, release_table, mode, handler_name + ): + _, registry, _ = release_table + with ( + patch.object(sys, "argv", ["run_eval.py", mode]), + patch.object(run_eval, "load_registry", return_value=registry), + patch.object(run_eval, "load_release_registry") as release_loader, + patch.object(run_eval, "_resolve_eval_target") as resolve_target, + patch.object(run_eval, "register_from_registry"), + patch.object(run_eval, handler_name) as handler, + ): + run_eval.main() + release_loader.assert_not_called() + resolve_target.assert_not_called() + handler.assert_called_once() + entries, args = handler.call_args.args + assert {entry.hf_id for entry in entries} == {entry.hf_id for entry in registry} + assert args.ep is None + assert args.device == "auto" + + @pytest.mark.parametrize("explicit", [False, True]) + def test_priority_filtering_preserves_legacy_registry( + self, run_eval, release_table, tmp_path, explicit + ): + _, registry, _ = release_table + listing = tmp_path / "legacy.json" + custom_registry = tmp_path / "custom.json" + priorities = list(run_eval._PRIORITY_RANK) + if explicit: + priorities = priorities[:2] + argv = ["run_eval.py", "--registry", str(custom_registry), "--list-json", str(listing)] + if explicit: + argv += ["--priority", *priorities] + with ( + patch.object(sys, "argv", argv), + patch.object(run_eval, "load_registry", return_value=registry) as loader, + patch.object(run_eval, "load_release_registry") as release_loader, + patch.object(run_eval, "register_from_registry"), + pytest.raises(SystemExit) as exc_info, + ): + run_eval.main() + assert exc_info.value.code == 0 + loader.assert_called_once_with(custom_registry) + release_loader.assert_not_called() + values = json.loads(listing.read_text(encoding="utf-8")) + assert {row["hf_id"] for row in values} == { + entry.hf_id for entry in registry if entry.priority in priorities + } + + def test_release_list_continue_uses_precision_suffix(self, run_eval, release_table, tmp_path): + path, registry, by_column = release_table + column = "intel/OpenVINOExecutionProvider_GPU" + labels = by_column[column] + entry = registry[labels.index("fp16")] + output = tmp_path / "results" + result_path = ( + run_eval.model_result_dir(output, entry.hf_id, entry.task, "fp16") / "eval_result.json" + ) + result_path.parent.mkdir(parents=True) + result_path.write_text(json.dumps({"perf": {"passed": True}}), encoding="utf-8") + listing = tmp_path / "pending.json" + argv = [ + "run_eval.py", + "--release", + "--ep", + "openvino", + "--device", + "gpu", + "--output-dir", + str(output), + "--continue", + "--list-json", + str(listing), + ] + with ( + patch.object(sys, "argv", argv), + patch.object(run_eval, "_RELEASE_MANIFEST", path), + patch.object(run_eval, "load_registry", return_value=registry), + patch.object(run_eval, "register_from_registry"), + pytest.raises(SystemExit) as exc_info, + ): + run_eval.main() + assert exc_info.value.code == 0 + values = json.loads(listing.read_text(encoding="utf-8")) + assert {row["hf_id"] for row in values} == {item.hf_id for item in registry} - {entry.hf_id} + + +def test_checked_in_recipe_filenames_match_quant_config(run_eval): + recipes_dir = Path(run_eval.__file__).resolve().parents[2] / "examples" / "recipes" + recipe_module = sys.modules["utils.recipes"] + mismatches = [] + config_paths = sorted(recipes_dir.rglob("*_config*.json")) + assert config_paths + for config_path in config_paths: + config = json.loads(config_path.read_text(encoding="utf-8")) + expected = ( + "fp32" if config["quant"] is None + else run_eval._precision_from_build_config(config_path) + ) + assert expected is not None, f"Unknown quant configuration: {config_path}" + group, _ = recipe_module.split_config_stem(config_path) + _, precision = recipe_module.split_task_precision(group) + if precision != expected: + mismatches.append(f"{config_path.relative_to(recipes_dir)}: {precision} != {expected}") + assert not mismatches, "\n".join(mismatches) + + +def test_release_keeps_renamed_recipe_configuration(run_eval, tmp_path): + testsets = Path(run_eval.__file__).parent / "testsets" + registry = run_eval.load_registry(testsets / "models_all.json") + entries, _ = run_eval.load_release_registry( + run_eval._RELEASE_MANIFEST, registry, ep="openvino", device="gpu" + ) + entry = next(entry for entry in entries if entry.hf_id == "microsoft/resnet-50") + jobs = run_eval._build_jobs( + [entry], testsets.parents[2] / "examples" / "recipes", + "gpu", ep="openvino", release=True, + ) + (job,) = jobs + assert job.precision == "fp32" + assert job.variant is not None + for component in job.variant.components: + config = json.loads(component.path.read_text(encoding="utf-8")) + assert config["quant"] is None + assert component.path.name == "image-classification_fp32_config.json" + + args = argparse.Namespace(ep="openvino", device="gpu", timeout=300) + with ( + patch.object( + run_eval, "_run_subprocess", + return_value={"exit_code": 0, "stdout": "", "stderr": ""}, + ) as subprocess_call, + patch.object(run_eval, "_extract_onnx_path", return_value=str(tmp_path / "model.onnx")), + ): + result, meta_config, _ = run_eval._build_for_job(job, args, tmp_path) + + assert result["success"] is True + assert result["precision"] == job.precision + assert meta_config == job.variant.components[0].path + for call in subprocess_call.call_args_list: + command = call.args[0] + assert command[command.index("-c") + 1] == str(job.variant.components[0].path) + assert "--precision" not in command + assert "--device" not in command + + +def test_checked_in_release_json_matches_markdown_and_job_plans(run_eval): + testsets = Path(run_eval.__file__).parent / "testsets" + manifest_path = testsets / "models_release_validation.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + rows = [ + [cell.strip() for cell in line.strip("|").split("|")] + for line in manifest_path.with_suffix(".md").read_text(encoding="utf-8").splitlines() + if line.startswith("|") + ] + headers, _, *data = rows + assert len(data) == len(manifest["models"]) + targets = list(manifest["models"][0]["targets"]) + assert set(headers[11:]) == set(targets) + assert len({model["hf_id"] for model in manifest["models"]}) == len(data) + for cells, model in zip(data, manifest["models"], strict=True): + assert cells[headers.index("Model")].startswith(f"[{model['hf_id']}](") + assert cells[headers.index("Task")] == model["task"] + assert cells[headers.index("Priority")] == model["priority"] + for target, values in model["targets"].items(): + assert set(values) == {"precision"} + cell = cells[headers.index(target)] + label, _link = cell[1:-1].split("](", 1) + assert label == values["precision"] + + registry = run_eval.load_registry(testsets / "models_all.json") + recipes = testsets.parents[2] / "examples" / "recipes" + for target in targets: + machine, ep_device = target.split("/", 1) + ep, device = ep_device.rsplit("_", 1) + entries, matched = run_eval.load_release_registry( + manifest_path, registry, ep=ep, device=device.lower(), machine=machine + ) + assert matched == target + expected = { + (model["hf_id"], model["task"]): model["targets"][target]["precision"] + for model in manifest["models"] + } + jobs = run_eval._build_jobs(entries, recipes, device.lower(), ep=ep, release=True) + assert len(jobs) == len(expected) + assert { + (job.entry.hf_id, job.entry.task): job.precision or "default" for job in jobs + } == expected + assert all(job.precision_locked for job in jobs) + + +class TestReleaseArgumentParsing: + @pytest.mark.parametrize("values", [[], ["--release"]]) + def test_priority_defaults_are_unchanged(self, run_eval, values): + with patch.object(sys, "argv", ["run_eval.py", *values]): + args = run_eval.parse_args() + assert args.priority == list(run_eval._PRIORITY_RANK) + assert args.release is bool(values) + + def test_explicit_priorities_remain_combinable(self, run_eval): + priorities = list(run_eval._PRIORITY_RANK) + with patch.object(sys, "argv", ["run_eval.py", "--priority", *priorities]): + assert run_eval.parse_args().priority == priorities + + def test_repeated_priorities_keep_last_value(self, run_eval): + with patch.object(sys, "argv", ["run_eval.py", "--priority", "P0", "--priority", "P1"]): + assert run_eval.parse_args().priority == ["P1"] + + def test_legacy_selection_options_remain_combinable(self, run_eval): + argv = [ + "run_eval.py", + "--priority", + "P0", + "--hf-model", + "release-test/model", + "--registry", + "custom.json", + "--task", + "text-classification", + "--group", + "test", + "--model-type", + "bert", + ] + with patch.object(sys, "argv", argv): + args = run_eval.parse_args() + assert args.release is False + assert args.priority == ["P0"] + assert args.hf_model == "release-test/model" + assert args.registry == Path("custom.json") + + @pytest.mark.parametrize( + "values", + [ + ["--priority", "P0", "P1", "P2", "P3"], + ["--priority=P0"], + ["--pri", "P0"], + ["--hf-model", "release-test/model"], + ["--registry", "custom.json"], + ["--task", "text-classification"], + ["--group", "test"], + ["--model-type", "bert"], + ["--build-only"], + ["--update-baseline"], + ], + ) + @pytest.mark.parametrize("release_first", [True, False]) + def test_release_rejects_explicit_conflicts(self, run_eval, capsys, values, release_first): + options = ["--release", *values] if release_first else [*values, "--release"] + with ( + patch.object(sys, "argv", ["run_eval.py", *options]), + pytest.raises(SystemExit) as exc_info, + ): + run_eval.parse_args() + assert exc_info.value.code == 2 + assert "--release cannot be combined" in capsys.readouterr().err + + def test_release_accepts_execution_controls(self, run_eval, tmp_path): + argv = [ + "run_eval.py", + "--release", + "--ep", + "dml", + "--device", + "gpu", + "--eval-type", + "both", + "--output-dir", + str(tmp_path / "nvidia"), + "--continue", + "--retry-failed", + "HF_FETCH_FAIL", + "--timeout", + "1200", + "--hf-download-stall-timeout", + "900", + "--clean-cache", + "winml", + "--list", + ] + with patch.object(sys, "argv", argv): + args = run_eval.parse_args() + assert args.release is True + assert args.continue_run is True + assert args.retry_failed == ["HF_FETCH_FAIL"] + assert args.timeout == 1200 + + def test_release_is_not_a_priority(self, run_eval): + with ( + patch.object(sys, "argv", ["run_eval.py", "--priority", "release"]), + pytest.raises(SystemExit) as exc_info, + ): + run_eval.parse_args() + assert exc_info.value.code == 2 + + class TestRetryFailedArgumentParsing: def test_accepts_and_normalizes_known_types(self, run_eval): argv = ["run_eval.py", "--retry-failed", "export_fail", "HF_FETCH_FAIL"] diff --git a/tests/unit/export/test_htp_exporter_attention_compat.py b/tests/unit/export/test_htp_exporter_attention_compat.py index 94f48a814..036254de5 100644 --- a/tests/unit/export/test_htp_exporter_attention_compat.py +++ b/tests/unit/export/test_htp_exporter_attention_compat.py @@ -7,7 +7,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest import torch @@ -360,3 +360,27 @@ def test_export_context_restores_sdpa_after_failure() -> None: assert torch.nn.functional.scaled_dot_product_attention is original_sdpa assert model.config._attn_implementation == "sdpa" + + +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64, torch.bool]) +@pytest.mark.parametrize("resolve_specs", [False, True]) +def test_io_resolution_preserves_generator_mask_dtype( + dtype: torch.dtype, resolve_specs: bool +) -> None: + from winml.modelkit.export import generate_dummy_inputs, resolve_io_specs + + mask = torch.randint(0, 2, (1, 8)).to(dtype) + onnx_config = MagicMock() + onnx_config.inputs = {"attention_mask": {0: "batch", 1: "sequence"}} + onnx_config.outputs = {"output": {0: "batch"}} + onnx_config.generate_dummy_inputs.return_value = {"attention_mask": mask} + hf_config = MagicMock(max_position_embeddings=8) + + with patch("winml.modelkit.export.io._get_onnx_config", return_value=onnx_config): + if resolve_specs: + specs = resolve_io_specs("fake", "feature-extraction", hf_config) + assert specs["input_dtypes"] == [str(dtype).removeprefix("torch.")] + else: + inputs = generate_dummy_inputs("fake", "feature-extraction", hf_config) + assert inputs["attention_mask"].dtype == dtype + torch.testing.assert_close(inputs["attention_mask"], mask) diff --git a/tests/unit/export/test_pytorch_export.py b/tests/unit/export/test_pytorch_export.py index bd036cfe9..7e245f3a0 100644 --- a/tests/unit/export/test_pytorch_export.py +++ b/tests/unit/export/test_pytorch_export.py @@ -145,6 +145,16 @@ def test_int64_tensor(self) -> None: assert t.dtype == torch.int64 assert (t == 1).all() + @pytest.mark.parametrize("value_range", [None, (0, 2)]) + def test_bool_tensor_preserves_dtype(self, value_range) -> None: + spec = InputTensorSpec( + name="selector", dtype="bool", shape=(1, 64), value_range=value_range + ) + tensor = spec.to_tensor() + assert tensor.shape == spec.shape + assert tensor.dtype == torch.bool + assert torch.all((tensor == 0) | (tensor == 1)) + def test_bbox_tensor_generates_ordered_coordinates_within_range(self) -> None: spec = InputTensorSpec( name="bbox", diff --git a/tests/unit/recipes/test_qnn_recipes.py b/tests/unit/recipes/test_qnn_recipes.py index f04120fa7..7365cffda 100644 --- a/tests/unit/recipes/test_qnn_recipes.py +++ b/tests/unit/recipes/test_qnn_recipes.py @@ -49,7 +49,7 @@ / "microsoft_swinv2-tiny-patch4-window16-256" / "qnn" / "npu" - / "image-classification_fp16_config.json", + / "image-classification_fp32_config.json", "loader_task": "image-classification", "optim_key": "matmul_transpose_fusion", "optim_value": True, diff --git a/tests/unit/test_quant_passes.py b/tests/unit/test_quant_passes.py index 862bf4eea..81e8802d6 100644 --- a/tests/unit/test_quant_passes.py +++ b/tests/unit/test_quant_passes.py @@ -28,6 +28,8 @@ if TYPE_CHECKING: from pathlib import Path + from onnx import GraphProto + # --------------------------------------------------------------------------- # Helpers @@ -351,7 +353,7 @@ def test_retries_without_shape_inference_when_proto_serialization_fails( ) -> None: """Large external-data models can exceed protobuf's in-memory serialize limit.""" calls: list[dict] = [] - model = SimpleNamespace(graph=SimpleNamespace(initializer=[], node=[])) + model = ModelProto() def fake_convert(model_arg, **kwargs): calls.append(kwargs) @@ -756,6 +758,178 @@ def test_rejects_directory_destinations_without_mutation( class TestStaticPassQuantizationRegionHints: + def test_external_16bit_opset_conversion_precedes_weight_loading( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from onnx import external_data_helper, version_converter + from onnxruntime.quantization import CalibrationDataReader + + class Reader(CalibrationDataReader): + def get_next(self) -> dict[str, np.ndarray] | None: + return next(self._iterator, None) + + def __init__(self) -> None: + self._iterator = iter( + [{"input": np.random.RandomState(7).randn(1, 4, 3).astype(np.float32)}] + ) + + model_path = tmp_path / "input.onnx" + output_path = tmp_path / "quantized.onnx" + save( + _make_static_grouped_conv_model(), + model_path, + save_as_external_data=True, + all_tensors_to_one_file=True, + location="input.onnx.data", + size_threshold=128, + ) + convert_version = version_converter.convert_version + conversions = [] + + def convert_compact_model(model: ModelProto, target_version: int) -> ModelProto: + external_weights = [ + tensor + for tensor in model.graph.initializer + if external_data_helper.uses_external_data(tensor) + ] + assert external_weights + assert all(not tensor.HasField("raw_data") for tensor in external_weights) + conversions.append(target_version) + return convert_version(model, target_version) + + monkeypatch.setattr(version_converter, "convert_version", convert_compact_model) + result = StaticPass( + WinMLQuantizationConfig( + mode="static", + calibration_data=Reader(), + activation_type="uint16", + weight_type="uint8", + per_channel=False, + ) + ).run(model_path, output_path) + + assert result.success + assert conversions == [21] + checker.check_model(str(output_path), full_check=True) + quantized = load(output_path) + qdq_nodes = [ + node for node in quantized.graph.node + if node.op_type in {"QuantizeLinear", "DequantizeLinear"} + ] + assert qdq_nodes + assert all(node.domain == "" for node in qdq_nodes) + + @pytest.mark.parametrize("depth", [1, 2]) + def test_external_subgraph_weights_preserve_scope( + self, + tmp_path: Path, + depth: int, + ) -> None: + from onnxruntime import InferenceSession + from onnxruntime.quantization import CalibrationDataReader + + generator = np.random.default_rng(7) + + def make_branch(level: int) -> GraphProto: + weights = numpy_helper.from_array( + generator.normal(size=(2, 2)).astype(np.float32), "weight" + ) + nodes = [ + helper.make_node( + "MatMul", ["input", "weight"], + ["product" if level > 1 else "branch_output"], + ) + ] + if level > 1: + nodes.extend([ + helper.make_node( + "If", ["condition"], ["nested_output"], + then_branch=make_branch(level - 1), + else_branch=make_branch(level - 1), + ), + helper.make_node("Add", ["product", "nested_output"], ["branch_output"]), + ]) + return helper.make_graph( + nodes, "branch", [], + [helper.make_tensor_value_info("branch_output", TensorProto.FLOAT, [1, 2])], + [weights], + ) + + graph = helper.make_graph( + [ + helper.make_node("MatMul", ["input", "weight"], ["aux_output"]), + helper.make_node( + "If", ["condition"], ["output"], + then_branch=make_branch(depth), else_branch=make_branch(depth), + ), + ], + "external_subgraphs", + [ + helper.make_tensor_value_info("input", TensorProto.FLOAT, [1, 2]), + helper.make_tensor_value_info("condition", TensorProto.BOOL, []), + ], + [ + helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, 2]), + helper.make_tensor_value_info("aux_output", TensorProto.FLOAT, [1, 2]), + ], + [numpy_helper.from_array(generator.normal(size=(2, 2)).astype(np.float32), "weight")], + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + model_path = tmp_path / "input.onnx" + output_path = tmp_path / "quantized.onnx" + save( + model, model_path, save_as_external_data=True, + all_tensors_to_one_file=True, location="input.onnx.data", size_threshold=0, + ) + checker.check_model(str(model_path), full_check=True) + samples = [ + {"input": np.ones((1, 2), dtype=np.float32), "condition": np.array(condition)} + for condition in (True, False) + ] + + class Reader(CalibrationDataReader): + def __init__(self) -> None: + self._iterator = iter(samples) + + def get_next(self) -> dict[str, np.ndarray] | None: + return next(self._iterator, None) + + result = StaticPass( + WinMLQuantizationConfig( + mode="static", calibration_data=Reader(), + activation_type="uint16", weight_type="uint8", per_channel=False, + ) + ).run(model_path, output_path) + + assert result.success + checker.check_model(str(output_path), full_check=True) + + def subgraph_weights(graph: GraphProto) -> list[np.ndarray]: + weights = [] + for node in graph.node: + for attribute in sorted(node.attribute, key=lambda attribute: attribute.name): + if attribute.HasField("g"): + weights.extend( + numpy_helper.to_array(tensor) for tensor in attribute.g.initializer + ) + weights.extend(subgraph_weights(attribute.g)) + return weights + + original_weights = subgraph_weights(load(model_path).graph) + restored_weights = subgraph_weights(load(output_path).graph) + assert original_weights + for original_weight, restored_weight in zip( + original_weights, restored_weights, strict=True + ): + np.testing.assert_array_equal(restored_weight, original_weight) + quantized = InferenceSession(str(output_path), providers=["CPUExecutionProvider"]) + for sample in samples: + output = quantized.run(["output"], sample)[0] + assert output.shape == sample["input"].shape + assert np.isfinite(output).all() + def test_restores_model_only_hint_and_canonicalizes_branch_outputs( self, tmp_path: Path, diff --git a/tests/unit/test_quantizer.py b/tests/unit/test_quantizer.py index 108fdcd32..b01b620e9 100644 --- a/tests/unit/test_quantizer.py +++ b/tests/unit/test_quantizer.py @@ -112,6 +112,8 @@ def test_quantize_onnx_publishes_relative_output_and_replaces_only_exact_sidecar monkeypatch: pytest.MonkeyPatch, ) -> None: """Publishing should replace only the exact output model and sidecar.""" + from onnx import load_model + monkeypatch.chdir(tmp_path) model_path = tmp_path / "model.onnx" _write_minimal_onnx_model(model_path) @@ -123,10 +125,10 @@ def test_quantize_onnx_publishes_relative_output_and_replaces_only_exact_sidecar # The quantizer hands the in-memory input model (not the path) to ORT so it # can tag it as pre-processed without mutating the user's input file. - input_model = SimpleNamespace() + input_model = load_model(model_path) def fake_quantize(*, model_input, model_output: str, quant_config) -> None: - assert model_input is input_model + assert model_input == input_model staged_output = Path(model_output) assert staged_output.is_absolute() assert staged_output.name == output_path.name @@ -143,7 +145,7 @@ def fake_quantize(*, model_input, model_output: str, quant_config) -> None: quantized_model = SimpleNamespace( graph=SimpleNamespace(node=[SimpleNamespace(op_type="QuantizeLinear")]) ) - load_results = [input_model, quantized_model] + load_results = [quantized_model] fake_onnx_module.capture_metadata = lambda _model: SimpleNamespace( model_prop_count=0, node_count=0, @@ -174,6 +176,7 @@ def fake_quantize(*, model_input, model_output: str, quant_config) -> None: assert result.success is True assert output_path.exists() + assert load_model(model_path) == input_model assert not exact_sidecar.exists() assert extra_suffix_sidecar.exists()