feat(eval): add translation evaluation and refresh opus-mt-en-ru recipes

examples/recipes/Helsinki-NLP_opus-mt-en-ru/cpu/cpu/translation_fp16_decoder_config.json

@@ -36,7 +36,10 @@
       {"name": "present_3_key"}, {"name": "present_3_value"},
       {"name": "present_4_key"}, {"name": "present_4_value"},
       {"name": "present_5_key"}, {"name": "present_5_value"}
-    ]
+    ],
+    "compatibility": {
+      "transformers_attention": "eager"
+    }
   },
   "optim": {
     "clamp_constant_values": true,

examples/recipes/Helsinki-NLP_opus-mt-en-ru/cpu/cpu/translation_fp16_encoder_config.json

@@ -13,7 +13,10 @@
       {"name": "input_ids", "dtype": "int32", "shape": [1, 512], "value_range": [0, 62518]},
       {"name": "attention_mask", "dtype": "int32", "shape": [1, 512], "value_range": [0, 2]}
     ],
-    "output_tensors": [{"name": "encoder_hidden_states"}]
+    "output_tensors": [{"name": "encoder_hidden_states"}],
+    "compatibility": {
+      "transformers_attention": "eager"
+    }
   },
   "optim": {
     "clamp_constant_values": true,

examples/recipes/Helsinki-NLP_opus-mt-en-ru/cpu/cpu/translation_fp32_decoder_config.json

@@ -36,7 +36,10 @@
       {"name": "present_3_key"}, {"name": "present_3_value"},
       {"name": "present_4_key"}, {"name": "present_4_value"},
       {"name": "present_5_key"}, {"name": "present_5_value"}
-    ]
+    ],
+    "compatibility": {
+      "transformers_attention": "eager"
+    }
   },
   "optim": {
     "clamp_constant_values": true,

examples/recipes/Helsinki-NLP_opus-mt-en-ru/cpu/cpu/translation_fp32_encoder_config.json

@@ -13,7 +13,10 @@
       {"name": "input_ids", "dtype": "int32", "shape": [1, 512], "value_range": [0, 62518]},
       {"name": "attention_mask", "dtype": "int32", "shape": [1, 512], "value_range": [0, 2]}
     ],
-    "output_tensors": [{"name": "encoder_hidden_states"}]
+    "output_tensors": [{"name": "encoder_hidden_states"}],
+    "compatibility": {
+      "transformers_attention": "eager"
+    }
   },
   "optim": {
     "clamp_constant_values": true,

src/winml/modelkit/eval/__init__.py

@@ -41,6 +41,7 @@
     from .text_classification_evaluator import WinMLTextClassificationEvaluator
     from .text_generation_evaluator import WinMLTextGenerationEvaluator
     from .token_classification_evaluator import WinMLTokenClassificationEvaluator
+    from .translation_evaluator import WinMLTranslationEvaluator
     from .zero_shot_classification_evaluator import WinMLZeroShotClassificationEvaluator
     from .zero_shot_image_classification_evaluator import WinMLZeroShotImageClassificationEvaluator
 
@@ -73,6 +74,7 @@
     "WinMLTokenClassificationEvaluator": (
         ".token_classification_evaluator:WinMLTokenClassificationEvaluator"
     ),
+    "WinMLTranslationEvaluator": ".translation_evaluator:WinMLTranslationEvaluator",
     "WinMLZeroShotClassificationEvaluator": (
         ".zero_shot_classification_evaluator:WinMLZeroShotClassificationEvaluator"
     ),
@@ -140,6 +142,7 @@ def __dir__() -> list[str]:
     "WinMLTextClassificationEvaluator",
     "WinMLTextGenerationEvaluator",
     "WinMLTokenClassificationEvaluator",
+    "WinMLTranslationEvaluator",
     "WinMLZeroShotClassificationEvaluator",
     "WinMLZeroShotImageClassificationEvaluator",
     "evaluate",

src/winml/modelkit/eval/evaluate.py

@@ -84,6 +84,8 @@ def _select_model_loader(config: WinMLEvaluationConfig) -> _ModelLoaderKind:
         "winml.modelkit.eval.image_feature_extraction_evaluator:WinMLImageFeatureExtractionEvaluator",
     "image-to-text":
         "winml.modelkit.eval.image_to_text_evaluator:WinMLImageToTextEvaluator",
+    "translation":
+        "winml.modelkit.eval.translation_evaluator:WinMLTranslationEvaluator",
     "fill-mask":
         "winml.modelkit.eval.fill_mask_evaluator:WinMLFillMaskEvaluator",
     "zero-shot-classification":

src/winml/modelkit/eval/metrics/translation.py

@@ -0,0 +1,53 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+
+"""Corpus-level machine-translation metrics."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class TranslationMetric:
+    """Aggregate corpus SacreBLEU-13a and chrF2 on a 0-100 scale."""
+
+    def __init__(self) -> None:
+        self._predictions: list[str] = []
+        self._references: list[list[str]] = []
+
+    def update(self, prediction: str, references: str | list[str]) -> None:
+        """Record one prediction and one or more non-empty references."""
+        refs = [references] if isinstance(references, str) else references
+        cleaned = [reference.strip() for reference in refs if reference and reference.strip()]
+        if not cleaned:
+            raise ValueError("at least one non-empty translation reference is required")
+        self._predictions.append((prediction or "").strip())
+        self._references.append(cleaned)
+
+    def compute(self) -> dict[str, Any]:
+        """Return corpus scores with names that state variant and scale."""
+        if not self._predictions:
+            return {
+                "sacrebleu_13a_0_100": None,
+                "chrf2_0_100": None,
+                "n_samples": 0,
+            }
+
+        from torchmetrics.text import CHRFScore, SacreBLEUScore
+
+        sacrebleu = SacreBLEUScore(tokenize="13a")(
+            self._predictions,
+            self._references,
+        )
+        chrf = CHRFScore(n_char_order=6, n_word_order=0, beta=2.0)(
+            self._predictions,
+            self._references,
+        )
+
+        return {
+            "sacrebleu_13a_0_100": round(float(sacrebleu) * 100, 4),
+            "chrf2_0_100": round(float(chrf) * 100, 4),
+            "n_samples": len(self._predictions),
+        }

src/winml/modelkit/eval/translation_evaluator.py

@@ -0,0 +1,213 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+
+"""Evaluator for text-to-text machine translation models."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any
+
+from .base_evaluator import WinMLEvaluator
+
+
+if TYPE_CHECKING:
+    from datasets import Dataset
+
+    from ..models.winml.composite_model import WinMLCompositeModel
+    from .config import DatasetConfig, WinMLEvaluationConfig
+
+
+logger = logging.getLogger(__name__)
+
+
+def _positive_int(mapping: Mapping[str, str], name: str, default: int) -> int:
+    from ..utils.eval_utils import DatasetValidationError
+
+    raw_value = mapping.get(name, str(default))
+    try:
+        value = int(raw_value)
+    except (TypeError, ValueError) as error:
+        raise DatasetValidationError(f"{name} must be a positive integer") from error
+    if value < 1:
+        raise DatasetValidationError(f"{name} must be a positive integer")
+    return value
+
+
+class WinMLTranslationEvaluator(WinMLEvaluator):
+    """Evaluate translations with bounded generation and corpus metrics."""
+
+    def __init__(
+        self,
+        config: WinMLEvaluationConfig,
+        model: WinMLCompositeModel,
+    ) -> None:
+        from ..utils.eval_utils import DatasetValidationError, get_default
+
+        mapping = config.dataset.columns_mapping
+        self._source_col = mapping.get(
+            "source_column",
+            get_default("translation", "source_column") or "translation",
+        )
+        self._reference_col = mapping.get(
+            "reference_column",
+            get_default("translation", "reference_column") or "translation",
+        )
+        self._source_lang = mapping.get("source_lang")
+        self._target_lang = mapping.get("target_lang")
+        self._tokenizer_source_lang = mapping.get("tokenizer_source_lang", self._source_lang)
+        self._tokenizer_target_lang = mapping.get("tokenizer_target_lang", self._target_lang)
+        self._source_prefix = mapping.get("source_prefix", "")
+        max_source_tokens = _positive_int(mapping, "max_source_tokens", 128)
+        max_new_tokens = _positive_int(mapping, "max_new_tokens", 64)
+        num_beams = _positive_int(mapping, "num_beams", 1)
+        num_return_sequences = _positive_int(mapping, "num_return_sequences", 1)
+        if num_beams != 1 or num_return_sequences != 1:
+            raise DatasetValidationError(
+                "translation evaluation requires num_beams=1 and num_return_sequences=1 "
+                "for the static batch-one generation contract"
+            )
+
+        super().__init__(config, model)
+        self._pipeline_kwargs: dict[str, Any] = {
+            "do_sample": False,
+            "max_new_tokens": max_new_tokens,
+            "num_beams": 1,
+            "num_return_sequences": 1,
+            "truncation": True,
+        }
+        if self._tokenizer_source_lang:
+            self._pipeline_kwargs["src_lang"] = self._tokenizer_source_lang
+        if self._tokenizer_target_lang:
+            self._pipeline_kwargs["tgt_lang"] = self._tokenizer_target_lang
+
+        max_encoder_length = getattr(model, "max_encoder_length", None)
+        if isinstance(max_encoder_length, int) and max_encoder_length > 0:
+            max_source_tokens = min(max_source_tokens, max_encoder_length)
+        if self.pipe.tokenizer is not None:
+            self.pipe.tokenizer.model_max_length = max_source_tokens
+
+        max_decode_length = getattr(model, "max_decode_length", None)
+        if isinstance(max_decode_length, int):
+            if max_decode_length <= 1:
+                raise DatasetValidationError(
+                    "decoder static cache must leave capacity for at least one generated token"
+                )
+            self._pipeline_kwargs["max_new_tokens"] = min(
+                max_new_tokens,
+                max_decode_length - 1,
+            )
+            self.pipe.generation_config.max_length = max_decode_length
+            self.pipe.generation_config.max_new_tokens = None
+            self.pipe.generation_config.num_beams = 1
+            self.pipe.generation_config.num_return_sequences = 1
+            self.pipe.generation_config.do_sample = False
+
+    def align_labels(self, dataset: Dataset, ds_config: DatasetConfig) -> Dataset:
+        """Return free-text references unchanged."""
+        return dataset
+
+    @staticmethod
+    def _extract_text(value: Any, language: str | None, field: str) -> str:
+        """Extract one text value from a flat string or language-keyed mapping."""
+        from ..utils.eval_utils import DatasetValidationError
+
+        if isinstance(value, Mapping):
+            if not language:
+                language_option = "source_lang" if field == "source" else "target_lang"
+                raise DatasetValidationError(
+                    f"{field} contains a translation dict; provide --column "
+                    f"{language_option}=<language key>"
+                )
+            if language not in value:
+                raise DatasetValidationError(
+                    f"{field} translation dict has no language key '{language}'; "
+                    f"available keys: {sorted(str(key) for key in value)}"
+                )
+            value = value[language]
+        if not isinstance(value, str) or not value.strip():
+            raise DatasetValidationError(f"{field} must resolve to a non-empty string")
+        return value.strip()
+
+    def _extract_references(self, value: Any) -> str | list[str]:
+        """Extract one or more reference translations."""
+        if isinstance(value, list):
+            if not value:
+                from ..utils.eval_utils import DatasetValidationError
+
+                raise DatasetValidationError("reference must contain at least one translation")
+            return [self._extract_text(item, self._target_lang, "reference") for item in value]
+        return self._extract_text(value, self._target_lang, "reference")
+
+    @staticmethod
+    def _prediction_text(output: Any) -> str:
+        """Normalize Hugging Face translation pipeline output shapes."""
+        from ..utils.eval_utils import DatasetValidationError
+
+        if isinstance(output, list):
+            if not output:
+                raise DatasetValidationError("pipeline returned no translations")
+            output = output[0]
+        if isinstance(output, Mapping):
+            prediction = output.get("translation_text", output.get("generated_text", ""))
+            if isinstance(prediction, str) and prediction.strip():
+                return prediction.strip()
+            raise DatasetValidationError("pipeline returned an empty translation")
+        if isinstance(output, str) and output.strip():
+            return output.strip()
+        raise DatasetValidationError("pipeline returned an unsupported translation result")
+
+    def _extract_sample(self, sample: Any) -> tuple[str, str | list[str]]:
+        """Extract and validate source/reference values from one dataset row."""
+        from ..utils.eval_utils import DatasetValidationError
+
+        if not isinstance(sample, Mapping):
+            raise DatasetValidationError(
+                f"dataset row must be a mapping, got {type(sample).__name__}"
+            )
+        source = self._extract_text(sample.get(self._source_col), self._source_lang, "source")
+        references = self._extract_references(sample.get(self._reference_col))
+        return f"{self._source_prefix}{source}", references
+
+    def compute(self) -> dict[str, Any]:
+        """Translate each valid row and compute corpus-level metrics."""
+        from tqdm.auto import tqdm
+
+        from ..utils.eval_utils import DatasetValidationError
+        from .metrics.translation import TranslationMetric
+
+        metric = TranslationMetric()
+        skipped = 0
+        first_error: str | None = None
+        attempted = 0
+
+        for sample in tqdm(self.data, desc="Evaluating", unit="sample"):
+            attempted += 1
+            try:
+                source, references = self._extract_sample(sample)
+            except DatasetValidationError as error:
+                logger.warning("Translation sample rejected: %s", error)
+                first_error = first_error or str(error)
+                skipped += 1
+                continue
+
+            output = self.pipe(source, **self._pipeline_kwargs)
+            try:
+                prediction = self._prediction_text(output)
+                metric.update(prediction, references)
+            except (DatasetValidationError, ValueError) as error:
+                logger.warning("Translation output rejected: %s", error)
+                first_error = first_error or str(error)
+                skipped += 1
+
+        result = metric.compute()
+        if result["n_samples"] == 0:
+            detail = f": {first_error}" if first_error else ""
+            raise DatasetValidationError(f"No valid translation samples were evaluated{detail}")
+        result["attempted"] = attempted
+        result["evaluated"] = result["n_samples"]
+        result["skipped"] = skipped
+        return result