From 2e0516d378111677492758aa522ff8135b13a990 Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Thu, 3 Sep 2026 07:47:54 +0800 Subject: [PATCH] feat(eval): support mms language identification --- .../cpu/audio-classification_fp16_config.json | 5 +- .../cpu/audio-classification_fp32_config.json | 5 +- src/winml/modelkit/commands/eval.py | 7 + src/winml/modelkit/eval/__init__.py | 8 + .../eval/audio_classification_evaluator.py | 898 +++++++++++++ src/winml/modelkit/eval/config.py | 14 + src/winml/modelkit/eval/evaluate.py | 2 + src/winml/modelkit/utils/eval_utils.py | 18 + .../test_audio_classification_evaluator.py | 1143 +++++++++++++++++ tests/unit/eval/test_eval.py | 23 + tests/unit/recipes/test_cpu_recipes.py | 43 +- 11 files changed, 2163 insertions(+), 3 deletions(-) create mode 100644 src/winml/modelkit/eval/audio_classification_evaluator.py create mode 100644 tests/unit/eval/test_audio_classification_evaluator.py diff --git a/examples/recipes/facebook_mms-lid-256/cpu/cpu/audio-classification_fp16_config.json b/examples/recipes/facebook_mms-lid-256/cpu/cpu/audio-classification_fp16_config.json index 47f8dc81c..929433888 100644 --- a/examples/recipes/facebook_mms-lid-256/cpu/cpu/audio-classification_fp16_config.json +++ b/examples/recipes/facebook_mms-lid-256/cpu/cpu/audio-classification_fp16_config.json @@ -27,7 +27,10 @@ { "name": "logits" } - ] + ], + "compatibility": { + "transformers_attention": "eager" + } }, "optim": {}, "quant": { diff --git a/examples/recipes/facebook_mms-lid-256/cpu/cpu/audio-classification_fp32_config.json b/examples/recipes/facebook_mms-lid-256/cpu/cpu/audio-classification_fp32_config.json index 1dce00fb3..bad088b73 100644 --- a/examples/recipes/facebook_mms-lid-256/cpu/cpu/audio-classification_fp32_config.json +++ b/examples/recipes/facebook_mms-lid-256/cpu/cpu/audio-classification_fp32_config.json @@ -27,7 +27,10 @@ { "name": "logits" } - ] + ], + "compatibility": { + "transformers_attention": "eager" + } }, "optim": {}, "quant": null, diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index adae4657d..6c254defb 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -131,6 +131,12 @@ show_default=True, help="Number of dataset samples.", ) +@click.option( + "--max-duration-seconds", + type=click.FloatRange(min=0, min_open=True), + default=None, + help="Maximum audio duration per sample. Omitted means unbounded.", +) @click.option( "--split", type=str, @@ -265,6 +271,7 @@ def eval( runtime: EvalRuntime, ep: EPNameOrAlias | None, samples: int, + max_duration_seconds: float | None, split: str, shuffle: bool, streaming: bool, diff --git a/src/winml/modelkit/eval/__init__.py b/src/winml/modelkit/eval/__init__.py index 435601a15..83b284c81 100644 --- a/src/winml/modelkit/eval/__init__.py +++ b/src/winml/modelkit/eval/__init__.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: + from .audio_classification_evaluator import WinMLAudioClassificationEvaluator from .depth_estimation_evaluator import WinMLDepthEstimationEvaluator from .feature_extraction_evaluator import WinMLFeatureExtractionEvaluator from .fill_mask_evaluator import WinMLFillMaskEvaluator @@ -45,8 +46,13 @@ from .zero_shot_image_classification_evaluator import WinMLZeroShotImageClassificationEvaluator +# Keep the key/value-per-line layout consistent with the evaluator registry. +# fmt: off _LAZY_ATTRS: dict[str, str] = { # Evaluators + "WinMLAudioClassificationEvaluator": ( + ".audio_classification_evaluator:WinMLAudioClassificationEvaluator" + ), "WinMLDepthEstimationEvaluator": ".depth_estimation_evaluator:WinMLDepthEstimationEvaluator", "WinMLFeatureExtractionEvaluator": ( ".feature_extraction_evaluator:WinMLFeatureExtractionEvaluator" @@ -92,6 +98,7 @@ "SpearmanCorrelationMetric": ".metrics.spearman_correlation:SpearmanCorrelationMetric", "TopKAccuracyMetric": ".metrics.top_k_accuracy:TopKAccuracyMetric", } +# fmt: on def __getattr__(name: str) -> Any: @@ -126,6 +133,7 @@ def __dir__() -> list[str]: "SpearmanCorrelationMetric", "TensorSimilarityEvaluator", "TopKAccuracyMetric", + "WinMLAudioClassificationEvaluator", "WinMLDepthEstimationEvaluator", "WinMLEvaluationConfig", "WinMLEvaluator", diff --git a/src/winml/modelkit/eval/audio_classification_evaluator.py b/src/winml/modelkit/eval/audio_classification_evaluator.py new file mode 100644 index 000000000..fcd5b26c7 --- /dev/null +++ b/src/winml/modelkit/eval/audio_classification_evaluator.py @@ -0,0 +1,898 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +"""Audio classification evaluation for scalar and multi-label targets. + +The evaluator deliberately has no built-in dataset. Audio-classification +labels can represent languages, speakers, emotions, intents, or arbitrary +acoustic events, so callers must provide a labeled dataset whose semantics +match the checkpoint. +""" + +from __future__ import annotations + +import logging +import math +import random +from collections import defaultdict +from collections.abc import Iterator, Mapping +from copy import copy +from dataclasses import dataclass +from io import BytesIO +from itertools import islice +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +import numpy as np +import torch +from scipy.signal import resample_poly + +from ..utils.eval_utils import DatasetValidationError +from .base_evaluator import WinMLEvaluator + + +if TYPE_CHECKING: + from datasets import Dataset + from numpy.typing import NDArray + + from ..models.winml.base import WinMLPreTrainedModel + from .config import DatasetConfig, WinMLEvaluationConfig + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _SelectedAudioSample(Mapping[str, Any]): + model_id: int + row: dict[str, Any] + + def __getitem__(self, key: str) -> Any: + return self.row[key] + + def __iter__(self) -> Iterator[str]: + return iter(self.row) + + def __len__(self) -> int: + return len(self.row) + + +class _AudioModelAdapter: + """Decode, preprocess, and run one audio row for native HF or WinML models.""" + + def __init__(self, config: WinMLEvaluationConfig, model: Any) -> None: + from transformers import AutoFeatureExtractor + + if not config.model_id: + raise ValueError("model_id is required to load the audio feature extractor.") + self._config = config + self.model = model + self._feature_extractor = AutoFeatureExtractor.from_pretrained( + config.model_id, + trust_remote_code=getattr(config, "trust_remote_code", False), + ) + self._input_contracts = self._resolve_input_contracts() + self._validate_output_contract() + self.last_was_truncated = False + self.last_window_count = 0 + + def __call__(self, raw_audio: Any) -> NDArray[np.float32]: + """Return utterance logits, averaging fixed-shape waveform windows.""" + if isinstance(raw_audio, (list, tuple, np.ndarray)): + waveform = np.asarray(raw_audio, dtype=np.float32) + sampling_rate = int(getattr(self._feature_extractor, "sampling_rate", 0)) + if waveform.ndim != 1: + raise ValueError( + f"pre-normalized waveform input must be 1D, got shape {waveform.shape}", + ) + else: + waveform, sampling_rate = WinMLAudioClassificationEvaluator._decode_audio(raw_audio) + + waveform = WinMLAudioClassificationEvaluator._to_mono(waveform) + if waveform.size == 0: + raise ValueError("audio waveform is empty") + target_rate = int( + getattr(self._feature_extractor, "sampling_rate", sampling_rate), + ) + if sampling_rate <= 0 or target_rate <= 0: + raise ValueError("audio sampling rate must be positive") + if sampling_rate != target_rate: + divisor = math.gcd(sampling_rate, target_rate) + waveform = resample_poly( + waveform, + target_rate // divisor, + sampling_rate // divisor, + ).astype(np.float32) + + self.last_was_truncated = False + max_duration = self._config.dataset.max_duration_seconds + if max_duration is not None: + max_samples = max(1, int(max_duration * target_rate)) + if waveform.size > max_samples: + waveform = waveform[:max_samples] + self.last_was_truncated = True + + waveform_contract = self._input_contracts.get("input_values") + if waveform_contract is not None and len(waveform_contract) == 2: + window_length = waveform_contract[1] + windows = [ + waveform[offset : offset + window_length] + for offset in range(0, waveform.size, window_length) + ] + else: + windows = [waveform] + self.last_window_count = len(windows) + return cast( + "NDArray[np.float32]", + np.mean(np.stack([self._run_window(window) for window in windows]), axis=0), + ) + + def _run_window(self, waveform: NDArray[np.float32]) -> NDArray[np.float32]: + """Preprocess and run one waveform window.""" + target_rate = int(getattr(self._feature_extractor, "sampling_rate", 0)) + extractor_kwargs: dict[str, Any] = { + "sampling_rate": target_rate, + "return_tensors": "pt", + } + waveform_contract = self._input_contracts.get("input_values") + if waveform_contract is not None and len(waveform_contract) == 2: + extractor_kwargs.update( + padding="max_length", + truncation=True, + max_length=waveform_contract[1], + ) + encoded = self._feature_extractor(waveform, **extractor_kwargs) + model_inputs = self._select_model_inputs(encoded) + device = ( + getattr(self._config, "pipeline_device", "cpu") + if getattr(self._config, "runtime", None) == "pytorch" + else "cpu" + ) + model_inputs = { + name: value.to(device) + if hasattr(value, "to") + else torch.as_tensor(value, device=device) + for name, value in model_inputs.items() + } + output = self.model(**model_inputs) + logits = ( + output.get("logits") + if isinstance(output, dict) + else getattr(output, "logits", None) + ) + if logits is None: + raise ValueError("audio-classification model output does not contain logits") + if hasattr(logits, "detach"): + logits = logits.detach().float().cpu().numpy() + array = np.asarray(logits, dtype=np.float32) + if array.ndim != 2 or array.shape[0] != 1: + raise ValueError(f"expected logits shape [1, classes], got {array.shape}") + return cast("NDArray[np.float32]", np.asarray(array[0], dtype=np.float32)) + + def _resolve_input_contracts(self) -> dict[str, list[int]]: + io_config = getattr(self.model, "io_config", None) or {} + names = io_config.get("input_names") or [] + shapes = io_config.get("input_shapes") or [] + if not names and not shapes: + return {} + if len(names) != len(shapes) or not names: + raise ValueError( + "audio-classification input names and shapes must have equal non-zero lengths", + ) + if len(names) != 1: + raise ValueError("audio-classification evaluation requires exactly one model input") + contracts: dict[str, list[int]] = {} + for name, raw_shape in zip(names, shapes, strict=True): + shape = list(raw_shape) + if len(shape) < 2 or any(not isinstance(value, int) for value in shape[1:]): + raise ValueError( + "audio-classification evaluation requires static non-batch input shapes", + ) + if isinstance(shape[0], int) and shape[0] != 1: + raise ValueError("audio-classification evaluation requires batch size 1") + contracts[str(name)] = [1, *(int(value) for value in shape[1:])] + return contracts + + def _validate_output_contract(self) -> None: + io_config = getattr(self.model, "io_config", None) or {} + names = io_config.get("output_names") or [] + shapes = io_config.get("output_shapes") or [] + if not names and not shapes: + return + if names != ["logits"] or len(shapes) != 1: + raise ValueError( + "audio-classification evaluation requires exactly one 'logits' output" + ) + shape = list(shapes[0]) + if ( + len(shape) != 2 + or (isinstance(shape[0], int) and shape[0] != 1) + or not isinstance(shape[1], int) + or shape[1] <= 0 + ): + raise ValueError( + "audio-classification evaluation requires logits shape [1, classes]" + ) + + def _select_model_inputs(self, encoded: Any) -> dict[str, Any]: + values = dict(encoded) + if not self._input_contracts: + if not values: + raise ValueError("audio feature extractor produced no model inputs") + return values + selected: dict[str, Any] = {} + for name, expected_shape in self._input_contracts.items(): + if name not in values: + raise ValueError( + f"audio feature extractor output must contain {name!r}; got {sorted(values)}", + ) + actual_shape = list(getattr(values[name], "shape", ())) + if actual_shape != expected_shape: + raise ValueError( + f"audio feature extractor produced {name} shape {actual_shape}; " + f"expected {expected_shape}", + ) + selected[name] = values[name] + return selected + + +class WinMLAudioClassificationEvaluator(WinMLEvaluator): + """Evaluate utterance-level audio classifiers using accuracy and macro-F1.""" + + def __init__( + self, + config: WinMLEvaluationConfig, + model: WinMLPreTrainedModel, + ) -> None: + from ..utils.eval_utils import get_default + + mapping = config.dataset.columns_mapping + task = "audio-classification" + audio_col = mapping.get("input_column", get_default(task, "input_column")) + label_col = mapping.get("label_column", get_default(task, "label_column")) + if audio_col is None or label_col is None: + raise DatasetValidationError( + "audio-classification requires input_column and label_column defaults", + ) + self._audio_col = audio_col + self._label_col = label_col + self._eligible_count = 0 + self._selected_count = 0 + self._dataset_label_to_model_id: dict[int | str, int] = {} + self._eligible_model_labels: list[str] = [] + self._indexed_dataset: Any = None + self._target_kind = "" + self._scalar_string_target = False + self._label_feature: Any = None + self._label_name_col = mapping.get("label_name_column", "human_labels") + self._model_id2label, self._model_label2id = self._model_labels(model) + super().__init__(config, model) + + def prepare_pipeline(self) -> Any: + """Create the shared native-HF/WinML callable audio adapter.""" + return _AudioModelAdapter(self.config, self.model) + + def prepare_data(self) -> list[_SelectedAudioSample] | list[dict[str, Any]]: + """Load, label-filter, then select a seeded stratified sample. + + Filtering happens before sampling so unsupported classes cannot consume + the requested sample budget. Shuffled streaming datasets use bounded + per-class reservoir sampling over the complete stream. Deterministic + streams stop once every authoritative overlapping class has supplied + its balanced quota. + """ + from datasets import load_dataset, load_from_disk + + ds = self.config.dataset + try: + ds_path = Path(ds.path).expanduser() if ds.path else None + if ds_path and ds_path.is_dir(): + dataset = load_from_disk(str(ds_path)) + else: + dataset = load_dataset( + ds.path, + name=ds.name, + split=ds.split, + streaming=ds.streaming, + revision=ds.revision, + ) + if hasattr(dataset, "keys") and ds.split in dataset: + dataset = dataset[ds.split] + elif hasattr(dataset, "keys"): + available = sorted(str(split) for split in dataset) + raise DatasetValidationError( + f"Dataset split '{ds.split}' was not found; available splits: {available}", + ) + except Exception as error: + if isinstance(error, DatasetValidationError): + raise + raise DatasetValidationError( + f"Failed to load dataset '{ds.path}' " + f"(name={ds.name!r}, split='{ds.split}'): {error}", + ) from error + + self._validate_target_schema(dataset, ds) + dataset = self._disable_backend_audio_decoding(dataset) + if ds.samples <= 0: + raise DatasetValidationError("samples must be greater than zero.") + + if self._target_kind == "multi-label": + if ds.streaming: + if ds.shuffle: + dataset = dataset.shuffle(seed=ds.seed) + selected_rows = list(islice(iter(dataset), ds.samples)) + self._eligible_count = len(selected_rows) + else: + if ds.shuffle: + dataset = dataset.shuffle(seed=ds.seed) + count = min(ds.samples, len(dataset)) + selected_rows = [dataset[index] for index in range(count)] + self._eligible_count = len(dataset) + self._selected_count = len(selected_rows) + if not selected_rows: + raise DatasetValidationError("No samples were selected for evaluation.") + return selected_rows + + if ds.streaming: + rows_by_label = self._streaming_reservoirs( + dataset, + ds.samples, + ds.seed, + ds.shuffle, + ) + else: + self._indexed_dataset = dataset + rows_by_label = self._indexed_rows(dataset, ds.seed, ds.shuffle) + + selected = self._balanced_take(rows_by_label, ds.samples) + self._selected_count = len(selected) + if not selected: + raise DatasetValidationError( + "No samples remain after label filtering. " + "Dataset and model labels have no overlap.", + ) + return selected + + def align_labels(self, dataset: Dataset, ds_config: DatasetConfig) -> Dataset: + """Keep base-class construction compatible; alignment is done before sampling.""" + return dataset + + def _validate_target_schema(self, dataset: Any, ds: DatasetConfig) -> None: + """Validate scalar or sequence label semantics before selecting rows.""" + from datasets import ClassLabel, Sequence, Value + + columns = set(dataset.column_names) + missing = [col for col in (self._audio_col, self._label_col) if col not in columns] + if missing: + raise DatasetValidationError( + f"missing required column(s) {missing}; dataset has {sorted(columns)}", + ) + feature = dataset.features[self._label_col] + self._label_feature = feature + if isinstance(feature, ClassLabel): + self._target_kind = "single-label" + self._validate_and_resolve_labels(dataset, ds) + elif isinstance(feature, Value) and feature.dtype == "string": + self._target_kind = "single-label" + self._scalar_string_target = True + self._validate_and_resolve_labels(dataset, ds) + elif isinstance(feature, Sequence) and isinstance(feature.feature, (ClassLabel, Value)): + self._target_kind = "multi-label" + else: + raise DatasetValidationError( + f"Column '{self._label_col}' must be ClassLabel, a string Value, or a " + f"sequence of ClassLabel/string values; got {feature!r}.", + ) + + @staticmethod + def _model_labels(model: Any) -> tuple[dict[int, str], dict[str, int]]: + config = getattr(model, "config", None) + raw_id2label = getattr(config, "id2label", None) or {} + id2label = {int(index): str(label) for index, label in raw_id2label.items()} + if not id2label or sorted(id2label) != list(range(len(id2label))): + raise DatasetValidationError( + "model.config.id2label must define contiguous class IDs starting at zero.", + ) + label2id = {label: index for index, label in id2label.items()} + if len(label2id) != len(id2label): + raise DatasetValidationError("model.config.id2label contains duplicate label names.") + return id2label, label2id + + def _validate_and_resolve_labels(self, dataset: Any, ds: DatasetConfig) -> None: + """Validate required columns and resolve exact dataset-name to model-ID mapping.""" + from datasets import ClassLabel, Value + + columns = set(dataset.column_names) + missing = [col for col in (self._audio_col, self._label_col) if col not in columns] + if missing: + raise DatasetValidationError( + f"missing required column(s) {missing}; dataset has {sorted(columns)}", + ) + + label_feature = dataset.features[self._label_col] + if not isinstance(label_feature, ClassLabel) and not ( + isinstance(label_feature, Value) and label_feature.dtype == "string" + ): + raise DatasetValidationError( + f"Column '{self._label_col}' must be a ClassLabel or string Value so " + "label semantics can be aligned explicitly.", + ) + + model_label2id = getattr(self.model.config, "label2id", None) or {} + model_id2label = getattr(self.model.config, "id2label", None) or {} + if not model_id2label: + raise DatasetValidationError("model.config.id2label is required for evaluation.") + if not model_label2id: + model_label2id = {str(name): int(model_id) for model_id, name in model_id2label.items()} + + # A user mapping is authoritative. Without one, only exact label-name + # identity is accepted; no locale, case, punctuation, or region inference. + label_mapping = ds.label_mapping + if label_mapping is None: + candidate_names = ( + label_feature.names + if isinstance(label_feature, ClassLabel) + else model_label2id.keys() + ) + label_mapping = { + name: int(model_label2id[name]) + for name in candidate_names + if name in model_label2id + } + + name_to_dataset_label: dict[str, int | str] = ( + {name: index for index, name in enumerate(label_feature.names)} + if isinstance(label_feature, ClassLabel) + else {str(name): str(name) for name in label_mapping} + ) + valid_model_ids = {int(key) for key in model_id2label} + resolved: dict[int | str, int] = {} + for dataset_name, model_id in label_mapping.items(): + if dataset_name not in name_to_dataset_label: + continue + target_id = int(model_id) + if target_id not in valid_model_ids: + raise DatasetValidationError( + f"Label mapping target {target_id} for '{dataset_name}' is not present " + "in model.config.id2label.", + ) + resolved[name_to_dataset_label[dataset_name]] = target_id + + if not resolved: + raise DatasetValidationError( + "No samples remain after label filtering. Dataset and model labels have " + "no exact overlap; provide --label-mapping with authoritative semantics.", + ) + + self._dataset_label_to_model_id = resolved + self._eligible_model_labels = sorted( + {self._decode_model_label(model_id) for model_id in resolved.values()}, + ) + + def _dataset_label_key(self, raw_label: Any) -> int | str: + return str(raw_label) if self._scalar_string_target else int(raw_label) + + def _disable_backend_audio_decoding(self, dataset: Any) -> Any: + """Keep dataset audio as bytes/path so decoding does not require TorchCodec.""" + from datasets import Audio, IterableDataset, Value + + audio_feature = dataset.features[self._audio_col] + if isinstance(audio_feature, Audio): + if isinstance(dataset, IterableDataset): + dataset = copy(dataset) + dataset._info = dataset._info.copy() + dataset._info.features = None + return dataset + return dataset.cast_column( + self._audio_col, + {"bytes": Value("binary"), "path": Value("string")}, + ) + return dataset + + def _indexed_rows( + self, + dataset: Any, + seed: int, + shuffle: bool, + ) -> dict[int, list[_SelectedAudioSample]]: + """Collect shuffled eligible row indices without decoding the audio column.""" + indices: dict[int, list[int]] = defaultdict(list) + for index, raw_label in enumerate(dataset[self._label_col]): + dataset_label = self._dataset_label_key(raw_label) + if dataset_label in self._dataset_label_to_model_id: + indices[self._dataset_label_to_model_id[dataset_label]].append(index) + self._eligible_count = sum(len(items) for items in indices.values()) + + rows: dict[int, list[_SelectedAudioSample]] = {} + for model_id, label_indices in sorted(indices.items()): + if shuffle: + random.Random(seed + model_id).shuffle(label_indices) + selected_indices = label_indices[: self.config.dataset.samples] + rows[model_id] = [ + _SelectedAudioSample(model_id=model_id, row=dataset[index]) + for index in selected_indices + ] + return rows + + def _streaming_reservoirs( + self, + dataset: Any, + limit: int, + seed: int, + shuffle: bool, + ) -> dict[int, list[_SelectedAudioSample]]: + """Select streaming rows without weakening requested shuffle semantics.""" + if not shuffle: + return self._streaming_balanced_prefix(dataset, limit) + + # True reservoir sampling requires visiting the complete stream. Keep + # this path separate from deterministic bounded selection so --shuffle + # never presents a prefix as a random sample. + reservoirs: dict[int, list[_SelectedAudioSample]] = defaultdict(list) + seen: dict[int, int] = defaultdict(int) + rng = random.Random(seed) + for row in dataset: + dataset_label = self._dataset_label_key(row[self._label_col]) + model_id = self._dataset_label_to_model_id.get(dataset_label) + if model_id is None: + continue + self._eligible_count += 1 + seen[model_id] += 1 + bucket = reservoirs[model_id] + selected = _SelectedAudioSample(model_id=model_id, row=row) + if len(bucket) < limit: + bucket.append(selected) + else: + replacement = rng.randrange(seen[model_id]) + if replacement < limit: + bucket[replacement] = selected + return dict(reservoirs) + + def _streaming_balanced_prefix( + self, + dataset: Any, + limit: int, + ) -> dict[int, list[_SelectedAudioSample]]: + """Collect a deterministic balanced prefix and stop when quotas are full. + + Quotas are based on the unique model classes resolved from the + authoritative exact label mapping, not on labels encountered so far. + Consequently an absent or short class forces stream exhaustion and a + short selection instead of silently redistributing its quota. + """ + quotas = self._balanced_quotas(self._dataset_label_to_model_id.values(), limit) + rows: dict[int, list[_SelectedAudioSample]] = defaultdict(list) + pending = sum(quota > 0 for quota in quotas.values()) + + for row in dataset: + dataset_label = self._dataset_label_key(row[self._label_col]) + model_id = self._dataset_label_to_model_id.get(dataset_label) + if model_id is None: + continue + self._eligible_count += 1 + bucket = rows[model_id] + quota = quotas[model_id] + if len(bucket) >= quota: + continue + bucket.append(_SelectedAudioSample(model_id=model_id, row=row)) + if len(bucket) == quota: + pending -= 1 + if pending == 0: + break + return dict(rows) + + @staticmethod + def _balanced_quotas(model_ids: Any, limit: int) -> dict[int, int]: + """Assign deterministic per-class quotas whose sum is ``limit``.""" + labels = sorted({int(model_id) for model_id in model_ids}) + if not labels: + return {} + base, remainder = divmod(limit, len(labels)) + return {label: base + (index < remainder) for index, label in enumerate(labels)} + + @staticmethod + def _balanced_take( + rows_by_label: dict[int, list[_SelectedAudioSample]], + limit: int, + ) -> list[_SelectedAudioSample]: + """Round-robin classes to form a balanced sample and redistribute shortages.""" + selected: list[_SelectedAudioSample] = [] + offsets = dict.fromkeys(rows_by_label, 0) + labels = sorted(rows_by_label) + while len(selected) < limit: + added = False + for label in labels: + offset = offsets[label] + if offset < len(rows_by_label[label]): + selected.append(rows_by_label[label][offset]) + offsets[label] += 1 + added = True + if len(selected) == limit: + break + if not added: + break + return selected + + def compute(self) -> dict[str, Any]: + """Run exactly one forward per selected row and report task metrics.""" + logits: list[np.ndarray] = [] + targets: list[Any] = [] + confusion: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + processed_by_label: dict[str, int] = defaultdict(int) + selected_by_label: dict[str, int] = defaultdict(int) + rejected_by_reason: dict[str, int] = defaultdict(int) + inference_windows = 0 + truncated_samples = 0 + + for selected in self.data: + if isinstance(selected, _SelectedAudioSample): + reference_id = selected.model_id + reference = self._decode_model_label(reference_id) + selected_by_label[reference] += 1 + else: + reference_id = None + reference = None + try: + if isinstance(selected, _SelectedAudioSample) and selected.row is not None: + sample = selected.row + elif isinstance(selected, dict): + sample = selected + else: + raise ValueError("selected audio sample has no row or dataset index") + target = ( + reference_id + if reference_id is not None + else self._target_for_row(sample) + ) + adapter = cast("_AudioModelAdapter", self.pipe) + prediction_logits = adapter(sample[self._audio_col]) + inference_windows += adapter.last_window_count + truncated_samples += int(adapter.last_was_truncated) + except (TypeError, ValueError, RuntimeError) as error: + rejected_by_reason[type(error).__name__] += 1 + logger.warning("Skipping audio sample: %s", error) + continue + logits.append(prediction_logits) + targets.append(target) + if reference is not None: + prediction = self._decode_model_label(int(np.argmax(prediction_logits))) + confusion[reference][prediction] += 1 + processed_by_label[reference] += 1 + + if not logits: + raise DatasetValidationError("No audio samples were successfully processed.") + + if self._target_kind == "single-label": + insufficient = { + label: (processed_by_label[label], min(5, selected_count)) + for label, selected_count in selected_by_label.items() + if processed_by_label[label] < min(5, selected_count) + } + if insufficient: + details = ", ".join( + f"{label}={actual}/{required} required" + for label, (actual, required) in sorted(insufficient.items()) + ) + raise DatasetValidationError( + f"Too few usable samples after audio decoding/preprocessing: {details}.", + ) + + scores = np.stack(logits) + if scores.shape[1] != len(self._model_id2label): + raise DatasetValidationError( + f"model returned {scores.shape[1]} classes but config defines " + f"{len(self._model_id2label)} labels.", + ) + metrics: dict[str, Any] = ( + self._multi_label_metrics(scores, targets) + if self._target_kind == "multi-label" + else self._single_label_metrics(scores, targets) + ) + processed = len(targets) + metrics.update({ + "requested_samples": self.config.dataset.samples, + "eligible_samples": self._eligible_count, + "selected_samples": self._selected_count, + "processed_samples": processed, + "inference_windows": inference_windows, + "truncated_samples": truncated_samples, + "rejected_samples": self._selected_count - processed, + "rejected_by_reason": dict(sorted(rejected_by_reason.items())), + "per_label_processed": dict(sorted(processed_by_label.items())), + "confusion_matrix": { + reference: dict(sorted(predictions.items())) + for reference, predictions in sorted(confusion.items()) + }, + }) + return metrics + + def _target_for_row(self, row: dict[str, Any]) -> int | list[int]: + from datasets import ClassLabel + + raw_target = row[self._label_col] + if self._target_kind == "single-label": + return self._resolve_label(self._label_feature.int2str(int(raw_target))) + + values = list(raw_target) + feature = self._label_feature.feature + decoded = ( + [feature.int2str(int(value)) for value in values] + if isinstance(feature, ClassLabel) + else [str(value) for value in values] + ) + parallel_names = row.get(self._label_name_col) + if parallel_names is not None and len(parallel_names) != len(decoded): + raise DatasetValidationError( + f"Columns '{self._label_col}' and '{self._label_name_col}' must contain " + "the same number of labels.", + ) + resolved = [ + self._resolve_label( + value, + fallback_name=parallel_names[index] if parallel_names is not None else None, + ) + for index, value in enumerate(decoded) + ] + if not resolved: + raise DatasetValidationError("Multi-label targets must contain at least one label.") + return sorted(set(resolved)) + + def _resolve_label(self, value: str, *, fallback_name: Any = None) -> int: + mapping = self.config.dataset.label_mapping or {} + if value in mapping: + model_id = int(mapping[value]) + elif value in self._model_label2id: + model_id = self._model_label2id[value] + elif fallback_name is not None and str(fallback_name) in self._model_label2id: + model_id = self._model_label2id[str(fallback_name)] + else: + raise DatasetValidationError( + f"Dataset label {value!r} has no exact model-label match; provide an " + "authoritative label mapping or parallel exact label-name column.", + ) + if model_id not in self._model_id2label: + raise DatasetValidationError( + f"Label mapping target {model_id} is absent from model.config.id2label.", + ) + return model_id + + def _single_label_metrics( + self, + logits: np.ndarray, + targets: list[int], + ) -> dict[str, Any]: + from .metrics.classification import ClassificationMetric + + predictions = [self._model_id2label[int(index)] for index in np.argmax(logits, axis=1)] + references = [self._model_id2label[int(index)] for index in targets] + represented = sorted(set(references)) + result = ClassificationMetric().compute(predictions, references, represented) + return { + "accuracy": result["accuracy"], + "macro_f1": result["f1"], + "represented_classes": len(represented), + "total_classes": len(self._model_id2label), + "class_coverage": len(represented) / len(self._model_id2label), + } + + @staticmethod + def _multi_label_metrics( + logits: np.ndarray, + targets: list[list[int]], + ) -> dict[str, float]: + from sklearn.metrics import average_precision_score + + references = np.zeros_like(logits, dtype=np.int8) + for row_index, model_ids in enumerate(targets): + references[row_index, model_ids] = 1 + probabilities = 1.0 / (1.0 + np.exp(-logits)) + sample_ap = float(average_precision_score(references, probabilities, average="samples")) + micro_ap = float(average_precision_score(references, probabilities, average="micro")) + if not np.isfinite(sample_ap) or not np.isfinite(micro_ap): + raise DatasetValidationError("Multi-label average precision must be finite.") + return { + "sample_average_precision": sample_ap, + "micro_average_precision": micro_ap, + } + + @staticmethod + def _decode_audio(audio: Any) -> tuple[np.ndarray, int]: + """Decode common datasets Audio values without assuming one backend version.""" + encoded_bytes = None + encoded_path = None + if isinstance(audio, dict): + if audio.get("array") is not None and audio.get("sampling_rate") is not None: + return np.asarray(audio["array"], dtype=np.float32), int( + audio["sampling_rate"] + ) + + encoded_bytes = audio.get("bytes") + encoded_path = audio.get("path") + if encoded_bytes is None and not encoded_path: + raise ValueError( + "audio dict requires array and sampling_rate, or encoded bytes/path" + ) + elif isinstance(audio, (bytes, bytearray, memoryview)): + encoded_bytes = bytes(audio) + elif isinstance(audio, (str, Path)): + encoded_path = str(audio) + + if encoded_bytes is not None or encoded_path: + try: + import soundfile as sf + from datasets.download.streaming_download_manager import xopen + except ImportError as error: + raise RuntimeError( + "Decoding encoded audio requires the 'audio' extra " + "(install winml-cli[audio])." + ) from error + try: + if encoded_bytes is not None: + waveform, sampling_rate = sf.read( + BytesIO(encoded_bytes), + dtype="float32", + always_2d=False, + ) + else: + with xopen(str(encoded_path), "rb") as source: + waveform, sampling_rate = sf.read( + source, + dtype="float32", + always_2d=False, + ) + except (OSError, RuntimeError) as error: + raise ValueError(f"failed to decode audio: {error}") from error + # SoundFile returns [frames, channels]. Normalize its known + # layout explicitly instead of guessing the channel axis for + # very short clips later in preprocessing. + if waveform.ndim == 2: + waveform = waveform.T + return np.asarray(waveform, dtype=np.float32), int(sampling_rate) + + get_all_samples = getattr(audio, "get_all_samples", None) + if callable(get_all_samples): + decoded = get_all_samples() + data = getattr(decoded, "data", getattr(decoded, "samples", None)) + rate = getattr(decoded, "sample_rate", getattr(decoded, "sampling_rate", None)) + if data is None or rate is None: + raise ValueError("decoded audio does not expose samples and sampling rate") + if hasattr(data, "detach"): + data = data.detach().cpu().numpy() + return np.asarray(data, dtype=np.float32), int(rate) + + array = getattr(audio, "array", None) + rate = getattr(audio, "sampling_rate", None) + if array is not None and rate is not None: + return np.asarray(array, dtype=np.float32), int(rate) + raise TypeError(f"Unsupported audio value: {type(audio).__name__}") + + @staticmethod + def _to_mono(waveform: np.ndarray) -> NDArray[np.float32]: + """Return one float32 channel from mono or common stereo layouts.""" + waveform = np.asarray(waveform, dtype=np.float32) + if waveform.ndim == 1: + return cast("NDArray[np.float32]", waveform) + if waveform.ndim != 2: + raise ValueError(f"audio must be 1D or 2D, got shape {waveform.shape}") + if waveform.shape[0] <= 8: + return cast( + "NDArray[np.float32]", + np.asarray(waveform.mean(axis=0), dtype=np.float32), + ) + if waveform.shape[1] <= 8: + return cast( + "NDArray[np.float32]", + np.asarray(waveform.mean(axis=1), dtype=np.float32), + ) + raise ValueError(f"cannot determine channel axis for audio shape {waveform.shape}") + + def _decode_model_label(self, model_id: int) -> str: + """Decode a class ID through checkpoint id2label.""" + model = cast("WinMLPreTrainedModel", self.model) + config = model.config + if config is None: + return str(model_id) + id2label = cast("dict[Any, Any]", config.id2label or {}) + label = id2label.get(model_id, id2label.get(str(model_id))) + return str(label) if label is not None else str(model_id) diff --git a/src/winml/modelkit/eval/config.py b/src/winml/modelkit/eval/config.py index 4c6b62e6d..d7887486c 100644 --- a/src/winml/modelkit/eval/config.py +++ b/src/winml/modelkit/eval/config.py @@ -8,6 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from math import isfinite from pathlib import Path from typing import Any, Literal @@ -40,6 +41,8 @@ class DatasetConfig: ``--output `` before the dataset is loaded. label_mapping_file: Path to a JSON file with label mapping. Resolved into ``label_mapping`` at eval time. + max_duration_seconds: Optional positive audio duration cap. When omitted, + evaluators preserve the full input duration. """ path: str | None = field(default=None, metadata={"cli_name": "dataset_path"}) @@ -54,6 +57,13 @@ class DatasetConfig: revision: str | None = field(default=None, metadata={"cli_name": "dataset_revision"}) build_script: str | None = field(default=None, metadata={"cli_name": "dataset_script"}) label_mapping_file: str | None = None + max_duration_seconds: float | None = None + + def __post_init__(self) -> None: + if self.max_duration_seconds is not None and ( + not isfinite(self.max_duration_seconds) or self.max_duration_seconds <= 0 + ): + raise ValueError("max_duration_seconds must be a finite value greater than zero.") def to_dict(self) -> dict[str, Any]: """Convert to dictionary for serialization.""" @@ -79,6 +89,8 @@ def to_dict(self) -> dict[str, Any]: result["build_script"] = self.build_script if self.label_mapping_file is not None: result["label_mapping_file"] = self.label_mapping_file + if self.max_duration_seconds is not None: + result["max_duration_seconds"] = self.max_duration_seconds return result @@ -267,10 +279,12 @@ def from_dict(cls, data: dict) -> WinMLEvaluationConfig: shuffle=ds_data.get("shuffle", True), seed=ds_data.get("seed", 42), columns_mapping=ds_data.get("columns_mapping", {}), + label_mapping=ds_data.get("label_mapping"), streaming=ds_data.get("streaming", False), revision=ds_data.get("revision"), build_script=ds_data.get("build_script"), label_mapping_file=ds_data.get("label_mapping_file"), + max_duration_seconds=ds_data.get("max_duration_seconds"), ) return cls( model_id=data.get("model_id"), diff --git a/src/winml/modelkit/eval/evaluate.py b/src/winml/modelkit/eval/evaluate.py index 42a9ab8a5..a94554414 100644 --- a/src/winml/modelkit/eval/evaluate.py +++ b/src/winml/modelkit/eval/evaluate.py @@ -60,6 +60,8 @@ def _select_model_loader(config: WinMLEvaluationConfig) -> _ModelLoaderKind: # default formatter layout) yields >100-char lines that trip E501. # fmt: off _EVALUATOR_REGISTRY: dict[str, str] = { + "audio-classification": + "winml.modelkit.eval.audio_classification_evaluator:WinMLAudioClassificationEvaluator", "image-classification": "winml.modelkit.eval.base_evaluator:WinMLEvaluator", "text-classification": diff --git a/src/winml/modelkit/utils/eval_utils.py b/src/winml/modelkit/utils/eval_utils.py index f85c672b4..ac657294e 100644 --- a/src/winml/modelkit/utils/eval_utils.py +++ b/src/winml/modelkit/utils/eval_utils.py @@ -57,6 +57,23 @@ class TaskSchema: ), ) +_AUDIO_CLASSIFICATION_SCHEMA = TaskSchema( + columns=( + SchemaItem( + "input_column", + "decoded audio with sampling rate, or pre-normalized 1D waveform", + default="audio", + remap_hint="", + ), + SchemaItem( + "label_column", + "audio class label (ClassLabel)", + default="label", + remap_hint="", + ), + ), +) + _TEXT_CLASSIFICATION_SCHEMA = TaskSchema( columns=( SchemaItem( @@ -449,6 +466,7 @@ class TaskSchema: ) TASK_SCHEMAS: dict[str, TaskSchema] = { + "audio-classification": _AUDIO_CLASSIFICATION_SCHEMA, "image-classification": _IMAGE_CLASSIFICATION_SCHEMA, "text-classification": _TEXT_CLASSIFICATION_SCHEMA, "sequence-classification": _TEXT_CLASSIFICATION_SCHEMA, diff --git a/tests/unit/eval/test_audio_classification_evaluator.py b/tests/unit/eval/test_audio_classification_evaluator.py new file mode 100644 index 000000000..16944b95e --- /dev/null +++ b/tests/unit/eval/test_audio_classification_evaluator.py @@ -0,0 +1,1143 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +from __future__ import annotations + +import json +from io import BytesIO +from types import SimpleNamespace +from typing import ClassVar +from unittest.mock import patch +from zipfile import ZipFile + +import numpy as np +import onnx +import pytest +import soundfile as sf +import torch +from click.testing import CliRunner +from datasets import ( + Audio, + ClassLabel, + Dataset, + DatasetDict, + Features, + IterableDataset, + Sequence, + Value, +) +from transformers import Wav2Vec2Config + +from winml.modelkit.commands.eval import eval as eval_command +from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig +from winml.modelkit.eval.audio_classification_evaluator import ( + WinMLAudioClassificationEvaluator, + _AudioModelAdapter, +) +from winml.modelkit.utils.eval_utils import DatasetValidationError + + +class _IdentityFeatureExtractor: + sampling_rate = 16_000 + padding_value = 0.0 + + def __call__(self, waveform, *, sampling_rate, return_tensors, **kwargs): + assert sampling_rate == self.sampling_rate + assert return_tensors in {"np", "pt"} + values = np.asarray(waveform, dtype=np.float32) + if kwargs.get("padding") == "max_length": + max_length = kwargs["max_length"] + values = values[:max_length] + values = np.pad(values, (0, max_length - values.size)) + tensor = values[None, :] + return { + "input_values": torch.from_numpy(tensor) if return_tensors == "pt" else tensor, + } + + +class _SignClassifier: + io_config: ClassVar = { + "input_names": ["input_values"], + "input_shapes": [[1, 8]], + "output_names": ["logits"], + "output_shapes": [[1, 3]], + } + config = SimpleNamespace( + label2id={"cat": 0, "dog": 1, "other": 2}, + id2label={0: "cat", 1: "dog", 2: "other"}, + ) + + def __call__(self, **kwargs): + values = kwargs["input_values"].cpu().numpy() + score = float(values.mean()) + return {"logits": torch.tensor([[score, -score, -10.0]])} + + +class _RecordingClassifier(_SignClassifier): + def __init__(self): + self.calls = [] + + def __call__(self, **kwargs): + self.calls.append(kwargs) + return super().__call__(**kwargs) + + +class _TwoInputFeatureExtractor: + sampling_rate = 16_000 + + def __init__(self): + self.kwargs = None + self.waveforms = [] + + def __call__(self, waveform, *, sampling_rate, return_tensors, **kwargs): + self.kwargs = kwargs + self.waveforms.append(np.asarray(waveform, dtype=np.float32)) + assert sampling_rate == self.sampling_rate + assert return_tensors == "pt" + length = kwargs.get("max_length", len(waveform)) + values = np.asarray(waveform, dtype=np.float32)[:length] + values = np.pad(values, (0, length - values.size)) + return { + "input_values": torch.from_numpy(values[None, :]), + "attention_mask": torch.ones((1, length), dtype=torch.int64), + "extra": torch.zeros((1, length), dtype=torch.int64), + } + + +class _TwoInputClassifier: + io_config: ClassVar = { + "input_names": ["input_values", "attention_mask"], + "input_shapes": [[1, 8], [1, 8]], + "output_names": ["logits"], + "output_shapes": [[1, 3]], + } + config = _SignClassifier.config + + def __init__(self): + self.calls = [] + + def __call__(self, **kwargs): + self.calls.append(kwargs) + return {"logits": torch.tensor([[5.0, 0.0, -5.0]])} + + +class _SpectrogramFeatureExtractor: + sampling_rate = 16_000 + + def __call__(self, waveform, *, sampling_rate, return_tensors): + assert waveform.size > 0 + assert sampling_rate == self.sampling_rate + assert return_tensors == "pt" + return {"input_values": torch.ones((1, 4, 3), dtype=torch.float32)} + + +class _SpectrogramClassifier: + io_config: ClassVar = { + "input_names": ["input_values"], + "input_shapes": [[1, 4, 3]], + "output_names": ["logits"], + "output_shapes": [[1, 3]], + } + config = _SignClassifier.config + + def __call__(self, **kwargs): + assert kwargs["input_values"].shape == (1, 4, 3) + return {"logits": torch.tensor([[1.0, 0.0, -1.0]])} + + +class _CountingStreamingDataset: + def __init__(self, rows): + self.column_names = ["audio", "label"] + self.features = { + "audio": Value("string"), + "label": ClassLabel(names=["cat", "dog", "en_us"]), + } + self._rows = rows + self.rows_yielded = 0 + + def __iter__(self): + for row in self._rows: + self.rows_yielded += 1 + yield row + + +def _audio_dataset(rows): + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "label": ClassLabel(names=["cat", "dog", "en_us"]), + } + ) + return Dataset.from_list(rows, features=features) + + +def _config( + samples=4, + label_mapping=None, + *, + streaming=False, + shuffle=True, + max_duration_seconds=None, +): + return WinMLEvaluationConfig( + model_id="example/audio-classifier", + task="audio-classification", + dataset=DatasetConfig( + path="example/audio-dataset", + split="test", + samples=samples, + shuffle=shuffle, + seed=42, + label_mapping=label_mapping, + streaming=streaming, + max_duration_seconds=max_duration_seconds, + ), + ) + + +def _save_sign_classifier(path): + input_info = onnx.helper.make_tensor_value_info( + "input_values", onnx.TensorProto.FLOAT, [1, 8] + ) + output_info = onnx.helper.make_tensor_value_info( + "logits", onnx.TensorProto.FLOAT, [1, 2] + ) + graph = onnx.helper.make_graph( + [ + onnx.helper.make_node( + "ReduceMean", ["input_values"], ["score"], axes=[1], keepdims=1 + ), + onnx.helper.make_node("Neg", ["score"], ["negative_score"]), + onnx.helper.make_node( + "Concat", ["score", "negative_score"], ["logits"], axis=1 + ), + ], + "sign_classifier", + [input_info], + [output_info], + ) + model = onnx.helper.make_model( + graph, + opset_imports=[onnx.helper.make_opsetid("", 17)], + ir_version=9, + ) + onnx.save(model, path) + + +class TestAudioPreprocessing: + def test_decodes_encoded_audio_bytes_without_torchcodec(self): + encoded = BytesIO() + sf.write(encoded, np.array([0.25, -0.5], dtype=np.float32), 16_000, format="WAV") + + waveform, sampling_rate = WinMLAudioClassificationEvaluator._decode_audio( + {"bytes": encoded.getvalue(), "path": None} + ) + + assert sampling_rate == 16_000 + np.testing.assert_allclose(waveform, [0.25, -0.5], atol=4e-5) + + def test_decodes_path_audio_through_dataset_opener(self, tmp_path): + audio_path = tmp_path / "clip.wav" + frames_first = np.array([[0.25, 0.5], [-0.25, -0.5]], dtype=np.float32) + sf.write(audio_path, frames_first, 16_000) + + waveform, sampling_rate = WinMLAudioClassificationEvaluator._decode_audio( + {"bytes": None, "path": str(audio_path)} + ) + + assert sampling_rate == 16_000 + assert waveform.shape == (2, 2) + np.testing.assert_allclose(waveform, frames_first.T, atol=4e-5) + + def test_mono_and_both_stereo_layouts(self): + mono = np.array([1.0, 3.0], dtype=np.float32) + channels_first = np.array([[1.0, 3.0], [3.0, 5.0]], dtype=np.float32) + channels_last = channels_first.T + one_frame_channels_first = np.array([[1.0], [0.5]], dtype=np.float32) + + np.testing.assert_array_equal( + WinMLAudioClassificationEvaluator._to_mono(mono), + mono, + ) + np.testing.assert_array_equal( + WinMLAudioClassificationEvaluator._to_mono(channels_first), + np.array([2.0, 4.0], dtype=np.float32), + ) + np.testing.assert_array_equal( + WinMLAudioClassificationEvaluator._to_mono(channels_last), + np.array([2.0, 4.0], dtype=np.float32), + ) + np.testing.assert_array_equal( + WinMLAudioClassificationEvaluator._to_mono(one_frame_channels_first), + np.array([0.75], dtype=np.float32), + ) + +class TestAudioLabelAlignmentAndSampling: + def test_fleurs_classlabel_maps_de_de_to_model_id_55(self): + encoded = BytesIO() + sf.write(encoded, np.ones(8, dtype=np.float32), 16_000, format="WAV") + language_names = [f"language_{index}" for index in range(17)] + ["de_de"] + dataset = Dataset.from_list( + [ + { + "audio": {"bytes": encoded.getvalue(), "path": "german.wav"}, + "lang_id": 17, + "language": "German", + } + ], + features=Features( + { + "audio": {"bytes": Value("binary"), "path": Value("string")}, + "lang_id": ClassLabel(names=language_names), + "language": Value("string"), + } + ), + ) + id2label = {index: f"model_{index}" for index in range(56)} + id2label[55] = "deu" + + class _LanguageClassifier: + io_config: ClassVar = { + "input_names": ["input_values"], + "input_shapes": [[1, 8]], + "output_names": ["logits"], + "output_shapes": [[1, 56]], + } + config = SimpleNamespace( + id2label=id2label, + label2id={label: index for index, label in id2label.items()}, + ) + + def __call__(self, **_kwargs): + logits = torch.zeros((1, 56)) + logits[0, 55] = 1.0 + return {"logits": logits} + + config = WinMLEvaluationConfig( + model_id="facebook/mms-lid-256", + task="audio-classification", + dataset=DatasetConfig( + path="google/fleurs", + name="de_de", + split="test", + samples=1, + shuffle=False, + columns_mapping={"label_column": "lang_id"}, + label_mapping={"de_de": 55}, + max_duration_seconds=1.0, + ), + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _LanguageClassifier()) + metrics = evaluator.compute() + + assert isinstance(evaluator._label_feature, ClassLabel) + assert evaluator._label_feature.int2str(17) == "de_de" + assert evaluator.data[0].model_id == 55 + assert metrics["accuracy"] == 1.0 + assert metrics["macro_f1"] == 1.0 + assert metrics["processed_samples"] == 1 + assert metrics["inference_windows"] == 1 + assert metrics["per_label_processed"] == {"deu": 1} + + def test_embedded_audio_with_scalar_string_label_mapping(self): + encoded = BytesIO() + sf.write(encoded, np.ones(8, dtype=np.float32), 16_000, format="WAV") + dataset = Dataset.from_list( + [{"audio": {"bytes": encoded.getvalue(), "path": "clip.wav"}, "genre": "feline"}], + features=Features( + { + "audio": {"bytes": Value("binary"), "path": Value("string")}, + "genre": Value("string"), + } + ), + ) + config = WinMLEvaluationConfig( + model_id="example/audio-classifier", + task="audio-classification", + dataset=DatasetConfig( + path="example/audio-dataset", + split="test", + samples=1, + shuffle=False, + columns_mapping={"label_column": "genre"}, + label_mapping={"feline": 0}, + ), + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _SignClassifier()) + metrics = evaluator.compute() + + assert evaluator.data[0].model_id == 0 + assert metrics["processed_samples"] == 1 + assert metrics["per_label_processed"] == {"cat": 1} + + def test_saved_dataset_dict_selects_requested_split(self, tmp_path): + train = _audio_dataset( + [{"audio": {"array": [-1.0] * 8, "sampling_rate": 16_000}, "label": 1}] + ) + test = _audio_dataset( + [{"audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, "label": 0}] + ) + dataset_path = tmp_path / "audio-dataset" + DatasetDict({"train": train, "test": test}).save_to_disk(dataset_path) + config = _config(samples=1, label_mapping={"cat": 0}) + config.dataset.path = str(dataset_path) + + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _SignClassifier()) + + assert evaluator._eligible_count == 1 + assert [sample.model_id for sample in evaluator.data] == [0] + + def test_saved_dataset_dict_missing_split_is_clear(self, tmp_path): + dataset_path = tmp_path / "audio-dataset" + DatasetDict( + { + "train": _audio_dataset( + [ + { + "audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, + "label": 0, + } + ] + ) + } + ).save_to_disk(dataset_path) + config = _config(samples=1) + config.dataset.path = str(dataset_path) + + with pytest.raises( + DatasetValidationError, + match=r"Dataset split 'test' was not found; available splits: \['train'\]", + ): + WinMLAudioClassificationEvaluator(config, _SignClassifier()) + + def test_filters_authoritative_labels_before_stratified_sampling(self): + rows = [ + {"audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, "label": 0}, + {"audio": {"array": [2.0] * 8, "sampling_rate": 16_000}, "label": 0}, + {"audio": {"array": [-1.0] * 8, "sampling_rate": 16_000}, "label": 1}, + {"audio": {"array": [-2.0] * 8, "sampling_rate": 16_000}, "label": 1}, + *[ + { + "audio": {"array": [9.0] * 8, "sampling_rate": 16_000}, + "label": 2, + } + for _ in range(20) + ], + ] + dataset = _audio_dataset(rows) + config = _config(samples=4, label_mapping={"cat": 0, "dog": 1}) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _SignClassifier()) + + assert evaluator._eligible_count == 4 + assert [sample.model_id for sample in evaluator.data].count(0) == 2 + assert [sample.model_id for sample in evaluator.data].count(1) == 2 + assert all(sample.model_id != 2 for sample in evaluator.data) + + def test_does_not_infer_cross_region_or_other_near_match(self): + dataset = _audio_dataset( + [ + { + "audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, + "label": 2, + } + ] + ) + model = _SignClassifier() + model.config = SimpleNamespace( + label2id={"en-IN": 0}, + id2label={0: "en-IN"}, + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + pytest.raises(DatasetValidationError, match="no exact overlap"), + ): + WinMLAudioClassificationEvaluator(_config(samples=1), model) + + def test_zero_overlap_mapping_fails_closed(self): + dataset = _audio_dataset( + [ + { + "audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, + "label": 0, + } + ] + ) + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + pytest.raises(DatasetValidationError, match="no exact overlap"), + ): + WinMLAudioClassificationEvaluator( + _config(samples=1, label_mapping={"missing": 0}), + _SignClassifier(), + ) + + def test_no_shuffle_streaming_stops_after_balanced_quota(self): + rows = [{"audio": f"cat-{index}", "label": 0} for index in range(3)] + [ + {"audio": f"dog-{index}", "label": 1} for index in range(20) + ] + dataset = _CountingStreamingDataset(rows) + config = _config( + samples=6, + label_mapping={"cat": 0, "dog": 1}, + streaming=True, + shuffle=False, + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _SignClassifier()) + + assert dataset.rows_yielded == 6 + assert dataset.rows_yielded < len(rows) + assert [sample.model_id for sample in evaluator.data] == [0, 1, 0, 1, 0, 1] + assert [sample.row["audio"] for sample in evaluator.data] == [ + "cat-0", + "dog-0", + "cat-1", + "dog-1", + "cat-2", + "dog-2", + ] + + @pytest.mark.parametrize( + ("labels", "expected_counts"), + [ + ([0, 0, 0, 1, 1, 1], {0: 2, 1: 2}), + ([0, 1, 1, 2, 2, 2], {0: 1, 1: 2, 2: 2}), + ], + ids=["absent-authoritative-class", "uneven-class"], + ) + def test_no_shuffle_streaming_exhausts_and_reports_short_quota( + self, + labels, + expected_counts, + ): + rows = [{"audio": str(index), "label": label} for index, label in enumerate(labels)] + dataset = _CountingStreamingDataset(rows) + config = _config( + samples=6, + label_mapping={"cat": 0, "dog": 1, "en_us": 2}, + streaming=True, + shuffle=False, + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _SignClassifier()) + + actual_counts = { + model_id: [sample.model_id for sample in evaluator.data].count(model_id) + for model_id in expected_counts + } + assert dataset.rows_yielded == len(rows) + assert actual_counts == expected_counts + assert evaluator._selected_count == sum(expected_counts.values()) + assert evaluator._selected_count < config.dataset.samples + + def test_shuffle_streaming_still_scans_complete_stream(self): + rows = [{"audio": f"cat-{index}", "label": 0} for index in range(20)] + [ + {"audio": f"dog-{index}", "label": 1} for index in range(20) + ] + dataset = _CountingStreamingDataset(rows) + config = _config( + samples=6, + label_mapping={"cat": 0, "dog": 1}, + streaming=True, + shuffle=True, + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _SignClassifier()) + + assert dataset.rows_yielded == len(rows) + assert evaluator._eligible_count == len(rows) + assert [sample.model_id for sample in evaluator.data].count(0) == 3 + assert [sample.model_id for sample in evaluator.data].count(1) == 3 + + +class TestAudioPredictionAndMetrics: + def test_pre_normalized_waveform_must_be_one_dimensional(self): + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ): + adapter = _AudioModelAdapter(_config(samples=1), _SignClassifier()) + with pytest.raises(ValueError, match="must be 1D"): + adapter(np.ones((1, 8), dtype=np.float32)) + + def test_rank_three_spectrogram_runs_through_adapter(self): + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_SpectrogramFeatureExtractor(), + ): + logits = _AudioModelAdapter(_config(samples=1), _SpectrogramClassifier())( + {"array": np.ones(16, dtype=np.float32), "sampling_rate": 16_000} + ) + + np.testing.assert_array_equal(logits, [1.0, 0.0, -1.0]) + + def test_overlength_waveform_aggregates_all_fixed_shape_windows_by_default(self): + model = _RecordingClassifier() + extractor = _TwoInputFeatureExtractor() + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=extractor, + ): + adapter = _AudioModelAdapter(_config(samples=1), model) + logits = adapter([1.0] * 24) + + assert adapter.model is model + assert len(model.calls) == 3 + assert all(set(call) == {"input_values"} for call in model.calls) + assert all(call["input_values"].shape == (1, 8) for call in model.calls) + assert adapter.last_was_truncated is False + assert adapter.last_window_count == 3 + assert extractor.kwargs == { + "padding": "max_length", + "truncation": True, + "max_length": 8, + } + np.testing.assert_array_equal(logits, [1.0, -1.0, -10.0]) + + def test_explicit_duration_cap_truncates_before_bounded_windowing(self): + model = _RecordingClassifier() + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_TwoInputFeatureExtractor(), + ): + adapter = _AudioModelAdapter( + _config(samples=1, max_duration_seconds=0.001), + model, + ) + adapter({"array": np.ones(40, dtype=np.float32), "sampling_rate": 16_000}) + + assert len(model.calls) == 2 + assert adapter.last_was_truncated is True + assert adapter.last_window_count == 2 + + def test_resamples_22050_hz_audio_to_feature_extractor_rate(self): + extractor = _TwoInputFeatureExtractor() + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=extractor, + ): + _AudioModelAdapter(_config(samples=1), _RecordingClassifier())( + {"array": np.ones(22_050, dtype=np.float32), "sampling_rate": 22_050} + ) + + assert sum(waveform.size for waveform in extractor.waveforms) == 16_000 + + def test_rejects_multiple_static_model_inputs(self): + with ( + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_TwoInputFeatureExtractor(), + ), + pytest.raises(ValueError, match="exactly one model input"), + ): + _AudioModelAdapter(_config(samples=1), _TwoInputClassifier()) + + def test_rejects_dynamic_audio_shape(self): + model = _SignClassifier() + model.io_config = { + **model.io_config, + "input_shapes": [[1, "audio_length"]], + } + with ( + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + pytest.raises(ValueError, match="static non-batch input shapes"), + ): + _AudioModelAdapter(_config(samples=1), model) + + @pytest.mark.parametrize( + ("output_names", "output_shapes", "message"), + [ + (["logits", "aux"], [[1, 3], [1, 3]], "exactly one 'logits' output"), + (["logits"], [[1, 3, 1]], r"logits shape \[1, classes\]"), + ], + ) + def test_rejects_invalid_output_contract(self, output_names, output_shapes, message): + model = _SignClassifier() + model.io_config = { + **model.io_config, + "output_names": output_names, + "output_shapes": output_shapes, + } + with ( + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + pytest.raises(ValueError, match=message), + ): + _AudioModelAdapter(_config(samples=1), model) + + def test_native_model_without_io_config_receives_all_extractor_outputs(self): + class _NativeModel: + config = _SignClassifier.config + + def __init__(self): + self.inputs = None + + def __call__(self, **kwargs): + self.inputs = kwargs + return SimpleNamespace(logits=torch.tensor([[1.0, 0.0, -1.0]])) + + model = _NativeModel() + extractor = _TwoInputFeatureExtractor() + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=extractor, + ): + logits = _AudioModelAdapter(_config(samples=1), model)([1.0] * 8) + + assert set(model.inputs) == {"input_values", "attention_mask", "extra"} + np.testing.assert_array_equal(logits, [1.0, 0.0, -1.0]) + + @pytest.mark.parametrize( + ("label_feature", "labels", "names", "label_mapping"), + [ + ( + Sequence(ClassLabel(names=["cat", "dog", "other"])), + [[0, 1], [1, 2]], + None, + None, + ), + ( + Sequence(Value("string")), + [["/cat", "/dog"], ["/dog", "/other"]], + [["cat", "dog"], ["dog", "other"]], + None, + ), + ], + ids=["sequence-classlabel", "sequence-value-with-label-names"], + ) + def test_multi_label_targets_have_finite_sigmoid_average_precision( + self, + label_feature, + labels, + names, + label_mapping, + ): + feature_map = { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "labels": label_feature, + } + rows = [ + { + "audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, + "labels": labels[0], + }, + { + "audio": {"array": [-1.0] * 8, "sampling_rate": 16_000}, + "labels": labels[1], + }, + ] + columns_mapping = {"label_column": "labels"} + if names is not None: + feature_map["names"] = Sequence(Value("string")) + rows[0]["names"] = names[0] + rows[1]["names"] = names[1] + columns_mapping["label_name_column"] = "names" + dataset = Dataset.from_list(rows, features=Features(feature_map)) + config = WinMLEvaluationConfig( + model_id="example/audio-classifier", + task="audio-classification", + dataset=DatasetConfig( + path="example/audio-dataset", + split="test", + samples=2, + shuffle=False, + columns_mapping=columns_mapping, + label_mapping=label_mapping, + ), + ) + model = _RecordingClassifier() + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_TwoInputFeatureExtractor(), + ), + ): + metrics = WinMLAudioClassificationEvaluator(config, model).compute() + + assert len(model.calls) == 2 + assert np.isfinite(metrics["sample_average_precision"]) + assert np.isfinite(metrics["micro_average_precision"]) + + def test_streaming_selection_preserves_raw_audio_without_reencoding(self): + encoded = BytesIO() + sf.write(encoded, np.ones(8, dtype=np.float32), 16_000, format="WAV") + raw_audio = {"bytes": encoded.getvalue(), "path": None} + dataset = IterableDataset.from_generator( + lambda: iter([{"audio": raw_audio, "labels": ["cat"]}]), + features=Features( + {"audio": Audio(), "labels": Sequence(Value("string"))} + ), + ) + config = WinMLEvaluationConfig( + model_id="example/audio-classifier", + task="audio-classification", + dataset=DatasetConfig( + path="example/audio-dataset", + split="test", + samples=1, + shuffle=False, + streaming=True, + columns_mapping={"label_column": "labels"}, + ), + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch.object( + Audio, + "encode_example", + side_effect=AssertionError("streaming audio must remain raw"), + ), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_TwoInputFeatureExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _SignClassifier()) + assert evaluator.data[0]["audio"]["bytes"] == raw_audio["bytes"] + metrics = evaluator.compute() + + assert metrics["processed_samples"] == 1 + assert np.isfinite(metrics["sample_average_precision"]) + + def test_streaming_selection_decodes_raw_virtual_archive_path(self, tmp_path): + encoded = BytesIO() + sf.write(encoded, np.ones(8, dtype=np.float32), 16_000, format="WAV") + archive_path = tmp_path / "audio.zip" + member_path = "genres/example.wav" + with ZipFile(archive_path, "w") as archive: + archive.writestr(member_path, encoded.getvalue()) + raw_audio = f"zip://{member_path}::{archive_path.as_posix()}" + dataset = IterableDataset.from_generator( + lambda: iter([{"audio": raw_audio, "labels": ["cat"]}]), + features=Features( + {"audio": Audio(), "labels": Sequence(Value("string"))} + ), + ) + config = WinMLEvaluationConfig( + model_id="example/audio-classifier", + task="audio-classification", + dataset=DatasetConfig( + path="example/audio-dataset", + split="test", + samples=1, + shuffle=False, + streaming=True, + columns_mapping={"label_column": "labels"}, + ), + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch.object( + Audio, + "encode_example", + side_effect=AssertionError("streaming audio must remain raw"), + ), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_TwoInputFeatureExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _SignClassifier()) + assert evaluator.data[0]["audio"] == raw_audio + metrics = evaluator.compute() + + assert metrics["processed_samples"] == 1 + assert np.isfinite(metrics["sample_average_precision"]) + + def test_end_to_end_fixed_shape_evaluator_reports_accounting(self): + rows = [ + {"audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, "label": 0}, + {"audio": {"array": [2.0] * 8, "sampling_rate": 16_000}, "label": 0}, + {"audio": {"array": [-1.0] * 8, "sampling_rate": 16_000}, "label": 1}, + {"audio": {"array": [-2.0] * 8, "sampling_rate": 16_000}, "label": 1}, + ] + dataset = _audio_dataset(rows) + config = _config(samples=4, label_mapping={"cat": 0, "dog": 1}) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + metrics = WinMLAudioClassificationEvaluator(config, _SignClassifier()).compute() + + assert metrics == { + "accuracy": 1.0, + "macro_f1": 1.0, + "represented_classes": 2, + "total_classes": 3, + "class_coverage": 2 / 3, + "requested_samples": 4, + "eligible_samples": 4, + "selected_samples": 4, + "processed_samples": 4, + "inference_windows": 4, + "truncated_samples": 0, + "rejected_samples": 0, + "rejected_by_reason": {}, + "per_label_processed": {"cat": 2, "dog": 2}, + "confusion_matrix": {"cat": {"cat": 2}, "dog": {"dog": 2}}, + } + + def test_duration_truncation_is_reported(self): + dataset = _audio_dataset( + [{"audio": {"array": [1.0] * 16, "sampling_rate": 16_000}, "label": 0}] + ) + config = _config( + samples=1, + label_mapping={"cat": 0}, + shuffle=False, + max_duration_seconds=0.0005, + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + metrics = WinMLAudioClassificationEvaluator(config, _SignClassifier()).compute() + + assert metrics["inference_windows"] == 1 + assert metrics["truncated_samples"] == 1 + + def test_rejected_audio_is_accounted_after_selection(self): + rows = [ + {"audio": {"array": [], "sampling_rate": 16_000}, "label": 0}, + *[ + {"audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, "label": 0} + for _ in range(5) + ], + *[ + {"audio": {"array": [-1.0] * 8, "sampling_rate": 16_000}, "label": 1} + for _ in range(6) + ], + ] + dataset = _audio_dataset(rows) + config = _config(samples=12, label_mapping={"cat": 0, "dog": 1}) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + metrics = WinMLAudioClassificationEvaluator(config, _SignClassifier()).compute() + + assert metrics["eligible_samples"] == 12 + assert metrics["selected_samples"] == 12 + assert metrics["processed_samples"] == 11 + assert metrics["rejected_samples"] == 1 + assert metrics["rejected_by_reason"] == {"ValueError": 1} + assert metrics["per_label_processed"] == {"cat": 5, "dog": 6} + + def test_prediction_outside_reference_labels_is_an_accuracy_error(self): + class _OtherClassifier(_SignClassifier): + def __call__(self, **_kwargs): + return {"logits": torch.tensor([[0.0, 0.0, 10.0]])} + + dataset = _audio_dataset( + [ + { + "audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, + "label": 0, + } + ] + ) + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + metrics = WinMLAudioClassificationEvaluator( + _config(samples=1, label_mapping={"cat": 0}), + _OtherClassifier(), + ).compute() + + assert metrics["accuracy"] == 0.0 + assert metrics["macro_f1"] == 0.0 + assert metrics["represented_classes"] == 1 + assert metrics["total_classes"] == 3 + assert metrics["class_coverage"] == pytest.approx(1 / 3) + assert metrics["confusion_matrix"] == {"cat": {"other": 1}} + + def test_cli_with_saved_dataset_and_fixed_shape_onnx_reports_metrics(self, tmp_path): + dataset_path = tmp_path / "dataset" + model_path = tmp_path / "classifier.onnx" + output_path = tmp_path / "result.json" + label_mapping_path = tmp_path / "labels.json" + rows = [ + {"audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, "label": 0}, + {"audio": {"array": [2.0] * 8, "sampling_rate": 16_000}, "label": 0}, + {"audio": {"array": [-1.0] * 8, "sampling_rate": 16_000}, "label": 1}, + {"audio": {"array": [-2.0] * 8, "sampling_rate": 16_000}, "label": 1}, + ] + _audio_dataset(rows).save_to_disk(dataset_path) + _save_sign_classifier(model_path) + label_mapping_path.write_text(json.dumps({"cat": 0, "dog": 1}), encoding="utf-8") + hf_config = Wav2Vec2Config( + id2label={0: "cat", 1: "dog"}, + label2id={"cat": 0, "dog": 1}, + ) + + with ( + patch("winml.modelkit.loader.load_hf_config", return_value=hf_config), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityFeatureExtractor(), + ), + ): + result = CliRunner().invoke( + eval_command, + [ + "-m", + str(model_path), + "--model-id", + "example/audio-classifier", + "--task", + "audio-classification", + "--dataset", + str(dataset_path), + "--split", + "train", + "--samples", + "4", + "--max-duration-seconds", + "0.00025", + "--device", + "cpu", + "--label-mapping", + str(label_mapping_path), + "--output", + str(output_path), + ], + obj={"debug": False}, + ) + + assert result.exit_code == 0, result.output + metrics = json.loads(output_path.read_text(encoding="utf-8"))["metrics"] + assert metrics["accuracy"] == 1.0 + assert metrics["macro_f1"] == 1.0 + assert metrics["requested_samples"] == 4 + assert metrics["eligible_samples"] == 4 + assert metrics["processed_samples"] == 4 + assert metrics["truncated_samples"] == 4 + assert metrics["rejected_samples"] == 0 + + def test_cli_rejects_nonpositive_max_duration(self): + result = CliRunner().invoke( + eval_command, + ["--max-duration-seconds", "0"], + obj={"debug": False}, + ) + + assert result.exit_code == 2 + assert "0 is not in the range x>0" in result.output + + +class TestAudioRegistryCompatibility: + def test_audio_schema_and_evaluator_registered_without_changing_sibling(self): + from winml.modelkit.eval.evaluate import _DEFAULT_DATASETS, get_evaluator_class + from winml.modelkit.eval.text_classification_evaluator import ( + WinMLTextClassificationEvaluator, + ) + from winml.modelkit.utils.eval_utils import TASK_SCHEMAS + + audio_schema = TASK_SCHEMAS["audio-classification"] + assert [item.default for item in audio_schema.columns] == ["audio", "label"] + assert "audio-classification" not in _DEFAULT_DATASETS + assert get_evaluator_class(_config()).__name__ == "WinMLAudioClassificationEvaluator" + assert ( + get_evaluator_class(WinMLEvaluationConfig(task="text-classification")) + is WinMLTextClassificationEvaluator + ) + + schema_result = CliRunner().invoke( + eval_command, + ["--schema", "--task", "audio-classification"], + obj={}, + ) + assert schema_result.exit_code == 0 + assert "default: audio" in schema_result.output + assert "default: label" in schema_result.output diff --git a/tests/unit/eval/test_eval.py b/tests/unit/eval/test_eval.py index c02379189..01f5d0198 100644 --- a/tests/unit/eval/test_eval.py +++ b/tests/unit/eval/test_eval.py @@ -72,12 +72,14 @@ def test_config_roundtrip(self): split="test", samples=20, columns_mapping={"label_column": "lbl"}, + label_mapping={"blues": 3}, ), ) restored = WinMLEvaluationConfig.from_dict(config.to_dict()) assert restored.model_id == config.model_id assert restored.dataset.path == config.dataset.path assert restored.dataset.columns_mapping == config.dataset.columns_mapping + assert restored.dataset.label_mapping == config.dataset.label_mapping def test_config_roundtrip_preserves_revision(self): """DatasetConfig.revision survives to_dict/from_dict roundtrip.""" @@ -98,6 +100,27 @@ def test_dataset_config_revision_default_is_none(self): assert ds.revision is None assert "revision" not in ds.to_dict() + def test_dataset_config_max_duration_seconds_roundtrip(self): + dataset = DatasetConfig(path="audio-dataset", max_duration_seconds=2.0) + + restored = WinMLEvaluationConfig.from_dict( + WinMLEvaluationConfig(dataset=dataset).to_dict() + ) + + assert restored.dataset.max_duration_seconds == 2.0 + assert dataset.to_dict()["max_duration_seconds"] == 2.0 + + def test_dataset_config_max_duration_seconds_defaults_to_unbounded(self): + dataset = DatasetConfig(path="audio-dataset") + + assert dataset.max_duration_seconds is None + assert "max_duration_seconds" not in dataset.to_dict() + + @pytest.mark.parametrize("value", [0.0, -1.0, float("nan"), float("inf")]) + def test_dataset_config_rejects_nonpositive_max_duration(self, value: float): + with pytest.raises(ValueError, match="finite value greater than zero"): + DatasetConfig(max_duration_seconds=value) + def test_input_data_default_is_none(self): """input_data defaults to None and is omitted from to_dict.""" config = WinMLEvaluationConfig(model_id="test/model") diff --git a/tests/unit/recipes/test_cpu_recipes.py b/tests/unit/recipes/test_cpu_recipes.py index bc3106c57..ea142faf1 100644 --- a/tests/unit/recipes/test_cpu_recipes.py +++ b/tests/unit/recipes/test_cpu_recipes.py @@ -14,6 +14,36 @@ REPO_ROOT = Path(__file__).resolve().parents[3] recipes = [ + { + "path": REPO_ROOT + / "examples" + / "recipes" + / "facebook_mms-lid-256" + / "cpu" + / "cpu" + / "audio-classification_fp32_config.json", + "loader_task": "audio-classification", + "loader_model_class": "AutoModelForAudioClassification", + "loader_model_type": "wav2vec2", + "opset_version": 17, + "quant_mode": None, + "transformers_attention": "eager", + }, + { + "path": REPO_ROOT + / "examples" + / "recipes" + / "facebook_mms-lid-256" + / "cpu" + / "cpu" + / "audio-classification_fp16_config.json", + "loader_task": "audio-classification", + "loader_model_class": "AutoModelForAudioClassification", + "loader_model_type": "wav2vec2", + "opset_version": 17, + "quant_mode": "fp16", + "transformers_attention": "eager", + }, { "path": REPO_ROOT / "examples" @@ -48,7 +78,12 @@ @pytest.mark.parametrize( "rec", recipes, - ids=["audeering-wav2vec2-emotion-fp32", "audeering-wav2vec2-emotion-fp16"], + ids=[ + "facebook-mms-lid-256-fp32", + "facebook-mms-lid-256-fp16", + "audeering-wav2vec2-emotion-fp32", + "audeering-wav2vec2-emotion-fp16", + ], ) def test_cpu_recipes(rec): path: Path = rec["path"] @@ -66,6 +101,12 @@ def test_cpu_recipes(rec): # export.opset_version exact assert config.export is not None assert config.export.opset_version == rec["opset_version"] + assert config.export.input_tensors[0].dtype == "float32" + if "transformers_attention" in rec: + assert ( + config.export.compatibility.transformers_attention + == rec["transformers_attention"] + ) # loader routes to the emotion-regression head assert config.loader.task == rec["loader_task"]