feat(asr): run the WhisperS2T presets offline from --artifacts-path

docling/.agents/skills/docling/references/cli.md

@@ -178,13 +178,16 @@ docling report.pdf --artifacts-path /models --output /tmp/
 
 For native Whisper ASR, prefetch a selected OpenAI-format checkpoint. Plain
 `--all` excludes ASR models; `--all --asr` includes all native Whisper sizes and
-the Distil-Whisper presets.
+the Distil-Whisper presets. WhisperS2T presets are prefetched only by name; each
+one downloads the `float16` and `float32` models and the word aligner it needs.
 
 ```bash
 docling-tools models download whisper_tiny --output-dir /models
 docling --asr-model whisper_tiny_native --artifacts-path /models audio.wav
 docling-tools models download whisper_distil_small_en --output-dir /models
 docling --asr-model whisper_distil_small_en_native --artifacts-path /models audio.wav
+docling-tools models download whisper_small_en_s2t --output-dir /models
+docling --asr-model whisper_small_en_s2t --artifacts-path /models audio.wav
 ```
 
 `--artifacts-path` overrides the default Hugging Face cache; you can also set

docling/cli/models.py

@@ -30,7 +30,9 @@
 
 from docling.datamodel.asr_model_specs import (
     _DISTIL_WHISPER_OPENAI_CHECKPOINTS,
+    _WHISPER_S2T_PRESETS,
     _distil_whisper_prefetch_name,
+    _whisper_s2t_prefetch_name,
 )
 from docling.datamodel.settings import settings
 from docling.exceptions import RapidOcrModelSizeNotSupportedError
@@ -93,6 +95,20 @@ class _AvailableModels(str, Enum):
     WHISPER_DISTIL_MEDIUM_EN = "whisper_distil_medium_en"
     WHISPER_DISTIL_LARGE_V3 = "whisper_distil_large_v3"
     WHISPER_DISTIL_LARGE_V3_5 = "whisper_distil_large_v3_5"
+    WHISPER_TINY_S2T = "whisper_tiny_s2t"
+    WHISPER_TINY_EN_S2T = "whisper_tiny_en_s2t"
+    WHISPER_BASE_S2T = "whisper_base_s2t"
+    WHISPER_BASE_EN_S2T = "whisper_base_en_s2t"
+    WHISPER_SMALL_S2T = "whisper_small_s2t"
+    WHISPER_SMALL_EN_S2T = "whisper_small_en_s2t"
+    WHISPER_DISTIL_SMALL_EN_S2T = "whisper_distil_small_en_s2t"
+    WHISPER_MEDIUM_S2T = "whisper_medium_s2t"
+    WHISPER_MEDIUM_EN_S2T = "whisper_medium_en_s2t"
+    WHISPER_DISTIL_MEDIUM_EN_S2T = "whisper_distil_medium_en_s2t"
+    WHISPER_LARGE_V3_S2T = "whisper_large_v3_s2t"
+    WHISPER_DISTIL_LARGE_V3_S2T = "whisper_distil_large_v3_s2t"
+    WHISPER_DISTIL_LARGE_V3_5_S2T = "whisper_distil_large_v3_5_s2t"
+    WHISPER_LARGE_V3_TURBO_S2T = "whisper_large_v3_turbo_s2t"
 
 
 _WHISPER_MODELS = {
@@ -113,6 +129,11 @@ class _AvailableModels(str, Enum):
     for model_name in _DISTIL_WHISPER_OPENAI_CHECKPOINTS
 }
 
+_WHISPER_S2T_MODELS = {
+    _AvailableModels(_whisper_s2t_prefetch_name(model_name)): model_name
+    for model_name in _WHISPER_S2T_PRESETS
+}
+
 
 _default_models = [
     _AvailableModels.LAYOUT,
@@ -159,7 +180,8 @@ def download(
             "--asr",
             help=(
                 "With --all, also download native Whisper ASR models, including "
-                "the Distil-Whisper presets."
+                "the Distil-Whisper presets. WhisperS2T models are downloaded "
+                "only by name."
             ),
         ),
     ] = False,
@@ -234,8 +256,13 @@ def download(
         [
             model
             for model in _AvailableModels
-            if asr
-            or (model not in _WHISPER_MODELS and model not in _WHISPER_DISTIL_MODELS)
+            if model not in _WHISPER_S2T_MODELS
+            and (
+                asr
+                or (
+                    model not in _WHISPER_MODELS and model not in _WHISPER_DISTIL_MODELS
+                )
+            )
         ]
         if all
         else _default_models
@@ -324,6 +351,11 @@ def download(
             for model in to_download
             if model in _WHISPER_DISTIL_MODELS
         ],
+        whisper_s2t_models=[
+            _WHISPER_S2T_MODELS[model]
+            for model in to_download
+            if model in _WHISPER_S2T_MODELS
+        ],
     )
 
     if quiet:

docling/datamodel/asr_model_specs.py

@@ -718,6 +718,43 @@ def _distil_whisper_prefetch_name(model_name: str) -> str:
     beam_size=1,
 )
 
+_WHISPER_S2T_PRESETS: dict[str, InlineAsrWhisperS2TOptions] = {
+    preset.repo_id: preset
+    for preset in (
+        WHISPER_TINY_S2T,
+        WHISPER_TINY_EN_S2T,
+        WHISPER_BASE_S2T,
+        WHISPER_BASE_EN_S2T,
+        WHISPER_SMALL_S2T,
+        WHISPER_SMALL_EN_S2T,
+        WHISPER_DISTIL_SMALL_EN_S2T,
+        WHISPER_MEDIUM_S2T,
+        WHISPER_MEDIUM_EN_S2T,
+        WHISPER_DISTIL_MEDIUM_EN_S2T,
+        WHISPER_LARGE_V3_S2T,
+        WHISPER_DISTIL_LARGE_V3_S2T,
+        WHISPER_DISTIL_LARGE_V3_5_S2T,
+        WHISPER_LARGE_V3_TURBO_S2T,
+    )
+}
+
+_WHISPER_S2T_COMPUTE_TYPES = ("float32", "float16", "bfloat16")
+
+
+def _whisper_s2t_compute_type(torch_dtype: str | None, device: str) -> str:
+    if (
+        device == "cpu"
+        or torch_dtype is None
+        or torch_dtype not in _WHISPER_S2T_COMPUTE_TYPES
+    ):
+        return "float32"
+    return torch_dtype
+
+
+def _whisper_s2t_prefetch_name(model_name: str) -> str:
+    return "whisper_" + model_name.replace("-", "_").replace(".", "_") + "_s2t"
+
+
 # =============================================================================
 # Note on auto-selecting models
 # =============================================================================

docling/pipeline/asr_transcriber.py

@@ -28,7 +28,10 @@
 )
 from docling.datamodel.asr_model_specs import (
     _DISTIL_WHISPER_OPENAI_CHECKPOINTS,
+    _WHISPER_S2T_PRESETS,
     _distil_whisper_prefetch_name,
+    _whisper_s2t_compute_type,
+    _whisper_s2t_prefetch_name,
 )
 from docling.datamodel.base_models import (
     ConversionStatus,
@@ -669,7 +672,7 @@ def __init__(
             except ImportError:
                 raise ImportError(
                     "whisper_s2t is not installed. Please install it via "
-                    "`pip install 'whisper-s2t-reborn[pyav]>=1.7.1'`."
+                    "`pip install 'whisper-s2t-reborn>=1.8.0'`."
                 )
 
             self.whisper_s2t = whisper_s2t
@@ -692,13 +695,15 @@ def __init__(
             # inference. Coerce to float32 when running on CPU so that the
             # explicit *_S2T presets (which default to float16 for CUDA
             # performance) do not fail at model load on CPU-only installs.
-            compute_type = asr_options.torch_dtype
-            if self.device == "cpu" and compute_type in ("float16", "bfloat16"):
+            compute_type = _whisper_s2t_compute_type(
+                asr_options.torch_dtype, self.device
+            )
+            if asr_options.torch_dtype not in (None, compute_type):
                 _log.warning(
-                    f"compute_type='{compute_type}' is not supported by "
-                    f"CTranslate2 on CPU; falling back to 'float32'."
+                    f"compute_type='{asr_options.torch_dtype}' is not supported "
+                    f"by WhisperS2T on {self.device}; falling back to "
+                    f"'{compute_type}'."
                 )
-                compute_type = "float32"
 
             # Build ASR options for whisper_s2t
             asr_opts = {
@@ -715,10 +720,34 @@ def __init__(
                 "asr_options": asr_opts,
             }
 
-            self.model = whisper_s2t.load_model(
-                model_identifier=self.model_identifier,
-                **model_kwargs,
-            )
+            if artifacts_path is None:
+                self.model = whisper_s2t.load_model(
+                    model_identifier=self.model_identifier,
+                    **model_kwargs,
+                )
+            else:
+                from huggingface_hub.utils import LocalEntryNotFoundError
+
+                try:
+                    self.model = whisper_s2t.load_model(
+                        model_identifier=self.model_identifier,
+                        cache_dir=str(artifacts_path),
+                        local_files_only=True,
+                        **model_kwargs,
+                    )
+                except LocalEntryNotFoundError as err:
+                    message = (
+                        f"artifacts_path ({artifacts_path}) does not contain the "
+                        f"{compute_type} WhisperS2T files required by ASR model "
+                        f"'{self.model_identifier}'."
+                    )
+                    if self.model_identifier in _WHISPER_S2T_PRESETS:
+                        message += (
+                            " Prefetch them with: docling-tools models download "
+                            f"{_whisper_s2t_prefetch_name(self.model_identifier)} "
+                            f'--output-dir "{artifacts_path}"'
+                        )
+                    raise FileNotFoundError(message) from err
 
             # Store options for transcription
             self.language = asr_options.language

docling/utils/model_downloader.py

@@ -10,7 +10,11 @@
 import requests
 from tqdm import tqdm
 
-from docling.datamodel.asr_model_specs import _DISTIL_WHISPER_OPENAI_CHECKPOINTS
+from docling.datamodel.asr_model_specs import (
+    _DISTIL_WHISPER_OPENAI_CHECKPOINTS,
+    _WHISPER_S2T_PRESETS,
+    _whisper_s2t_compute_type,
+)
 from docling.datamodel.pipeline_options import (
     LayoutObjectDetectionOptions,
     granite_picture_description,
@@ -146,6 +150,47 @@ def _download_distil_whisper_models(
         )
 
 
+def _download_whisper_s2t_models(
+    model_names: list[str], output_dir: Path, *, force: bool, progress: bool
+) -> None:
+    try:
+        import whisper_s2t  # type: ignore
+    except ImportError as exc:
+        raise ImportError(
+            "WhisperS2T prefetch requires whisper-s2t-reborn. Install it via "
+            "`pip install 'whisper-s2t-reborn>=1.8.0'`."
+        ) from exc
+    from huggingface_hub.utils import disable_progress_bars
+
+    unknown = set(model_names) - _WHISPER_S2T_PRESETS.keys()
+    if unknown:
+        raise ValueError(f"Unknown WhisperS2T models: {sorted(unknown)}")
+    if not progress:
+        disable_progress_bars()
+
+    for model_name in model_names:
+        torch_dtype = _WHISPER_S2T_PRESETS[model_name].torch_dtype
+        compute_types = sorted(
+            {
+                _whisper_s2t_compute_type(torch_dtype, device)
+                for device in ("cuda", "cpu")
+            }
+        )
+        aligner = whisper_s2t.default_word_aligner(model_name)
+        names = [model_name] if aligner is None else [model_name, aligner]
+        for compute_type in compute_types:
+            _log.info(
+                "Downloading WhisperS2T model %s (%s)...", model_name, compute_type
+            )
+            for name in names:
+                whisper_s2t.download_model(
+                    name,
+                    compute_type=compute_type,
+                    cache_dir=str(output_dir),
+                    force_download=force,
+                )
+
+
 def download_models(
     output_dir: Optional[Path] = None,
     *,
@@ -174,6 +219,7 @@ def download_models(
     with_nemotron_ocr: bool = False,
     whisper_models: Optional[list[str]] = None,
     whisper_distil_models: Optional[list[str]] = None,
+    whisper_s2t_models: Optional[list[str]] = None,
 ):
     if easyocr_languages is not None and not with_easyocr:
         raise ValueError("easyocr_languages requires with_easyocr=True")
@@ -202,6 +248,11 @@ def download_models(
             whisper_distil_models, output_dir, force=force, progress=progress
         )
 
+    if whisper_s2t_models:
+        _download_whisper_s2t_models(
+            whisper_s2t_models, output_dir, force=force, progress=progress
+        )
+
     if with_layout:
         _log.info("Downloading layout model...")
         layout_spec = LayoutObjectDetectionOptions().model_spec

docs/usage/advanced_options.md

@@ -45,6 +45,16 @@ docling-tools models download whisper_distil_small_en --output-dir /models
 docling --asr-model whisper_distil_small_en_native --artifacts-path /models audio.wav
 ```
 
+The WhisperS2T presets use their `--asr-model` names. Each one downloads its
+CTranslate2 model in `float16` (for CUDA) and `float32` (for CPU), plus the
+small word-alignment model that word timestamps use if the preset needs one,
+into the output directory in the Hugging Face cache layout:
+
+```sh
+docling-tools models download whisper_small_en_s2t --output-dir /models
+docling --asr-model whisper_small_en_s2t --artifacts-path /models audio.wav
+```
+
 `docling-tools models download --all` does not include ASR models. Add `--asr`
 to prefetch all native Whisper model sizes, English-only variants, and
 Distil-Whisper presets:
@@ -54,7 +64,8 @@ docling-tools models download --all --asr --output-dir /models
 ```
 
 This requires the `asr` extra and downloads OpenAI-format checkpoints. It does
-not prefetch Transformers-format, MLX, or WhisperS2T models.
+not prefetch Transformers-format, MLX, or WhisperS2T models. Prefetch a
+WhisperS2T preset by its name, as shown above.
 
 Alternatively, models can be programmatically downloaded using `docling.utils.model_downloader.download_models()`.
 

pyproject.toml

@@ -181,7 +181,7 @@ format-audio = [
   'mlx-whisper>=0.4.3 ; python_version >= "3.10" and sys_platform == "darwin" and platform_machine == "arm64"',
   'openai-whisper>=20250625',
   'numba>=0.63.0',
-  'whisper-s2t-reborn>=1.7.1,<2.0.0 ; sys_platform != "darwin" or platform_machine != "arm64"',
+  'whisper-s2t-reborn>=1.8.0,<2.0.0 ; sys_platform != "darwin" or platform_machine != "arm64"',
 ]
 
 feat-asr-transformers = [