diff --git a/.gitignore b/.gitignore
index f258900c0d..4ea1c3c9cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -454,3 +454,6 @@ pip-selfcheck.json
# AI agents
**/.bob/
+
+# Local ASR model validation output
+/asr-validation/
diff --git a/docling/.agents/skills/docling/references/cli.md b/docling/.agents/skills/docling/references/cli.md
index 7b8ebbb31c..5408deab20 100644
--- a/docling/.agents/skills/docling/references/cli.md
+++ b/docling/.agents/skills/docling/references/cli.md
@@ -1,5 +1,9 @@
# Docling CLI reference
+For Transformers ASR, install `docling-slim[feat-asr-transformers]` and select
+`--asr-model granite_speech_5_0_2b` (or another speech preset). The same ASR
+settings apply to video.
+
The `docling` CLI is the quickest way to understand the content of any supported
document. It ships with the `docling` package (`pip install docling`) and needs
no code. Use it any time you need to read, convert, or extract content from a
diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md
index 254f59f383..3472ad42ff 100644
--- a/docling/.agents/skills/docling/references/python-sdk.md
+++ b/docling/.agents/skills/docling/references/python-sdk.md
@@ -394,8 +394,36 @@ result = converter.convert("interview.wav")
print(result.document.export_to_markdown()) # timestamped transcript
```
+For a Transformers-format Whisper checkpoint, install
+`docling-slim[feat-asr-transformers]` and select an explicit preset such as
+`asr_model_specs.WHISPER_TINY_TRANSFORMERS`. The presets cover tiny, base,
+small, medium, large, and large-v3-turbo. Set
+`pipeline_options.artifacts_path` to a directory that contains `config.json`
+and the Transformers model files, or to its parent directory with a
+`openai--whisper-tiny` subdirectory. The local directory is used without a
+Hugging Face lookup. For MLX Whisper, use `WHISPER_TINY_MLX` and provide an
+MLX checkpoint with `config.json` and `weights.safetensors` or `weights.npz`.
+
+Other explicit Transformers presets are `GRANITE_4_0_1B_SPEECH` and
+`NEMOTRON_3_5_ASR_STREAMING`. These models return
+one transcript item that covers the full audio duration. The Transformers
+Whisper presets provide model segment timestamps.
+
The CLI can also transcribe: `docling interview.wav --to md --output /tmp/`.
+Additional presets include `GRANITE_SPEECH_5_0_2B`,
+`GRANITE_SPEECH_5_0_470M_TURBOCTC`, its `_NC` variant,
+`GRANITE_SPEECH_4_1_2B`, its `_PLUS` variant, and
+`NEMOTRON_SPEECH_STREAMING_EN_0_6B`. Copy a preset with
+`.model_copy(deep=True)` before changing `revision`, `language`, or `prompt`.
+Granite and TurboCTC use FP32 weights on MPS to avoid BF16 normalization failures.
+Plus defaults to plain transcription; speaker and timestamp tags are not parsed
+into tracks.
+The `_NC` preset uses a noncommercial license.
+For real decoding checks, use `scripts/validate_asr_models.py` in a repository
+checkout. Repeat `--model` to select presets; without it, the script runs every
+ASR preset on the same fragment. It is outside normal pytest collection.
+
Video files (`.mp4`, `.mov`, `.mkv`, `.webm`, `.avi`) use `VideoPipeline` with
`VideoPipelineOptions`, which also samples frames. `max_sampled_frames` defaults
to `200` (set `None` for no limit). When `document_timeout` is set, all FFmpeg
diff --git a/docling/backend/md_backend.py b/docling/backend/md_backend.py
index 7e77cb8054..874e137815 100644
--- a/docling/backend/md_backend.py
+++ b/docling/backend/md_backend.py
@@ -1182,20 +1182,22 @@ def convert(self) -> DoclingDocument:
)
html_str = html_payload.text
- def _restore_original_html(txt, regex):
- _txt, count = re.subn(regex, "", txt)
- if count != self._html_blocks:
- raise RuntimeError(
- "An internal error has occurred during Markdown conversion."
- )
- return _txt
-
- # restore original HTML by removing previously added markers
- for regex in [
- rf"
\s*\s*{_START_MARKER}",
- rf"{_STOP_MARKER}\s*\s*
",
- ]:
- html_str = _restore_original_html(txt=html_str, regex=regex)
+ # Restore only the marked HTML blocks. The HTML serializer
+ # escapes code content, so decode each block before parsing it.
+ pattern = (
+ rf"\s*\s*{re.escape(_START_MARKER)}"
+ rf"(.*?){re.escape(_STOP_MARKER)}\s*\s*
"
+ )
+ html_str, count = re.subn(
+ pattern,
+ lambda match: unescape(match.group(1)),
+ html_str,
+ flags=re.DOTALL,
+ )
+ if count != self._html_blocks:
+ raise RuntimeError(
+ "An internal error has occurred during Markdown conversion."
+ )
self._html_blocks = 0
# delegate to HTML backend
stream = BytesIO(bytes(html_str, encoding="utf-8"))
diff --git a/docling/cli/main.py b/docling/cli/main.py
index 9434598b54..ca835aabf2 100644
--- a/docling/cli/main.py
+++ b/docling/cli/main.py
@@ -66,12 +66,16 @@
)
from docling.datamodel.accelerator_options import AcceleratorDevice, AcceleratorOptions
from docling.datamodel.asr_model_specs import (
+ ADDITIONAL_ASR_PRESETS,
+ GRANITE_4_0_1B_SPEECH,
+ NEMOTRON_3_5_ASR_STREAMING,
WHISPER_BASE,
WHISPER_BASE_EN_NATIVE,
WHISPER_BASE_EN_S2T,
WHISPER_BASE_MLX,
WHISPER_BASE_NATIVE,
WHISPER_BASE_S2T,
+ WHISPER_BASE_TRANSFORMERS,
WHISPER_DISTIL_LARGE_V3_5_NATIVE,
WHISPER_DISTIL_LARGE_V3_5_S2T,
WHISPER_DISTIL_LARGE_V3_NATIVE,
@@ -83,6 +87,7 @@
WHISPER_LARGE,
WHISPER_LARGE_MLX,
WHISPER_LARGE_NATIVE,
+ WHISPER_LARGE_TRANSFORMERS,
WHISPER_LARGE_V3_S2T,
WHISPER_LARGE_V3_TURBO_S2T,
WHISPER_MEDIUM,
@@ -91,21 +96,25 @@
WHISPER_MEDIUM_MLX,
WHISPER_MEDIUM_NATIVE,
WHISPER_MEDIUM_S2T,
+ WHISPER_MEDIUM_TRANSFORMERS,
WHISPER_SMALL,
WHISPER_SMALL_EN_NATIVE,
WHISPER_SMALL_EN_S2T,
WHISPER_SMALL_MLX,
WHISPER_SMALL_NATIVE,
WHISPER_SMALL_S2T,
+ WHISPER_SMALL_TRANSFORMERS,
WHISPER_TINY,
WHISPER_TINY_EN_NATIVE,
WHISPER_TINY_EN_S2T,
WHISPER_TINY_MLX,
WHISPER_TINY_NATIVE,
WHISPER_TINY_S2T,
+ WHISPER_TINY_TRANSFORMERS,
WHISPER_TURBO,
WHISPER_TURBO_MLX,
WHISPER_TURBO_NATIVE,
+ WHISPER_TURBO_TRANSFORMERS,
AsrModelType,
)
from docling.datamodel.backend_options import (
@@ -148,7 +157,10 @@
VlmPipelineOptions,
normalize_pdf_backend,
)
-from docling.datamodel.pipeline_options_asr_model import InlineAsrOptions
+from docling.datamodel.pipeline_options_asr_model import (
+ InlineAsrOptions,
+ InlineAsrTransformersOptions,
+)
from docling.datamodel.settings import DEFAULT_PAGE_RANGE, settings
from docling.datamodel.vlm_engine_options import (
AutoInlineVlmEngineOptions,
@@ -406,13 +418,23 @@ def _resolve_asr_options(asr_model: AsrModelType) -> InlineAsrOptions:
Shared mapping so both audio and (later) video CLI setup resolve
ASR presets the same way.
"""
+ if asr_model.value in ADDITIONAL_ASR_PRESETS:
+ return ADDITIONAL_ASR_PRESETS[asr_model.value].to_options()
mapping: dict[AsrModelType, InlineAsrOptions] = {
+ AsrModelType.GRANITE_4_0_1B_SPEECH: GRANITE_4_0_1B_SPEECH,
+ AsrModelType.NEMOTRON_3_5_ASR_STREAMING: NEMOTRON_3_5_ASR_STREAMING,
AsrModelType.WHISPER_TINY: WHISPER_TINY,
AsrModelType.WHISPER_SMALL: WHISPER_SMALL,
AsrModelType.WHISPER_MEDIUM: WHISPER_MEDIUM,
AsrModelType.WHISPER_BASE: WHISPER_BASE,
AsrModelType.WHISPER_LARGE: WHISPER_LARGE,
AsrModelType.WHISPER_TURBO: WHISPER_TURBO,
+ AsrModelType.WHISPER_TINY_TRANSFORMERS: WHISPER_TINY_TRANSFORMERS,
+ AsrModelType.WHISPER_BASE_TRANSFORMERS: WHISPER_BASE_TRANSFORMERS,
+ AsrModelType.WHISPER_SMALL_TRANSFORMERS: WHISPER_SMALL_TRANSFORMERS,
+ AsrModelType.WHISPER_MEDIUM_TRANSFORMERS: WHISPER_MEDIUM_TRANSFORMERS,
+ AsrModelType.WHISPER_LARGE_TRANSFORMERS: WHISPER_LARGE_TRANSFORMERS,
+ AsrModelType.WHISPER_TURBO_TRANSFORMERS: WHISPER_TURBO_TRANSFORMERS,
AsrModelType.WHISPER_TINY_MLX: WHISPER_TINY_MLX,
AsrModelType.WHISPER_SMALL_MLX: WHISPER_SMALL_MLX,
AsrModelType.WHISPER_MEDIUM_MLX: WHISPER_MEDIUM_MLX,
@@ -1001,6 +1023,13 @@ def convert( # noqa: C901
AsrModelType,
typer.Option(..., help="Choose the ASR model to use with audio/video files."),
] = AsrModelType.WHISPER_TINY,
+ asr_language: Annotated[
+ str | None,
+ typer.Option(
+ "--asr-language",
+ help="Source language for Transformers ASR.",
+ ),
+ ] = None,
video_sampling_mode: Annotated[
Literal["fixed", "scene"],
typer.Option(..., help="frame sampling mode."),
@@ -2020,6 +2049,12 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
# Auto-selecting models (choose best implementation for hardware)
asr_pipeline_options.asr_options = _resolve_asr_options(asr_model)
+ if isinstance(asr_pipeline_options.asr_options, InlineAsrTransformersOptions):
+ asr_pipeline_options.asr_options = (
+ asr_pipeline_options.asr_options.model_copy(
+ update={"language": asr_language}
+ )
+ )
_log.debug(f"ASR pipeline_options: {asr_pipeline_options}")
@@ -2053,7 +2088,9 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
video_pipeline_options = VideoPipelineOptions()
video_pipeline_options.enable_diarization = video_diarization
video_pipeline_options.document_timeout = document_timeout
- video_pipeline_options.asr_options = _resolve_asr_options(asr_model)
+ video_pipeline_options.asr_options = (
+ asr_pipeline_options.asr_options.model_copy(deep=True)
+ )
if video_sampling_mode == "scene":
video_pipeline_options.frame_sampling_mode = (
VideoFrameSamplingMode.SCENE_CHANGE
diff --git a/docling/datamodel/asr_model_specs.py b/docling/datamodel/asr_model_specs.py
index 005d113215..4609754881 100644
--- a/docling/datamodel/asr_model_specs.py
+++ b/docling/datamodel/asr_model_specs.py
@@ -9,6 +9,11 @@
)
from docling.datamodel.accelerator_options import AcceleratorDevice
+from docling.datamodel.asr_presets import (
+ AsrEngineConfig,
+ AsrModelPreset,
+ AsrModelSpec,
+)
from docling.datamodel.pipeline_options_asr_model import (
# AsrResponseFormat,
# ApiAsrOptions,
@@ -729,7 +734,190 @@ def _distil_whisper_prefetch_name(model_name: str) -> str:
# =============================================================================
+_WHISPER_EXPORTS = {
+ "tiny": "mlx-community/whisper-tiny-mlx",
+ "base": "mlx-community/whisper-base-mlx",
+ "small": "mlx-community/whisper-small-mlx",
+ "medium": "mlx-community/whisper-medium-mlx-8bit",
+ "large": "mlx-community/whisper-large-mlx-8bit",
+ "turbo": "mlx-community/whisper-turbo",
+}
+
+WHISPER_ASR_PRESETS: dict[str, AsrModelPreset] = {
+ size: AsrModelPreset(
+ preset_id=f"whisper_{size}",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name=f"Whisper {size}",
+ default_repo_id=(
+ "openai/whisper-large-v3-turbo"
+ if size == "turbo"
+ else f"openai/whisper-{size}"
+ ),
+ supported_engines={
+ InferenceAsrFramework.TRANSFORMERS,
+ InferenceAsrFramework.MLX,
+ },
+ engine_overrides={
+ InferenceAsrFramework.MLX: AsrEngineConfig(repo_id=mlx_repo_id)
+ },
+ ),
+ )
+ for size, mlx_repo_id in _WHISPER_EXPORTS.items()
+}
+
+WHISPER_TINY_TRANSFORMERS = WHISPER_ASR_PRESETS["tiny"].to_options()
+WHISPER_BASE_TRANSFORMERS = WHISPER_ASR_PRESETS["base"].to_options()
+WHISPER_SMALL_TRANSFORMERS = WHISPER_ASR_PRESETS["small"].to_options()
+WHISPER_MEDIUM_TRANSFORMERS = WHISPER_ASR_PRESETS["medium"].to_options()
+WHISPER_LARGE_TRANSFORMERS = WHISPER_ASR_PRESETS["large"].to_options()
+WHISPER_TURBO_TRANSFORMERS = WHISPER_ASR_PRESETS["turbo"].to_options()
+
+GRANITE_4_0_1B_SPEECH_PRESET = AsrModelPreset(
+ preset_id="granite_4_0_1b_speech",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Granite 4.0 1B Speech",
+ default_repo_id="ibm-granite/granite-4.0-1b-speech",
+ model_family="granite",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+NEMOTRON_3_5_ASR_STREAMING_PRESET = AsrModelPreset(
+ preset_id="nemotron_3_5_asr_streaming_0_6b",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Nemotron 3.5 ASR Streaming 0.6B",
+ default_repo_id="nvidia/nemotron-3.5-asr-streaming-0.6b",
+ model_family="nemotron",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+
+GRANITE_4_0_1B_SPEECH = GRANITE_4_0_1B_SPEECH_PRESET.to_options()
+NEMOTRON_3_5_ASR_STREAMING = NEMOTRON_3_5_ASR_STREAMING_PRESET.to_options()
+
+
+GRANITE_SPEECH_5_0_2B_PRESET = AsrModelPreset(
+ preset_id="granite_speech_5_0_2b",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Granite Speech 5.0 2B",
+ default_repo_id="ibm-granite/granite-speech-5.0-2b",
+ revision="f4e68169fa2a5751acc4b5da9957855e5e8fc3f1",
+ model_family="granite",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+GRANITE_SPEECH_5_0_2B = GRANITE_SPEECH_5_0_2B_PRESET.to_options()
+
+GRANITE_SPEECH_5_0_470M_TURBOCTC_PRESET = AsrModelPreset(
+ preset_id="granite_speech_5_0_470m_turboctc",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Granite Speech 5.0 470M TurboCTC",
+ default_repo_id="ibm-granite/granite-speech-5.0-470m-turboctc",
+ revision="1dd23f46f6d4e6eae2531982ba2e33220086dd97",
+ model_family="granite_ctc",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+GRANITE_SPEECH_5_0_470M_TURBOCTC = GRANITE_SPEECH_5_0_470M_TURBOCTC_PRESET.to_options()
+
+GRANITE_SPEECH_5_0_470M_TURBOCTC_NC_PRESET = AsrModelPreset(
+ preset_id="granite_speech_5_0_470m_turboctc_nc",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Granite Speech 5.0 470M TurboCTC NC",
+ default_repo_id="ibm-granite/granite-speech-5.0-470m-turboctc-nc",
+ revision="af0a8e8663defc902944ebdae0d29a5219be11a3",
+ model_family="granite_ctc",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+GRANITE_SPEECH_5_0_470M_TURBOCTC_NC = (
+ GRANITE_SPEECH_5_0_470M_TURBOCTC_NC_PRESET.to_options()
+)
+
+GRANITE_SPEECH_4_1_2B_PRESET = AsrModelPreset(
+ preset_id="granite_speech_4_1_2b",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Granite Speech 4.1 2B",
+ default_repo_id="ibm-granite/granite-speech-4.1-2b",
+ revision="de575db64086f84fdc79da4932d1076e965bc546",
+ model_family="granite",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+GRANITE_SPEECH_4_1_2B = GRANITE_SPEECH_4_1_2B_PRESET.to_options()
+
+GRANITE_SPEECH_4_1_2B_PLUS_PRESET = AsrModelPreset(
+ preset_id="granite_speech_4_1_2b_plus",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Granite Speech 4.1 2B Plus",
+ default_repo_id="ibm-granite/granite-speech-4.1-2b-plus",
+ revision="e69efe37f97b6283c61f5a8dadda2b3eb67765d6",
+ model_family="granite",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+GRANITE_SPEECH_4_1_2B_PLUS = GRANITE_SPEECH_4_1_2B_PLUS_PRESET.to_options()
+
+NEMOTRON_SPEECH_STREAMING_EN_0_6B_PRESET = AsrModelPreset(
+ preset_id="nemotron_speech_streaming_en_0_6b",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Nemotron Speech Streaming English 0.6B",
+ default_repo_id="nvidia/nemotron-speech-streaming-en-0.6b",
+ revision="ebe59e5a817142986528bbbee5dba8db7b38ed50",
+ model_family="nemotron",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+NEMOTRON_SPEECH_STREAMING_EN_0_6B = (
+ NEMOTRON_SPEECH_STREAMING_EN_0_6B_PRESET.to_options()
+)
+
+PARAKEET_CTC_1_1B_PRESET = AsrModelPreset(
+ preset_id="parakeet_ctc_1_1b",
+ default_engine_type=InferenceAsrFramework.TRANSFORMERS,
+ model_spec=AsrModelSpec(
+ name="Parakeet CTC 1.1B",
+ default_repo_id="nvidia/parakeet-ctc-1.1b",
+ revision="20e63a0fed6aedba145b74b826dbd41df0941730",
+ model_family="parakeet",
+ supported_engines={InferenceAsrFramework.TRANSFORMERS},
+ ),
+)
+PARAKEET_CTC_1_1B = PARAKEET_CTC_1_1B_PRESET.to_options()
+
+
+ADDITIONAL_ASR_PRESETS: dict[str, AsrModelPreset] = {
+ preset.preset_id: preset
+ for preset in (
+ GRANITE_SPEECH_5_0_2B_PRESET,
+ GRANITE_SPEECH_5_0_470M_TURBOCTC_PRESET,
+ GRANITE_SPEECH_5_0_470M_TURBOCTC_NC_PRESET,
+ GRANITE_SPEECH_4_1_2B_PRESET,
+ GRANITE_SPEECH_4_1_2B_PLUS_PRESET,
+ NEMOTRON_SPEECH_STREAMING_EN_0_6B_PRESET,
+ PARAKEET_CTC_1_1B_PRESET,
+ )
+}
+
+
class AsrModelType(str, Enum):
+ GRANITE_SPEECH_5_0_2B = "granite_speech_5_0_2b"
+ GRANITE_SPEECH_5_0_470M_TURBOCTC = "granite_speech_5_0_470m_turboctc"
+ GRANITE_SPEECH_5_0_470M_TURBOCTC_NC = "granite_speech_5_0_470m_turboctc_nc"
+ GRANITE_SPEECH_4_1_2B = "granite_speech_4_1_2b"
+ GRANITE_SPEECH_4_1_2B_PLUS = "granite_speech_4_1_2b_plus"
+ NEMOTRON_SPEECH_STREAMING_EN_0_6B = "nemotron_speech_streaming_en_0_6b"
+ PARAKEET_CTC_1_1B = "parakeet_ctc_1_1b"
+ GRANITE_4_0_1B_SPEECH = "granite_4_0_1b_speech"
+ NEMOTRON_3_5_ASR_STREAMING = "nemotron_3_5_asr_streaming"
# Auto-selecting models (choose best implementation for hardware)
WHISPER_TINY = "whisper_tiny"
WHISPER_SMALL = "whisper_small"
@@ -738,6 +926,14 @@ class AsrModelType(str, Enum):
WHISPER_LARGE = "whisper_large"
WHISPER_TURBO = "whisper_turbo"
+ # Transformers-format Whisper checkpoints
+ WHISPER_TINY_TRANSFORMERS = "whisper_tiny_transformers"
+ WHISPER_BASE_TRANSFORMERS = "whisper_base_transformers"
+ WHISPER_SMALL_TRANSFORMERS = "whisper_small_transformers"
+ WHISPER_MEDIUM_TRANSFORMERS = "whisper_medium_transformers"
+ WHISPER_LARGE_TRANSFORMERS = "whisper_large_transformers"
+ WHISPER_TURBO_TRANSFORMERS = "whisper_turbo_transformers"
+
# Explicit MLX models (force MLX implementation)
WHISPER_TINY_MLX = "whisper_tiny_mlx"
WHISPER_SMALL_MLX = "whisper_small_mlx"
diff --git a/docling/datamodel/asr_presets.py b/docling/datamodel/asr_presets.py
new file mode 100644
index 0000000000..1aa554977b
--- /dev/null
+++ b/docling/datamodel/asr_presets.py
@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: The Docling Contributors
+# SPDX-License-Identifier: MIT
+
+"""ASR model specifications with engine-specific checkpoint exports."""
+
+from pydantic import BaseModel, Field
+
+from docling.datamodel.pipeline_options_asr_model import (
+ AsrModelFamily,
+ InferenceAsrFramework,
+ InlineAsrMlxWhisperOptions,
+ InlineAsrOptions,
+ InlineAsrTransformersOptions,
+)
+
+
+class AsrEngineConfig(BaseModel):
+ """Checkpoint settings for one ASR engine."""
+
+ repo_id: str | None = None
+ revision: str | None = None
+ torch_dtype: str | None = None
+
+ def merge_with(self, repo_id: str, revision: str) -> "AsrEngineConfig":
+ return AsrEngineConfig(
+ repo_id=self.repo_id or repo_id,
+ revision=self.revision or revision,
+ torch_dtype=self.torch_dtype,
+ )
+
+
+class AsrModelSpec(BaseModel):
+ """Checkpoint locations supported by an ASR model family."""
+
+ name: str
+ default_repo_id: str
+ revision: str = "main"
+ model_family: AsrModelFamily = "whisper"
+ supported_engines: set[InferenceAsrFramework]
+ engine_overrides: dict[InferenceAsrFramework, AsrEngineConfig] = Field(
+ default_factory=dict
+ )
+
+ def get_engine_config(self, engine: InferenceAsrFramework) -> AsrEngineConfig:
+ if engine not in self.supported_engines:
+ raise ValueError(f"{self.name} does not support the {engine.value} engine")
+ override = self.engine_overrides.get(engine, AsrEngineConfig())
+ return override.merge_with(self.default_repo_id, self.revision)
+
+
+class AsrModelPreset(BaseModel):
+ """A named ASR model and its default engine."""
+
+ preset_id: str
+ model_spec: AsrModelSpec
+ default_engine_type: InferenceAsrFramework
+
+ def to_options(
+ self, engine: InferenceAsrFramework | None = None
+ ) -> InlineAsrOptions:
+ selected = engine or self.default_engine_type
+ config = self.model_spec.get_engine_config(selected)
+ if config.repo_id is None:
+ raise ValueError(f"No repository ID for {self.preset_id}")
+ if selected == InferenceAsrFramework.TRANSFORMERS:
+ return InlineAsrTransformersOptions(
+ repo_id=config.repo_id,
+ revision=config.revision or "main",
+ torch_dtype=config.torch_dtype,
+ model_family=self.model_spec.model_family,
+ )
+ if selected == InferenceAsrFramework.MLX:
+ return InlineAsrMlxWhisperOptions(repo_id=config.repo_id)
+ raise ValueError(f"{selected.value} is not configured by this ASR preset")
diff --git a/docling/datamodel/pipeline_options_asr_model.py b/docling/datamodel/pipeline_options_asr_model.py
index 80214156c1..11edd5e4e9 100644
--- a/docling/datamodel/pipeline_options_asr_model.py
+++ b/docling/datamodel/pipeline_options_asr_model.py
@@ -9,6 +9,7 @@
- ``InlineAsrNativeWhisperOptions``: OpenAI's native ``whisper`` (PyTorch), the
default backend, supported on CPU and CUDA.
- ``InlineAsrMlxWhisperOptions``: ``mlx-whisper``, optimized for Apple Silicon.
+- ``InlineAsrTransformersOptions``: Transformers checkpoints for speech models.
- ``InlineAsrWhisperS2TOptions``: WhisperS2T (CTranslate2), an optional,
experimental high-throughput backend installed via the ``format-audio`` extra.
@@ -53,7 +54,7 @@ class BaseAsrOptions(BaseModel):
class InferenceAsrFramework(str, Enum):
MLX = "mlx"
- # TRANSFORMERS = "transformers" # disabled for now
+ TRANSFORMERS = "transformers"
WHISPER = "whisper"
WHISPER_S2T = "whisper_s2t"
@@ -158,10 +159,7 @@ class InlineAsrOptions(BaseAsrOptions):
repo_id: Annotated[
str,
Field(
- description=(
- "HuggingFace model repository ID for the ASR model. Must be a "
- "Whisper-compatible model for automatic speech recognition."
- ),
+ description=("Hugging Face model repository ID for the ASR model."),
examples=["openai/whisper-tiny", "openai/whisper-base"],
),
]
@@ -316,6 +314,34 @@ class InlineAsrNativeWhisperOptions(InlineAsrOptions):
] = None
+AsrModelFamily = Literal[
+ "whisper",
+ "granite",
+ "granite_ctc",
+ "parakeet",
+ "nemotron",
+]
+
+
+class InlineAsrTransformersOptions(InlineAsrOptions):
+ """ASR with a Hugging Face Transformers checkpoint."""
+
+ inference_framework: Literal[InferenceAsrFramework.TRANSFORMERS] = (
+ InferenceAsrFramework.TRANSFORMERS
+ )
+ revision: str = "main"
+ model_family: AsrModelFamily = "whisper"
+ prompt: str | None = None
+ language: str | None = None
+ task: Literal["transcribe", "translate"] = "transcribe"
+ supported_devices: list[AcceleratorDevice] = [
+ AcceleratorDevice.CPU,
+ AcceleratorDevice.CUDA,
+ AcceleratorDevice.MPS,
+ AcceleratorDevice.XPU,
+ ]
+
+
class InlineAsrMlxWhisperOptions(InlineAsrOptions):
"""MLX Whisper options for Apple Silicon optimization.
diff --git a/docling/pipeline/asr_transcriber.py b/docling/pipeline/asr_transcriber.py
index 1eec06ed05..8203c29713 100644
--- a/docling/pipeline/asr_transcriber.py
+++ b/docling/pipeline/asr_transcriber.py
@@ -1,8 +1,10 @@
# SPDX-FileCopyrightText: The Docling Contributors
# SPDX-License-Identifier: MIT
+import json
import logging
import shutil
+import subprocess
import sys
import tempfile
from io import BytesIO
@@ -42,9 +44,11 @@
InlineAsrMlxWhisperOptions,
InlineAsrNativeWhisperOptions,
InlineAsrOptions,
+ InlineAsrTransformersOptions,
InlineAsrWhisperS2TOptions,
)
from docling.exceptions import OperationNotAllowed
+from docling.pipeline.asr_transformers import TransformersAsrDecoder
from docling.pipeline.base_pipeline import BasePipeline
from docling.utils.accelerator_utils import decide_device
from docling.utils.profiling import ProfilingScope, TimeRecorder
@@ -187,6 +191,41 @@ def to_string(self) -> str:
return result
+def _resolve_asr_artifacts(artifacts_path: Path, repo_id: str) -> Path:
+ """Find a local checkpoint, either directly or under its repo cache folder."""
+ roots = (
+ artifacts_path,
+ artifacts_path / repo_id.replace("/", "--"),
+ )
+ for candidate in roots:
+ if (candidate / "config.json").is_file():
+ return candidate
+ raise FileNotFoundError(
+ f"ASR model {repo_id} was not found in {artifacts_path}. "
+ "Expected config.json in that directory or its repo cache folder."
+ )
+
+
+def _audio_duration_seconds(fpath: Path) -> float:
+ """Read audio duration for ASR models that return text without timestamps."""
+ result = subprocess.run(
+ [
+ "ffprobe",
+ "-v",
+ "error",
+ "-show_entries",
+ "format=duration",
+ "-of",
+ "json",
+ str(fpath),
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ return float(json.loads(result.stdout)["format"]["duration"])
+
+
class _NativeWhisperModel:
def __init__(
self,
@@ -403,8 +442,18 @@ def __init__(
self.model_name = asr_options.repo_id
_log.info(f"loading _MlxWhisperModel({self.model_name})")
- # MLX Whisper models are loaded differently - they use HuggingFace repos
- self.model_path = self.model_name
+ self.model_path = (
+ str(_resolve_asr_artifacts(artifacts_path, self.model_name))
+ if artifacts_path is not None
+ else self.model_name
+ )
+ if artifacts_path is not None and not any(
+ (Path(self.model_path) / name).is_file()
+ for name in ("weights.safetensors", "weights.npz")
+ ):
+ raise FileNotFoundError(
+ f"MLX Whisper weights are missing from {self.model_path}"
+ )
# Store MLX-specific options
self.language = asr_options.language
@@ -454,7 +503,7 @@ def run(self, conv_res: ConversionResult) -> ConversionResult:
return conv_res
except Exception as exc:
- _log.error(f"MLX Audio transcription has an error: {exc}")
+ _log.error(f"ASR transcription has an error: {exc}")
conv_res.status = ConversionStatus.FAILURE
return conv_res
@@ -514,6 +563,91 @@ def transcribe(self, fpath: Path) -> list[_ConversationItem]:
return convo
+class _TransformersAsrModel(_MlxWhisperModel):
+ """Transcribe speech with a Transformers ASR pipeline."""
+
+ def __init__(
+ self,
+ enabled: bool,
+ artifacts_path: Path | None,
+ accelerator_options: AcceleratorOptions,
+ asr_options: InlineAsrTransformersOptions,
+ ):
+ self.enabled = enabled
+ if not enabled:
+ return
+ self.asr_options = asr_options
+ self.device = decide_device(
+ accelerator_options.device,
+ supported_devices=asr_options.supported_devices,
+ )
+ model = (
+ str(_resolve_asr_artifacts(artifacts_path, asr_options.repo_id))
+ if artifacts_path is not None
+ else asr_options.repo_id
+ )
+ self.decoder: TransformersAsrDecoder | None = None
+ if asr_options.model_family in {"granite", "granite_ctc"}:
+ self.decoder = TransformersAsrDecoder(
+ asr_options, self.device, model, artifacts_path is not None
+ )
+ return
+
+ try:
+ from transformers import pipeline
+ except ImportError as exc:
+ raise ImportError(
+ "Transformers ASR requires transformers. Install the "
+ "`feat-asr-transformers` extra."
+ ) from exc
+ pipeline_kwargs = {
+ "task": "automatic-speech-recognition",
+ "model": model,
+ "device": self.device,
+ }
+ if artifacts_path is not None:
+ pipeline_kwargs["model_kwargs"] = {"local_files_only": True}
+ else:
+ pipeline_kwargs["revision"] = asr_options.revision
+ if asr_options.torch_dtype is not None:
+ pipeline_kwargs["dtype"] = asr_options.torch_dtype
+ self.asr_pipeline = pipeline(**pipeline_kwargs)
+
+ def transcribe(self, fpath: Path) -> list[_ConversationItem]:
+ if self.asr_options.model_family == "whisper":
+ generate_kwargs: dict[str, str] = {"task": self.asr_options.task}
+ if self.asr_options.language is not None:
+ generate_kwargs["language"] = self.asr_options.language
+ result = self.asr_pipeline(
+ str(fpath),
+ return_timestamps=True,
+ generate_kwargs=generate_kwargs,
+ )
+ return [
+ _ConversationItem(
+ start_time=chunk["timestamp"][0],
+ end_time=chunk["timestamp"][1],
+ text=chunk["text"].strip(),
+ )
+ for chunk in result["chunks"]
+ ]
+
+ if self.decoder is not None:
+ text = self.decoder.transcribe(fpath)
+ else:
+ result = self.asr_pipeline(str(fpath))
+ text = result["text"].strip()
+ if not text:
+ return []
+ return [
+ _ConversationItem(
+ start_time=0.0,
+ end_time=_audio_duration_seconds(fpath),
+ text=text,
+ )
+ ]
+
+
class _WhisperS2TModel:
"""Transcriber using WhisperS2T with CTranslate2 backend for high-speed inference."""
@@ -930,6 +1064,13 @@ def create(
accelerator_options=accelerator_options,
asr_options=asr_options,
)
+ elif isinstance(asr_options, InlineAsrTransformersOptions):
+ return _TransformersAsrModel(
+ enabled=True,
+ artifacts_path=artifacts_path,
+ accelerator_options=accelerator_options,
+ asr_options=asr_options,
+ )
elif isinstance(asr_options, InlineAsrWhisperS2TOptions):
return _WhisperS2TModel(
enabled=True,
diff --git a/docling/pipeline/asr_transformers.py b/docling/pipeline/asr_transformers.py
new file mode 100644
index 0000000000..23e72f0372
--- /dev/null
+++ b/docling/pipeline/asr_transformers.py
@@ -0,0 +1,121 @@
+# SPDX-FileCopyrightText: The Docling Contributors
+# SPDX-License-Identifier: MIT
+
+"""Model-specific speech decoding through documented Transformers interfaces."""
+
+from pathlib import Path
+from typing import Any
+
+import torch
+
+from docling.datamodel.pipeline_options_asr_model import InlineAsrTransformersOptions
+
+
+def _expand_granite_queries(
+ _module: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]
+) -> tuple[tuple[Any, ...], dict[str, Any]] | None:
+ """Match Q-Former queries to the projector's batch of audio windows."""
+ queries = kwargs["query_embeds"]
+ window_batch = kwargs["encoder_hidden_states"].shape[0]
+ if queries.shape[0] == 1 and window_batch > 1:
+ kwargs["query_embeds"] = queries.expand(window_batch, -1, -1)
+ return args, kwargs
+ return None
+
+
+class TransformersAsrDecoder:
+ """Decode models that need a processor or custom model method."""
+
+ def __init__(
+ self,
+ options: InlineAsrTransformersOptions,
+ device: str,
+ checkpoint: str,
+ local: bool,
+ ) -> None:
+ # These imports belong to the optional Transformers ASR feature.
+ from transformers import (
+ AutoModelForCTC,
+ AutoModelForSpeechSeq2Seq,
+ AutoProcessor,
+ )
+
+ self.options = options
+ self.device = device
+ load_kwargs: dict[str, Any] = {
+ "local_files_only": local,
+ }
+ if not local:
+ load_kwargs["revision"] = options.revision
+
+ model_kwargs = dict(load_kwargs)
+ if options.torch_dtype is not None:
+ model_kwargs["dtype"] = options.torch_dtype
+ # Granite's BF16 normalization can abort the MPS process. Set the
+ # loading dtype explicitly; catching an exception cannot recover it.
+ if device == "mps" and options.model_family in {"granite", "granite_ctc"}:
+ model_kwargs["dtype"] = "float32"
+
+ self.processor = AutoProcessor.from_pretrained(checkpoint, **load_kwargs)
+ if options.model_family == "granite_ctc":
+ self.model = AutoModelForCTC.from_pretrained(checkpoint, **model_kwargs)
+ else:
+ self.model = AutoModelForSpeechSeq2Seq.from_pretrained(
+ checkpoint, **model_kwargs
+ )
+ self.model.to(device).eval()
+ if options.model_family == "granite":
+ # Transformers 5.16.1 passes a single query batch for all audio
+ # windows. Its Q-Former can return only one window, so the projector
+ # then fails to reshape the output. Scope this correction to the
+ # loaded instance; already expanded queries are left intact.
+ self.model.model.projector.qformer.register_forward_pre_hook(
+ _expand_granite_queries, with_kwargs=True
+ )
+
+ def transcribe(self, path: Path) -> str:
+ from transformers.pipelines.audio_utils import ffmpeg_read
+
+ waveform = ffmpeg_read(path.read_bytes(), sampling_rate=16000).copy()
+ if waveform.size == 0:
+ return ""
+ with torch.inference_mode():
+ if self.options.model_family == "granite_ctc":
+ inputs = self.processor(
+ [waveform],
+ sampling_rate=16000,
+ device=self.device,
+ return_tensors="pt",
+ ).to(self.device)
+ output = self.model.generate(**inputs)
+ return self.processor.batch_decode(output, skip_special_tokens=True)[
+ 0
+ ].strip()
+ # Granite's decoder is a language model: it needs an audio prompt,
+ # and generate() includes the input tokens in the returned sequence.
+ prompt = self.options.prompt or (
+ "translate the speech into English."
+ if self.options.task == "translate"
+ else "transcribe the speech with proper punctuation and capitalization."
+ )
+ formatted = self.processor.tokenizer.apply_chat_template(
+ [{"role": "user", "content": f"<|audio|>{prompt}"}],
+ tokenize=False,
+ add_generation_prompt=True,
+ )
+ inputs = self.processor(
+ formatted,
+ torch.from_numpy(waveform).unsqueeze(0),
+ device=self.device,
+ return_tensors="pt",
+ ).to(self.device)
+ output = self.model.generate(
+ **inputs,
+ max_new_tokens=self.options.max_new_tokens,
+ do_sample=False,
+ num_beams=1,
+ )
+ generated = output[:, inputs["input_ids"].shape[-1] :]
+ return self.processor.tokenizer.batch_decode(
+ generated, skip_special_tokens=True
+ )[0].strip()
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index 2820d19e8d..32d88bfe4f 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -55,7 +55,8 @@ docling convert [OPTIONS] source
| `--trust-remote-code` | `boolean` | `false` | Allow the local VLM engine to run code from the model repository. Some --vlm-model presets require it. Review the model's code at the preset's pinned revision first. |
| `--vlm-write-native-output` | `boolean` | `false` | Write each page's unparsed VLM response under `