feat(ASR): add transformers and MLX presets for the ASR pipelines

.gitignore

@@ -454,3 +454,6 @@ pip-selfcheck.json
 
 # AI agents
 **/.bob/
+
+# Local ASR model validation output
+/asr-validation/

docling/.agents/skills/docling/references/cli.md

@@ -1,5 +1,9 @@
 # Docling CLI reference
 
+For Transformers ASR, install `docling-slim[feat-asr-transformers]` and select
+`--asr-model granite_speech_5_0_2b` (or another speech preset). The same ASR
+settings apply to video.
+
 The `docling` CLI is the quickest way to understand the content of any supported
 document. It ships with the `docling` package (`pip install docling`) and needs
 no code. Use it any time you need to read, convert, or extract content from a

docling/.agents/skills/docling/references/python-sdk.md

@@ -394,8 +394,36 @@ result = converter.convert("interview.wav")
 print(result.document.export_to_markdown())   # timestamped transcript
 ```
 
+For a Transformers-format Whisper checkpoint, install
+`docling-slim[feat-asr-transformers]` and select an explicit preset such as
+`asr_model_specs.WHISPER_TINY_TRANSFORMERS`. The presets cover tiny, base,
+small, medium, large, and large-v3-turbo. Set
+`pipeline_options.artifacts_path` to a directory that contains `config.json`
+and the Transformers model files, or to its parent directory with a
+`openai--whisper-tiny` subdirectory. The local directory is used without a
+Hugging Face lookup. For MLX Whisper, use `WHISPER_TINY_MLX` and provide an
+MLX checkpoint with `config.json` and `weights.safetensors` or `weights.npz`.
+
+Other explicit Transformers presets are `GRANITE_4_0_1B_SPEECH` and
+`NEMOTRON_3_5_ASR_STREAMING`. These models return
+one transcript item that covers the full audio duration. The Transformers
+Whisper presets provide model segment timestamps.
+
 The CLI can also transcribe: `docling interview.wav --to md --output /tmp/`.
 
+Additional presets include `GRANITE_SPEECH_5_0_2B`,
+`GRANITE_SPEECH_5_0_470M_TURBOCTC`, its `_NC` variant,
+`GRANITE_SPEECH_4_1_2B`, its `_PLUS` variant, and
+`NEMOTRON_SPEECH_STREAMING_EN_0_6B`. Copy a preset with
+`.model_copy(deep=True)` before changing `revision`, `language`, or `prompt`.
+Granite and TurboCTC use FP32 weights on MPS to avoid BF16 normalization failures.
+Plus defaults to plain transcription; speaker and timestamp tags are not parsed
+into tracks.
+The `_NC` preset uses a noncommercial license.
+For real decoding checks, use `scripts/validate_asr_models.py` in a repository
+checkout. Repeat `--model` to select presets; without it, the script runs every
+ASR preset on the same fragment. It is outside normal pytest collection.
+
 Video files (`.mp4`, `.mov`, `.mkv`, `.webm`, `.avi`) use `VideoPipeline` with
 `VideoPipelineOptions`, which also samples frames. `max_sampled_frames` defaults
 to `200` (set `None` for no limit). When `document_timeout` is set, all FFmpeg

docling/backend/md_backend.py

@@ -1182,20 +1182,22 @@ def convert(self) -> DoclingDocument:
                 )
                 html_str = html_payload.text
 
-                def _restore_original_html(txt, regex):
-                    _txt, count = re.subn(regex, "", txt)
-                    if count != self._html_blocks:
-                        raise RuntimeError(
-                            "An internal error has occurred during Markdown conversion."
-                        )
-                    return _txt
-
-                # restore original HTML by removing previously added markers
-                for regex in [
-                    rf"<pre>\s*<code>\s*{_START_MARKER}",
-                    rf"{_STOP_MARKER}\s*</code>\s*</pre>",
-                ]:
-                    html_str = _restore_original_html(txt=html_str, regex=regex)
+                # Restore only the marked HTML blocks. The HTML serializer
+                # escapes code content, so decode each block before parsing it.
+                pattern = (
+                    rf"<pre>\s*<code>\s*{re.escape(_START_MARKER)}"
+                    rf"(.*?){re.escape(_STOP_MARKER)}\s*</code>\s*</pre>"
+                )
+                html_str, count = re.subn(
+                    pattern,
+                    lambda match: unescape(match.group(1)),
+                    html_str,
+                    flags=re.DOTALL,
+                )
+                if count != self._html_blocks:
+                    raise RuntimeError(
+                        "An internal error has occurred during Markdown conversion."
+                    )
                 self._html_blocks = 0
                 # delegate to HTML backend
                 stream = BytesIO(bytes(html_str, encoding="utf-8"))

docling/cli/main.py

@@ -66,12 +66,16 @@
 )
 from docling.datamodel.accelerator_options import AcceleratorDevice, AcceleratorOptions
 from docling.datamodel.asr_model_specs import (
+    ADDITIONAL_ASR_PRESETS,
+    GRANITE_4_0_1B_SPEECH,
+    NEMOTRON_3_5_ASR_STREAMING,
     WHISPER_BASE,
     WHISPER_BASE_EN_NATIVE,
     WHISPER_BASE_EN_S2T,
     WHISPER_BASE_MLX,
     WHISPER_BASE_NATIVE,
     WHISPER_BASE_S2T,
+    WHISPER_BASE_TRANSFORMERS,
     WHISPER_DISTIL_LARGE_V3_5_NATIVE,
     WHISPER_DISTIL_LARGE_V3_5_S2T,
     WHISPER_DISTIL_LARGE_V3_NATIVE,
@@ -83,6 +87,7 @@
     WHISPER_LARGE,
     WHISPER_LARGE_MLX,
     WHISPER_LARGE_NATIVE,
+    WHISPER_LARGE_TRANSFORMERS,
     WHISPER_LARGE_V3_S2T,
     WHISPER_LARGE_V3_TURBO_S2T,
     WHISPER_MEDIUM,
@@ -91,21 +96,25 @@
     WHISPER_MEDIUM_MLX,
     WHISPER_MEDIUM_NATIVE,
     WHISPER_MEDIUM_S2T,
+    WHISPER_MEDIUM_TRANSFORMERS,
     WHISPER_SMALL,
     WHISPER_SMALL_EN_NATIVE,
     WHISPER_SMALL_EN_S2T,
     WHISPER_SMALL_MLX,
     WHISPER_SMALL_NATIVE,
     WHISPER_SMALL_S2T,
+    WHISPER_SMALL_TRANSFORMERS,
     WHISPER_TINY,
     WHISPER_TINY_EN_NATIVE,
     WHISPER_TINY_EN_S2T,
     WHISPER_TINY_MLX,
     WHISPER_TINY_NATIVE,
     WHISPER_TINY_S2T,
+    WHISPER_TINY_TRANSFORMERS,
     WHISPER_TURBO,
     WHISPER_TURBO_MLX,
     WHISPER_TURBO_NATIVE,
+    WHISPER_TURBO_TRANSFORMERS,
     AsrModelType,
 )
 from docling.datamodel.backend_options import (
@@ -148,7 +157,10 @@
     VlmPipelineOptions,
     normalize_pdf_backend,
 )
-from docling.datamodel.pipeline_options_asr_model import InlineAsrOptions
+from docling.datamodel.pipeline_options_asr_model import (
+    InlineAsrOptions,
+    InlineAsrTransformersOptions,
+)
 from docling.datamodel.settings import DEFAULT_PAGE_RANGE, settings
 from docling.datamodel.vlm_engine_options import (
     AutoInlineVlmEngineOptions,
@@ -406,13 +418,23 @@ def _resolve_asr_options(asr_model: AsrModelType) -> InlineAsrOptions:
     Shared mapping so both audio and (later) video CLI setup resolve
     ASR presets the same way.
     """
+    if asr_model.value in ADDITIONAL_ASR_PRESETS:
+        return ADDITIONAL_ASR_PRESETS[asr_model.value].to_options()
     mapping: dict[AsrModelType, InlineAsrOptions] = {
+        AsrModelType.GRANITE_4_0_1B_SPEECH: GRANITE_4_0_1B_SPEECH,
+        AsrModelType.NEMOTRON_3_5_ASR_STREAMING: NEMOTRON_3_5_ASR_STREAMING,
         AsrModelType.WHISPER_TINY: WHISPER_TINY,
         AsrModelType.WHISPER_SMALL: WHISPER_SMALL,
         AsrModelType.WHISPER_MEDIUM: WHISPER_MEDIUM,
         AsrModelType.WHISPER_BASE: WHISPER_BASE,
         AsrModelType.WHISPER_LARGE: WHISPER_LARGE,
         AsrModelType.WHISPER_TURBO: WHISPER_TURBO,
+        AsrModelType.WHISPER_TINY_TRANSFORMERS: WHISPER_TINY_TRANSFORMERS,
+        AsrModelType.WHISPER_BASE_TRANSFORMERS: WHISPER_BASE_TRANSFORMERS,
+        AsrModelType.WHISPER_SMALL_TRANSFORMERS: WHISPER_SMALL_TRANSFORMERS,
+        AsrModelType.WHISPER_MEDIUM_TRANSFORMERS: WHISPER_MEDIUM_TRANSFORMERS,
+        AsrModelType.WHISPER_LARGE_TRANSFORMERS: WHISPER_LARGE_TRANSFORMERS,
+        AsrModelType.WHISPER_TURBO_TRANSFORMERS: WHISPER_TURBO_TRANSFORMERS,
         AsrModelType.WHISPER_TINY_MLX: WHISPER_TINY_MLX,
         AsrModelType.WHISPER_SMALL_MLX: WHISPER_SMALL_MLX,
         AsrModelType.WHISPER_MEDIUM_MLX: WHISPER_MEDIUM_MLX,
@@ -1001,6 +1023,13 @@ def convert(  # noqa: C901
         AsrModelType,
         typer.Option(..., help="Choose the ASR model to use with audio/video files."),
     ] = AsrModelType.WHISPER_TINY,
+    asr_language: Annotated[
+        str | None,
+        typer.Option(
+            "--asr-language",
+            help="Source language for Transformers ASR.",
+        ),
+    ] = None,
     video_sampling_mode: Annotated[
         Literal["fixed", "scene"],
         typer.Option(..., help="frame sampling mode."),
@@ -2020,6 +2049,12 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
 
         # Auto-selecting models (choose best implementation for hardware)
         asr_pipeline_options.asr_options = _resolve_asr_options(asr_model)
+        if isinstance(asr_pipeline_options.asr_options, InlineAsrTransformersOptions):
+            asr_pipeline_options.asr_options = (
+                asr_pipeline_options.asr_options.model_copy(
+                    update={"language": asr_language}
+                )
+            )
 
         _log.debug(f"ASR pipeline_options: {asr_pipeline_options}")
 
@@ -2053,7 +2088,9 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
             video_pipeline_options = VideoPipelineOptions()
             video_pipeline_options.enable_diarization = video_diarization
             video_pipeline_options.document_timeout = document_timeout
-            video_pipeline_options.asr_options = _resolve_asr_options(asr_model)
+            video_pipeline_options.asr_options = (
+                asr_pipeline_options.asr_options.model_copy(deep=True)
+            )
             if video_sampling_mode == "scene":
                 video_pipeline_options.frame_sampling_mode = (
                     VideoFrameSamplingMode.SCENE_CHANGE