feat(asr): transcribe with a remote OpenAI-compatible server

docling/datamodel/pipeline_options.py

@@ -53,7 +53,10 @@
 from docling.datamodel.picture_classification_options import (
     DocumentPictureClassifierOptions,
 )
-from docling.datamodel.pipeline_options_asr_model import InlineAsrOptions
+from docling.datamodel.pipeline_options_asr_model import (
+    ApiAsrOptions,
+    InlineAsrOptions,
+)
 from docling.datamodel.pipeline_options_vlm_model import (
     ApiVlmOptions,
     InferenceFramework,
@@ -1803,11 +1806,13 @@ class AsrPipelineOptions(PipelineOptions):
     """
 
     asr_options: Annotated[
-        InlineAsrOptions,
+        InlineAsrOptions | ApiAsrOptions,
         Field(
             description=(
                 "Automatic Speech Recognition (ASR) model configuration for audio transcription. Specifies which "
-                "ASR model to use (e.g., Whisper variants) and model-specific parameters for speech-to-text conversion."
+                "ASR model to use (e.g., Whisper variants) and model-specific parameters for speech-to-text conversion. "
+                "Use `ApiAsrOptions` (with `enable_remote_services=True`) to transcribe with an OpenAI-compatible "
+                "server instead of a model loaded in this process."
             )
         ),
     ] = asr_model_specs.WHISPER_TINY
@@ -1829,7 +1834,7 @@ class VideoPipelineOptions(PipelineOptions):
     """
 
     asr_options: Annotated[
-        InlineAsrOptions,
+        InlineAsrOptions | ApiAsrOptions,
         Field(description="ASR model configuration for the video audio track."),
     ] = asr_model_specs.WHISPER_TINY
 

docling/datamodel/pipeline_options_asr_model.py

@@ -12,6 +12,10 @@
 - ``InlineAsrWhisperS2TOptions``: WhisperS2T (CTranslate2), an optional,
   experimental high-throughput backend installed via the ``format-audio`` extra.
 
+``ApiAsrOptions`` is the alternative to all of them: nothing runs locally, the audio
+is sent to an OpenAI-compatible ``/v1/audio/transcriptions`` endpoint (a Whisper
+server that is deployed on its own, or a hosted service).
+
 The concrete model presets, the ``AsrModelType`` enum, and the hardware-based
 auto-selection live in ``docling.datamodel.asr_model_specs``. The auto-selecting
 ``WHISPER_*`` presets (the ASR pipeline default) use MLX Whisper on Apple Silicon
@@ -54,6 +58,99 @@ class InferenceAsrFramework(str, Enum):
     WHISPER_S2T = "whisper_s2t"
 
 
+class ApiAsrOptions(BaseAsrOptions):
+    """Configuration for transcription by a remote, OpenAI-compatible service.
+
+    The audio file is posted as ``multipart/form-data`` to an
+    ``/audio/transcriptions`` endpoint with ``response_format=verbose_json``, and the
+    timed segments of the answer become the text items of the document. No model is
+    loaded in the Docling process, and FFmpeg is not needed for audio files (the
+    server decodes them).
+
+    Notes:
+        Requires ``enable_remote_services=True`` on the parent pipeline options to
+        permit external API calls.
+    """
+
+    kind: Literal["api_model_options"] = "api_model_options"
+    url: Annotated[
+        AnyUrl,
+        Field(
+            description=(
+                "Transcription endpoint of an OpenAI-compatible server, for example "
+                "`http://whisper:8000/v1/audio/transcriptions`. The default points to a "
+                "local server; update it for a deployed or cloud service."
+            ),
+        ),
+    ] = AnyUrl("http://localhost:8000/v1/audio/transcriptions")
+    headers: Annotated[
+        dict[str, str],
+        Field(
+            description=(
+                "HTTP headers to include in the request. Use for authentication."
+            ),
+            examples=[{"Authorization": "Bearer TOKEN"}],
+        ),
+    ] = {}
+    model: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "Value of the `model` form field, for servers that serve several "
+                "models. Omitted when empty."
+            ),
+            examples=["whisper-1", "Systran/faster-whisper-small"],
+        ),
+    ] = None
+    language: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "ISO 639-1 language code of the audio (`en`, `fr`...). Omitted when "
+                "empty, so that the server detects the language."
+            ),
+            examples=["en", "fr", "de"],
+        ),
+    ] = None
+    initial_prompt: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "Text sent as the `prompt` form field, to condition the transcription "
+                "style or give domain vocabulary (names, call-signs, jargon)."
+            )
+        ),
+    ] = None
+    word_timestamps: Annotated[
+        bool,
+        Field(
+            description=(
+                "Ask for word-level timestamps (`timestamp_granularities[]=word`). "
+                "The server must support them."
+            )
+        ),
+    ] = False
+    params: Annotated[
+        dict[str, Any],
+        Field(
+            description=(
+                "Additional form fields to send, for service-specific settings that "
+                "are not covered above (for example `temperature` or `vad_filter`)."
+            ),
+            examples=[{"temperature": 0.0}],
+        ),
+    ] = {}
+    timeout: Annotated[
+        float,
+        Field(
+            description=(
+                "Seconds to wait for the answer. Transcribing a long recording on a "
+                "busy server can take minutes."
+            )
+        ),
+    ] = 600.0
+
+
 class InlineAsrOptions(BaseAsrOptions):
     """Configuration for inline ASR models running locally."""
 

docling/pipeline/asr_pipeline.py

@@ -40,6 +40,7 @@ def __init__(self, pipeline_options: AsrPipelineOptions):
             asr_options=self.pipeline_options.asr_options,
             artifacts_path=self.artifacts_path,
             accelerator_options=pipeline_options.accelerator_options,
+            enable_remote_services=pipeline_options.enable_remote_services,
         )
 
     def _has_text(self, document: "DoclingDocument") -> bool:

docling/pipeline/asr_transcriber.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from typing import Final, Protocol
 
+import requests
 from docling_core.types.doc import (
     ContentLayer,
     DocItemLabel,
@@ -33,11 +34,13 @@
     AsrPipelineOptions,
 )
 from docling.datamodel.pipeline_options_asr_model import (
+    ApiAsrOptions,
     InlineAsrMlxWhisperOptions,
     InlineAsrNativeWhisperOptions,
     InlineAsrOptions,
     InlineAsrWhisperS2TOptions,
 )
+from docling.exceptions import OperationNotAllowed
 from docling.pipeline.base_pipeline import BasePipeline
 from docling.utils.accelerator_utils import decide_device
 from docling.utils.profiling import ProfilingScope, TimeRecorder
@@ -696,6 +699,137 @@ def transcribe(self, fpath: Path) -> list[_ConversationItem]:
 # ============================================================
 
 
+class _ApiAsrModel:
+    """Transcriber that sends the audio to an OpenAI-compatible transcription API.
+
+    Nothing is loaded in this process: the file is posted to ``/audio/transcriptions``
+    (a Whisper server of its own, or a hosted service) with
+    ``response_format=verbose_json`` and the timed segments of the answer become the
+    conversation. FFmpeg is not needed, the server decodes the audio.
+    """
+
+    def __init__(self, asr_options: ApiAsrOptions):
+        self.asr_options = asr_options
+
+    def _form_fields(self) -> list[tuple[str, str]]:
+        options = self.asr_options
+        fields: list[tuple[str, str]] = [("response_format", "verbose_json")]
+        if options.model:
+            fields.append(("model", options.model))
+        if options.language:
+            fields.append(("language", options.language))
+        if options.initial_prompt:
+            fields.append(("prompt", options.initial_prompt))
+        if options.word_timestamps:
+            fields.append(("timestamp_granularities[]", "segment"))
+            fields.append(("timestamp_granularities[]", "word"))
+        for key, value in options.params.items():
+            # `response_format` stays verbose_json: the timed segments are needed
+            if key != "response_format":
+                fields.append(
+                    (key, str(value).lower() if isinstance(value, bool) else str(value))
+                )
+        return fields
+
+    def _post(self, name: str, content: bytes) -> dict:
+        options = self.asr_options
+        _log.info(f"transcribing {name} with {options.url}")
+        response = requests.post(
+            str(options.url),
+            headers=options.headers,
+            data=self._form_fields(),
+            files={"file": (name, content, _audio_mimetype(name))},
+            timeout=options.timeout,
+        )
+        if not response.ok:
+            detail = " ".join(response.text.split())[:300]
+            raise RuntimeError(
+                f"The transcription service answered HTTP {response.status_code}"
+                + (f": {detail}" if detail else "")
+            )
+        try:
+            return response.json()
+        except ValueError as err:
+            raise RuntimeError(
+                "The transcription service did not answer with JSON; it must "
+                "support response_format=verbose_json"
+            ) from err
+
+    @staticmethod
+    def _to_conversation(answer: dict) -> list[_ConversationItem]:
+        segments = answer.get("segments")
+        if not isinstance(segments, list):
+            raise RuntimeError(
+                "The transcription service gave no timed segments; it must "
+                "support response_format=verbose_json"
+            )
+        # OpenAI puts the words in a list of their own, other servers inside each
+        # segment: take both, and give a word to the segment it falls in.
+        top_level_words = [
+            _ConversationWord(
+                start_time=w.get("start"), end_time=w.get("end"), text=w.get("word", "")
+            )
+            for w in answer.get("words") or []
+        ]
+        convo: list[_ConversationItem] = []
+        for seg in segments:
+            start, end = seg.get("start"), seg.get("end")
+            words = [
+                _ConversationWord(
+                    start_time=w.get("start"),
+                    end_time=w.get("end"),
+                    text=w.get("word", ""),
+                )
+                for w in seg.get("words") or []
+            ]
+            if not words and top_level_words and start is not None and end is not None:
+                words = [
+                    w
+                    for w in top_level_words
+                    if w.start_time is not None and start <= w.start_time < end
+                ]
+            convo.append(
+                _ConversationItem(
+                    start_time=start,
+                    end_time=end,
+                    text=str(seg.get("text", "")).strip(),
+                    words=words,
+                )
+            )
+        return convo
+
+    def transcribe(self, fpath: Path) -> list[_ConversationItem]:
+        return self._to_conversation(self._post(fpath.name, fpath.read_bytes()))
+
+    def run(self, conv_res: ConversionResult) -> ConversionResult:
+        path_or_stream = conv_res.input._backend.path_or_stream
+        try:
+            if isinstance(path_or_stream, BytesIO):
+                conversation = self._to_conversation(
+                    self._post(conv_res.input.file.name, path_or_stream.getvalue())
+                )
+            elif isinstance(path_or_stream, Path):
+                conversation = self.transcribe(path_or_stream)
+            else:
+                raise RuntimeError(
+                    f"ASR pipeline requires a file path or BytesIO stream, "
+                    f"but got {type(path_or_stream)}"
+                )
+            _process_conversation(conversation, conv_res)
+            conv_res.status = ConversionStatus.SUCCESS
+        except Exception as exc:
+            _log.error(f"Audio transcription by the API has an error: {exc}")
+            conv_res.errors.append(
+                ErrorItem(
+                    component_type=DoclingComponentType.PIPELINE,
+                    module_name="AsrPipeline",
+                    error_message=str(exc),
+                )
+            )
+            conv_res.status = ConversionStatus.FAILURE
+        return conv_res
+
+
 # ============================================================
 # Transcriber protocol + factory (new; consumed by AsrPipeline and VideoPipeline)
 # ============================================================
@@ -773,10 +907,19 @@ class _AsrModelFactory:
 
     @staticmethod
     def create(
-        asr_options: InlineAsrOptions,
+        asr_options: InlineAsrOptions | ApiAsrOptions,
         artifacts_path: Path | None,
         accelerator_options: AcceleratorOptions,
+        enable_remote_services: bool = False,
     ) -> _AsrTranscriber:
+        if isinstance(asr_options, ApiAsrOptions):
+            if not enable_remote_services:
+                raise OperationNotAllowed(
+                    "Transcribing with a remote API is not allowed by default. "
+                    "Set `enable_remote_services=True` in the pipeline options to "
+                    "send the audio to the configured service."
+                )
+            return _ApiAsrModel(asr_options=asr_options)
         if isinstance(asr_options, InlineAsrNativeWhisperOptions):
             return _NativeWhisperModel(
                 enabled=True,

docling/pipeline/video_pipeline.py

@@ -115,6 +115,7 @@ def __init__(self, pipeline_options: VideoPipelineOptions):
             asr_options=self.pipeline_options.asr_options,
             artifacts_path=self.artifacts_path,
             accelerator_options=pipeline_options.accelerator_options,
+            enable_remote_services=pipeline_options.enable_remote_services,
         )
 
     @classmethod