feat(pptx,odf): keep slide video and audio and record their media file

docling/.agents/skills/docling/references/python-sdk.md

@@ -372,6 +372,14 @@ to `200` (set `None` for no limit). When `document_timeout` is set, all FFmpeg
 calls share that budget; running out yields `PARTIAL_SUCCESS` with a `TIMEOUT`
 error (`result.has_timeout_errors()`).
 
+Video and audio on PPTX and ODP slides become pictures whose `meta` has a
+`docling__video` or `docling__audio` field with the path of the media file. Set
+`ConvertPipelineOptions(do_media_conversion=True)` on `PowerpointFormatOption` or
+`OdpFormatOption` to transcribe them too: the transcript (and the frames of a
+video) goes in a group right after the picture. `media_asr_options` picks the
+Whisper model. Linked media files load only with the backend options
+`enable_remote_fetch` / `enable_local_fetch`.
+
 ## Exporting images and tables
 
 To keep and export page/figure images, tell the pipeline to generate them, then

docling/backend/abstract_backend.py

@@ -87,3 +87,20 @@ def __init__(
     @abstractmethod
     def convert(self) -> DoclingDocument:
         pass
+
+    def load_media(self, location: str) -> Optional[bytes]:
+        """Return the bytes of a video or audio file that the document plays.
+
+        A backend that records media on pictures (see
+        ``docling.backend.utils.media``) overrides this, so that a pipeline can
+        convert the media. The default knows no media and returns None.
+
+        Args:
+            location: The value of a ``docling__video`` or ``docling__audio``
+                meta field: a path inside the document package, or a link.
+
+        Raises:
+            OperationNotAllowed: If the file is linked and the backend options
+                do not allow fetching it.
+        """
+        return None

docling/backend/mspowerpoint_backend.py

@@ -26,6 +26,7 @@
     PictureClassificationLabel,
     PictureClassificationMetaField,
     PictureClassificationPrediction,
+    PictureItem,
     PictureMeta,
     ProvenanceItem,
     Size,
@@ -49,6 +50,7 @@
 )
 from docling.backend.docx.latex.omml import oMath2Latex
 from docling.backend.utils.image import normalize_image_for_png
+from docling.backend.utils.media import MediaKind, load_linked_media, set_media_meta
 from docling.datamodel.backend_options import MsPowerpointBackendOptions
 from docling.datamodel.base_models import FormatToMimeType, InputFormat
 from docling.datamodel.document import InputDocument
@@ -66,7 +68,7 @@
     from pptx.oxml.text import CT_TextLineBreak
     from pptx.shapes.autoshape import Shape
     from pptx.shapes.base import BaseShape
-    from pptx.shapes.picture import Picture
+    from pptx.shapes.picture import Movie, Picture
 
     _PPTX_AVAILABLE = True
 except ImportError as e:  # pragma: no cover - import-time guard
@@ -128,6 +130,17 @@
 _DRAWING_2010_NAMESPACE: Final = "http://schemas.microsoft.com/office/drawing/2010/main"
 _MC_NAMESPACE: Final = "http://schemas.openxmlformats.org/markup-compatibility/2006"
 
+# The media elements that turn a picture shape into a video or audio shape.
+# ``a:audioCd`` is left out: it names CD tracks, not a file.
+_MEDIA_FILE_KINDS: Final[dict[str, MediaKind]] = {
+    "videoFile": "video",
+    "quickTimeFile": "video",
+    "audioFile": "audio",
+    "wavAudioFile": "audio",
+}
+# PowerPoint 2010 and later also point at the media file from this extension.
+_P14_MEDIA: Final = "{http://schemas.microsoft.com/office/powerpoint/2010/main}media"
+
 
 def _iter_paragraph_elements(element: etree._Element) -> Iterator[etree._Element]:
     """Read text and OMML in order, selecting one markup-compatibility branch."""
@@ -304,6 +317,11 @@ def __init__(
     ) -> None:
         if not _PPTX_AVAILABLE:
             raise ImportError(_INSTALL_HINT) from _PPTX_IMPORT_ERROR
+        # Linked media is resolved against the file the user gave, not against
+        # the PPTX that a legacy PPT is converted to.
+        self._base_path: Optional[str] = (
+            str(path_or_stream) if isinstance(path_or_stream, Path) else None
+        )
         if in_doc.format == InputFormat.PPT:
             path_or_stream = convert_to_modern_format(path_or_stream, "ppt", "pptx")
         if options is None:
@@ -386,6 +404,14 @@ def convert(self) -> DoclingDocument:
 
         return doc
 
+    @override
+    def load_media(self, location: str) -> Optional[bytes]:
+        if self.pptx_obj is not None:
+            for part in self.pptx_obj.part.package.iter_parts():
+                if part.partname.membername == location:
+                    return part.blob
+        return load_linked_media(location, self.options, self._base_path)
+
     def _generate_prov(
         self, shape, slide_ind, text="", slide_size=Size(width=1, height=1)
     ):
@@ -1040,7 +1066,10 @@ def _handle_pictures(self, shape, parent_slide, slide_ind, doc, slide_size):
         # Read the picture out of the package. A shape that slips past other
         # tools' parsers can still fail here, and there is nothing to record.
         try:
-            image = shape.image
+            # A video shows its poster frame where a picture shows its image.
+            image = shape.poster_frame if isinstance(shape, Movie) else shape.image
+            if image is None:
+                raise ValueError("video has no poster frame")
             image_bytes = image.blob
             im_dpi, _ = image.dpi
             prov = self._generate_prov(shape, slide_ind, "", slide_size)
@@ -1083,14 +1112,48 @@ def _handle_pictures(self, shape, parent_slide, slide_ind, doc, slide_size):
                 # 72 matches the other LibreOffice-rendered images.
                 image_ref = ImageRef.from_pil(image=rasterized, dpi=72)
 
-        doc.add_picture(
+        picture = doc.add_picture(
             parent=parent_slide,
             image=image_ref,
             caption=None,
             prov=prov,
         )
+        self._add_media_meta(shape, picture)
         return
 
+    @staticmethod
+    def _add_media_meta(shape, picture: PictureItem) -> None:
+        """Record the video or audio file that a picture shape plays.
+
+        PowerPoint draws a video or audio object as a picture: its poster frame
+        or a speaker icon. A media element in the shape's ``p:nvPr`` gives the
+        kind, and its relationship (or the ``p14:media`` one) gives the file:
+        a part inside the package, or an external link.
+        """
+        media_files = [
+            element
+            for element in shape._element.xpath("./p:nvPicPr/p:nvPr/a:*")
+            if etree.QName(element).localname in _MEDIA_FILE_KINDS
+        ]
+        if not media_files:
+            return
+        kind = _MEDIA_FILE_KINDS[etree.QName(media_files[0]).localname]
+
+        rels = shape.part.rels
+        r_ids = [
+            r_id
+            for element in (*shape._element.iter(_P14_MEDIA), media_files[0])
+            for attribute in (qn("r:embed"), qn("r:link"))
+            if (r_id := element.get(attribute)) is not None and r_id in rels
+        ]
+        if not r_ids:
+            return
+        rel = rels[r_ids[0]]
+        location = (
+            rel.target_ref if rel.is_external else rel.target_part.partname.membername
+        )
+        set_media_meta(picture, kind, location)
+
     def _handle_tables(self, shape, parent_slide, slide_ind, doc, slide_size):
         # Handling tables, images, charts
         if shape.has_table:
@@ -1666,8 +1729,9 @@ def handle_shape_content(shape, parent_slide, slide_ind, doc, slide_size):
                     self._handle_tables(shape, parent_slide, slide_ind, doc, slide_size)
                 if shape.has_chart:
                     self._handle_chart(shape, parent_slide, slide_ind, doc, slide_size)
-                if isinstance(shape, Picture):
-                    # Handle Pictures, including those inserted into a picture placeholder
+                if isinstance(shape, (Picture, Movie)):
+                    # Handle Pictures, including those inserted into a picture
+                    # placeholder, and video shapes, shown by their poster frame
                     self._handle_pictures(
                         shape, parent_slide, slide_ind, doc, slide_size
                     )

docling/backend/opendocument_backend.py

@@ -43,6 +43,7 @@
     PictureClassificationLabel,
     PictureClassificationMetaField,
     PictureClassificationPrediction,
+    PictureItem,
     PictureMeta,
     ProvenanceItem,
     RichTableCell,
@@ -63,6 +64,7 @@
 )
 from docling.backend.utils.image import normalize_image_for_png
 from docling.backend.utils.image_resource_loader import ImageResourceLoader
+from docling.backend.utils.media import load_linked_media, media_kind, set_media_meta
 from docling.datamodel.backend_options import OdsBackendOptions
 from docling.datamodel.base_models import InputFormat
 from docling.datamodel.document import InputDocument
@@ -1059,6 +1061,37 @@ def _add_odf_images(
     return image_count
 
 
+def _add_odf_media_meta(
+    doc: DoclingDocument,
+    frame: Frame,
+    parent: NodeItem,
+    odf_obj: OdfDocument,
+    pictures: list[PictureItem],
+) -> None:
+    """Record the video or audio file that a frame plays.
+
+    The media is a ``draw:plugin`` in the frame, next to a ``draw:image``
+    preview that is already a picture. The first such picture records it. A
+    frame without one gets an empty picture, so the media keeps its place.
+    """
+    plugin = frame.get_element("draw:plugin")
+    if plugin is None:
+        return
+    location = (plugin.get_attribute_string("xlink:href") or "").removeprefix("./")
+    if not location:
+        return
+    # LibreOffice may declare its generic "application/vnd.sun.star.media" type.
+    # The manifest entry of an embedded file, or else the extension, then tells
+    # video from audio.
+    kind = media_kind(plugin.get_attribute_string("draw:mime-type")) or media_kind(
+        odf_obj.manifest.get_media_type(location), location
+    )
+    if kind is None:
+        return
+    picture = pictures[0] if pictures else doc.add_picture(parent=parent)
+    set_media_meta(picture, kind, location)
+
+
 def _add_odf_child(
     doc: DoclingDocument,
     element: Any,
@@ -1869,6 +1902,17 @@ def convert(self) -> DoclingDocument:
                 self._hide_slide(doc, slide_group)
         return doc
 
+    @override
+    def load_media(self, location: str) -> bytes | None:
+        if location in self.odf_obj.get_parts():
+            part = self.odf_obj.get_part(location)
+            if isinstance(part, bytes):
+                return part
+        base_path = (
+            str(self.path_or_stream) if isinstance(self.path_or_stream, Path) else None
+        )
+        return load_linked_media(location, self.options, base_path)
+
     @staticmethod
     def _hide_slide(doc: DoclingDocument, slide_group: NodeItem) -> None:
         """Move the content of a hidden slide to the invisible layer.
@@ -1981,6 +2025,7 @@ def _walk_slide_frame(
                 image_loader=self._image_loader,
             )
 
+        first_picture = len(doc.pictures)
         _add_odf_images(
             doc,
             frame.get_images(),
@@ -1990,6 +2035,9 @@ def _walk_slide_frame(
             image_loader=self._image_loader,
             skip_object_replacements=chart_count > 0,
         )
+        _add_odf_media_meta(
+            doc, frame, parent, self.odf_obj, doc.pictures[first_picture:]
+        )
 
         for textbox in frame.get_elements("descendant::draw:text-box"):
             self._walk_textbox_children(

docling/backend/utils/media.py

@@ -0,0 +1,123 @@
+# SPDX-FileCopyrightText: The Docling Contributors
+# SPDX-License-Identifier: MIT
+
+"""Helpers to record and load the video and audio that a presentation plays.
+
+A slide shows a video or audio object as a picture: a poster frame or a
+speaker icon. The backends keep that picture and record on it where the media
+file is. A pipeline can then load the file through the backend and convert it
+with the audio or video pipeline.
+"""
+
+from pathlib import PurePosixPath
+from typing import Final, Literal, Optional
+from urllib.parse import unquote, urlparse
+
+from docling_core.types.doc import PictureItem, PictureMeta
+
+from docling.backend.utils.image_resource_loader import ImageResourceLoader
+from docling.datamodel.backend_options import BaseBackendOptions
+from docling.datamodel.base_models import FormatToExtensions, InputFormat
+from docling.exceptions import OperationNotAllowed
+
+MediaKind = Literal["video", "audio"]
+
+MEDIA_META_NAMESPACE: Final = "docling"
+"""The media file is in the ``docling__video`` or ``docling__audio`` meta field."""
+
+MAX_LINKED_MEDIA_BYTES: Final = 512 * 1024 * 1024
+"""Size limit for a linked media file that is downloaded from a remote URL."""
+
+_MEDIA_KINDS: Final[tuple[MediaKind, ...]] = ("video", "audio")
+
+
+def media_kind(mimetype: Optional[str], location: str = "") -> Optional[MediaKind]:
+    """Tell video from audio by the MIME type, or else by the file extension.
+
+    The extensions are the ones the audio and video input formats accept.
+    """
+    major = (mimetype or "").partition("/")[0]
+    if major == "video":
+        return "video"
+    if major == "audio":
+        return "audio"
+    suffix = PurePosixPath(location).suffix.lower().removeprefix(".")
+    if suffix in FormatToExtensions[InputFormat.VIDEO]:
+        return "video"
+    if suffix in FormatToExtensions[InputFormat.AUDIO]:
+        return "audio"
+    return None
+
+
+def set_media_meta(picture: PictureItem, kind: MediaKind, location: str) -> None:
+    """Record on a picture the video or audio file that it stands for.
+
+    The value is the path of the file inside the document package when the
+    file is embedded, and the link target that the document gives when the file
+    is linked. It is a plain string, so the Markdown export shows it under the
+    picture.
+
+    Args:
+        picture: The picture that shows the media on the slide.
+        kind: ``"video"`` or ``"audio"``; it names the meta field.
+        location: The package path or the link target of the media file.
+    """
+    if picture.meta is None:
+        picture.meta = PictureMeta()
+    picture.meta.set_custom_field(
+        namespace=MEDIA_META_NAMESPACE, name=kind, value=location
+    )
+
+
+def get_media_meta(picture: PictureItem) -> Optional[tuple[MediaKind, str]]:
+    """Return the kind and the location of the media file that a picture stands for."""
+    if picture.meta is None:
+        return None
+    fields = picture.meta.get_custom_part()
+    for kind in _MEDIA_KINDS:
+        location = fields.get(f"{MEDIA_META_NAMESPACE}__{kind}")
+        if isinstance(location, str) and location:
+            return kind, location
+    return None
+
+
+def media_file_name(location: str) -> str:
+    """Return the file name in a package path, a URL or a Windows path."""
+    return PurePosixPath(urlparse(location).path.replace("\\", "/")).name
+
+
+def load_linked_media(
+    location: str, options: BaseBackendOptions, base_path: Optional[str]
+) -> Optional[bytes]:
+    """Load a linked media file with the fetch rules of the backend options.
+
+    A remote URL needs ``enable_remote_fetch`` and gets the same address checks
+    as a remote image, with a limit of ``MAX_LINKED_MEDIA_BYTES``. A local file
+    needs ``enable_local_fetch`` and a document that was read from a file
+    (``base_path``). The media file must be in the folder of the document or
+    below it, so an absolute link is refused.
+
+    Raises:
+        OperationNotAllowed: If the backend options do not allow the fetch.
+        ValueError: If the file is outside the folder of the document, cannot
+            be read, or is too large.
+        requests.RequestException, urllib3.exceptions.HTTPError: If a download
+            fails.
+    """
+    loader = ImageResourceLoader(
+        enable_local_fetch=options.enable_local_fetch,
+        enable_remote_fetch=options.enable_remote_fetch,
+        max_remote_image_bytes=MAX_LINKED_MEDIA_BYTES,
+    )
+    if loader.is_remote_url(location):
+        return loader.fetch_remote(location).content
+    # Check the option first: the path checks below would otherwise report a
+    # link that is not allowed as a broken one.
+    if not options.enable_local_fetch:
+        raise OperationNotAllowed(
+            "Fetching local resources is only allowed when set explicitly. "
+            "Set options.enable_local_fetch=True."
+        )
+    # A link target is a URI reference, so a space in a file name is "%20".
+    path = loader.resolve_relative_path(unquote(location), base_path)
+    return loader.load_image_data(path, base_path)