diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md index 54cc35d6b2..c129b8fb2f 100644 --- a/docling/.agents/skills/docling/references/python-sdk.md +++ b/docling/.agents/skills/docling/references/python-sdk.md @@ -372,6 +372,14 @@ to `200` (set `None` for no limit). When `document_timeout` is set, all FFmpeg calls share that budget; running out yields `PARTIAL_SUCCESS` with a `TIMEOUT` error (`result.has_timeout_errors()`). +Video and audio on PPTX and ODP slides become pictures whose `meta` has a +`docling__video` or `docling__audio` field with the path of the media file. Set +`ConvertPipelineOptions(do_media_conversion=True)` on `PowerpointFormatOption` or +`OdpFormatOption` to transcribe them too: the transcript (and the frames of a +video) goes in a group right after the picture. `media_asr_options` picks the +Whisper model. Linked media files load only with the backend options +`enable_remote_fetch` / `enable_local_fetch`. + ## Exporting images and tables To keep and export page/figure images, tell the pipeline to generate them, then diff --git a/docling/backend/abstract_backend.py b/docling/backend/abstract_backend.py index 0c84a1467a..bc2262933f 100644 --- a/docling/backend/abstract_backend.py +++ b/docling/backend/abstract_backend.py @@ -87,3 +87,20 @@ def __init__( @abstractmethod def convert(self) -> DoclingDocument: pass + + def load_media(self, location: str) -> Optional[bytes]: + """Return the bytes of a video or audio file that the document plays. + + A backend that records media on pictures (see + ``docling.backend.utils.media``) overrides this, so that a pipeline can + convert the media. The default knows no media and returns None. + + Args: + location: The value of a ``docling__video`` or ``docling__audio`` + meta field: a path inside the document package, or a link. + + Raises: + OperationNotAllowed: If the file is linked and the backend options + do not allow fetching it. + """ + return None diff --git a/docling/backend/mspowerpoint_backend.py b/docling/backend/mspowerpoint_backend.py index cdd6d85540..1c43ecc404 100644 --- a/docling/backend/mspowerpoint_backend.py +++ b/docling/backend/mspowerpoint_backend.py @@ -26,6 +26,7 @@ PictureClassificationLabel, PictureClassificationMetaField, PictureClassificationPrediction, + PictureItem, PictureMeta, ProvenanceItem, Size, @@ -49,6 +50,7 @@ ) from docling.backend.docx.latex.omml import oMath2Latex from docling.backend.utils.image import normalize_image_for_png +from docling.backend.utils.media import MediaKind, load_linked_media, set_media_meta from docling.datamodel.backend_options import MsPowerpointBackendOptions from docling.datamodel.base_models import FormatToMimeType, InputFormat from docling.datamodel.document import InputDocument @@ -66,7 +68,7 @@ from pptx.oxml.text import CT_TextLineBreak from pptx.shapes.autoshape import Shape from pptx.shapes.base import BaseShape - from pptx.shapes.picture import Picture + from pptx.shapes.picture import Movie, Picture _PPTX_AVAILABLE = True except ImportError as e: # pragma: no cover - import-time guard @@ -128,6 +130,17 @@ _DRAWING_2010_NAMESPACE: Final = "http://schemas.microsoft.com/office/drawing/2010/main" _MC_NAMESPACE: Final = "http://schemas.openxmlformats.org/markup-compatibility/2006" +# The media elements that turn a picture shape into a video or audio shape. +# ``a:audioCd`` is left out: it names CD tracks, not a file. +_MEDIA_FILE_KINDS: Final[dict[str, MediaKind]] = { + "videoFile": "video", + "quickTimeFile": "video", + "audioFile": "audio", + "wavAudioFile": "audio", +} +# PowerPoint 2010 and later also point at the media file from this extension. +_P14_MEDIA: Final = "{http://schemas.microsoft.com/office/powerpoint/2010/main}media" + def _iter_paragraph_elements(element: etree._Element) -> Iterator[etree._Element]: """Read text and OMML in order, selecting one markup-compatibility branch.""" @@ -304,6 +317,11 @@ def __init__( ) -> None: if not _PPTX_AVAILABLE: raise ImportError(_INSTALL_HINT) from _PPTX_IMPORT_ERROR + # Linked media is resolved against the file the user gave, not against + # the PPTX that a legacy PPT is converted to. + self._base_path: Optional[str] = ( + str(path_or_stream) if isinstance(path_or_stream, Path) else None + ) if in_doc.format == InputFormat.PPT: path_or_stream = convert_to_modern_format(path_or_stream, "ppt", "pptx") if options is None: @@ -386,6 +404,14 @@ def convert(self) -> DoclingDocument: return doc + @override + def load_media(self, location: str) -> Optional[bytes]: + if self.pptx_obj is not None: + for part in self.pptx_obj.part.package.iter_parts(): + if part.partname.membername == location: + return part.blob + return load_linked_media(location, self.options, self._base_path) + def _generate_prov( self, shape, slide_ind, text="", slide_size=Size(width=1, height=1) ): @@ -1040,7 +1066,10 @@ def _handle_pictures(self, shape, parent_slide, slide_ind, doc, slide_size): # Read the picture out of the package. A shape that slips past other # tools' parsers can still fail here, and there is nothing to record. try: - image = shape.image + # A video shows its poster frame where a picture shows its image. + image = shape.poster_frame if isinstance(shape, Movie) else shape.image + if image is None: + raise ValueError("video has no poster frame") image_bytes = image.blob im_dpi, _ = image.dpi prov = self._generate_prov(shape, slide_ind, "", slide_size) @@ -1083,14 +1112,48 @@ def _handle_pictures(self, shape, parent_slide, slide_ind, doc, slide_size): # 72 matches the other LibreOffice-rendered images. image_ref = ImageRef.from_pil(image=rasterized, dpi=72) - doc.add_picture( + picture = doc.add_picture( parent=parent_slide, image=image_ref, caption=None, prov=prov, ) + self._add_media_meta(shape, picture) return + @staticmethod + def _add_media_meta(shape, picture: PictureItem) -> None: + """Record the video or audio file that a picture shape plays. + + PowerPoint draws a video or audio object as a picture: its poster frame + or a speaker icon. A media element in the shape's ``p:nvPr`` gives the + kind, and its relationship (or the ``p14:media`` one) gives the file: + a part inside the package, or an external link. + """ + media_files = [ + element + for element in shape._element.xpath("./p:nvPicPr/p:nvPr/a:*") + if etree.QName(element).localname in _MEDIA_FILE_KINDS + ] + if not media_files: + return + kind = _MEDIA_FILE_KINDS[etree.QName(media_files[0]).localname] + + rels = shape.part.rels + r_ids = [ + r_id + for element in (*shape._element.iter(_P14_MEDIA), media_files[0]) + for attribute in (qn("r:embed"), qn("r:link")) + if (r_id := element.get(attribute)) is not None and r_id in rels + ] + if not r_ids: + return + rel = rels[r_ids[0]] + location = ( + rel.target_ref if rel.is_external else rel.target_part.partname.membername + ) + set_media_meta(picture, kind, location) + def _handle_tables(self, shape, parent_slide, slide_ind, doc, slide_size): # Handling tables, images, charts if shape.has_table: @@ -1666,8 +1729,9 @@ def handle_shape_content(shape, parent_slide, slide_ind, doc, slide_size): self._handle_tables(shape, parent_slide, slide_ind, doc, slide_size) if shape.has_chart: self._handle_chart(shape, parent_slide, slide_ind, doc, slide_size) - if isinstance(shape, Picture): - # Handle Pictures, including those inserted into a picture placeholder + if isinstance(shape, (Picture, Movie)): + # Handle Pictures, including those inserted into a picture + # placeholder, and video shapes, shown by their poster frame self._handle_pictures( shape, parent_slide, slide_ind, doc, slide_size ) diff --git a/docling/backend/opendocument_backend.py b/docling/backend/opendocument_backend.py index 6932f04639..699d44fc33 100644 --- a/docling/backend/opendocument_backend.py +++ b/docling/backend/opendocument_backend.py @@ -43,6 +43,7 @@ PictureClassificationLabel, PictureClassificationMetaField, PictureClassificationPrediction, + PictureItem, PictureMeta, ProvenanceItem, RichTableCell, @@ -63,6 +64,7 @@ ) from docling.backend.utils.image import normalize_image_for_png from docling.backend.utils.image_resource_loader import ImageResourceLoader +from docling.backend.utils.media import load_linked_media, media_kind, set_media_meta from docling.datamodel.backend_options import OdsBackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument @@ -1059,6 +1061,37 @@ def _add_odf_images( return image_count +def _add_odf_media_meta( + doc: DoclingDocument, + frame: Frame, + parent: NodeItem, + odf_obj: OdfDocument, + pictures: list[PictureItem], +) -> None: + """Record the video or audio file that a frame plays. + + The media is a ``draw:plugin`` in the frame, next to a ``draw:image`` + preview that is already a picture. The first such picture records it. A + frame without one gets an empty picture, so the media keeps its place. + """ + plugin = frame.get_element("draw:plugin") + if plugin is None: + return + location = (plugin.get_attribute_string("xlink:href") or "").removeprefix("./") + if not location: + return + # LibreOffice may declare its generic "application/vnd.sun.star.media" type. + # The manifest entry of an embedded file, or else the extension, then tells + # video from audio. + kind = media_kind(plugin.get_attribute_string("draw:mime-type")) or media_kind( + odf_obj.manifest.get_media_type(location), location + ) + if kind is None: + return + picture = pictures[0] if pictures else doc.add_picture(parent=parent) + set_media_meta(picture, kind, location) + + def _add_odf_child( doc: DoclingDocument, element: Any, @@ -1869,6 +1902,17 @@ def convert(self) -> DoclingDocument: self._hide_slide(doc, slide_group) return doc + @override + def load_media(self, location: str) -> bytes | None: + if location in self.odf_obj.get_parts(): + part = self.odf_obj.get_part(location) + if isinstance(part, bytes): + return part + base_path = ( + str(self.path_or_stream) if isinstance(self.path_or_stream, Path) else None + ) + return load_linked_media(location, self.options, base_path) + @staticmethod def _hide_slide(doc: DoclingDocument, slide_group: NodeItem) -> None: """Move the content of a hidden slide to the invisible layer. @@ -1981,6 +2025,7 @@ def _walk_slide_frame( image_loader=self._image_loader, ) + first_picture = len(doc.pictures) _add_odf_images( doc, frame.get_images(), @@ -1990,6 +2035,9 @@ def _walk_slide_frame( image_loader=self._image_loader, skip_object_replacements=chart_count > 0, ) + _add_odf_media_meta( + doc, frame, parent, self.odf_obj, doc.pictures[first_picture:] + ) for textbox in frame.get_elements("descendant::draw:text-box"): self._walk_textbox_children( diff --git a/docling/backend/utils/media.py b/docling/backend/utils/media.py new file mode 100644 index 0000000000..7b5e9b307f --- /dev/null +++ b/docling/backend/utils/media.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Helpers to record and load the video and audio that a presentation plays. + +A slide shows a video or audio object as a picture: a poster frame or a +speaker icon. The backends keep that picture and record on it where the media +file is. A pipeline can then load the file through the backend and convert it +with the audio or video pipeline. +""" + +from pathlib import PurePosixPath +from typing import Final, Literal, Optional +from urllib.parse import unquote, urlparse + +from docling_core.types.doc import PictureItem, PictureMeta + +from docling.backend.utils.image_resource_loader import ImageResourceLoader +from docling.datamodel.backend_options import BaseBackendOptions +from docling.datamodel.base_models import FormatToExtensions, InputFormat +from docling.exceptions import OperationNotAllowed + +MediaKind = Literal["video", "audio"] + +MEDIA_META_NAMESPACE: Final = "docling" +"""The media file is in the ``docling__video`` or ``docling__audio`` meta field.""" + +MAX_LINKED_MEDIA_BYTES: Final = 512 * 1024 * 1024 +"""Size limit for a linked media file that is downloaded from a remote URL.""" + +_MEDIA_KINDS: Final[tuple[MediaKind, ...]] = ("video", "audio") + + +def media_kind(mimetype: Optional[str], location: str = "") -> Optional[MediaKind]: + """Tell video from audio by the MIME type, or else by the file extension. + + The extensions are the ones the audio and video input formats accept. + """ + major = (mimetype or "").partition("/")[0] + if major == "video": + return "video" + if major == "audio": + return "audio" + suffix = PurePosixPath(location).suffix.lower().removeprefix(".") + if suffix in FormatToExtensions[InputFormat.VIDEO]: + return "video" + if suffix in FormatToExtensions[InputFormat.AUDIO]: + return "audio" + return None + + +def set_media_meta(picture: PictureItem, kind: MediaKind, location: str) -> None: + """Record on a picture the video or audio file that it stands for. + + The value is the path of the file inside the document package when the + file is embedded, and the link target that the document gives when the file + is linked. It is a plain string, so the Markdown export shows it under the + picture. + + Args: + picture: The picture that shows the media on the slide. + kind: ``"video"`` or ``"audio"``; it names the meta field. + location: The package path or the link target of the media file. + """ + if picture.meta is None: + picture.meta = PictureMeta() + picture.meta.set_custom_field( + namespace=MEDIA_META_NAMESPACE, name=kind, value=location + ) + + +def get_media_meta(picture: PictureItem) -> Optional[tuple[MediaKind, str]]: + """Return the kind and the location of the media file that a picture stands for.""" + if picture.meta is None: + return None + fields = picture.meta.get_custom_part() + for kind in _MEDIA_KINDS: + location = fields.get(f"{MEDIA_META_NAMESPACE}__{kind}") + if isinstance(location, str) and location: + return kind, location + return None + + +def media_file_name(location: str) -> str: + """Return the file name in a package path, a URL or a Windows path.""" + return PurePosixPath(urlparse(location).path.replace("\\", "/")).name + + +def load_linked_media( + location: str, options: BaseBackendOptions, base_path: Optional[str] +) -> Optional[bytes]: + """Load a linked media file with the fetch rules of the backend options. + + A remote URL needs ``enable_remote_fetch`` and gets the same address checks + as a remote image, with a limit of ``MAX_LINKED_MEDIA_BYTES``. A local file + needs ``enable_local_fetch`` and a document that was read from a file + (``base_path``). The media file must be in the folder of the document or + below it, so an absolute link is refused. + + Raises: + OperationNotAllowed: If the backend options do not allow the fetch. + ValueError: If the file is outside the folder of the document, cannot + be read, or is too large. + requests.RequestException, urllib3.exceptions.HTTPError: If a download + fails. + """ + loader = ImageResourceLoader( + enable_local_fetch=options.enable_local_fetch, + enable_remote_fetch=options.enable_remote_fetch, + max_remote_image_bytes=MAX_LINKED_MEDIA_BYTES, + ) + if loader.is_remote_url(location): + return loader.fetch_remote(location).content + # Check the option first: the path checks below would otherwise report a + # link that is not allowed as a broken one. + if not options.enable_local_fetch: + raise OperationNotAllowed( + "Fetching local resources is only allowed when set explicitly. " + "Set options.enable_local_fetch=True." + ) + # A link target is a URI reference, so a space in a file name is "%20". + path = loader.resolve_relative_path(unquote(location), base_path) + return loader.load_image_data(path, base_path) diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index 0da23a7be2..ce900a1b83 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -1513,6 +1513,26 @@ class ConvertPipelineOptions(PipelineOptions): ) ), ] = _default_chart_extraction_options + do_media_conversion: Annotated[ + bool, + Field( + description=( + "Convert the video and audio files that slides play (PPTX, ODP) with the audio or video " + "pipeline. The transcript, and the sampled frames of a video, go in a group after the " + "picture of each media object. Needs the `asr` extra and ffmpeg. A linked file " + "is fetched only when the backend options `enable_remote_fetch` or `enable_local_fetch` " + "allow it. Only formats that the simple pipeline converts use this option." + ) + ), + ] = False + media_asr_options: Annotated[ + InlineAsrOptions, + Field( + description=( + "Speech recognition model for the media files of `do_media_conversion`." + ) + ), + ] = asr_model_specs.WHISPER_TINY class PaginatedPipelineOptions(ConvertPipelineOptions): diff --git a/docling/pipeline/simple_pipeline.py b/docling/pipeline/simple_pipeline.py index 0e0c8f7a07..ea7fd7784c 100644 --- a/docling/pipeline/simple_pipeline.py +++ b/docling/pipeline/simple_pipeline.py @@ -2,15 +2,39 @@ # SPDX-License-Identifier: MIT import logging +from io import BytesIO + +from docling_core.types.doc import ( + ContentLayer, + DocItem, + DoclingDocument, + GroupLabel, + PictureItem, + TextItem, +) from docling.backend.abstract_backend import ( AbstractDocumentBackend, DeclarativeDocumentBackend, ) -from docling.datamodel.base_models import ConversionStatus -from docling.datamodel.document import ConversionResult -from docling.datamodel.pipeline_options import ConvertPipelineOptions -from docling.pipeline.base_pipeline import ConvertPipeline +from docling.backend.noop_backend import NoOpBackend +from docling.backend.utils.media import MediaKind, get_media_meta, media_file_name +from docling.datamodel.base_models import ( + ConversionStatus, + DoclingComponentType, + ErrorItem, + InputFormat, +) +from docling.datamodel.document import ConversionResult, InputDocument +from docling.datamodel.pipeline_options import ( + AsrPipelineOptions, + ConvertPipelineOptions, + VideoPipelineOptions, +) +from docling.exceptions import OperationNotAllowed +from docling.pipeline.asr_pipeline import AsrPipeline +from docling.pipeline.base_pipeline import BasePipeline, ConvertPipeline +from docling.pipeline.video_pipeline import VideoPipeline from docling.utils.profiling import ProfilingScope, TimeRecorder _log = logging.getLogger(__name__) @@ -25,11 +49,14 @@ class SimplePipeline(ConvertPipeline): def __init__(self, pipeline_options: ConvertPipelineOptions): super().__init__(pipeline_options) + # Built on first use, so a document without media loads no ASR model. + self._media_pipelines: dict[MediaKind, BasePipeline] = {} def _build_document(self, conv_res: ConversionResult) -> ConversionResult: - if not isinstance(conv_res.input._backend, DeclarativeDocumentBackend): + backend = conv_res.input._backend + if not isinstance(backend, DeclarativeDocumentBackend): raise RuntimeError( - f"The selected backend {type(conv_res.input._backend).__name__} for {conv_res.input.file} is not a declarative backend. " + f"The selected backend {type(backend).__name__} for {conv_res.input.file} is not a declarative backend. " f"Can not convert this with simple pipeline. " f"Please check your format configuration on DocumentConverter." ) @@ -40,9 +67,137 @@ def _build_document(self, conv_res: ConversionResult) -> ConversionResult: # the backend is expected to be of type DeclarativeDocumentBackend, which can output # a DoclingDocument straight. with TimeRecorder(conv_res, "doc_build", scope=ProfilingScope.DOCUMENT): - conv_res.document = conv_res.input._backend.convert() + conv_res.document = backend.convert() + + if self.pipeline_options.do_media_conversion: + with TimeRecorder(conv_res, "media_convert", scope=ProfilingScope.DOCUMENT): + self._convert_media(conv_res, backend) return conv_res + def _convert_media( + self, conv_res: ConversionResult, backend: DeclarativeDocumentBackend + ) -> None: + """Convert the video and audio files that the document plays. + + The backend records each media file on the picture that shows it, and + loads the file again on request. The audio or video pipeline converts + it, and its document goes in a group right after that picture. + """ + doc = conv_res.document + for picture in list(doc.pictures): + media = get_media_meta(picture) + if media is None: + continue + kind, location = media + try: + data = backend.load_media(location) + except OperationNotAllowed as exc: + # The fetch rules of the backend options decide this, as for images. + _log.warning("Media file %s is not loaded: %s", location, exc) + continue + except Exception as exc: + self._add_media_error(conv_res, location, f"cannot be loaded: {exc}") + continue + if not data: + continue + + try: + pipeline = self._get_media_pipeline(kind) + except ImportError as exc: + # Like a missing ffmpeg, a missing ASR extra fails the media only. + self._add_media_error(conv_res, location, f"cannot be converted: {exc}") + continue + media_input = InputDocument( + path_or_stream=BytesIO(data), + format=InputFormat.VIDEO if kind == "video" else InputFormat.AUDIO, + backend=NoOpBackend, + filename=media_file_name(location), + limits=conv_res.input.limits, + ) + result = pipeline.execute(media_input, raises_on_error=False) + if result.status not in { + ConversionStatus.SUCCESS, + ConversionStatus.PARTIAL_SUCCESS, + }: + reasons = "; ".join(error.error_message for error in result.errors) + self._add_media_error( + conv_res, location, f"cannot be converted: {reasons}" + ) + continue + self._add_media_document(doc, picture, location, result.document) + + def _get_media_pipeline(self, kind: MediaKind) -> BasePipeline: + if kind not in self._media_pipelines: + options = self.pipeline_options + pipeline: BasePipeline + if kind == "video": + pipeline = VideoPipeline( + VideoPipelineOptions( + asr_options=options.media_asr_options, + accelerator_options=options.accelerator_options, + artifacts_path=options.artifacts_path, + document_timeout=options.document_timeout, + ) + ) + else: + pipeline = AsrPipeline( + AsrPipelineOptions( + asr_options=options.media_asr_options, + accelerator_options=options.accelerator_options, + artifacts_path=options.artifacts_path, + document_timeout=options.document_timeout, + ) + ) + self._media_pipelines[kind] = pipeline + return self._media_pipelines[kind] + + @staticmethod + def _add_media_document( + doc: DoclingDocument, + picture: PictureItem, + location: str, + media_doc: DoclingDocument, + ) -> None: + """Add the converted media after its picture. + + The items take the content layer of the picture, so the media of a + hidden shape stays hidden, and its provenance: the media plays in the + area of the picture on the slide. + """ + group = doc.insert_group( + sibling=picture, + label=GroupLabel.SECTION, + name=f"media: {location}", + content_layer=picture.content_layer, + ) + doc.add_document(media_doc, parent=group) + for item, _ in doc.iterate_items( + root=group, + with_groups=True, + traverse_pictures=True, + included_content_layers=set(ContentLayer), + ): + item.content_layer = picture.content_layer + if isinstance(item, DocItem): + length = len(item.text) if isinstance(item, TextItem) else 0 + item.prov = [ + prov.model_copy(update={"charspan": (0, length)}) + for prov in picture.prov + ] + + @staticmethod + def _add_media_error( + conv_res: ConversionResult, location: str, message: str + ) -> None: + _log.warning("Media file %s %s", location, message) + conv_res.errors.append( + ErrorItem( + component_type=DoclingComponentType.PIPELINE, + module_name=SimplePipeline.__name__, + error_message=f"Media file {location} {message}", + ) + ) + def _determine_status(self, conv_res: ConversionResult) -> ConversionStatus: # This is called only if the previous steps didn't raise. # Since we don't have anything else to evaluate, we can diff --git a/docs/usage/processing_audio_media.md b/docs/usage/processing_audio_media.md index 46d77d75d2..ddd0ef1f23 100644 --- a/docs/usage/processing_audio_media.md +++ b/docs/usage/processing_audio_media.md @@ -258,6 +258,40 @@ docling --to md --video-sampling-mode scene --video-diarization video.mp4 See the [CLI reference](../reference/cli.md) for the complete flag list. +## Video and audio in slides + +PowerPoint (PPTX) and OpenDocument (ODP) slides can play video and audio. By default, Docling keeps each media object as a picture of its poster frame or icon. The `docling__video` or `docling__audio` field of the picture `meta` tells where the media file is: its path inside the presentation package, or the link of a linked file. + +Set `do_media_conversion=True` to also convert the media. Docling sends each media file through the ASR pipeline (audio) or the video pipeline (video). The result goes in a group right after the picture: the transcript with its timestamps and, for a video, the sampled frames. These items get the provenance of the picture, so you can see on which slide the media plays. + +```python +from pathlib import Path + +from docling.datamodel.base_models import InputFormat +from docling.datamodel.pipeline_options import ConvertPipelineOptions +from docling.document_converter import ( + DocumentConverter, + OdpFormatOption, + PowerpointFormatOption, +) + +pipeline_options = ConvertPipelineOptions(do_media_conversion=True) +converter = DocumentConverter( + format_options={ + InputFormat.PPTX: PowerpointFormatOption(pipeline_options=pipeline_options), + InputFormat.ODP: OdpFormatOption(pipeline_options=pipeline_options), + } +) +result = converter.convert(Path("slides.pptx")) +print(result.document.export_to_markdown()) +``` + +`media_asr_options` selects the Whisper model (default: `WHISPER_TINY`). Frames are sampled with the default settings of the video pipeline. + +Docling loads a linked media file only when the backend options allow it: `enable_remote_fetch=True` for a URL, and `enable_local_fetch=True` for a local file. A local file must be in the folder of the presentation or below it, and the presentation must be given as a `Path`. PowerPoint stores links to local files as absolute paths, so Docling does not load them. + +If a media file cannot be converted, the picture stays, the result status is `PARTIAL_SUCCESS`, and the reason is in `ConversionResult.errors`. + ## Choosing an ASR model and backend Docling ships three interchangeable ASR backends, all installed by the `asr` extra shown above: diff --git a/docs/usage/supported_formats.md b/docs/usage/supported_formats.md index 188f1a679f..e05724207b 100644 --- a/docs/usage/supported_formats.md +++ b/docs/usage/supported_formats.md @@ -34,6 +34,12 @@ Below you can find a listing of all supported input and output formats. PowerPoint equations stored as Office Math (OMML) in text shapes are converted to LaTeX formula items. Equations mixed with text or list content retain their order. +Video and audio on PPTX and ODP slides are kept as pictures of their poster frame +or icon. The `docling__video` or `docling__audio` field of the picture `meta` gives +the path of the media file inside the package, or the link of a linked file. Set +`do_media_conversion=True` in the pipeline options to also transcribe the media +(see [Video and audio in slides](processing_audio_media.md#video-and-audio-in-slides)). + Schema-specific support: | Format | Description | diff --git a/tach.toml b/tach.toml index 0b2160d683..8dbf95d4d8 100644 --- a/tach.toml +++ b/tach.toml @@ -448,7 +448,10 @@ layer = "pipeline" depends_on = [ "docling.backend", "docling.datamodel", + "docling.exceptions", + "docling.pipeline.asr_pipeline", "docling.pipeline.base_pipeline", + "docling.pipeline.video_pipeline", "docling.utils", ] diff --git a/tests/test_backend_odp_media.py b/tests/test_backend_odp_media.py new file mode 100644 index 0000000000..343fcc233a --- /dev/null +++ b/tests/test_backend_odp_media.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Tests for video and audio objects on OpenDocument presentation slides.""" + +from pathlib import Path + +import pytest +from docling_core.types.doc import DoclingDocument +from PIL import Image + +from docling.datamodel.base_models import InputFormat +from docling.document_converter import DocumentConverter + +pytest.importorskip("odfdo") +from odfdo import Document as OdfDocument, DrawPage, Element, Frame + + +def _plugin(href: str, mime_type: str) -> Element: + """Return a media object the way LibreOffice Impress writes it.""" + return Element.from_tag( + f'' + '' + ) + + +def _preview(odf: OdfDocument, tmp_path: Path) -> str: + image = tmp_path / "preview.png" + Image.new("RGB", (32, 18), "blue").save(image) + return odf.add_file(str(image)) + + +def _convert(odf: OdfDocument, frame: Frame, path: Path) -> DoclingDocument: + odf.body.clear() + page = DrawPage("page1", name="Slide One") + page.append(frame) + odf.body.append(page) + odf.save(str(path)) + converter = DocumentConverter(allowed_formats=[InputFormat.ODP]) + return converter.convert(path).document + + +def test_odp_embedded_video_records_its_media_path(tmp_path: Path): + """The preview of an embedded video records where the video is.""" + odf = OdfDocument("presentation") + video = tmp_path / "clip.mp4" + video.write_bytes(b"not decoded by the backend") + media_path = odf.add_file(str(video)) + frame = Frame.image_frame(_preview(odf, tmp_path), size=("8cm", "4.5cm")) + frame.insert(_plugin(media_path, "video/mp4"), position=0) + doc = _convert(odf, frame, tmp_path / "video.odp") + + assert len(doc.pictures) == 1 + picture = doc.pictures[0] + assert picture.image is not None + assert picture.meta is not None + assert picture.meta.get_custom_part() == {"docling__video": media_path} + + +def test_odp_generic_media_type_falls_back_to_the_extension(tmp_path: Path): + """LibreOffice's generic media type still gives a kind, from the file name. + + A linked file is recorded by its link, and a frame without a preview still + gets a picture, so the media keeps its place on the slide. + """ + odf = OdfDocument("presentation") + frame = Frame(size=("2cm", "2cm"), position=("1cm", "1cm")) + frame.append(_plugin("../sounds/tone.mp3", "application/vnd.sun.star.media")) + doc = _convert(odf, frame, tmp_path / "audio.odp") + + assert len(doc.pictures) == 1 + assert doc.pictures[0].image is None + assert doc.pictures[0].meta is not None + assert doc.pictures[0].meta.get_custom_part() == { + "docling__audio": "../sounds/tone.mp3" + } + + +def test_odp_plugin_that_is_not_media_is_ignored(tmp_path: Path): + """A plugin that is neither video nor audio adds nothing.""" + odf = OdfDocument("presentation") + frame = Frame.image_frame(_preview(odf, tmp_path), size=("8cm", "4.5cm")) + frame.append(_plugin("Objects/applet.swf", "application/x-shockwave-flash")) + doc = _convert(odf, frame, tmp_path / "plugin.odp") + + assert len(doc.pictures) == 1 + assert doc.pictures[0].meta is None diff --git a/tests/test_backend_pptx_media.py b/tests/test_backend_pptx_media.py new file mode 100644 index 0000000000..d546096f47 --- /dev/null +++ b/tests/test_backend_pptx_media.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Tests for video and audio shapes on PowerPoint slides.""" + +import zipfile +from pathlib import Path + +from docling_core.types.doc import DoclingDocument, PictureItem + +from .test_backend_pptx import get_converter + +_P14_MEDIA = "{http://schemas.microsoft.com/office/powerpoint/2010/main}media" + + +def _add_media(tmp_path: Path, file_name: str, mime_type: str): + """Add a media shape the way PowerPoint stores it, and return the deck and shape. + + python-pptx writes the same XML as PowerPoint: an ``a:videoFile`` link plus a + ``p14:media`` extension, both pointing at one part under ``ppt/media``. + """ + from pptx import Presentation + from pptx.util import Inches + + media_path = tmp_path / file_name + media_path.write_bytes(b"not decoded by the backend") + prs = Presentation() + slide = prs.slides.add_slide(prs.slide_layouts[6]) + title = slide.shapes.add_textbox(Inches(1), Inches(0.5), Inches(6), Inches(1)) + title.text_frame.text = "Slide with media" + shape = slide.shapes.add_movie( + str(media_path), + Inches(1), + Inches(2), + Inches(4), + Inches(2.25), + mime_type=mime_type, + ) + return prs, slide, shape + + +def _media_meta(picture: PictureItem) -> dict: + assert picture.meta is not None + return picture.meta.get_custom_part() + + +def _convert(prs, path: Path) -> DoclingDocument: + prs.save(path) + return get_converter().convert(path).document + + +def test_pptx_embedded_video_is_a_picture_with_its_media_path(tmp_path: Path): + """An embedded video becomes a picture of its poster frame on its slide. + + The picture records the path of the video inside the package, so the video + can be read from the deck, e.g. to transcribe it. + """ + prs, _, _ = _add_media(tmp_path, "clip.mp4", "video/mp4") + deck = tmp_path / "video.pptx" + doc = _convert(prs, deck) + + assert len(doc.pictures) == 1 + picture = doc.pictures[0] + assert picture.image is not None + assert picture.prov[0].page_no == 1 + assert _media_meta(picture) == {"docling__video": "ppt/media/media1.mp4"} + with zipfile.ZipFile(deck) as package: + assert package.read("ppt/media/media1.mp4") == b"not decoded by the backend" + assert doc.export_to_markdown() == ( + "Slide with media\n\n\n\nppt/media/media1.mp4" + ) + + +def test_pptx_audio_picture_records_its_media_path(tmp_path: Path): + """An audio shape keeps its speaker icon and now says which file it plays. + + PowerPoint stores audio like video, with ``a:audioFile`` in place of + ``a:videoFile``; python-pptx reads it as a plain picture. + """ + from pptx.oxml.ns import qn + + prs, _, shape = _add_media(tmp_path, "tone.mp3", "audio/mpeg") + shape._element.xpath("./p:nvPicPr/p:nvPr/a:videoFile")[0].tag = qn("a:audioFile") + doc = _convert(prs, tmp_path / "audio.pptx") + + assert len(doc.pictures) == 1 + assert doc.pictures[0].image is not None + assert _media_meta(doc.pictures[0]) == {"docling__audio": "ppt/media/media1.mp3"} + + +def test_pptx_linked_video_records_its_link(tmp_path: Path): + """A video that is linked, not embedded, records the link target.""" + from pptx.opc.constants import RELATIONSHIP_TYPE as RT + from pptx.oxml.ns import qn + + prs, slide, shape = _add_media(tmp_path, "clip.mp4", "video/mp4") + link = "https://example.com/talk.mp4" + video_rid = slide.part.relate_to(link, RT.VIDEO, is_external=True) + media_rid = slide.part.relate_to(link, RT.MEDIA, is_external=True) + shape._element.xpath("./p:nvPicPr/p:nvPr/a:videoFile")[0].set( + qn("r:link"), video_rid + ) + p14_media = next(shape._element.iter(_P14_MEDIA)) + del p14_media.attrib[qn("r:embed")] + p14_media.set(qn("r:link"), media_rid) + doc = _convert(prs, tmp_path / "linked.pptx") + + assert len(doc.pictures) == 1 + assert _media_meta(doc.pictures[0]) == {"docling__video": link} + + +def test_pptx_picture_without_media_has_no_media_meta(tmp_path: Path): + """A plain picture is unchanged: it carries no media field.""" + from io import BytesIO + + from PIL import Image + from pptx import Presentation + from pptx.util import Inches + + image = BytesIO() + Image.new("RGB", (40, 20), "red").save(image, format="PNG") + prs = Presentation() + slide = prs.slides.add_slide(prs.slide_layouts[6]) + slide.shapes.add_picture(image, Inches(1), Inches(1)) + doc = _convert(prs, tmp_path / "picture.pptx") + + assert len(doc.pictures) == 1 + assert doc.pictures[0].meta is None diff --git a/tests/test_simple_pipeline_media.py b/tests/test_simple_pipeline_media.py new file mode 100644 index 0000000000..9516dee7df --- /dev/null +++ b/tests/test_simple_pipeline_media.py @@ -0,0 +1,293 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Tests for the conversion of slide video and audio in the simple pipeline. + +A stub stands in for the audio and video pipelines, so these tests need no +speech model. ``test_simple_pipeline_media_asr.py`` runs the real models. +""" + +from pathlib import Path +from typing import Optional + +import pytest +from docling_core.types.doc import ( + ContentLayer, + DocItemLabel, + DoclingDocument, + GroupItem, + GroupLabel, + PictureItem, + TextItem, + TrackSource, +) + +from docling.datamodel.backend_options import MsPowerpointBackendOptions +from docling.datamodel.base_models import ( + ConversionStatus, + DoclingComponentType, + ErrorItem, + InputFormat, +) +from docling.datamodel.document import ConversionResult, InputDocument +from docling.datamodel.pipeline_options import ConvertPipelineOptions +from docling.document_converter import DocumentConverter, PowerpointFormatOption +from docling.pipeline.simple_pipeline import SimplePipeline + +_MEDIA_BYTES = b"bytes of the media file" +_TRANSCRIPT = "Spoken words" + + +class _StubMediaPipeline: + """Stands in for the audio and video pipelines: no model, a fixed transcript.""" + + def __init__(self, status: ConversionStatus = ConversionStatus.SUCCESS) -> None: + self.status = status + self.calls: list[tuple[str, str, bytes]] = [] + + def execute(self, in_doc: InputDocument, raises_on_error: bool) -> ConversionResult: + stream = in_doc._backend.path_or_stream + self.calls.append((in_doc.format.value, in_doc.file.name, stream.getvalue())) + result = ConversionResult(input=in_doc, status=self.status) + result.document = DoclingDocument(name=in_doc.file.stem) + result.document.add_text( + label=DocItemLabel.TEXT, + text=_TRANSCRIPT, + source=TrackSource(start_time=0.0, end_time=1.0), + ) + if self.status == ConversionStatus.FAILURE: + result.errors.append( + ErrorItem( + component_type=DoclingComponentType.PIPELINE, + module_name="StubMediaPipeline", + error_message="no audio track", + ) + ) + return result + + +@pytest.fixture +def stub(monkeypatch: pytest.MonkeyPatch) -> _StubMediaPipeline: + stub = _StubMediaPipeline() + monkeypatch.setattr(SimplePipeline, "_get_media_pipeline", lambda self, kind: stub) + return stub + + +def _deck(tmp_path: Path, file_name: str = "clip.mp4"): + """Return a deck with a title and a video, and the video shape.""" + from pptx import Presentation + from pptx.util import Inches + + media_path = tmp_path / file_name + media_path.write_bytes(_MEDIA_BYTES) + prs = Presentation() + slide = prs.slides.add_slide(prs.slide_layouts[6]) + title = slide.shapes.add_textbox(Inches(1), Inches(0.5), Inches(6), Inches(1)) + title.text_frame.text = "Slide with media" + video = slide.shapes.add_movie( + str(media_path), + Inches(1), + Inches(2), + Inches(4), + Inches(2.25), + mime_type="video/mp4", + ) + return prs, slide, video + + +def _convert( + path: Path, + do_media_conversion: bool = True, + backend_options: Optional[MsPowerpointBackendOptions] = None, +) -> ConversionResult: + converter = DocumentConverter( + allowed_formats=[InputFormat.PPTX], + format_options={ + InputFormat.PPTX: PowerpointFormatOption( + pipeline_options=ConvertPipelineOptions( + do_media_conversion=do_media_conversion + ), + backend_options=backend_options, + ) + }, + ) + return converter.convert(path, raises_on_error=False) + + +def _media_group(doc: DoclingDocument, picture: PictureItem) -> Optional[GroupItem]: + """Return the group right after the picture, if it holds converted media.""" + assert picture.parent is not None + siblings = picture.parent.resolve(doc).children + index = [ref.cref for ref in siblings].index(picture.self_ref) + if index + 1 == len(siblings): + return None + sibling = siblings[index + 1].resolve(doc) + if isinstance(sibling, GroupItem) and sibling.label == GroupLabel.SECTION: + return sibling + return None + + +def test_media_document_goes_after_its_picture(tmp_path: Path, stub): + """The converted media follows the picture that shows it, on the same slide. + + Its items get the provenance of the picture: the media plays there. + Without ``do_media_conversion`` nothing is converted. + """ + prs, _, _ = _deck(tmp_path) + deck = tmp_path / "deck.pptx" + prs.save(deck) + + plain = _convert(deck, do_media_conversion=False).document + assert stub.calls == [] + assert _media_group(plain, plain.pictures[0]) is None + + result = _convert(deck) + assert result.status == ConversionStatus.SUCCESS + assert stub.calls == [("video", "media1.mp4", _MEDIA_BYTES)] + doc = result.document + picture = doc.pictures[0] + group = _media_group(doc, picture) + assert group is not None + assert group.name == "media: ppt/media/media1.mp4" + transcript = group.children[0].resolve(doc) + assert isinstance(transcript, TextItem) + assert transcript.text == _TRANSCRIPT + assert transcript.source[0].start_time == 0.0 + assert transcript.prov[0].page_no == 1 + assert transcript.prov[0].bbox == picture.prov[0].bbox + assert transcript.prov[0].charspan == (0, len(_TRANSCRIPT)) + assert doc.export_to_markdown() == ( + "Slide with media\n\n\n\nppt/media/media1.mp4\n\nSpoken words" + ) + + +def test_media_of_a_hidden_shape_stays_hidden(tmp_path: Path, stub): + """The converted media takes the content layer of its picture.""" + prs, _, video = _deck(tmp_path) + video._element.xpath("./*/p:cNvPr")[0].set("hidden", "1") + deck = tmp_path / "hidden.pptx" + prs.save(deck) + + doc = _convert(deck).document + + group = _media_group(doc, doc.pictures[0]) + assert group is not None + assert group.content_layer == ContentLayer.INVISIBLE + assert group.children[0].resolve(doc).content_layer == ContentLayer.INVISIBLE + assert _TRANSCRIPT not in doc.export_to_markdown() + + +def test_linked_media_follows_the_fetch_options(tmp_path: Path, stub): + """A linked file is loaded only when the backend options allow it. + + A file that is not allowed is skipped, and the conversion still succeeds. + A local file must also be in the folder of the deck: PowerPoint links a + local file by its absolute path, and such a link is refused and reported. + """ + from pptx.opc.constants import RELATIONSHIP_TYPE as RT + from pptx.oxml.ns import qn + + def link_video(target: str, path: Path) -> None: + prs, slide, video = _deck(tmp_path) + r_id = slide.part.relate_to(target, RT.VIDEO, is_external=True) + video._element.xpath("./p:nvPicPr/p:nvPr/a:videoFile")[0].set( + qn("r:link"), r_id + ) + # Drop the p14:media extension, which points at the embedded copy. + ext_lst = video._element.xpath("./p:nvPicPr/p:nvPr/p:extLst")[0] + ext_lst.getparent().remove(ext_lst) + prs.save(path) + + (tmp_path / "talk.mp4").write_bytes(b"linked video next to the deck") + local_deck = tmp_path / "local.pptx" + link_video("talk.mp4", local_deck) + absolute_deck = tmp_path / "absolute.pptx" + link_video((tmp_path / "talk.mp4").as_uri(), absolute_deck) + remote_deck = tmp_path / "remote.pptx" + link_video("https://example.com/talk.mp4", remote_deck) + local_fetch = MsPowerpointBackendOptions(enable_local_fetch=True) + + for deck in (local_deck, absolute_deck, remote_deck): + assert _convert(deck).status == ConversionStatus.SUCCESS + assert stub.calls == [] + + allowed = _convert(local_deck, backend_options=local_fetch) + assert allowed.status == ConversionStatus.SUCCESS + assert stub.calls == [("video", "talk.mp4", b"linked video next to the deck")] + assert _media_group(allowed.document, allowed.document.pictures[0]) is not None + + absolute = _convert(absolute_deck, backend_options=local_fetch) + assert absolute.status == ConversionStatus.PARTIAL_SUCCESS + assert len(stub.calls) == 1 + + +@pytest.mark.parametrize("missing_extra", [False, True]) +def test_media_that_cannot_be_converted_is_reported( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, missing_extra: bool +): + """A failed media conversion keeps the picture and makes a partial success. + + A missing ASR extra fails only the media too, the same way a missing ffmpeg + fails the video pipeline. + """ + failing = _StubMediaPipeline(status=ConversionStatus.FAILURE) + + def get_media_pipeline(self, kind): + if missing_extra: + raise ImportError("whisper is not installed") + return failing + + monkeypatch.setattr(SimplePipeline, "_get_media_pipeline", get_media_pipeline) + prs, _, _ = _deck(tmp_path) + deck = tmp_path / "deck.pptx" + prs.save(deck) + + result = _convert(deck) + + reason = "whisper is not installed" if missing_extra else "no audio track" + assert result.status == ConversionStatus.PARTIAL_SUCCESS + assert [error.error_message for error in result.errors] == [ + f"Media file ppt/media/media1.mp4 cannot be converted: {reason}" + ] + assert len(result.document.pictures) == 1 + assert _media_group(result.document, result.document.pictures[0]) is None + + +def test_odp_audio_is_converted(tmp_path: Path, stub): + """The ODP backend loads its embedded media for the pipeline too.""" + pytest.importorskip("odfdo") + from odfdo import Document as OdfDocument, DrawPage, Element, Frame + + from docling.document_converter import OdpFormatOption + + audio = tmp_path / "note.mp3" + audio.write_bytes(_MEDIA_BYTES) + odf = OdfDocument("presentation") + media_path = odf.add_file(str(audio)) + frame = Frame(size=("2cm", "2cm"), position=("1cm", "1cm")) + frame.append( + Element.from_tag( + f'' + ) + ) + odf.body.clear() + page = DrawPage("page1", name="Slide One") + page.append(frame) + odf.body.append(page) + path = tmp_path / "audio.odp" + odf.save(str(path)) + + converter = DocumentConverter( + allowed_formats=[InputFormat.ODP], + format_options={ + InputFormat.ODP: OdpFormatOption( + pipeline_options=ConvertPipelineOptions(do_media_conversion=True) + ) + }, + ) + doc = converter.convert(path).document + + assert stub.calls == [("audio", Path(media_path).name, _MEDIA_BYTES)] + group = _media_group(doc, doc.pictures[0]) + assert group is not None + assert group.children[0].resolve(doc).text == _TRANSCRIPT diff --git a/tests/test_simple_pipeline_media_asr.py b/tests/test_simple_pipeline_media_asr.py new file mode 100644 index 0000000000..b8f1ec1237 --- /dev/null +++ b/tests/test_simple_pipeline_media_asr.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Slide video and audio converted with the real audio and video pipelines.""" + +import shutil +from pathlib import Path + +import pytest +from docling_core.types.doc import GroupItem, PictureItem, TextItem + +from docling.datamodel.base_models import ConversionStatus, InputFormat +from docling.datamodel.pipeline_options import ConvertPipelineOptions +from docling.document_converter import DocumentConverter, PowerpointFormatOption + +pytestmark = pytest.mark.ml_asr + +_SOURCES = Path("./tests/data/audio/sources") + + +@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") +def test_slide_video_and_audio_are_transcribed(tmp_path: Path): + """The transcript of each media file follows its picture on the slide. + + A video also gives its sampled frames. + """ + from pptx import Presentation + from pptx.oxml.ns import qn + from pptx.util import Inches + + prs = Presentation() + slide = prs.slides.add_slide(prs.slide_layouts[6]) + slide.shapes.add_movie( + str(_SOURCES / "sample_10s_video-mp4.mp4"), + Inches(1), + Inches(1), + Inches(4), + Inches(2.25), + mime_type="video/mp4", + ) + audio = slide.shapes.add_movie( + str(_SOURCES / "sample_10s.mp3"), + Inches(6), + Inches(1), + Inches(1), + Inches(1), + mime_type="audio/mpeg", + ) + audio._element.xpath("./p:nvPicPr/p:nvPr/a:videoFile")[0].tag = qn("a:audioFile") + deck = tmp_path / "media.pptx" + prs.save(deck) + + converter = DocumentConverter( + allowed_formats=[InputFormat.PPTX], + format_options={ + InputFormat.PPTX: PowerpointFormatOption( + pipeline_options=ConvertPipelineOptions(do_media_conversion=True) + ) + }, + ) + result = converter.convert(deck) + + assert result.status == ConversionStatus.SUCCESS + doc = result.document + media = [item for item in doc.groups if item.name.startswith("media: ")] + assert [group.name for group in media] == [ + "media: ppt/media/media1.mp4", + "media: ppt/media/media2.mp3", + ] + for group in media: + items = [ref.resolve(doc) for ref in group.children] + texts = [item for item in items if isinstance(item, TextItem)] + assert any(text.text.strip() for text in texts) + assert all(text.source and text.prov[0].page_no == 1 for text in texts) + video_items = [ref.resolve(doc) for ref in media[0].children] + assert any(isinstance(item, PictureItem) for item in video_items) + assert not any(isinstance(item, GroupItem) for item in video_items)