diff --git a/docling/backend/msexcel_backend.py b/docling/backend/msexcel_backend.py index 37e9be5eb7..41f24f8c3d 100644 --- a/docling/backend/msexcel_backend.py +++ b/docling/backend/msexcel_backend.py @@ -13,13 +13,14 @@ from io import BytesIO from pathlib import Path from tempfile import mkdtemp -from typing import Annotated, Any, Callable, Final, cast +from typing import Annotated, Any, Callable, Final, NamedTuple, cast from zipfile import ZipFile from docling_core.types.doc import ( BoundingBox, ContentLayer, CoordOrigin, + DescriptionMetaField, DocItem, DocItemLabel, DoclingDocument, @@ -27,12 +28,15 @@ GroupItem, GroupLabel, ImageRef, + NodeItem, PictureClassificationLabel, PictureClassificationMetaField, PictureClassificationPrediction, + PictureItem, PictureMeta, ProvenanceItem, RefItem, + RichTableCell, Size, TableCell, TableData, @@ -98,10 +102,15 @@ SpreadsheetDrawing, TwoCellAnchor, ) - from openpyxl.packaging.relationship import get_dependents, get_rels_path + from openpyxl.packaging.relationship import ( + Relationship, + RelationshipList, + get_dependents, + get_rels_path, + ) from openpyxl.reader.excel import ExcelReader from openpyxl.styles import PatternFill - from openpyxl.utils.cell import range_boundaries + from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries from openpyxl.worksheet.merge import MergedCellRange from openpyxl.worksheet.worksheet import Worksheet from openpyxl.xml.constants import IMAGE_NS @@ -153,6 +162,194 @@ def _has_unsafe_zip_paths(namelist: list[str]) -> bool: return any(m.startswith("/") or ".." in m for m in namelist) +# Defined here, not imported from openpyxl, because the module must import +# without openpyxl (see the import guard above). +_NS_MAIN: Final = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +_NS_REL: Final = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +_NS_RICH_DATA: Final = "http://schemas.microsoft.com/office/spreadsheetml/2017/richdata" +_NS_RICH_VALUE_REL: Final = ( + "http://schemas.microsoft.com/office/spreadsheetml/2022/richvaluerel" +) +_RICH_DATA_NAMESPACES: Final = { + "m": _NS_MAIN, + "rd": _NS_RICH_DATA, + "rvrel": _NS_RICH_VALUE_REL, +} +_R_ID: Final = f"{{{_NS_REL}}}id" + +# Workbook parts that hold the images placed in cells, by the last segment of +# their relationship type: the namespace part differs between Excel versions. +_RICH_DATA_PARTS: Final = frozenset( + {"sheetMetadata", "rdRichValue", "rdRichValueStructure", "richValueRel"} +) + + +class ExcelCellImage(NamedTuple): + """Represents an image placed in a cell ("Place in Cell" in Excel). + + Attributes: + target: The archive member of the image, e.g. ``xl/media/image1.png``. + alt_text: The alternative text of the image, if any. + """ + + target: str + alt_text: str | None + + +def _read_xml(zip_file: ZipFile, member: str, max_bytes: int) -> etree._Element: + """Read and parse an XML member of the archive with the safe parser.""" + return etree.fromstring( + read_member(zip_file, member, max_bytes), parser=_SAFE_XML_PARSER + ) + + +def _rich_data_parts( + workbook_rels: RelationshipList | dict[str, Relationship], +) -> dict[str, str] | None: + """Return the rich-data parts of a workbook, or None if any is missing. + + Args: + workbook_rels: openpyxl's relationships of the workbook part. At + runtime `WorkbookParser.rels` is a dict by id, although the + openpyxl stub names a `RelationshipList`. + """ + relationships = ( + workbook_rels.values() if isinstance(workbook_rels, dict) else workbook_rels + ) + parts = { + name: rel.target + for rel in relationships + if (name := rel.Type.rsplit("/", 1)[-1]) in _RICH_DATA_PARTS + } + return parts if parts.keys() == _RICH_DATA_PARTS else None + + +def _rich_value_images( + zip_file: ZipFile, parts: dict[str, str], max_bytes: int +) -> dict[int, ExcelCellImage]: + """Return the images placed in cells, by value metadata index. + + Excel stores an image placed in a cell as a rich value, not as a drawing. + The cell keeps the error value ``#VALUE!`` for older readers and refers to + the image through its ``vm`` attribute:: + + -> metadata valueMetadata -> futureMetadata XLRICHVALUE + -> rdrichvalue rv -> rdrichvaluestructure "_localImage" + -> richValueRel rel -> image member + + Only rich values with the ``_localImage`` structure are images. Other rich + values, such as linked data types, are not returned. + + Args: + zip_file: The open XLSX archive. + parts: The rich-data parts, from `_rich_data_parts`. + max_bytes: The largest decompressed size accepted for each XML part. + + Returns: + A dict from the 1-based ``vm`` index of a cell to its image. + """ + ns = _RICH_DATA_NAMESPACES + metadata = _read_xml(zip_file, parts["sheetMetadata"], max_bytes) + type_names = [ + node.get("name") + for node in metadata.iterfind("m:metadataTypes/m:metadataType", ns) + ] + if "XLRICHVALUE" not in type_names: + return {} + # Metadata types are 1-based in `rc/@t`. + rich_value_type = str(type_names.index("XLRICHVALUE") + 1) + future_blocks = [ + rvb.get("i") if (rvb := block.find(".//rd:rvb", ns)) is not None else None + for block in metadata.iterfind("m:futureMetadata[@name='XLRICHVALUE']/m:bk", ns) + ] + + structures = [ + (node.get("t"), [key.get("n") for key in node.iterfind("rd:k", ns)]) + for node in _read_xml( + zip_file, parts["rdRichValueStructure"], max_bytes + ).iterfind("rd:s", ns) + ] + rel_ids = [ + node.get(_R_ID) + for node in _read_xml(zip_file, parts["richValueRel"], max_bytes).iterfind( + "rvrel:rel", ns + ) + ] + image_targets: dict[str, str] = {} + rel_list_rels = get_rels_path(parts["richValueRel"]) + if rel_list_rels in zip_file.NameToInfo: + image_targets = { + rel.Id: rel.target + for rel in get_dependents(zip_file, rel_list_rels) + if rel.Type == IMAGE_NS and rel.Id is not None + } + + images: dict[int, ExcelCellImage] = {} + rich_values = _read_xml(zip_file, parts["rdRichValue"], max_bytes) + for index, node in enumerate(rich_values.iterfind("rd:rv", ns)): + structure = int(node.get("s", "-1")) + if not 0 <= structure < len(structures): + continue + structure_type, keys = structures[structure] + if structure_type != "_localImage": + continue + fields = dict(zip(keys, (v.text or "" for v in node.iterfind("rd:v", ns)))) + rel_index = int(fields.get("_rvRel:LocalImageIdentifier", "-1")) + if not 0 <= rel_index < len(rel_ids): + continue + target = image_targets.get(rel_ids[rel_index] or "") + if target is not None: + images[index] = ExcelCellImage(target, fields.get("Text") or None) + + by_vm: dict[int, ExcelCellImage] = {} + for vm, record in enumerate( + metadata.iterfind("m:valueMetadata/m:bk/m:rc", ns), start=1 + ): + future_index = int(record.get("v", "-1")) + if record.get("t") != rich_value_type or not ( + 0 <= future_index < len(future_blocks) + ): + continue + image = images.get(int(future_blocks[future_index] or "-1")) + if image is not None: + by_vm[vm] = image + return by_vm + + +def _cells_with_rich_values( + zip_file: ZipFile, sheet_part: str, max_bytes: int +) -> dict[tuple[int, int], int]: + """Return the ``vm`` index of each cell that has one, by 0-based (row, col). + + openpyxl does not keep the ``vm`` attribute, so the sheet XML is read again. + """ + cells: dict[tuple[int, int], int] = {} + source = BytesIO(read_member(zip_file, sheet_part, max_bytes)) + for _, cell in etree.iterparse( + source, + events=("end",), + tag=f"{{{_NS_MAIN}}}c", + resolve_entities=False, + load_dtd=False, + no_network=True, + ): + vm = cell.get("vm") + coordinate = cell.get("r") + if vm is not None and coordinate is not None: + row, col = coordinate_to_tuple(coordinate) + cells[(row - 1, col - 1)] = int(vm) + # Drop the parsed cells and rows, so that the tree does not grow with + # the sheet. + cell.clear() + while cell.getprevious() is not None: + del cell.getparent()[0] + row_element = cell.getparent() + if row_element is not None: + while row_element.getprevious() is not None: + del row_element.getparent()[0] + return cells + + def _order_comment_thread( entries: list[tuple[str | None, str | None, tuple[str, str, datetime | None]]], ) -> list[tuple[str, str, datetime | None]]: @@ -496,7 +693,7 @@ def create_bounded_sheet( ) -def _load_workbook(filename: str | BytesIO) -> Workbook: +def _load_workbook(filename: str | BytesIO) -> ExcelReader: """Load a workbook like ``load_workbook(filename, data_only=True)``. Merged cell and hyperlink ranges are clipped so that they cover a bounded @@ -510,7 +707,7 @@ def _load_workbook(filename: str | BytesIO) -> Workbook: reader.wb, read_worksheets ) reader.read() - return reader.wb + return reader class ExcelCell(BaseModel): @@ -522,6 +719,7 @@ class ExcelCell(BaseModel): text: The text content of the cell. row_span: The number of rows the cell spans. col_span: The number of columns the cell spans. + image: The image placed in the cell, if any. """ row: int @@ -529,6 +727,7 @@ class ExcelCell(BaseModel): text: str row_span: int col_span: int + image: ExcelCellImage | None = None class ExcelTable(BaseModel): @@ -619,6 +818,14 @@ def __init__( # Last used row of each sheet that a chart range reads, by sheet title self._chart_sheet_rows: dict[str, int] = {} + # Images placed in cells, by `vm` index, and the worksheet parts, by + # sheet name. Both stay empty for a workbook without such images. + self._rich_value_images: dict[int, ExcelCellImage] = {} + self._sheet_parts: dict[str, str] = {} + # Images placed in cells, by archive member, so that an image used in + # several cells is decoded once. + self._cell_image_refs: dict[str, ImageRef | None] = {} + self.workbook = None try: self._check_archive_limits() @@ -637,10 +844,14 @@ def __init__( category=UserWarning, module=r"openpyxl\.reader\.drawings", ) + reader: ExcelReader | None = None if isinstance(self.path_or_stream, BytesIO): - self.workbook = _load_workbook(self.path_or_stream) + reader = _load_workbook(self.path_or_stream) elif isinstance(self.path_or_stream, Path): - self.workbook = _load_workbook(str(self.path_or_stream)) + reader = _load_workbook(str(self.path_or_stream)) + if reader is not None: + self.workbook = reader.wb + self._read_rich_value_images(reader) self.valid = self.workbook is not None except ArchiveLimitError: @@ -969,6 +1180,20 @@ def _top_row(ref: RefItem) -> float: sheet_group.children.sort(key=_top_row) + @staticmethod + def _cell_prov( + page_no: int, col: int, row: int, col_span: int, row_span: int + ) -> ProvenanceItem: + """Return the provenance of a cell range, in 0-based cell units.""" + return ProvenanceItem( + page_no=page_no, + charspan=(0, 0), + bbox=BoundingBox.from_tuple( + (col, row, col + col_span, row + row_span), + origin=CoordOrigin.TOPLEFT, + ), + ) + def _find_tables_in_sheet( self, doc: DoclingDocument, sheet: Worksheet, page_no: int ) -> DoclingDocument: @@ -1005,48 +1230,46 @@ def _find_tables_in_sheet( text=title_cell.text, label=DocItemLabel.TEXT, parent=self.parent, - prov=ProvenanceItem( - page_no=page_no, - charspan=(0, 0), - bbox=BoundingBox.from_tuple( - ( - origin_col + title_cell.col, - origin_row - 1, - origin_col + title_cell.col + title_cell.col_span, - origin_row, - ), - origin=CoordOrigin.TOPLEFT, - ), + prov=self._cell_prov( + page_no, + origin_col + title_cell.col, + origin_row - 1, + title_cell.col_span, + 1, ), content_layer=content_layer, ) + table_prov = self._cell_prov( + page_no, origin_col, origin_row, num_cols, num_rows + ) if treat_singleton_as_text and len(excel_table.data) == 1: - doc.add_text( - text=excel_table.data[0].text, - label=DocItemLabel.TEXT, - parent=self.parent, - prov=ProvenanceItem( - page_no=page_no, - charspan=(0, 0), - bbox=BoundingBox.from_tuple( - ( - origin_col, - origin_row, - origin_col + num_cols, - origin_row + num_rows, - ), - origin=CoordOrigin.TOPLEFT, - ), + single_cell = excel_table.data[0] + if single_cell.image is not None: + self._add_cell_image( + doc, + single_cell.image, + self.parent, + table_prov, + content_layer, + ) + else: + doc.add_text( + text=single_cell.text, + label=DocItemLabel.TEXT, + parent=self.parent, + prov=table_prov, + content_layer=content_layer, + ) + else: + table_item = doc.add_table( + data=TableData( + num_rows=num_rows, num_cols=num_cols, table_cells=[] ), + parent=self.parent, + prov=table_prov, content_layer=content_layer, ) - else: - table_data = TableData( - num_rows=num_rows, - num_cols=num_cols, - table_cells=[], - ) for excel_cell in excel_table.data: cell = TableCell( @@ -1060,26 +1283,24 @@ def _find_tables_in_sheet( column_header=excel_cell.row == 0, row_header=False, ) - table_data.table_cells.append(cell) - - doc.add_table( - data=table_data, - parent=self.parent, - prov=ProvenanceItem( - page_no=page_no, - charspan=(0, 0), - bbox=BoundingBox.from_tuple( - ( - origin_col, - origin_row, - origin_col + num_cols, - origin_row + num_rows, + if excel_cell.image is not None: + picture = self._add_cell_image( + doc, + excel_cell.image, + table_item, + self._cell_prov( + page_no, + origin_col + excel_cell.col, + origin_row + excel_cell.row, + excel_cell.col_span, + excel_cell.row_span, ), - origin=CoordOrigin.TOPLEFT, - ), - ), - content_layer=content_layer, - ) + content_layer, + ) + cell = RichTableCell( + **cell.model_dump(), ref=picture.get_ref() + ) + doc.add_table_cell(table_item=table_item, cell=cell) # Extract comments and link them to cells for (row, col), thread in comment_map.items(): @@ -1128,7 +1349,11 @@ def _split_leading_section_label( return None, table first_row_cells = [cell for cell in table.data if cell.row == 0] - first_row_text_cells = [cell for cell in first_row_cells if cell.text.strip()] + first_row_text_cells = [ + cell + for cell in first_row_cells + if cell.text.strip() or cell.image is not None + ] if len(first_row_text_cells) != 1: return None, table @@ -1150,13 +1375,7 @@ def _split_leading_section_label( return None, table data = [ - ExcelCell( - row=cell.row - 1, - col=cell.col, - text=cell.text, - row_span=cell.row_span, - col_span=cell.col_span, - ) + cell.model_copy(update={"row": cell.row - 1}) for cell in table.data if cell.row > 0 ] @@ -1250,6 +1469,7 @@ def _find_data_tables( # Parse threaded comments from XML (Excel 365+ format with proper author names and timestamps) threaded_comments = self._parse_threaded_comments(sheet.title) + cell_images = self._find_cell_images(sheet) # Limit scan to actual data bounds for ri, row in enumerate( @@ -1299,6 +1519,7 @@ def _find_data_tables( max_row=bounds.max_row, max_col=bounds.max_col, merged_cell_index=merged_cell_index, + cell_images=cell_images, ) visited.update(visited_cells) # Mark these cells as visited tables.append(table_bounds) @@ -1314,6 +1535,7 @@ def _find_table_bounds( max_row: int, max_col: int, merged_cell_index: _MergedCellIndex, + cell_images: dict[tuple[int, int], ExcelCellImage], ) -> tuple[ExcelTable, set[tuple[int, int]]]: """Determine table bounds using a Flood Fill (BFS) strategy. @@ -1333,6 +1555,7 @@ def _find_table_bounds( start_col: The starting column index (0-based) for the flood fill. max_row: The exclusive row bound to consider in the worksheet. max_col: The exclusive column bound to consider in the worksheet. + cell_images: The images placed in cells, by 0-based (row, col). Returns: A tuple containing: @@ -1423,7 +1646,12 @@ def has_content(r: int, c: int) -> bool: if merged_cell_index.is_shadow(cell): continue - cell_text = str(cell.value) if cell.value is not None else "" + # An image placed in a cell has the error value "#VALUE!" only + # as a fallback for readers without rich-data support. + image = cell_images.get((ri, rj)) if cell.data_type == "e" else None + cell_text = ( + str(cell.value) if cell.value is not None and image is None else "" + ) # Compute Spans row_span, col_span = merged_cell_index.span_at(ri, rj) @@ -1435,6 +1663,7 @@ def has_content(r: int, c: int) -> bool: text=cell_text, row_span=row_span, col_span=col_span, + image=image, ) ) @@ -1677,6 +1906,134 @@ def _process_drawing_for_unsupported_images( return doc + def _open_archive(self) -> ZipFile: + """Open the XLSX archive again, from the start of a stream input.""" + if self.path_or_stream is None: + raise RuntimeError("The backend was unloaded; the archive is closed.") + if isinstance(self.path_or_stream, BytesIO): + self.path_or_stream.seek(0) + return ZipFile(self.path_or_stream, "r") + + def _read_rich_value_images(self, reader: ExcelReader) -> None: + """Read the index of the images placed in cells ("Place in Cell"). + + Excel 365 stores such an image as a rich value in ``xl/richData``, + which openpyxl does not read: the cell value is only the ``#VALUE!`` + fallback. The archive is opened only when the workbook has rich data. + + Args: + reader: The openpyxl reader that loaded the workbook. + """ + parts = _rich_data_parts(reader.parser.rels) + if parts is None: + return + try: + with self._open_archive() as zip_file: + if _has_unsafe_zip_paths(zip_file.namelist()): + _log.warning( + "Skipping images in cells: XLSX archive contains unsafe " + "ZIP paths" + ) + return + self._rich_value_images = _rich_value_images( + zip_file, parts, self._archive_limits.max_file_bytes + ) + except Exception as exc: + _log.warning("Could not read the images in cells: %s", exc) + return + if self._rich_value_images: + self._sheet_parts = { + sheet.name: rel.target for sheet, rel in reader.parser.find_sheets() + } + + def _find_cell_images( + self, sheet: Worksheet + ) -> dict[tuple[int, int], ExcelCellImage]: + """Find the images placed in the cells of a worksheet. + + The images are decoded here, once for each archive member, so that the + archive is opened once for each sheet. + + Args: + sheet: The Excel worksheet to be parsed. + + Returns: + A dict from the 0-based (row, col) of a cell to its image. + """ + sheet_part = self._sheet_parts.get(sheet.title) + # An image cell holds an error value, which openpyxl has already read. + if ( + not self._rich_value_images + or sheet_part is None + or not any(cell.data_type == "e" for cell in sheet._cells.values()) + ): + return {} + cell_images: dict[tuple[int, int], ExcelCellImage] = {} + try: + with self._open_archive() as zip_file: + cells = _cells_with_rich_values( + zip_file, sheet_part, self._archive_limits.max_file_bytes + ) + for coordinate, vm in cells.items(): + cell_image = self._rich_value_images.get(vm) + if cell_image is None: + continue + if cell_image.target not in self._cell_image_refs: + self._cell_image_refs[cell_image.target] = ( + self._read_cell_image(zip_file, cell_image.target) + ) + cell_images[coordinate] = cell_image + except Exception as exc: + _log.warning( + "Could not read the images in the cells of %s: %s", sheet.title, exc + ) + return {} + return cell_images + + def _read_cell_image(self, zip_file: ZipFile, target: str) -> ImageRef | None: + """Read and decode an image placed in a cell.""" + try: + image_bytes = read_member(zip_file, target, self._MAX_IMAGE_BYTES) + except (ArchiveLimitError, KeyError) as exc: + _log.warning("Skipping image %s in a cell: %s", target, exc) + return None + try: + pil_image: PILImage.Image | None = normalize_image_for_png( + PILImage.open(BytesIO(image_bytes)) + ) + except (UnidentifiedImageError, OSError, ValueError): + pil_image = self._convert_emf_to_pil(image_bytes) + if pil_image is None: + _log.warning("Could not decode the image %s in a cell", target) + return None + return ImageRef.from_pil(image=pil_image, dpi=72) + + def _add_cell_image( + self, + doc: DoclingDocument, + cell_image: ExcelCellImage, + parent: NodeItem | None, + prov: ProvenanceItem, + content_layer: ContentLayer | None, + ) -> PictureItem: + """Add an image placed in a cell as a picture. + + Excel's alternative text of the image becomes the picture description. + """ + image_ref = self._cell_image_refs.get(cell_image.target) + picture = doc.add_picture( + parent=parent, + image=image_ref.model_copy() if image_ref is not None else None, + caption=None, + prov=prov, + content_layer=content_layer, + ) + if cell_image.alt_text: + picture.meta = PictureMeta( + description=DescriptionMetaField(text=cell_image.alt_text) + ) + return picture + def _find_images_in_sheet( self, doc: DoclingDocument, sheet: Worksheet, page_no: int ) -> DoclingDocument: diff --git a/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.itxt b/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.itxt new file mode 100644 index 0000000000..fb752fc7b0 --- /dev/null +++ b/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.itxt @@ -0,0 +1,8 @@ +item-0 at level 0: unspecified: group _root_ + item-1 at level 1: sheet: group Products + item-2 at level 2: table with [3x4] + item-3 at level 3: picture + item-4 at level 3: picture + item-5 at level 1: sheet: group Logo + item-6 at level 2: table with [1x1] + item-7 at level 3: picture \ No newline at end of file diff --git a/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.json b/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.json new file mode 100644 index 0000000000..a25440add7 --- /dev/null +++ b/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.json @@ -0,0 +1,650 @@ +{ + "schema_name": "DoclingDocument", + "version": "1.10.0", + "name": "xlsx_10_place_in_cell", + "origin": { + "mimetype": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "binary_hash": 12306546942560803529, + "filename": "xlsx_10_place_in_cell.xlsx" + }, + "furniture": { + "self_ref": "#/furniture", + "children": [], + "content_layer": "furniture", + "name": "_root_", + "label": "unspecified" + }, + "body": { + "self_ref": "#/body", + "children": [ + { + "$ref": "#/groups/0" + }, + { + "$ref": "#/groups/1" + } + ], + "content_layer": "body", + "name": "_root_", + "label": "unspecified" + }, + "groups": [ + { + "self_ref": "#/groups/0", + "parent": { + "$ref": "#/body" + }, + "children": [ + { + "$ref": "#/tables/0" + } + ], + "content_layer": "body", + "name": "Products", + "label": "sheet" + }, + { + "self_ref": "#/groups/1", + "parent": { + "$ref": "#/body" + }, + "children": [ + { + "$ref": "#/tables/1" + } + ], + "content_layer": "body", + "name": "Logo", + "label": "sheet" + } + ], + "texts": [], + "pictures": [ + { + "self_ref": "#/pictures/0", + "parent": { + "$ref": "#/tables/0" + }, + "children": [], + "content_layer": "body", + "meta": { + "description": { + "text": "A red square" + } + }, + "label": "picture", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 1.0, + "t": 1.0, + "r": 2.0, + "b": 2.0, + "coord_origin": "TOPLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ], + "captions": [], + "references": [], + "footnotes": [], + "image": { + "mimetype": "image/png", + "dpi": 72, + "size": { + "width": 16.0, + "height": 16.0 + }, + "uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAF0lEQVQokWP8z0AaYCJR/aiGUQ1DSAMAQC4BH5CRCM8AAAAASUVORK5CYII=" + }, + "annotations": [] + }, + { + "self_ref": "#/pictures/1", + "parent": { + "$ref": "#/tables/0" + }, + "children": [], + "content_layer": "body", + "meta": { + "description": { + "text": "A green square" + } + }, + "label": "picture", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 1.0, + "t": 2.0, + "r": 2.0, + "b": 3.0, + "coord_origin": "TOPLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ], + "captions": [], + "references": [], + "footnotes": [], + "image": { + "mimetype": "image/png", + "dpi": 72, + "size": { + "width": 16.0, + "height": 16.0 + }, + "uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAGElEQVQokWNkaGAgCTCRpnxUw6iGoaQBALsfAKAiyANaAAAAAElFTkSuQmCC" + }, + "annotations": [] + }, + { + "self_ref": "#/pictures/2", + "parent": { + "$ref": "#/tables/1" + }, + "children": [], + "content_layer": "body", + "label": "picture", + "prov": [ + { + "page_no": 2, + "bbox": { + "l": 0.0, + "t": 0.0, + "r": 1.0, + "b": 1.0, + "coord_origin": "TOPLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ], + "captions": [], + "references": [], + "footnotes": [], + "image": { + "mimetype": "image/png", + "dpi": 72, + "size": { + "width": 16.0, + "height": 16.0 + }, + "uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAGElEQVQokWNkaGAgCTCRpnxUw6iGoaQBALsfAKAiyANaAAAAAElFTkSuQmCC" + }, + "annotations": [] + } + ], + "tables": [ + { + "self_ref": "#/tables/0", + "parent": { + "$ref": "#/groups/0" + }, + "children": [ + { + "$ref": "#/pictures/0" + }, + { + "$ref": "#/pictures/1" + } + ], + "content_layer": "body", + "label": "table", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 0.0, + "t": 0.0, + "r": 4.0, + "b": 3.0, + "coord_origin": "TOPLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ], + "captions": [], + "references": [], + "footnotes": [], + "data": { + "table_cells": [ + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 0, + "end_col_offset_idx": 1, + "text": "name", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 1, + "end_col_offset_idx": 2, + "text": "photo", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 2, + "end_col_offset_idx": 3, + "text": "price", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 3, + "end_col_offset_idx": 4, + "text": "stock", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 1, + "end_row_offset_idx": 2, + "start_col_offset_idx": 0, + "end_col_offset_idx": 1, + "text": "apple", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 1, + "end_row_offset_idx": 2, + "start_col_offset_idx": 1, + "end_col_offset_idx": 2, + "text": "", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false, + "ref": { + "$ref": "#/pictures/0" + } + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 1, + "end_row_offset_idx": 2, + "start_col_offset_idx": 2, + "end_col_offset_idx": 3, + "text": "100", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 1, + "end_row_offset_idx": 2, + "start_col_offset_idx": 3, + "end_col_offset_idx": 4, + "text": "#N/A", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 2, + "end_row_offset_idx": 3, + "start_col_offset_idx": 0, + "end_col_offset_idx": 1, + "text": "pear", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 2, + "end_row_offset_idx": 3, + "start_col_offset_idx": 1, + "end_col_offset_idx": 2, + "text": "", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false, + "ref": { + "$ref": "#/pictures/1" + } + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 2, + "end_row_offset_idx": 3, + "start_col_offset_idx": 2, + "end_col_offset_idx": 3, + "text": "200", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 2, + "end_row_offset_idx": 3, + "start_col_offset_idx": 3, + "end_col_offset_idx": 4, + "text": "7", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + } + ], + "num_rows": 3, + "num_cols": 4, + "orientation": "rot_0", + "grid": [ + [ + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 0, + "end_col_offset_idx": 1, + "text": "name", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 1, + "end_col_offset_idx": 2, + "text": "photo", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 2, + "end_col_offset_idx": 3, + "text": "price", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 3, + "end_col_offset_idx": 4, + "text": "stock", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + } + ], + [ + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 1, + "end_row_offset_idx": 2, + "start_col_offset_idx": 0, + "end_col_offset_idx": 1, + "text": "apple", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 1, + "end_row_offset_idx": 2, + "start_col_offset_idx": 1, + "end_col_offset_idx": 2, + "text": "", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 1, + "end_row_offset_idx": 2, + "start_col_offset_idx": 2, + "end_col_offset_idx": 3, + "text": "100", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 1, + "end_row_offset_idx": 2, + "start_col_offset_idx": 3, + "end_col_offset_idx": 4, + "text": "#N/A", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + } + ], + [ + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 2, + "end_row_offset_idx": 3, + "start_col_offset_idx": 0, + "end_col_offset_idx": 1, + "text": "pear", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 2, + "end_row_offset_idx": 3, + "start_col_offset_idx": 1, + "end_col_offset_idx": 2, + "text": "", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 2, + "end_row_offset_idx": 3, + "start_col_offset_idx": 2, + "end_col_offset_idx": 3, + "text": "200", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + }, + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 2, + "end_row_offset_idx": 3, + "start_col_offset_idx": 3, + "end_col_offset_idx": 4, + "text": "7", + "column_header": false, + "row_header": false, + "row_section": false, + "fillable": false + } + ] + ] + }, + "annotations": [] + }, + { + "self_ref": "#/tables/1", + "parent": { + "$ref": "#/groups/1" + }, + "children": [ + { + "$ref": "#/pictures/2" + } + ], + "content_layer": "body", + "label": "table", + "prov": [ + { + "page_no": 2, + "bbox": { + "l": 0.0, + "t": 0.0, + "r": 1.0, + "b": 1.0, + "coord_origin": "TOPLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ], + "captions": [], + "references": [], + "footnotes": [], + "data": { + "table_cells": [ + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 0, + "end_col_offset_idx": 1, + "text": "", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false, + "ref": { + "$ref": "#/pictures/2" + } + } + ], + "num_rows": 1, + "num_cols": 1, + "orientation": "rot_0", + "grid": [ + [ + { + "row_span": 1, + "col_span": 1, + "start_row_offset_idx": 0, + "end_row_offset_idx": 1, + "start_col_offset_idx": 0, + "end_col_offset_idx": 1, + "text": "", + "column_header": true, + "row_header": false, + "row_section": false, + "fillable": false + } + ] + ] + }, + "annotations": [] + } + ], + "key_value_items": [], + "form_items": [], + "pages": { + "1": { + "size": { + "width": 4.0, + "height": 3.0 + }, + "page_no": 1 + }, + "2": { + "size": { + "width": 1.0, + "height": 1.0 + }, + "page_no": 2 + } + } +} \ No newline at end of file diff --git a/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.md b/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.md new file mode 100644 index 0000000000..4583a38028 --- /dev/null +++ b/tests/data/xlsx/groundtruth/xlsx_10_place_in_cell.xlsx.md @@ -0,0 +1,11 @@ +## Products + +| name | photo | price | stock | +| - | - | - | - | +| apple | A red square | 100 | #N/A | +| pear | A green square | 200 | 7 | + +## Logo + +| | +| - | \ No newline at end of file diff --git a/tests/data/xlsx/sources/xlsx_10_place_in_cell.xlsx b/tests/data/xlsx/sources/xlsx_10_place_in_cell.xlsx new file mode 100644 index 0000000000..216335d2d6 Binary files /dev/null and b/tests/data/xlsx/sources/xlsx_10_place_in_cell.xlsx differ diff --git a/tests/test_backend_msexcel.py b/tests/test_backend_msexcel.py index f35085da07..8158007546 100644 --- a/tests/test_backend_msexcel.py +++ b/tests/test_backend_msexcel.py @@ -19,6 +19,7 @@ GroupLabel, PictureClassificationLabel, PictureItem, + RichTableCell, TableItem, TextItem, ) @@ -26,6 +27,7 @@ from openpyxl import Workbook, load_workbook from openpyxl.comments import Comment from openpyxl.worksheet.merge import MergedCellRange +from PIL import Image from docling.backend.docx.drawingml.utils import get_libreoffice_cmd from docling.backend.msexcel_backend import ( @@ -1145,6 +1147,87 @@ def test_one_cell_anchor_image(): assert prov.bbox.b == 2.0, f"Image bottom should be 2.0, got {prov.bbox.b}" +def test_images_placed_in_cells(documents) -> None: + """Test that images placed in cells become pictures in rich table cells. + + Excel stores an image placed in a cell ("Place in Cell") as a rich value in + xl/richData. The cell itself only holds the fallback error value #VALUE!. + """ + doc = next(doc for path, doc in documents if path.stem == "xlsx_10_place_in_cell") + + table = doc.tables[0] + cells = { + (cell.start_row_offset_idx, cell.start_col_offset_idx): cell + for cell in table.data.table_cells + } + for row, alt_text in ((1, "A red square"), (2, "A green square")): + cell = cells[(row, 1)] + assert isinstance(cell, RichTableCell) + assert cell.text == "" + picture = cell.ref.resolve(doc) + assert isinstance(picture, PictureItem) + assert picture.parent is not None + assert picture.parent.cref == table.self_ref + assert picture.image is not None + assert picture.meta is not None + assert picture.meta.description is not None + assert picture.meta.description.text == alt_text + + # An error value without an image is cell content and keeps its text. + assert not isinstance(cells[(1, 3)], RichTableCell) + assert cells[(1, 3)].text == "#N/A" + + +def test_image_placed_in_a_singleton_cell() -> None: + """Test that a lone image cell becomes a picture with treat_singleton_as_text.""" + path = next( + item for item in get_excel_paths() if item.stem == "xlsx_10_place_in_cell" + ) + options = MsExcelBackendOptions(treat_singleton_as_text=True) + format_options = {InputFormat.XLSX: ExcelFormatOption(backend_options=options)} + converter = DocumentConverter( + allowed_formats=[InputFormat.XLSX], format_options=format_options + ) + doc = converter.convert(path).document + + # The "Logo" sheet (page 2) holds only one image cell. + assert not [table for table in doc.tables if table.prov[0].page_no == 2] + assert not [text for text in doc.texts if text.prov[0].page_no == 2] + pictures = [picture for picture in doc.pictures if picture.prov[0].page_no == 2] + assert len(pictures) == 1 + assert pictures[0].image is not None + + +def test_image_cell_next_to_a_section_label(tmp_path: Path) -> None: + """Test that an image cell keeps a merged label in its table.""" + xlsxwriter = pytest.importorskip("xlsxwriter") + + image_path = tmp_path / "icon.png" + Image.new("RGB", (16, 16), "red").save(image_path) + file_path = tmp_path / "section_image.xlsx" + workbook = xlsxwriter.Workbook(str(file_path)) + sheet = workbook.add_worksheet() + sheet.merge_range("A1:B1", "Fruit list") + sheet.embed_image("C1", str(image_path), {"description": "Section icon"}) + sheet.write_row("A2", ["name", "qty", "note"]) + sheet.write_row("A3", ["apple", 1, "x"]) + sheet.write_row("A4", ["pear", 2, "y"]) + workbook.close() + + doc = DocumentConverter().convert(file_path).document + + # The image is content in the first row, so the label is not split off. + assert not doc.texts + [table] = doc.tables + assert (table.data.num_rows, table.data.num_cols) == (4, 3) + [picture] = doc.pictures + cell = table.data.grid[0][2] + assert isinstance(cell, RichTableCell) + assert cell.ref.cref == picture.self_ref + assert picture.parent is not None + assert picture.parent.cref == table.self_ref + + def test_find_data_tables_handles_a_filled_last_excel_row(tmp_path): workbook = Workbook() sheet = workbook.active