diff --git a/.github/max-lines-ignore b/.github/max-lines-ignore index aeb750a24b..d01d4f7c19 100644 --- a/.github/max-lines-ignore +++ b/.github/max-lines-ignore @@ -21,6 +21,7 @@ docling/backend/opendocument_backend.py docling/datamodel/pipeline_options.py docling/datamodel/service/options.py docling/datamodel/stage_model_specs.py +docling/models/stages/reading_order/readingorder_model.py docling/pipeline/standard_pdf_pipeline.py docling/service_client/_async_client.py docling/service_client/client.py @@ -29,4 +30,5 @@ tests/test_service_client_sdk_unit.py tests/test_backend_opendocument.py tests/test_backend_html.py tests/test_backend_msword.py -tests/test_cli.py \ No newline at end of file +tests/test_cli.py +tests/test_form_extraction.py \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7ed8aa5de7..c8f3befd70 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ model_artifacts/ scratch/ ds_convert_models/ +output/ # Created by https://www.toptal.com/developers/gitignore/api/python,macos,virtualenv,pycharm,visualstudiocode,emacs,vim,jupyternotebooks # Edit at https://www.toptal.com/developers/gitignore?templates=python,macos,virtualenv,pycharm,visualstudiocode,emacs,vim,jupyternotebooks diff --git a/docling/.agents/skills/docling/references/cli.md b/docling/.agents/skills/docling/references/cli.md index c432f5cd3b..8b7dc6c657 100644 --- a/docling/.agents/skills/docling/references/cli.md +++ b/docling/.agents/skills/docling/references/cli.md @@ -111,6 +111,7 @@ docling report.pdf --no-tables --output /tmp/ # skip table structure docling report.pdf --table-mode accurate --output /tmp/ # vs. fast docling report.pdf --layout-engine docling_layout_default --output /tmp/ # choose layout engine docling report.pdf --table-structure-engine docling_tableformer_v2 --output /tmp/ # choose table engine +docling form.pdf --extract-form-fields --output /tmp/ # AcroForm widgets as fields with keys from printed text (not with --to doctags) docling report.pdf --enrich-code --output /tmp/ # code understanding docling report.pdf --enrich-formula --output /tmp/ # formula understanding docling report.pdf --enrich-picture-classes --output /tmp/ diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md index 6b764830bb..be17238dad 100644 --- a/docling/.agents/skills/docling/references/python-sdk.md +++ b/docling/.agents/skills/docling/references/python-sdk.md @@ -61,6 +61,7 @@ Useful `PdfPipelineOptions` / base fields: |---|---| | `do_ocr` | Run OCR (default engine EasyOCR) | | `do_table_structure` | Detect table structure | +| `extract_form_fields` | Convert docling-parse PDF widgets (AcroForm) into fillable fields with keys from printed text | | `do_code_enrichment` / `do_formula_enrichment` | Enrich code / formulas | | `code_formula_options` | Code/formula model, crop margin (`expansion_factor`), and stopping repeated output (`stop_on_repetition`, on by default) | | `ocr_options` | Choose/parametrize the OCR engine (see below) | @@ -72,6 +73,20 @@ Useful `PdfPipelineOptions` / base fields: | `artifacts_path` | Use pre-downloaded model artifacts (offline) | | `enable_remote_services` | Gate all outbound HTTP (required for any remote model) | +`extract_form_fields=True` associates each widget with a field key from printed +text, using page geometry. The key can be text in or beside its cell, or a +paragraph that contains the widget. Checkbox and radio option text becomes an +option key; a shared prompt stays as ordinary text. Additional context becomes +a field hint. A section header or formula used as a key stays inside the key. +Fields keep their widget order and sit among the surrounding text by page +geometry. A widget in a detected table goes into its table cell, with a key +only from text in that same cell. PDF field +names are not used as keys, and PDF-specific widget metadata is not stored in +the `DoclingDocument`. Use `generate_parsed_pages=True` to retain raw widgets +on parsed pages when needed. Backends without page-local widgets produce no +fields; table cells need `do_table_structure=True`. Not available in the +service client, and not supported by the DocTags export. + ### Choosing an OCR engine ```python diff --git a/docling/backend/docling_parse_backend.py b/docling/backend/docling_parse_backend.py index 555a4663c8..f071e84098 100644 --- a/docling/backend/docling_parse_backend.py +++ b/docling/backend/docling_parse_backend.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional, Union -from docling_core.types.doc import BoundingBox, Size +from docling_core.types.doc import BoundingBox, CoordOrigin, Size from docling_core.types.doc.page import ( PdfCellRenderingMode, PdfTextCell, @@ -243,6 +243,31 @@ def get_connected_shape_bounding_boxes( ) ] + def get_thin_shape_boxes( + self, *, max_thickness: float + ) -> Optional[list[BoundingBox]]: + if not self.is_valid(): + return [] + + page_height = self.get_size().height + boxes = [] + # docling-parse exposes the individual painted paths only through its + # page decoder; get_connected_shape_bounding_boxes merges them first. + for shape in self._result._require_page_decoder().get_page_shapes(): + xs, ys = list(shape.get_x()), list(shape.get_y()) + if not xs or not ys: + continue + box = BoundingBox( + l=min(xs), + b=min(ys), + r=max(xs), + t=max(ys), + coord_origin=CoordOrigin.BOTTOMLEFT, + ) + if min(box.width, box.height) <= max_thickness: + boxes.append(box.to_top_left_origin(page_height)) + return boxes + def get_page_image( self, scale: float = 1, cropbox: Optional[BoundingBox] = None ) -> Image.Image: diff --git a/docling/backend/pdf_backend.py b/docling/backend/pdf_backend.py index 4a93f3e84e..9635ccb546 100644 --- a/docling/backend/pdf_backend.py +++ b/docling/backend/pdf_backend.py @@ -88,6 +88,17 @@ def get_connected_shape_bounding_boxes( """ return None + def get_thin_shape_boxes( + self, *, max_thickness: float + ) -> Optional[list[BoundingBox]]: + """Return the boxes of painted shapes no thicker than `max_thickness`. + + Rules drawn as thin filled rectangles, which `get_shape_lines` (strokes + only) misses. Boxes use top-left origin. `None` means this backend cannot + answer the query at all. + """ + return None + @abstractmethod def get_page_image( self, scale: float = 1, cropbox: Optional[BoundingBox] = None diff --git a/docling/cli/main.py b/docling/cli/main.py index ec6d123ccb..8fc1d88e38 100644 --- a/docling/cli/main.py +++ b/docling/cli/main.py @@ -969,6 +969,17 @@ def convert( # noqa: C901 help="If enabled, the table structure model will be used to extract table information.", ), ] = True, + extract_form_fields: Annotated[ + bool, + typer.Option( + ..., + help=( + "If enabled, native PDF form widgets (AcroForm) become fillable field " + "values with field keys from printed text. Standard and legacy pipelines " + "with the docling-parse backend only." + ), + ), + ] = False, reading_order_separators: Annotated[ bool, typer.Option( @@ -1409,6 +1420,18 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] ) raise typer.Abort() + if to_formats is None: + to_formats = [OutputFormat.MARKDOWN] + if ( + extract_form_fields + and pipeline in {ProcessingPipeline.STANDARD, ProcessingPipeline.LEGACY} + and OutputFormat.DOCTAGS in to_formats + ): + # DocTags has no tokens for form fields; the export raises on them. + raise typer.BadParameter( + "--extract-form-fields cannot be combined with `--to doctags`." + ) + if profiling or save_profiling: settings.debug.profile_pipeline_timings = True @@ -1474,9 +1497,6 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] _log.info(err) # will print more details if verbose is activated raise typer.Abort() - if to_formats is None: - to_formats = [OutputFormat.MARKDOWN] - if output_file is not None: if len(input_doc_paths) != 1: err_console.print( @@ -1565,6 +1585,7 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] do_ocr=ocr, ocr_options=ocr_options, do_table_structure=tables, + extract_form_fields=extract_form_fields, use_reading_order_separators=reading_order_separators, layout_options=layout_options, table_structure_options=table_structure_options, diff --git a/docling/datamodel/base_models.py b/docling/datamodel/base_models.py index 3180af13bd..c33a4bc024 100644 --- a/docling/datamodel/base_models.py +++ b/docling/datamodel/base_models.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Annotated, Any, Optional, Type, Union +from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Type, Union import numpy as np from docling_core.types.doc import ( @@ -429,18 +429,69 @@ class ApiImageStreamingRequestResult: stop_reason: VlmStopReason = VlmStopReason.UNSPECIFIED +class FieldValuePrediction(BaseModel): + text: str + orig: str + bbox: BoundingBox + # Selection state for a checkbox/radio widget; None for text-like fields. + # When set, the value is materialized as an empty field value that nests a + # CHECKBOX_SELECTED/UNSELECTED child (state lives on the child, not the text). + checkbox: ( + Literal[DocItemLabel.CHECKBOX_SELECTED, DocItemLabel.CHECKBOX_UNSELECTED] | None + ) = None + + +class FieldItemPrediction(BaseModel): + # A keyed item groups one or more values under one field key chosen from + # printed text beside a widget, in its table cell, or in a paragraph that + # contains it. key_bbox is the provenance of the key. Keyless items carry + # a single value. hint_text is extra context emitted as a field hint: + # outside detected tables, an aligned header along the other axis of a + # grid of like values. Items in a table cell never carry a hint. + key_text: str = "" + key_bbox: BoundingBox | None = None + values: list[FieldValuePrediction] = [] + hint_text: str = "" + + +class FormFieldPrediction(BaseModel): + # One entry per placement route of the form field stage. Items in table + # cells are on Table.field_cells instead. + inline_items: dict[int, FieldItemPrediction] = {} # host paragraph id + form_items: dict[int, list[FieldItemPrediction]] = {} # FORM cluster id + loose_items: list[FieldItemPrediction] = [] # placed by geometry + + +class TableFieldPrediction(BaseModel): + # Field items that live inside one cell of a structured table. The table's + # own row and column headers carry the association, so the items only hold + # a key printed in that same cell. Offsets follow TableCell's convention. + start_row_offset_idx: int + end_row_offset_idx: int + start_col_offset_idx: int + end_col_offset_idx: int + items: list[FieldItemPrediction] = [] + + class ContainerElement( BasePageElement ): # Used for Form and Key-Value-Regions, only for typing. pass +class FieldRegionElement(ContainerElement): + items: list[FieldItemPrediction] = [] + + class Table(BasePageElement): otsl_seq: list[str] num_rows: int = 0 num_cols: int = 0 orientation: Orientation = Orientation.ROT_0 table_cells: list[TableCell] + # AcroForm field items placed in this table's cells (see TableFieldPrediction); + # written by the form field stage, empty unless form fields are extracted. + field_cells: list[TableFieldPrediction] = [] class TableStructurePrediction(BaseModel): @@ -450,6 +501,12 @@ class TableStructurePrediction(BaseModel): class TextElement(BasePageElement): text: str hyperlink: Optional[Union[AnyUrl, Path]] = None + # Set when this paragraph inlines AcroForm widgets (e.g. a sentence with an + # inline checkbox and a fillable amount). The paragraph then materializes as + # a field_item -- its text becomes the key, the widgets its values -- in the + # paragraph's own place in the reading order (its list, its container), rather + # than being pulled out into a separate field_region. + field_item: Optional["FieldItemPrediction"] = None class FigureElement(BasePageElement): @@ -481,13 +538,16 @@ class EquationPrediction(BaseModel): class PagePredictions(BaseModel): layout: LayoutPrediction | None = None + form_fields: FormFieldPrediction | None = None tablestructure: TableStructurePrediction | None = None figures_classification: FigureClassificationPrediction | None = None equations_prediction: EquationPrediction | None = None vlm_response: VlmPrediction | None = None -PageElement = Union[TextElement, Table, FigureElement, ContainerElement] +PageElement = Union[ + TextElement, Table, FigureElement, ContainerElement, FieldRegionElement +] class AssembledUnit(BaseModel): diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index 6928fe5fc7..26ea1caee4 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -2108,6 +2108,21 @@ class PdfPipelineOptions(PaginatedPipelineOptions): ) ), ] = True + extract_form_fields: Annotated[ + bool, + Field( + description=( + "Extract native interactive PDF widgets (AcroForm) as format-neutral fillable field values, " + "each with a field key from printed text associated by page geometry: text in or beside " + "its cell, or a whole paragraph that inlines the widget. A widget in a detected table goes " + "into its table cell, keyed only by text printed in that same cell; row and column " + "headers are not used as keys or hints. Requires the docling-parse backend, which supplies " + "page widgets; table cells need table structure. Raw widget metadata remains available only " + "on retained parsed pages. Scanned or flattened forms and backends without page widgets are " + "unaffected." + ) + ), + ] = False do_ocr: Annotated[ bool, Field( diff --git a/docling/models/stages/form_field/__init__.py b/docling/models/stages/form_field/__init__.py new file mode 100644 index 0000000000..c4d8f5f986 --- /dev/null +++ b/docling/models/stages/form_field/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT diff --git a/docling/models/stages/form_field/form_field_model.py b/docling/models/stages/form_field/form_field_model.py new file mode 100644 index 0000000000..afe67da6a5 --- /dev/null +++ b/docling/models/stages/form_field/form_field_model.py @@ -0,0 +1,625 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Field keys and hints for native AcroForm widget values. + +The stage runs after table structure. For every page with widgets it passes +the widgets, layout clusters and detected table cells to ``keying.assign``, +which chooses the key of each value, and turns the result into a +``FormFieldPrediction``: one item per key-value association. +Outside detected tables, a value in a grid of like values also keeps the +aligned header along its other axis as a field hint. Items are placed as the +original stage placed its values: a paragraph that inlines all of an item's widgets +hosts the item in place, otherwise the enclosing FORM region does, otherwise a +page-wide region. A value in a cell of a detected table goes into that cell +instead (``Table.field_cells``), keyed only by text printed in the +same cell and without hints: the table's headers already carry the +association. Text used as a field key leaves the body, also when it shares a +text block with text that did not, and text blocks that merely re-render a filled value are +dropped. A page whose keying fails keeps its values, without keys. +""" + +import logging +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass, field + +from docling_core.types.doc import BoundingBox, DocItemLabel +from docling_core.types.doc.page import PdfWidget + +from docling.datamodel.base_models import ( + Cluster, + FieldItemPrediction, + FieldValuePrediction, + FormFieldPrediction, + Page, + TableFieldPrediction, +) +from docling.datamodel.document import ConversionResult +from docling.models.base_layout_model import PAGE_HEADER_LABELS, TEXT_ELEM_LABELS +from docling.models.base_model import BasePageModel +from docling.models.stages.form_field.keying import ( + PUSHBUTTON_FLAG, + WIDGET_COVERAGE, + Assignment, + KeyText, + assign, + is_skipped, + regions, +) +from docling.models.stages.form_field.keying.rules import THIN +from docling.utils.form_fields import without_whitespace +from docling.utils.profiling import TimeRecorder + +_log = logging.getLogger(__name__) + +# Paragraphs that may host an item in place. Code, and captions or footnotes +# attached to a table or picture, take reading-order paths that ignore +# TextElement.field_item. A section header or formula that is a key goes to a +# field region instead. +INLINE_HOSTS = set(TEXT_ELEM_LABELS) - { + DocItemLabel.CODE, + DocItemLabel.CAPTION, + DocItemLabel.FOOTNOTE, + DocItemLabel.SECTION_HEADER, + DocItemLabel.FORMULA, + *PAGE_HEADER_LABELS, +} + + +@dataclass +class _Unit: + """One field item built from one association, or from one unkeyed value.""" + + item: FieldItemPrediction + positions: list[int] # indices into Assignment.values + consumed: KeyText | None # key whose text leaves the body + + +@dataclass +class _Plan: + """Everything the stage writes to a page, computed before any edit.""" + + regions: FormFieldPrediction + dropped: set[int] # clusters whose text all became keys + trimmed: dict[int, set[int]] # cluster id -> its cells used as keys or values + hosts: set[int] # paragraphs that carry an item in place; never dropped + values: list[FieldValuePrediction] + # table cluster id -> its cells with items, in order + table_fields: dict[int, list[TableFieldPrediction]] = field(default_factory=dict) + + +def _walk(clusters: list[Cluster]) -> dict[int, Cluster]: + """Every cluster by id, children included.""" + return {cluster.id: cluster for cluster in regions(clusters)} + + +def _printable(cluster: Cluster) -> set[int]: + return {cell.index for cell in cluster.cells if cell.text.strip()} + + +def _assign(page: Page) -> Assignment: + """Key the page's widgets from its layout, table structure and printed rules.""" + assert page.size is not None and page.parsed_page is not None + assert page.predictions.layout is not None + structure = page.predictions.tablestructure + table_cells = ( + {} + if structure is None + else { + table_id: table.table_cells + for table_id, table in structure.table_map.items() + } + ) + # Printed rules only place values in the cells of detected tables. Both + # queries are needed: stroked segments give the edges of a cell border + # drawn as one rectangle (its own box is not thin), and thin boxes give + # rules drawn as filled rectangles. Only ThreadedDoclingParsePageBackend, + # the default backend, answers the second; elsewhere it returns None. + rules: list[BoundingBox] = [] + if table_cells and page._backend is not None: + rules = [ + *(page._backend.get_shape_lines() or []), + *(page._backend.get_thin_shape_boxes(max_thickness=THIN) or []), + ] + return assign( + page.parsed_page.widgets, + page.predictions.layout.clusters, + table_cells, + page.size.height, + rules, + ) + + +def _atom_sources( + assignment: Assignment, +) -> tuple[dict[int, tuple[int, int]], set[int]]: + """Atom -> (cluster id, cell index), plus clusters that must stay in the body. + + A cell the layout put in two clusters leaves both clusters in place + ("unsure"). + """ + sources = {atom: min(found) for atom, found in assignment.sources.items()} + unsure = { + cluster_id + for found in assignment.sources.values() + if len({cluster_id for cluster_id, _ in found}) > 1 + for cluster_id, _ in found + } + return sources, unsure + + +def _build_units( + assignment: Assignment, values: list[FieldValuePrediction] +) -> list[_Unit]: + """One field item per selected association, plus one per unkeyed value.""" + units: list[_Unit] = [] + owned: set[int] = set() + for candidate in (assignment.candidates[i] for i in assignment.selected): + # A shared prompt stays as ordinary text; each option keeps its own key. + if candidate.kind == "choice_group": + continue + key_text = assignment.key_texts[candidate.key] + units.append( + _Unit( + FieldItemPrediction( + key_text=key_text.text, + key_bbox=key_text.bbox, + values=[values[m] for m in candidate.members], + hint_text="" + if candidate.hint is None + else assignment.key_texts[candidate.hint].text, + ), + list(candidate.members), + # Table-cell keys have no atoms: the table keeps its text. + key_text if key_text.atoms else None, + ) + ) + owned.update(candidate.members) + units += [ + _Unit(FieldItemPrediction(values=[value]), [m], None) + for m, value in enumerate(values) + if m not in owned + ] + return sorted(units, key=lambda u: min(u.positions)) + + +def _cell_units( + units: list[_Unit], assignment: Assignment +) -> tuple[dict[int, list[TableFieldPrediction]], list[_Unit]]: + """Units whose values all sit in one table cell, grouped into that cell. + + The table's own row and column headers carry the association there, and + keying.tables keys such a value only by text of its own cell, so the item + keeps its key and drops any hint. Returns the cells by table id, then the + units placed as before. + """ + cells: dict[tuple[int, tuple[int, int], tuple[int, int]], list[_Unit]] = ( + defaultdict(list) + ) + rest: list[_Unit] = [] + for unit in units: + slots = {assignment.slots.get(m) for m in unit.positions} + if len(slots) != 1 or None in slots: + rest.append(unit) + continue + (slot,) = slots + assert slot is not None + unit.item = unit.item.model_copy(update={"hint_text": ""}) + cells[slot.table, slot.rows, slot.columns].append(unit) + placed: dict[int, list[TableFieldPrediction]] = defaultdict(list) + for (table, rows, columns), members in cells.items(): + placed[table].append( + TableFieldPrediction( + start_row_offset_idx=rows[0], + end_row_offset_idx=rows[1], + start_col_offset_idx=columns[0], + end_col_offset_idx=columns[1], + items=[unit.item for unit in members], + ) + ) + return placed, rest + + +def _inline_hosts( + units: list[_Unit], + assignment: Assignment, + sources: dict[int, tuple[int, int]], + page: Page, + unsure: set[int], +) -> dict[int, Cluster]: + """Units whose key is one whole paragraph holding all their widgets. + + Such a paragraph stays in place as a field item (see TextElement.field_item); + page assembly attaches one item per paragraph. + """ + assert page.predictions.layout is not None + top = {c.id: c for c in page.predictions.layout.clusters} + users: dict[int, set[int]] = defaultdict(set) + for k, unit in enumerate(units): + if unit.consumed is not None: + for atom in unit.consumed.atoms: + users[sources[atom][0]].add(k) + hosts: dict[int, Cluster] = {} + for k, unit in enumerate(units): + if unit.consumed is None: + continue + clusters = {sources[atom][0] for atom in unit.consumed.atoms} + if len(clusters) != 1: + continue + (cluster_id,) = clusters + host = top.get(cluster_id) + if ( + host is not None + and cluster_id not in unsure + and host.label in INLINE_HOSTS + and users[cluster_id] == {k} + and {sources[atom][1] for atom in unit.consumed.atoms} == _printable(host) + and all( + assignment.values[m].bbox.intersection_over_self(host.bbox) + >= WIDGET_COVERAGE + for m in unit.positions + ) + ): + hosts[k] = host + return hosts + + +def _place( + units: list[_Unit], + hosts: dict[int, Cluster], + assignment: Assignment, + page: Page, +) -> FormFieldPrediction: + """The three placement routes: inline paragraph, FORM region, page-wide.""" + inline_items: dict[int, FieldItemPrediction] = {} + rest: list[tuple[BoundingBox, FieldItemPrediction]] = [] + for k, unit in enumerate(units): + host = hosts.get(k) + if host is not None: + inline_items[host.id] = unit.item + continue + rest.append((assignment.values[min(unit.positions)].bbox, unit.item)) + fields = _form_regions(rest, page) + fields.inline_items = inline_items + return fields + + +def _form_regions( + items: list[tuple[BoundingBox, FieldItemPrediction]], page: Page +) -> FormFieldPrediction: + """Items grouped by the FORM region holding their box; the rest page-wide.""" + assert page.predictions.layout is not None + forms = [ + c for c in page.predictions.layout.clusters if c.label == DocItemLabel.FORM + ] + by_form: dict[int, list[FieldItemPrediction]] = defaultdict(list) + loose: list[FieldItemPrediction] = [] + for bbox, item in items: + form = PdfFormFieldModel._match_form(bbox, forms) + (loose if form is None else by_form[form.id]).append(item) + return FormFieldPrediction(form_items=dict(by_form), loose_items=loose) + + +def _table_children(page: Page) -> set[int]: + """Text blocks inside a structured table: its TableFormer cells show the text.""" + assert page.predictions.layout is not None + return { + child.id + for cluster in _walk(page.predictions.layout.clusters).values() + if cluster.label in {DocItemLabel.TABLE, DocItemLabel.DOCUMENT_INDEX} + for child in cluster.children + } + + +def _consumed_clusters( + units: list[_Unit], + sources: dict[int, tuple[int, int]], + assignment: Assignment, + page: Page, + keep: set[int], +) -> tuple[set[int], dict[int, set[int]]]: + """Clusters whose text became key text, wholly or in part. + + Returns the clusters whose every printable cell became a key, which leave + the body, then the clusters only partly used, with the cells that became + keys: those cells leave the cluster and the rest stays in the body. In a + cluster that gives a key, a cell recorded as a painted value during input + preparation counts as used too, so no paragraph is left holding + only the values of those fields. + """ + assert page.size is not None and page.predictions.layout is not None + clusters = _walk(page.predictions.layout.clusters) + used: dict[int, set[int]] = defaultdict(set) + for unit in units: + if unit.consumed is not None: + for atom in unit.consumed.atoms: + cluster_id, cell_index = sources[atom] + used[cluster_id].add(cell_index) + dropped: set[int] = set() + trimmed: dict[int, set[int]] = {} + for cluster_id, cells in used.items(): + if cluster_id in keep: + continue + cluster = clusters[cluster_id] + cells = cells | { + cell.index + for cell in cluster.cells + if (cluster_id, cell.index) in assignment.painted_cells + } + if cells >= _printable(cluster): + dropped.add(cluster_id) + else: + trimmed[cluster_id] = cells + return dropped, trimmed + + +class PdfFormFieldModel(BasePageModel): + # A rendered-text cluster counts as a widget's duplicate only when this much + # of it sits inside the widget rect. Guards against deleting ordinary printed + # text that merely equals a field value by coincidence. + _DUPLICATE_CONTAINMENT_THRESHOLD = 0.6 + + def __init__(self, *, enabled: bool) -> None: + self.enabled = enabled + + @classmethod + def _cluster_text(cls, cluster: Cluster) -> str: + return without_whitespace( + " ".join(cell.text for cell in cluster.cells if cell.text.strip()) + ) + + @classmethod + def _normalize_widget( + cls, widget: PdfWidget, bbox: BoundingBox + ) -> FieldValuePrediction: + source_value = widget.widget_text or "" + if ( + widget.widget_field_type == "/Btn" + and not widget.widget_field_flags & PUSHBUTTON_FLAG + ): + # A checkbox/radio widget carries state, not text. Encode the state + # as a nested checkbox child (see FieldValuePrediction.checkbox) and + # keep the value text empty -- the serializer only inlines the + # token when the hosting value has no text of its own. + source_value = widget.widget_appearance_state or source_value + off = source_value in {"", "/Off", "Off"} + return FieldValuePrediction( + text="", + orig=source_value, + bbox=bbox, + checkbox=DocItemLabel.CHECKBOX_UNSELECTED + if off + else DocItemLabel.CHECKBOX_SELECTED, + ) + + return FieldValuePrediction(text=source_value, orig=source_value, bbox=bbox) + + @classmethod + def _match_form( + cls, widget_bbox: BoundingBox, forms: list[Cluster] + ) -> Cluster | None: + matches = [ + (widget_bbox.intersection_over_self(form.bbox), form) + for form in forms + if widget_bbox.intersection_over_self(form.bbox) >= WIDGET_COVERAGE + ] + if not matches: + return None + return max( + matches, + key=lambda match: ( + match[0], + -match[1].bbox.area(), + -match[1].id, + ), + )[1] + + def __call__( + self, conv_res: ConversionResult, page_batch: Iterable[Page] + ) -> Iterable[Page]: + for page in page_batch: + if ( + not self.enabled + or page.parsed_page is None + or not page.parsed_page.widgets + ): + yield page + continue + + with TimeRecorder(conv_res, "form_field"): + self._key_page(page) + + yield page + + def _key_page(self, page: Page) -> None: + """Key the page's widgets; on any failure keep the values without keys. + + A stage exception would mark the page as failed and the document as + partially converted, so the widgets' values are worth more than the + keys: the fallback plan places them unkeyed, and if even that fails + the page passes through untouched. + """ + try: + plan = self._plan_page(page) + except Exception: + _log.warning( + "Form field keying failed on page %d; its values are kept without keys", + page.page_no, + exc_info=True, + ) + try: + plan = self._keyless_plan(page) + except Exception: + _log.warning( + "Form field extraction failed on page %d; page left unchanged", + page.page_no, + exc_info=True, + ) + return + self._commit(page, plan) + + def _plan_page(self, page: Page) -> _Plan: + """Compute the page's field items and edits without touching the page.""" + assert page.size is not None and page.parsed_page is not None + assert page.predictions.layout is not None + assignment = _assign(page) + if assignment.solver_status != "optimal": + _log.warning( + "Form field keying abstained on page %d (%s); free-form values are " + "kept without keys", + page.page_no, + assignment.solver_status, + ) + values = [self._normalize_widget(v.native, v.bbox) for v in assignment.values] + sources, unsure = _atom_sources(assignment) + units = _build_units(assignment, values) + in_cells, rest = _cell_units(units, assignment) + hosts = _inline_hosts(rest, assignment, sources, page, unsure) + regions = _place(rest, hosts, assignment, page) + host_ids = {host.id for host in hosts.values()} + keep = unsure | _table_children(page) | host_ids + dropped, trimmed = _consumed_clusters(units, sources, assignment, page, keep) + return _Plan(regions, dropped, trimmed, host_ids, values, in_cells) + + def _keyless_plan(self, page: Page) -> _Plan: + """Every retained widget as an unkeyed value, placed by FORM region.""" + assert page.size is not None and page.parsed_page is not None + values: list[FieldValuePrediction] = [] + for widget in page.parsed_page.widgets: + bbox = widget.rect.to_bounding_box().to_top_left_origin(page.size.height) + if not is_skipped(widget, bbox): + values.append(self._normalize_widget(widget, bbox)) + items = [(value.bbox, FieldItemPrediction(values=[value])) for value in values] + return _Plan(_form_regions(items, page), set(), {}, set(), values) + + def _commit(self, page: Page, plan: _Plan) -> None: + """Write the plan to the page; restore the layout if that fails.""" + assert page.predictions.layout is not None + layout = page.predictions.layout + clusters = list(layout.clusters) + children = {cluster.id: cluster.children for cluster in clusters} + trimmed = [ + (cluster, cluster.cells, cluster.bbox) + for cluster_id, cluster in _walk(clusters).items() + if cluster_id in plan.trimmed + ] + # table_fields keys come from table_map (see _assign), so it is set. + structure = page.predictions.tablestructure + tables = ( + [] + if structure is None + else [ + (structure.table_map[table_id], cells) + for table_id, cells in plan.table_fields.items() + ] + ) + try: + page.predictions.form_fields = plan.regions + for table, cells in tables: + table.field_cells = cells + self._drop_clusters(page, plan.dropped) + self._trim_clusters(page, plan.trimmed) + self._suppress_duplicate_text(page, plan.values, keep=frozenset(plan.hosts)) + except Exception: + _log.warning( + "Form field extraction failed on page %d; page left unchanged", + page.page_no, + exc_info=True, + ) + page.predictions.form_fields = None + for table, _ in tables: + table.field_cells = [] + for cluster in clusters: + cluster.children = children[cluster.id] + for cluster, cells, bbox in trimmed: + cluster.cells = cells + cluster.bbox = bbox + layout.clusters = clusters + + @classmethod + def _suppress_duplicate_text( + cls, + page: Page, + values: list[FieldValuePrediction], + *, + keep: frozenset[int] = frozenset(), + ) -> None: + """Drop plain text clusters that merely re-render a native field value. + + A filled widget's appearance stream is painted into the page raster, so + the layout model also detects it as an ordinary text cluster -- producing + a duplicate of the widget's native ``/V``. Suppress such a cluster only + when its text equals a field value's *and* it sits inside that widget's + rect; the containment gate keeps ordinary printed text that coincidentally + matches a value from being deleted. A paragraph that hosts a field + item in place (``keep``) is the item's key, not a duplicate. + + A value the layout model glued onto a neighbouring key (a substring + of a larger line, not an equal twin) is left in place: cutting it out + of the line risks removing printed text. + """ + assert page.predictions.layout is not None + by_text: dict[str, list[BoundingBox]] = {} + for value in values: + text = without_whitespace(value.text) + if text: + by_text.setdefault(text, []).append(value.bbox) + + if not by_text: + return + + def is_duplicate(cluster: Cluster) -> bool: + return cluster.label in TEXT_ELEM_LABELS and any( + cluster.bbox.intersection_over_self(widget_bbox) + > cls._DUPLICATE_CONTAINMENT_THRESHOLD + for widget_bbox in by_text.get(cls._cluster_text(cluster), []) + ) + + dropped_ids = { + cluster.id + for cluster in page.predictions.layout.clusters + if cluster.id not in keep and is_duplicate(cluster) + } + cls._drop_clusters(page, dropped_ids) + + @staticmethod + def _trim_clusters(page: Page, trimmed: dict[int, set[int]]) -> None: + """Remove the given cells from their clusters and refit their boxes. + + The cells left in a cluster always include printable text (a cluster + whose every printable cell became a key is dropped instead), and the + box encloses them as the layout postprocessor fits text clusters. + """ + if not trimmed: + return + assert page.predictions.layout is not None + for cluster_id, cluster in _walk(page.predictions.layout.clusters).items(): + used = trimmed.get(cluster_id) + if not used: + continue + cluster.cells = [cell for cell in cluster.cells if cell.index not in used] + cluster.bbox = BoundingBox.enclosing_bbox( + [cell.rect.to_bounding_box() for cell in cluster.cells] + ) + + @staticmethod + def _drop_clusters(page: Page, dropped_ids: set[int]) -> None: + """Remove clusters (and any container child refs to them) from the page. + + Shared by raster-duplicate suppression and key promotion so + reading-order assembly never references a removed cluster. + """ + if not dropped_ids: + return + assert page.predictions.layout is not None + for cluster in page.predictions.layout.clusters: + if cluster.children: + cluster.children = [ + child for child in cluster.children if child.id not in dropped_ids + ] + page.predictions.layout.clusters = [ + cluster + for cluster in page.predictions.layout.clusters + if cluster.id not in dropped_ids + ] diff --git a/docling/models/stages/form_field/keying/__init__.py b/docling/models/stages/form_field/keying/__init__.py new file mode 100644 index 0000000000..af27cbb268 --- /dev/null +++ b/docling/models/stages/form_field/keying/__init__.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Geometric association of native AcroForm widget values with printed keys. + +``assign`` takes one page's widgets, layout clusters and detected table cells +and returns an ``Assignment``: for every value, its field key, +chosen by a small integer program over geometric candidates. A value inside a +detected table is keyed only by text of its own cell. The module reads no +files, keeps no state and does not modify its inputs. +""" + +from docling.models.stages.form_field.keying.inputs import ( + regions, + scope_of, +) +from docling.models.stages.form_field.keying.solver import assign +from docling.models.stages.form_field.keying.types import ( + PUSHBUTTON_FLAG, + WIDGET_COVERAGE, + Assignment, + Candidate, + KeyText, + Scope, + is_skipped, +) + +__all__ = [ + "PUSHBUTTON_FLAG", + "WIDGET_COVERAGE", + "Assignment", + "Candidate", + "KeyText", + "Scope", + "assign", + "is_skipped", + "regions", + "scope_of", +] diff --git a/docling/models/stages/form_field/keying/candidates.py b/docling/models/stages/form_field/keying/candidates.py new file mode 100644 index 0000000000..120470afb9 --- /dev/null +++ b/docling/models/stages/form_field/keying/candidates.py @@ -0,0 +1,367 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Candidate key-value associations and their costs.""" + +from __future__ import annotations + +import math +from collections import defaultdict +from collections.abc import Callable + +import numpy as np +from docling_core.types.doc import BoundingBox, DocItemLabel + +from docling.models.stages.form_field.keying.geometry import ( + anchors, + corridor, + gap, + lettered, + obstructs, + side_of, +) +from docling.models.stages.form_field.keying.types import ( + WIDGET_COVERAGE, + Candidate, + KeyText, + Scope, + Side, + Value, +) + + +def local_features( + key_text: KeyText, members: tuple[int, ...], values: list[Value], h: float +) -> dict[str, float]: + distance, misalignment, obstacles = 0.0, 0.0, 0.0 + for i in members: + value = values[i] + a, b = key_text.bbox, value.bbox + distance += math.log1p(gap(a, b) / h) + xover = a.x_overlap_with(b) / max(1e-6, min(a.width, b.width)) + yover = a.y_overlap_with(b) / max(1e-6, min(a.height, b.height)) + misalignment += 0.5 * (1 - max(xover, yover)) + start, end = anchors(a, b) + obstacles += 4 * sum( + obstructs(start, end, other.bbox, h * 0.05) + for j, other in enumerate(values) + if j not in members + ) + return { + "distance": distance, + "alignment": misalignment, + "obstruction": obstacles, + "role": len(members) * role_cost(key_text, values, h), + "fragment": 0.5 * len(members) * key_text.fragment, + } + + +def role_cost(key_text: KeyText, values: list[Value], h: float) -> float: + role = 3.0 * (not lettered(key_text.text)) + role += 3.0 * ( + key_text.layout_label + in {DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER, DocItemLabel.FOOTNOTE} + ) + # A short printed component between two text boxes is weak key evidence. + # The rule uses position and length, never a fixture/token blacklist. + component = ( + len(key_text.text.strip()) <= 3 + and any( + not v.checkbox + and v.bbox.r <= key_text.bbox.l + and gap(v.bbox, key_text.bbox) < h + for v in values + ) + and any( + not v.checkbox + and v.bbox.l >= key_text.bbox.r + and gap(v.bbox, key_text.bbox) < h + for v in values + ) + ) + return role + 3.0 * component + + +def slot_features( + key_text: KeyText, + value: Value, + side: Side, + alignment: float, + values: list[Value], + h: float, +) -> dict[str, float]: + """Cost of one candidate key for one value: cell, then alignment, then distance. + + A key in the value's own printed cell (inside, or touching it above or + below) is preferred over an aligned key along the same line, which is + preferred over an aligned key farther up or down. Keys precede + their value in reading order: a key below costs a little more than one + above (in stacked boxes the next box's key touches from below), and + text after the value on its line only keys it when adjacent, as an + option key follows its checkbox; farther right it starts the next cell. + Sharing half of the band counts as aligned; below that, misalignment grows + to the cost of a cell difference. Proximity only breaks ties. + """ + gap_h = gap(key_text.bbox, value.bbox) / h + if side == "inside" or (side in ("up", "down") and gap_h <= 0.5): + cell = 0.0 + else: + cell = 1.0 if side in ("left", "right") else 1.5 + after = 0.5 * (side == "down") + 1.0 * (side == "right" and gap_h > 1.0) + return { + "cell": cell + after, + "misaligned": max(0.0, 1.0 - alignment / 0.5), + "distance": 0.25 * min(math.log1p(gap_h), 2.0), + "role": role_cost(key_text, values, h), + "fragment": 0.5 * key_text.fragment, + } + + +def blockers(key_texts: list[KeyText]) -> Callable[[KeyText, BoundingBox | None], bool]: + """Whether other lettered text lies in a corridor, hiding a key behind it. + + Only the first text met on each side of a value is a candidate: the value's + own cell, or the neighbouring cell on that side. Digits, codes and symbols + are transparent, as are other values: line numbers, arithmetic signs and + sibling fields sit between many keys and their values. + """ + atoms = { + next(iter(key_text.atoms)): key_text.bbox + for key_text in key_texts + if len(key_text.atoms) == 1 and lettered(key_text.text) + } + ids = np.array(list(atoms), dtype=int) + boxes = np.array([[b.l, b.t, b.r, b.b] for b in atoms.values()]).reshape(-1, 4) + widths, heights = boxes[:, 2] - boxes[:, 0], boxes[:, 3] - boxes[:, 1] + + def blocked(key_text: KeyText, lane: BoundingBox | None) -> bool: + if lane is None or not len(ids): + return False + width = np.minimum(boxes[:, 2], lane.r) - np.maximum(boxes[:, 0], lane.l) + height = np.minimum(boxes[:, 3], lane.b) - np.maximum(boxes[:, 1], lane.t) + # Text inside the corridor, or a line crossing it: the overlap covers + # half of the smaller extent along both axes. + across = (width >= 0.5 * np.minimum(widths, lane.width)) & (width > 0) + across &= (height >= 0.5 * np.minimum(heights, lane.height)) & (height > 0) + return bool(np.any(across & ~np.isin(ids, list(key_text.atoms)))) + + return blocked + + +def foreign(i: int, lane: BoundingBox | None, side: Side, values: list[Value]) -> bool: + """Whether a different kind of field sits in a column corridor. + + Walking up or down a column passes only the value's siblings (a repeated + column of like fields under its header); another field there is another + cell, with its own key. Rows are different: operand boxes and flags + legitimately sit between a line key and its amount. + """ + if lane is None or side not in ("up", "down"): + return False + return any( + j != i + and v.bbox.x_overlap_with(lane) >= 0.5 * min(v.bbox.width, lane.width) + and v.bbox.y_overlap_with(lane) >= 0.5 * min(v.bbox.height, lane.height) + and not siblings(values[i], v, side) + for j, v in enumerate(values) + ) + + +def candidates_for( + values: list[Value], key_texts: list[KeyText], h: float +) -> list[Candidate]: + candidates: list[Candidate] = [] + blocked = blockers(key_texts) + for li, key_text in enumerate(key_texts): + eligible = [ + i + for i, v in enumerate(values) + if v.scope.eligible and v.scope == key_text.scope + ] + contained = tuple( + i + for i in eligible + if values[i].bbox.intersection_over_self(key_text.bbox) >= WIDGET_COVERAGE + ) + for i in eligible: + facing = side_of(key_text.bbox, values[i].bbox) + if facing is None or not lettered(key_text.text): + continue + lane = corridor(key_text.bbox, values[i].bbox, facing[0]) + if blocked(key_text, lane) or foreign(i, lane, facing[0], values): + continue + kind = "option_key" if values[i].checkbox else "field_key" + features = slot_features(key_text, values[i], *facing, values, h) + candidates.append(Candidate((i,), li, kind, features, facing[0])) + clause = tuple( + i + for i in contained + if i not in answers(contained, key_text, key_texts, values, h) + ) + if len(clause) > 1: + candidates.append( + Candidate( + clause, + li, + "inline_clause", + local_features(key_text, clause, values, h), + ) + ) + # A common key over adjacent components, with no full intervening + # key. Do not make arbitrary runs of checkboxes into one field. + text_values = [i for i in eligible if not values[i].checkbox] + for start in range(len(text_values)): + members = [text_values[start]] + for nxt in text_values[start + 1 :]: + prev, curr = values[members[-1]].bbox, values[nxt].bbox + if abs(prev.t - curr.t) > h or gap(prev, curr) > 2 * h: + break + members.append(nxt) + union = BoundingBox.enclosing_bbox([values[i].bbox for i in members]) + if ( + key_text.bbox.b <= union.t + and union.t - key_text.bbox.b <= 5 * h + and BoundingBox( + l=union.l, r=union.r, t=key_text.bbox.t, b=key_text.bbox.b + ).intersection_area_with(key_text.bbox) + > 0 + ): + # Any member with its own key above it is a separate field. + sibling_key = any( + other.scope == key_text.scope + and other.atoms.isdisjoint(key_text.atoms) + and lettered(other.text) + and other.bbox.b <= member.t + and other.bbox.t >= key_text.bbox.t - h + and other.bbox.x_overlap_with(member) >= 0.5 * other.bbox.width + for other in key_texts + for member in (values[i].bbox for i in members) + ) + if sibling_key: + break + features = local_features(key_text, tuple(members), values, h) + # The key describes the composite envelope. Charge its + # distance once PER VALUE, so a big group is not a free link. + features["distance"] = len(members) * math.log1p( + gap(key_text.bbox, union) / h + ) + features["alignment"] = 0.0 + features["group"] = 0.5 + candidates.append( + Candidate(tuple(members), li, "composite_field", features) + ) + + # Shared PDF names are corroboration, not an assertion of field identity. + # Also retain contiguous native checkbox runs as alternatives when keys + # form a repeated, compact arrangement (including a column reset). + groups: set[tuple[int, ...]] = set() + names: dict[tuple[Scope, str], list[int]] = defaultdict(list) + run: list[int] = [] + for i, value in enumerate(values): + if value.checkbox and value.scope.eligible: + column_reset = bool(run) and value.bbox.t <= values[run[0]].bbox.t + h + if run and ( + value.scope != values[run[-1]].scope + or (gap(value.bbox, values[run[-1]].bbox) > 4 * h and not column_reset) + ): + if len(run) > 1: + groups.add(tuple(run)) + run = [] + run.append(i) + if value.native.widget_field_name: + names[value.scope, value.native.widget_field_name].append(i) + else: + if len(run) > 1: + groups.add(tuple(run)) + run = [] + if len(run) > 1: + groups.add(tuple(run)) + groups.update(tuple(g) for g in names.values() if len(g) > 1) + for members in sorted(groups): + scope = values[members[0]].scope + union = BoundingBox.enclosing_bbox([values[i].bbox for i in members]) + for li, key_text in enumerate(key_texts): + if key_text.scope != scope or not any(c.isalpha() for c in key_text.text): + continue + # Common prompts sit above the group or alongside its first option; + # individual short option keys are not plausible common prompts. + first_box = values[members[0]].bbox + above = key_text.bbox.b <= union.t - 0.2 * h + beside = ( + key_text.bbox.r <= first_box.l - h and key_text.bbox.t <= first_box.b + ) + if not (above or beside) or gap(key_text.bbox, first_box) > 10 * h: + continue + if len(key_text.text.split()) < 2: + continue + features = { + "prompt_distance": math.log1p(gap(key_text.bbox, union) / h), + "prompt_prior": -2.5, + "fragment": 0.5 * key_text.fragment, + } + candidates.append(Candidate(members, li, "choice_group", features)) + # Within a key, the whole text is the key, not one of its lines: a + # line seen from the same side as its whole stacked key pays extra. + whole: dict[tuple[tuple[int, ...], Side], list[frozenset[int]]] = defaultdict(list) + for c in candidates: + if c.side is not None and key_texts[c.key].stack: + whole[c.members, c.side].append(key_texts[c.key].atoms) + for c in candidates: + if c.side is not None and any( + key_texts[c.key].atoms < atoms for atoms in whole[c.members, c.side] + ): + c.features["partial"] = 0.5 + return candidates + + +def answers( + members: tuple[int, ...], + clause: KeyText, + key_texts: list[KeyText], + values: list[Value], + h: float, +) -> set[int]: + """Checkboxes in a clause that are options of it, not blanks within it. + + An option has its own key: a separate piece of the clause's text on its + line, beside it within a text line. Two or more such checkboxes answer the + clause ("Business income [ ] Yes [ ] No"), so each keeps its own key. A + single checkbox, or a clause printed as one piece of text, stays a blank of + the clause ("check here [ ] and enter the amount"). + """ + found = set() + for i in members: + if not values[i].checkbox: + continue + box = values[i].bbox + if any( + len(key_text.atoms) == 1 + and key_text.atoms < clause.atoms + and lettered(key_text.text) + and key_text.bbox.y_overlap_with(box) + >= 0.5 * min(key_text.bbox.height, box.height) + and (key_text.bbox.r <= box.l or key_text.bbox.l >= box.r) + and gap(key_text.bbox, box) <= h + for key_text in key_texts + ): + found.add(i) + return found if len(found) > 1 else set() + + +def siblings(a: Value, b: Value, side: Side) -> bool: + """Same kind and size, aligned along the line a shared key runs along. + + A row key keys every like-sized value of its row, a column key + every like-sized value of its column. Operand boxes of a different size in + the same row (multipliers, rates) do not inherit the row key. + """ + if side == "inside" or a.checkbox != b.checkbox: + return False + x, y = a.bbox, b.bbox + if min(x.width, y.width) < 0.8 * max(x.width, y.width): + return False + if min(x.height, y.height) < 0.8 * max(x.height, y.height): + return False + if side in ("left", "right"): + return x.y_overlap_with(y) >= 0.5 * min(x.height, y.height) + return x.x_overlap_with(y) >= 0.5 * min(x.width, y.width) diff --git a/docling/models/stages/form_field/keying/geometry.py b/docling/models/stages/form_field/keying/geometry.py new file mode 100644 index 0000000000..10c6b75731 --- /dev/null +++ b/docling/models/stages/form_field/keying/geometry.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Box geometry shared by the keying steps.""" + +from __future__ import annotations + +from docling_core.types.doc import BoundingBox + +from docling.models.stages.form_field.keying.types import Side + + +def contains(outer: BoundingBox, inner: BoundingBox) -> bool: + """Whether ``outer`` encloses ``inner``, edges included. + + A boolean with a small tolerance, so a box that touches an edge counts as + inside; BoundingBox.intersection_over_self gives a ratio instead. + """ + return ( + outer.l <= inner.l + 1e-6 + and outer.t <= inner.t + 1e-6 + and outer.r >= inner.r - 1e-6 + and outer.b >= inner.b - 1e-6 + ) + + +def gap(a: BoundingBox, b: BoundingBox) -> float: + return max(0.0, a.l - b.r, b.l - a.r) + max(0.0, a.t - b.b, b.t - a.b) + + +def span_overlap(lo: float, hi: float, other_lo: float, other_hi: float) -> float: + """Overlap of two intervals, for row and column bands that are no boxes. + + Two boxes use BoundingBox.x_overlap_with and y_overlap_with. + """ + return max(0.0, min(hi, other_hi) - max(lo, other_lo)) + + +def lettered(text: str) -> bool: + return any(c.isalpha() for c in text) + + +def side_of(key_text: BoundingBox, value: BoundingBox) -> tuple[Side, float] | None: + """Side from which a key faces a value, and how well it is aligned. + + Left/right keys share the value's row band; up/down keys its column + band. Alignment is the shared fraction of the smaller perpendicular extent. + Purely diagonal keys return None: only group candidates use them. + """ + if key_text.intersection_area_with(value) >= 0.5 * min( + key_text.area(), value.area() + ): + return "inside", 1.0 + rows = key_text.y_overlap_with(value) + columns = key_text.x_overlap_with(value) + if rows > 0 and columns > 0: + # A partial overlap, typically a key straddling the value's top + # edge inside the same printed box: use the axis it sticks out along. + vertical = rows / key_text.height <= columns / key_text.width + else: + vertical = columns > 0 + if vertical: + up = key_text.t + key_text.b < value.t + value.b + return ("up" if up else "down"), columns / min(key_text.width, value.width) + if rows > 0: + left = key_text.l + key_text.r < value.l + value.r + return ("left" if left else "right"), rows / min(key_text.height, value.height) + return None + + +def corridor( + key_text: BoundingBox, value: BoundingBox, side: Side +) -> BoundingBox | None: + """Space between a facing key and its value, within their shared band.""" + if side == "left": + box = (key_text.r, max(key_text.t, value.t), value.l, min(key_text.b, value.b)) + elif side == "right": + box = (value.r, max(key_text.t, value.t), key_text.l, min(key_text.b, value.b)) + elif side == "up": + box = (max(key_text.l, value.l), key_text.b, min(key_text.r, value.r), value.t) + elif side == "down": + box = (max(key_text.l, value.l), value.b, min(key_text.r, value.r), key_text.t) + else: + return None + left, top, right, bottom = box + if right <= left or bottom <= top: + return None + return BoundingBox(l=left, t=top, r=right, b=bottom) + + +def anchors( + a: BoundingBox, b: BoundingBox +) -> tuple[tuple[float, float], tuple[float, float]]: + """Facing boundary points; use the shared band when boxes are aligned.""" + + def axis( + lo: float, hi: float, other_lo: float, other_hi: float + ) -> tuple[float, float]: + if hi < other_lo: + return hi, other_lo + if other_hi < lo: + return lo, other_hi + mid = (max(lo, other_lo) + min(hi, other_hi)) / 2 + return mid, mid + + ax, bx = axis(a.l, a.r, b.l, b.r) + ay, by = axis(a.t, a.b, b.t, b.b) + return (ax, ay), (bx, by) + + +def obstructs( + a: tuple[float, float], b: tuple[float, float], box: BoundingBox, tolerance: float +) -> bool: + """Segment intersects the rectangle interior; mere border touches do not count.""" + lower, upper = 0.0, 1.0 + for start, end, lo, hi in zip(a, b, (box.l, box.t), (box.r, box.b)): + lo, hi = lo + tolerance, hi - tolerance + if lo >= hi: + return False + delta = end - start + if abs(delta) < 1e-12: + if not lo < start < hi: + return False + continue + near, far = sorted(((lo - start) / delta, (hi - start) / delta)) + lower, upper = max(lower, near), min(upper, far) + if lower >= upper: + return False + return lower < upper + + +def band_index(lo: float, hi: float, bands: dict[int, tuple[float, float]]) -> int: + """The band sharing most of [lo, hi], else the one with the nearest centre.""" + shared = {k: span_overlap(lo, hi, *band) for k, band in bands.items()} + best = max(shared, key=lambda k: (shared[k], -k)) + if shared[best] > 0: + return best + middle = (lo + hi) / 2 + return min(bands, key=lambda k: (abs(sum(bands[k]) / 2 - middle), k)) diff --git a/docling/models/stages/form_field/keying/inputs.py b/docling/models/stages/form_field/keying/inputs.py new file mode 100644 index 0000000000..e0bd85b598 --- /dev/null +++ b/docling/models/stages/form_field/keying/inputs.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Values and candidate keys of one page: widgets, text atoms and joined key text.""" + +from __future__ import annotations + +import statistics +from collections import defaultdict + +from docling_core.types.doc import BoundingBox, DocItemLabel, TableCell +from docling_core.types.doc.page import PdfWidget + +from docling.datamodel.base_models import Cluster +from docling.models.stages.form_field.keying.geometry import ( + contains, + gap, + lettered, +) +from docling.models.stages.form_field.keying.types import ( + Inputs, + KeyText, + Scope, + Value, + is_skipped, +) +from docling.utils.form_fields import without_whitespace + +# Pages with more text atoms than this skip key joining and abstain from +# free-form keying. +MAX_KEY_TEXTS = 3000 +# Layout regions whose cells never become keys. +NOT_KEYS = { + DocItemLabel.FORM, + DocItemLabel.KEY_VALUE_REGION, + DocItemLabel.TABLE, + DocItemLabel.DOCUMENT_INDEX, + DocItemLabel.PICTURE, +} + + +def regions(clusters: list[Cluster]) -> list[Cluster]: + """Every layout cluster once, children included, ordered by id.""" + found: dict[int, Cluster] = {} + pending = list(clusters) + while pending: + region = pending.pop() + if region.id not in found: + found[region.id] = region + pending.extend(region.children) + return sorted(found.values(), key=lambda r: r.id) + + +def scope_of( + bbox: BoundingBox, found: list[Cluster], table_cells: dict[int, list[TableCell]] +) -> Scope: + """The detected table, and its cell, that a top-left box belongs to. + + ``found`` is the flat list of layout clusters (see ``regions``) and + ``table_cells`` the detected cells of each table cluster, by cluster id. + """ + tables = [ + r + for r in found + if r.label in {DocItemLabel.TABLE, DocItemLabel.DOCUMENT_INDEX} + and r.bbox.intersection_area_with(bbox) > 0 + ] + if not tables: + return Scope() + # Any intersection excludes a box from the outside pool. A local exception + # requires complete containment in one unambiguous detected cell. + table = min(tables, key=lambda r: (r.bbox.area(), r.id)) + if not contains(table.bbox, bbox) or any( + not contains(other.bbox, table.bbox) for other in tables if other.id != table.id + ): + return Scope(table.id) + cells = [ + i + for i, cell in enumerate(table_cells.get(table.id, [])) + if cell.bbox is not None and contains(cell.bbox, bbox) + ] + return Scope(table.id, cells[0] if len(cells) == 1 else None) + + +def paints_value(box: BoundingBox, text: str, values: list[Value]) -> bool: + """Whether the text is a filled value its widget paints into the page. + + Such text sits inside the widget's box and equals the widget's value; it + is the value itself, never a key. + """ + return any( + contains(v.bbox, box) + and without_whitespace(text) == without_whitespace(v.native.widget_text or "") + for v in values + ) + + +def inputs( + widgets: list[PdfWidget], + found: list[Cluster], + table_cells: dict[int, list[TableCell]], + page_height: float, +) -> Inputs: + """Values, key texts, median text height, atom sources, and painted source cells.""" + values = [] + for native in widgets: + bbox = native.rect.to_bounding_box().to_top_left_origin(page_height) + if is_skipped(native, bbox): + continue + values.append(Value(native, bbox, scope_of(bbox, found, table_cells))) + + # Atom identity is source-backed and shared by all overlapping span choices. + atoms: dict[tuple, int] = {} + sources: dict[int, set[tuple[int, int]]] = defaultdict(set) + key_texts: dict[tuple[frozenset[int], Scope], KeyText] = {} + heights = [] + painted_cells: set[tuple[int, int]] = set() + for region in found: + if region.label in NOT_KEYS: + continue + by_scope: dict[Scope, list[tuple[int, str, BoundingBox]]] = defaultdict(list) + for cell in region.cells: + text = cell.text.strip() + box = cell.rect.to_bounding_box().to_top_left_origin(page_height) + if not text: + continue + # Cleanup also reads cells excluded by candidate area or table scope. + if paints_value(box, text, values): + painted_cells.add((region.id, cell.index)) + if box.area() <= 0 or (region.id, cell.index) in painted_cells: + continue + scope = scope_of(box, found, table_cells) + if not scope.eligible: + continue + atom = atoms.setdefault( + (cell.index, tuple(box.as_tuple()), text), len(atoms) + ) + sources[atom].add((region.id, cell.index)) + heights.append(box.height) + by_scope[scope].append((atom, text, box)) + for scope, cells in by_scope.items(): + for bundle in [cells, *[[cell] for cell in cells]]: + ids = frozenset(c[0] for c in bundle) + bbox = BoundingBox.enclosing_bbox([c[2] for c in bundle]) + if scope_of(bbox, found, table_cells) != scope: + continue + key_texts[ids, scope] = KeyText( + " ".join(c[1] for c in bundle), + bbox, + ids, + scope, + region.label, + len(bundle) < len(cells), + ) + h = statistics.median(heights) if heights else 1.0 + if len(key_texts) > MAX_KEY_TEXTS: + # Joining keys compares every pair of blocks; a page this large + # abstains in assign() anyway. + return Inputs(values, list(key_texts.values()), h, sources, painted_cells) + # Rebuild whole keys the layout split into pieces: first the pieces of + # one text line, then the lines of one key. Joined keys compete + # with their pieces; they never replace them. + blocks = lines( + [key_text for key_text in key_texts.values() if not key_text.fragment], + values, + h, + ) + for key_text in [*blocks, *stacks(blocks, values, h)]: + key_texts.setdefault((key_text.atoms, key_text.scope), key_text) + return Inputs(values, list(key_texts.values()), h, sources, painted_cells) + + +def joined(parts: list[KeyText]) -> KeyText: + return KeyText( + " ".join(part.text for part in parts), + BoundingBox.enclosing_bbox([part.bbox for part in parts]), + frozenset().union(*(part.atoms for part in parts)), + parts[0].scope, + parts[0].layout_label, + stack=True, + ) + + +def nested(a: frozenset[int], b: frozenset[int]) -> bool: + return a <= b or b <= a + + +def lines(blocks: list[KeyText], values: list[Value], h: float) -> list[KeyText]: + """Text lines the layout split into side-by-side blocks, joined back. + + Two blocks of one line join when they are each other's nearest neighbour, + have the same height, are under a text line apart and have no value + between them. Pieces of one line share the lines just above and below it; + blocks with different neighbouring lines, or that each key their own + value directly above or below (sub-keys over separate boxes), are + separate cells and stay apart. Returns the blocks with every joined run + replaced by its line. + """ + + def boxes_at(b: KeyText) -> frozenset[int]: + return frozenset( + i + for i, v in enumerate(values) + if v.scope == b.scope + and b.bbox.x_overlap_with(v.bbox) >= 0.5 * min(b.bbox.width, v.bbox.width) + and gap(b.bbox, v.bbox) <= h + ) + + def around(b: KeyText) -> frozenset[int]: + return frozenset( + k + for k, other in enumerate(blocks) + if other.scope == b.scope + and b.bbox.x_overlap_with(other.bbox) > 0 + and 0 < max(other.bbox.t - b.bbox.b, b.bbox.t - other.bbox.b) <= 0.75 * h + ) + + below = [boxes_at(b) for b in blocks] + neighbours = [around(b) for b in blocks] + nearest: dict[int, int] = {} + for i, a in enumerate(blocks): + right = [ + j + for j, b in enumerate(blocks) + if j != i + and b.scope == a.scope + and min(a.bbox.height, b.bbox.height) + >= 0.8 * max(a.bbox.height, b.bbox.height) + and a.bbox.y_overlap_with(b.bbox) >= 0.8 * min(a.bbox.height, b.bbox.height) + and -0.25 * h <= b.bbox.l - a.bbox.r <= h + ] + if right: + nearest[i] = min(right, key=lambda j: (blocks[j].bbox.l, j)) + links: dict[int, int] = {} + for i, j in nearest.items(): + a, b = blocks[i].bbox, blocks[j].bbox + between = BoundingBox( + l=min(a.r, b.l), t=max(a.t, b.t), r=max(a.r, b.l), b=min(a.b, b.b) + ) + if ( + [k for k, n in nearest.items() if n == j] == [i] + and nested(below[i], below[j]) + and neighbours[i] == neighbours[j] + and not any(v.bbox.intersection_area_with(between) > 0 for v in values) + ): + links[i] = j + result = [] + for start, block in enumerate(blocks): + if start in links.values(): + continue + run = [start] + while run[-1] in links: + run.append(links[run[-1]]) + result.append(block if len(run) == 1 else joined([blocks[k] for k in run])) + return result + + +def stacks(blocks: list[KeyText], values: list[Value], h: float) -> list[KeyText]: + """Keys the layout split into one block per line, joined back. + + Two single-line blocks of the same line height (one font size, so never a + heading over a key) join when they are each other's only neighbour + across a gap of under three quarters of a text line, share a left edge or + a centre, and the upper block aligns with no value the lower one misses: a + key ends at its value's row, so a line below a key that already + sits beside its value starts another key. A heading over several + sub-keys has several + neighbours and never joins one of them; a line with a value inside it is an + inline key of its own. A key merely touching its value's edge + still joins. Chains stop at four blocks. + """ + joinable = [ + b + for b in blocks + if lettered(b.text) + and b.bbox.height <= 1.5 * h + and b.layout_label + not in { + DocItemLabel.SECTION_HEADER, + DocItemLabel.PAGE_HEADER, + DocItemLabel.PAGE_FOOTER, + DocItemLabel.TITLE, + } + and not any( + b.bbox.intersection_area_with(v.bbox) > 0 + and b.bbox.y_overlap_with(v.bbox) >= 0.5 * min(b.bbox.height, v.bbox.height) + for v in values + ) + ] + + def adjacent(upper: BoundingBox, lower: BoundingBox) -> bool: + return ( + lower.t > upper.t + and -0.25 * h <= lower.t - upper.b <= 0.75 * h + and upper.x_overlap_with(lower) >= 0.5 * min(upper.width, lower.width) + and min(upper.height, lower.height) + >= 0.95 * max(upper.height, lower.height) + ) + + rows = [ + frozenset( + i + for i, v in enumerate(values) + if v.scope == b.scope + and b.bbox.y_overlap_with(v.bbox) >= 0.5 * min(b.bbox.height, v.bbox.height) + ) + for b in joinable + ] + below: dict[int, int] = {} + for i, upper in enumerate(joinable): + under = [ + j + for j, lower in enumerate(joinable) + if lower.scope == upper.scope and adjacent(upper.bbox, lower.bbox) + ] + if len(under) != 1: + continue + (j,) = under + u, lower = upper.bbox, joinable[j].bbox + over = [ + k + for k, other in enumerate(joinable) + if other.scope == upper.scope and adjacent(other.bbox, lower) + ] + if ( + over == [i] + and (abs(lower.l - u.l) <= h or abs(lower.l + lower.r - u.l - u.r) <= 2 * h) + and rows[i] <= rows[j] + ): + below[i] = j + keys = [] + for start in range(len(joinable)): + chain = [joinable[start]] + index = start + while index in below and len(chain) < 4: + index = below[index] + chain.append(joinable[index]) + keys.append(joined(chain)) + return keys diff --git a/docling/models/stages/form_field/keying/rules.py b/docling/models/stages/form_field/keying/rules.py new file mode 100644 index 0000000000..4db01e7fc6 --- /dev/null +++ b/docling/models/stages/form_field/keying/rules.py @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Printed rules of a page and the printed cell they draw around a value.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from docling_core.types.doc import BoundingBox + +# A painted shape at most this thick is a rule. +THIN = 3.5 +# Shorter marks are ticks, dots or glyph parts, not cell borders. +MIN_LENGTH = 5.0 +# Collinear pieces this far off the same line are one rule: many forms draw a +# border cell by cell. Along the line they may be apart by up to THIN, where a +# crossing rule interrupts them. +MERGE = 1.0 +# A rule this close to a value's edge is its own printed box, unless it runs +# past the value: then it is the border of the row or column the value sits in. +OUTLINE = 2.0 + + +@dataclass(frozen=True) +class Rule: + horizontal: bool + at: float # y of a horizontal rule, x of a vertical one (top-left points) + start: float + end: float + + +def rules_of(boxes: list[BoundingBox]) -> list[Rule]: + """Thin boxes as horizontal or vertical rules, collinear pieces joined. + + Pieces of one line join across gaps no wider than a rule, the joints where + crossing rules cut them. + """ + pieces = [] + for box in boxes: + if box.height <= THIN and box.width >= MIN_LENGTH: + pieces.append((True, (box.t + box.b) / 2, box.l, box.r)) + elif box.width <= THIN and box.height >= MIN_LENGTH: + pieces.append((False, (box.l + box.r) / 2, box.t, box.b)) + merged: list[list] = [] + for horizontal, at, start, end in sorted(pieces): + last = next( + ( + rule + for rule in reversed(merged) + if rule[0] == horizontal + and abs(rule[1] - at) <= MERGE + and start <= rule[3] + THIN + ), + None, + ) + if last is None: + merged.append([horizontal, at, start, end]) + else: + last[2], last[3] = min(last[2], start), max(last[3], end) + return [Rule(*rule) for rule in merged] + + +def printed_cell( + box: BoundingBox, rules: list[Rule], frame: BoundingBox +) -> BoundingBox | None: + """The cell printed around a value: the nearest rule on each side within frame. + + A rule along the value's own edge is its printed outline and bounds nothing, + unless it runs past the value. None when a side has no rule. + """ + middle_x, middle_y = (box.l + box.r) / 2, (box.t + box.b) / 2 + found: dict[str, tuple[float, float]] = {} + + def offer(side: str, distance: float, at: float) -> None: + if side not in found or distance < found[side][0]: + found[side] = (distance, at) + + for rule in rules: + if rule.horizontal: + if not ( + rule.start <= middle_x <= rule.end + and frame.t - OUTLINE <= rule.at <= frame.b + OUTLINE + ): + continue + longer = rule.start < box.l - OUTLINE or rule.end > box.r + OUTLINE + for side, edge, beyond in (("up", box.t, -1), ("down", box.b, 1)): + near = abs(rule.at - edge) <= OUTLINE + if (near and longer) or (not near and (rule.at - edge) * beyond > 0): + offer(side, abs(rule.at - edge), rule.at) + else: + if not ( + rule.start <= middle_y <= rule.end + and frame.l - OUTLINE <= rule.at <= frame.r + OUTLINE + ): + continue + longer = rule.start < box.t - OUTLINE or rule.end > box.b + OUTLINE + for side, edge, beyond in (("left", box.l, -1), ("right", box.r, 1)): + near = abs(rule.at - edge) <= OUTLINE + if (near and longer) or (not near and (rule.at - edge) * beyond > 0): + offer(side, abs(rule.at - edge), rule.at) + if len(found) < 4: + return None + return BoundingBox( + l=found["left"][1], t=found["up"][1], r=found["right"][1], b=found["down"][1] + ) diff --git a/docling/models/stages/form_field/keying/solver.py b/docling/models/stages/form_field/keying/solver.py new file mode 100644 index 0000000000..523c74b45e --- /dev/null +++ b/docling/models/stages/form_field/keying/solver.py @@ -0,0 +1,520 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Global choice of associations: pairwise penalties and the MILP.""" + +from __future__ import annotations + +import math +from collections import Counter, defaultdict +from collections.abc import Callable +from itertools import combinations + +import numpy as np +from docling_core.types.doc import BoundingBox, DocItemLabel, TableCell +from docling_core.types.doc.page import PdfWidget + +from docling.datamodel.base_models import Cluster +from docling.models.stages.form_field.keying.candidates import ( + candidates_for, + siblings, +) +from docling.models.stages.form_field.keying.geometry import ( + anchors, + lettered, + side_of, +) +from docling.models.stages.form_field.keying.inputs import ( + MAX_KEY_TEXTS, + inputs, + regions, +) +from docling.models.stages.form_field.keying.symmetry import ( + SIDE_COST, + mark_sequences, + sequences, + structures, +) +from docling.models.stages.form_field.keying.tables import add_hints, table_fields +from docling.models.stages.form_field.keying.types import ( + WIDGET_COVERAGE, + Assignment, + Candidate, + KeyText, + TableSlot, + Value, +) +from docling.utils.layout_postprocessor import UnionFind + +# Page size limits. Candidate construction scans every key against every +# value, and the pairwise penalties every pair of surviving candidates, both in +# Python; a page beyond these limits would hold the stage's thread for tens of +# seconds. Such a page keeps its table keys and leaves free-form values +# unkeyed, as a solver timeout does. +MAX_KEY_VALUE_PAIRS = 1_000_000 +MAX_ACTIVE_CANDIDATES = 1000 +# Cost of leaving a value without a key. +NULL_COST = 3.0 +# Seconds the MILP solver may spend on one page. +TIME_LIMIT = 10.0 + + +def crossing( + a: Candidate, b: Candidate, values: list[Value], key_texts: list[KeyText] +) -> bool: + if ( + a.kind == "choice_group" + or b.kind == "choice_group" + or set(a.members) & set(b.members) + ): + return False + + def turn( + p: tuple[float, float], q: tuple[float, float], r: tuple[float, float] + ) -> float: + return (q[0] - p[0]) * (r[1] - p[1]) - (q[1] - p[1]) * (r[0] - p[0]) + + for i in a.members: + for j in b.members: + p, q = anchors(key_texts[a.key].bbox, values[i].bbox) + r, s = anchors(key_texts[b.key].bbox, values[j].bbox) + if ( + turn(p, q, r) * turn(p, q, s) < -1e-9 + and turn(r, s, p) * turn(r, s, q) < -1e-9 + ): + return True + return False + + +def transition_penalty( + a: Candidate, b: Candidate, values: list[Value], key_texts: list[KeyText], h: float +) -> float: + """Compare key movement to the FIXED adjacent value movement in one lane. + + A two-column reset is not comparable within a lane and incurs no penalty. + This never proposes or chooses a different native value order. + """ + if a.kind == "choice_group" or b.kind == "choice_group": + return 0.0 + if a.members[0] > b.members[0]: + a, b = b, a + if a.members[-1] + 1 != b.members[0]: + return 0.0 + va, vb = values[a.members[-1]].bbox, values[b.members[0]].bbox + la, lb = key_texts[a.key].bbox, key_texts[b.key].bbox + if abs(va.t - vb.t) <= h: + return 3.0 if (vb.l - va.l) * (lb.l - la.l) < -(h * h) else 0.0 + if min(va.r, vb.r) > max(va.l, vb.l): + return 3.0 if (vb.t - va.t) * (lb.t - la.t) < -(h * h) else 0.0 + return 0.0 + + +def penalized_pairs( + active: list[int], + proposed: list[Candidate], + values: list[Value], + key_texts: list[KeyText], + h: float, +) -> list[tuple[int, int, float]]: + """Pairs of active columns with a crossing or transition penalty, in order. + + Gives the same pairs as scoring every pair, sorted the same way, without + scoring every pair. Every anchor segment lies in the box around its + candidate's key and values, so two candidates can only cross where those + boxes meet; a transition needs one candidate's first value right after + the other's last. Choice groups take neither penalty. + """ + scored = [ + column + for column, index in enumerate(active) + if proposed[index].kind != "choice_group" + ] + boxes = { + column: BoundingBox.enclosing_bbox( + [ + key_texts[proposed[active[column]].key].bbox, + *(values[i].bbox for i in proposed[active[column]].members), + ] + ) + for column in scored + } + pairs: set[tuple[int, int]] = set() + by_left = sorted(scored, key=lambda column: boxes[column].l) + for n, x in enumerate(by_left): + for y in by_left[n + 1 :]: + if boxes[y].l > boxes[x].r: + break + if boxes[y].t <= boxes[x].b and boxes[x].t <= boxes[y].b: + pairs.add((min(x, y), max(x, y))) + by_first: dict[int, list[int]] = defaultdict(list) + for column in scored: + by_first[proposed[active[column]].members[0]].append(column) + for x in scored: + for y in by_first.get(proposed[active[x]].members[-1] + 1, []): + if x != y: + pairs.add((min(x, y), max(x, y))) + found = [] + for a, b in sorted(pairs): + ca, cb = proposed[active[a]], proposed[active[b]] + if key_texts[ca.key].scope != key_texts[cb.key].scope: + continue + penalty = 3.0 * crossing(ca, cb, values, key_texts) + transition_penalty( + ca, cb, values, key_texts, h + ) + if penalty: + found.append((a, b, penalty)) + return found + + +def share_groups( + columns: list[int], + proposed: list[Candidate], + active: list[int], + values: list[Value], +) -> list[list[int]]: + """Partition the candidates using one text atom into groups that may co-own it. + + Single-value keys reaching the same key from the same side join a + group when their values are siblings (transitively, along the line). + Every other candidate is a group of its own. + """ + uf = UnionFind(columns) + for x, y in combinations(columns, 2): + cx, cy = proposed[active[x]], proposed[active[y]] + if ( + cx.side is not None + and cx.side == cy.side + and cx.key == cy.key + and siblings(values[cx.members[0]], values[cy.members[0]], cx.side) + ): + uf.union(x, y) + return list(uf.get_groups().values()) + + +def co_owners(a: Candidate, b: Candidate, values: list[Value]) -> bool: + """A checkbox and a text value of one row reading the same key. + + A row's flags and amounts form two lines under one row key, and a + printed cell's key keys both its text box (below) and its checkbox + (beside). The same key in a column keys both kinds from the same side. + """ + if a.side is None or b.side is None or a.key != b.key: + return False + x, y = values[a.members[0]].bbox, values[b.members[0]].bbox + if values[a.members[0]].checkbox == values[b.members[0]].checkbox: + return False + if x.y_overlap_with(y) >= 0.5 * min(x.height, y.height): + return True + return ( + a.side == b.side + and a.side in ("up", "down") + and x.x_overlap_with(y) >= 0.5 * min(x.width, y.width) + ) + + +def abstained( + values: list[Value], + key_texts: list[KeyText], + candidates: list[Candidate], + status: str, + slots: dict[int, TableSlot], +) -> Assignment: + """No free-form key on this page; the table keys and cells still hold.""" + fixed = [i for i, c in enumerate(candidates) if c.kind == "table_cell"] + return Assignment(values, key_texts, candidates, fixed, status, slots) + + +def _sparse(rows: list[dict[int, float]], columns: int): + """The constraint matrix, rows and columns in their construction order.""" + # SciPy is optional for converters that do not process PDFs. + from scipy.sparse import coo_matrix + + rr, cc, data = [], [], [] + for r, row in enumerate(rows): + for c, coefficient in row.items(): + rr.append(r) + cc.append(c) + data.append(coefficient) + return coo_matrix((data, (rr, cc)), shape=(len(rows), columns)).tocsc() + + +def _one_side_per_structure( + groups: list[list[int]], + facing: dict[int, list[int]], + proposed: list[Candidate], + active: list[int], + costs: list[float], + constraint: Callable[[dict[int, float], float, float], None], +) -> None: + """A repeated structure of like fields reads its keys from one side. + + The structure, along rows and columns at once, picks one side (one switch + per side seen, exactly one on), and each field reading another side pays + SIDE_COST. The side is not fixed in advance: the majority sets it. + """ + for group in groups: + by_side: dict[str, list[int]] = defaultdict(list) + for i in group: + for x in facing[i]: + by_side[str(proposed[active[x]].side)].append(x) + if len(by_side) < 2: + continue + switches = {} + for side in sorted(by_side): + switches[side] = len(costs) + costs.append(0.0) + constraint(dict.fromkeys(switches.values(), 1.0), 1, 1) + for side, columns in sorted(by_side.items()): + for x in columns: + constraint({x: 1, switches[side]: -1, len(costs): -1}, -math.inf, 0) + costs.append(SIDE_COST) + + +def assign( + widgets: list[PdfWidget], + clusters: list[Cluster], + table_cells: dict[int, list[TableCell]], + page_height: float, + rules: list[BoundingBox] | None = None, +) -> Assignment: + """Choose the key of every widget value on one page. + + ``clusters`` are the page's layout clusters and ``table_cells`` the + detected cells of each table cluster, by cluster id. ``rules`` are the + boxes of printed rules (top-left origin); they are only read for values in + detected tables. Nothing passed in is modified. + """ + found = regions(clusters) + read = inputs(widgets, found, table_cells, page_height) + + def keyed_in_tables() -> tuple[list[Candidate], dict[int, TableSlot]]: + return table_fields( + found, table_cells, rules or [], page_height, read.values, read.key_texts + ) + + assignment = _solve( + read.values, + read.key_texts, + read.h, + keyed_in_tables, + in_checkbox_blocks(read.values, read.key_texts, found, read.sources), + ) + assignment.sources = read.sources + assignment.painted_cells = read.painted_cells + return assignment + + +def in_checkbox_blocks( + values: list[Value], + key_texts: list[KeyText], + found: list[Cluster], + sources: dict[int, set[tuple[int, int]]], +) -> list[Candidate]: + """Checkboxes keyed by the text of the checkbox block drawn around them. + + The layout boxes a checkbox together with its label; a block holding + exactly one value, a checkbox outside detected tables, is keyed by all of + its text. The checkbox takes part in no other association, but the text + stays in the search, under the sharing rules of any key, so it can also + key a field of the same row. A block with several values, text the layout + also put in another block, or text without letters is left to the + geometric candidates. + """ + keyed = [] + for region in found: + if region.label not in { + DocItemLabel.CHECKBOX_SELECTED, + DocItemLabel.CHECKBOX_UNSELECTED, + }: + continue + # Held as a host paragraph holds its widgets: most of the value inside. + inside = [ + i + for i, v in enumerate(values) + if v.bbox.intersection_over_self(region.bbox) >= WIDGET_COVERAGE + ] + if len(inside) != 1: + continue + (i,) = inside + if not values[i].checkbox or values[i].scope.table is not None: + continue + atoms = { + atom + for atom, found_in in sources.items() + if any(cluster_id == region.id for cluster_id, _ in found_in) + } + if not atoms or any(len(sources[atom]) > 1 for atom in atoms): + continue + for k, key_text in enumerate(key_texts): + if ( + key_text.atoms != atoms + or key_text.stack + or key_text.scope != values[i].scope + or not lettered(key_text.text) + ): + continue + # The side lets the sharing rules apply, as to any single key. + facing = side_of(key_text.bbox, values[i].bbox) + keyed.append( + Candidate( + (i,), k, "field_key", {}, None if facing is None else facing[0] + ) + ) + once = Counter(c.members[0] for c in keyed) + return [c for c in keyed if once[c.members[0]] == 1] + + +def _solve( + values: list[Value], + key_texts: list[KeyText], + h: float, + keyed_in_tables: Callable[[], tuple[list[Candidate], dict[int, TableSlot]]], + in_blocks: list[Candidate], +) -> Assignment: + if ( + len(key_texts) > MAX_KEY_TEXTS + or len(values) * len(key_texts) > MAX_KEY_VALUE_PAIRS + ): + tables, slots = keyed_in_tables() + return abstained( + values, + key_texts, + tables, + f"skipped: {len(values)} values x {len(key_texts)} key texts exceed the page limit", + slots, + ) + # A checkbox keyed by its block takes part in no other association. + taken = {c.members[0] for c in in_blocks} + found = candidates_for(values, key_texts, h) + proposed = [c for c in found if taken.isdisjoint(c.members)] + # A decided alternating run fixes which key each of its values reads. + mark_sequences(proposed, sequences(values, key_texts, proposed, NULL_COST, h)) + # Added once the runs are decided: a run's side does not apply to a block. + proposed += in_blocks + # Values in detected tables are keyed from their own cells, outside the + # free-form search: table text never competes with free-form keys. + tables, slots = keyed_in_tables() + fixed = list(range(len(proposed), len(proposed) + len(tables))) + # A candidate worse than leaving all its members blank cannot help: all + # remaining interactions are penalties. Keep the full list for diagnostics. + active = [ + i + for i, c in enumerate(proposed) + if c.cost < (0 if c.kind == "choice_group" else NULL_COST * len(c.members)) + ] + if len(active) > MAX_ACTIVE_CANDIDATES: + return abstained( + values, + key_texts, + proposed + tables, + f"skipped: {len(active)} candidates exceed the page limit", + slots, + ) + costs = [proposed[i].cost for i in active] + rows: list[dict[int, float]] = [] + lower, upper = [], [] + + def constraint(coefficients: dict[int, float], lo: float, hi: float) -> None: + rows.append(coefficients) + lower.append(lo) + upper.append(hi) + + owners: dict[int, dict[int, float]] = defaultdict(dict) + prompts: dict[int, dict[int, float]] = defaultdict(dict) + spans: dict[int, dict[int, float]] = defaultdict(dict) + for column, index in enumerate(active): + candidate = proposed[index] + for i in candidate.members: + (prompts if candidate.kind == "choice_group" else owners)[i][column] = 1.0 + for atom in key_texts[candidate.key].atoms: + spans[atom][column] = 1.0 + for i, value in enumerate(values): + if not value.scope.eligible: + continue + # A checkbox keyed by its block has no blank option: its block's text + # is its only key. + if i not in taken: + owners[i][len(costs)] = 1.0 + costs.append(NULL_COST) + constraint(owners[i], 1, 1) + for coefficients in prompts.values(): + constraint(coefficients, 0, 1) + # Each text atom keys one line of sibling values, so each shareable group + # gets one switch. A row's flags and amounts form two lines under the same + # key; any other pair of groups excludes each other. + for columns in spans.values(): + groups = share_groups(sorted(columns), proposed, active, values) + switches = [] + for group in groups: + if len(group) == 1: + switches.append(group[0]) + continue + switches.append(len(costs)) + costs.append(0.0) + for column in group: + constraint({column: 1, switches[-1]: -1}, -math.inf, 0) + for (ga, sa), (gb, sb) in combinations(zip(groups, switches), 2): + first, second = proposed[active[ga[0]]], proposed[active[gb[0]]] + if not co_owners(first, second, values): + constraint({sa: 1, sb: 1}, 0, 1) + for a, b, penalty in penalized_pairs(active, proposed, values, key_texts, h): + auxiliary = len(costs) + costs.append(penalty) + constraint({a: 1, b: 1, auxiliary: -1}, -math.inf, 1) + constraint({auxiliary: 1, a: -1}, -math.inf, 0) + constraint({auxiliary: 1, b: -1}, -math.inf, 0) + # Aligned like-sized neighbours in the native order usually read their + # keys from the same side: a soft cost, never a rule. It stays below + # any cell or alignment difference, so it only settles close calls. + facing: dict[int, list[int]] = defaultdict(list) + for column, index in enumerate(active): + if proposed[index].side is not None: + facing[proposed[index].members[0]].append(column) + for i in range(len(values) - 1): + a, b = values[i], values[i + 1] + if not (siblings(a, b, "left") or siblings(a, b, "up")): + continue + for x in facing[i]: + for y in facing[i + 1]: + if proposed[active[x]].side != proposed[active[y]].side: + constraint({x: 1, y: 1, len(costs): -1}, -math.inf, 1) + costs.append(0.5) + _one_side_per_structure( + structures(values, key_texts, proposed, NULL_COST), + facing, + proposed, + active, + costs, + constraint, + ) + if not costs: + return Assignment(values, key_texts, proposed + tables, fixed, "optimal", slots) + # Load the optional solver only when a page needs it. + from scipy.optimize import Bounds, LinearConstraint, milp + + matrix = _sparse(rows, len(costs)) + result = milp( + np.array(costs), + integrality=np.ones(len(costs)), + bounds=Bounds(0, 1), + constraints=LinearConstraint(matrix, lower, upper), + options={"time_limit": TIME_LIMIT, "mip_rel_gap": 0.0}, + ) + if result.status != 0: + # Abstain on timeout: no unsupported confidence claims about a partial + # solution and no missing native values. + return abstained( + values, key_texts, proposed + tables, str(result.message), slots + ) + selected = [index for column, index in enumerate(active) if result.x[column] > 0.5] + # A hint never moves a key: a boxed checkbox's header comes from its + # geometric candidates, even though they took no part in the search. + add_hints([proposed[i] for i in selected], found + in_blocks, values, NULL_COST) + return Assignment( + values, + key_texts, + proposed + tables, + selected + fixed, + "optimal", + slots, + ) diff --git a/docling/models/stages/form_field/keying/symmetry.py b/docling/models/stages/form_field/keying/symmetry.py new file mode 100644 index 0000000000..1faf0bd869 --- /dev/null +++ b/docling/models/stages/form_field/keying/symmetry.py @@ -0,0 +1,311 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Symmetry of repeated fields: alternating runs and one side per structure. + +Both rules read the geometry of like fields (same kind, similar size) laid out +along rows and columns, and both only add costs: strong local evidence can +still win. The weights are set from the other costs, not fitted to data. +""" + +from __future__ import annotations + +import math +from collections import defaultdict +from dataclasses import dataclass +from itertools import pairwise +from typing import Literal + +from docling_core.types.doc import BoundingBox + +from docling.models.stages.form_field.keying.candidates import siblings +from docling.models.stages.form_field.keying.geometry import gap, lettered +from docling.models.stages.form_field.keying.types import ( + Candidate, + KeyText, + Side, + Value, +) +from docling.utils.layout_postprocessor import UnionFind + +# Above the usual gap between a key before and one after a value (a cell +# difference, 1) and below leaving a value blank (3): a decided run turns +# close calls, and never forces a value to stay unkeyed. +SEQUENCE_COST = 1.5 +# One cell difference: a single field breaking a repeated structure's side +# must have clearly better local evidence than its siblings. +SIDE_COST = 1.0 + +Axis = Literal["row", "column"] +# The side that faces the next field along the axis, then the previous one. +FACING: dict[Axis, tuple[Side, Side]] = { + "row": ("right", "left"), + "column": ("down", "up"), +} + + +@dataclass(frozen=True) +class Reach: + """The first text a value sees on one side: its atoms and nearest gap.""" + + atoms: frozenset[int] + gap: float + + +def reaches( + candidates: list[Candidate], + key_texts: list[KeyText], + values: list[Value], + null_cost: float, +) -> dict[tuple[int, Side], Reach]: + """What every value sees on each side, from its plausible single keys. + + Text too costly to key a value on its own (a page header, a code) is not a + key, so it never shapes a run or a structure. + """ + atoms: dict[tuple[int, Side], set[int]] = defaultdict(set) + gaps: dict[tuple[int, Side], float] = {} + for c in candidates: + if ( + c.side is None + or c.side == "inside" + or len(c.members) != 1 + or c.cost >= null_cost + ): + continue + key = (c.members[0], c.side) + key_text = key_texts[c.key] + atoms[key] |= key_text.atoms + distance = gap(key_text.bbox, values[c.members[0]].bbox) + gaps[key] = min(gaps.get(key, distance), distance) + return {key: Reach(frozenset(atoms[key]), gaps[key]) for key in atoms} + + +def _between(a: BoundingBox, b: BoundingBox, axis: Axis) -> BoundingBox: + """The corridor between two aligned boxes, a before b along the axis.""" + if axis == "row": + return BoundingBox(l=a.r, t=max(a.t, b.t), r=b.l, b=min(a.b, b.b)) + return BoundingBox(l=max(a.l, b.l), t=a.b, r=min(a.r, b.r), b=b.t) + + +def neighbours(values: list[Value], axis: Axis) -> dict[int, int]: + """Each value's next like field along a row or column, when mutual. + + The next field is the nearest sibling after it on the axis with no other + field in between; a pair counts only when each is the other's nearest. + """ + side: Side = "left" if axis == "row" else "up" + + def start(v: Value) -> float: + return v.bbox.l if axis == "row" else v.bbox.t + + def end(v: Value) -> float: + return v.bbox.r if axis == "row" else v.bbox.b + + def nearest(i: int, forward: bool) -> int | None: + found = [ + j + for j, v in enumerate(values) + if j != i + and v.scope == values[i].scope + and siblings(values[i], v, side) + and ( + start(v) >= end(values[i]) - 1e-6 + if forward + else end(v) <= start(values[i]) + 1e-6 + ) + ] + if not found: + return None + j = ( + min(found, key=lambda k: (start(values[k]), k)) + if forward + else max(found, key=lambda k: (end(values[k]), -k)) + ) + a, b = (i, j) if forward else (j, i) + corridor = _between(values[a].bbox, values[b].bbox, axis) + if any( + k not in (a, b) and v.bbox.intersection_area_with(corridor) > 0 + for k, v in enumerate(values) + ): + return None + return j + + pairs = {} + for i, v in enumerate(values): + if not v.scope.eligible: + continue + j = nearest(i, True) + if j is not None and nearest(j, False) == i: + pairs[i] = j + return pairs + + +def _texts_between( + a: Value, b: Value, axis: Axis, key_texts: list[KeyText] +) -> frozenset[int]: + """Lettered text atoms whose centre lies in the corridor between a and b.""" + corridor = _between(a.bbox, b.bbox, axis) + found = set() + for key_text in key_texts: + if len(key_text.atoms) != 1 or not lettered(key_text.text): + continue + x = (key_text.bbox.l + key_text.bbox.r) / 2 + y = (key_text.bbox.t + key_text.bbox.b) / 2 + if corridor.l <= x <= corridor.r and corridor.t <= y <= corridor.b: + found |= key_text.atoms + return frozenset(found) + + +def _leftover( + run: list[int], + reach: dict[tuple[int, Side], Reach], + axis: Axis, + values: list[Value], + h: float, +) -> Side | None: + """The key side of a run with text at both ends, when two signs agree. + + One text is left over: a shared prompt before the fields, or the start of the + next field after them. Neither sign alone settles it, so both must agree: + the inner texts all sitting nearer the field before them (keys after) + or after them (keys before), and the layout convention of the fields. + Along a row, options read the key after them and text boxes the one + before; down a column, text boxes read the key above and options have + no convention. An inner text within half a text line of both its fields + touches both, and says nothing about which one it keys. + """ + after, before = FACING[axis] + pairs = list(pairwise(run)) + if any( + reach[i, after].gap <= 0.5 * h and reach[j, before].gap <= 0.5 * h + for i, j in pairs + ): + return None + if all(reach[i, after].gap < reach[j, before].gap for i, j in pairs): + lean = after + elif all(reach[i, after].gap > reach[j, before].gap for i, j in pairs): + lean = before + else: + return None + checkbox = values[run[0]].checkbox + if axis == "row": + convention: Side | None = after if checkbox else before + else: + convention = None if checkbox else before + return lean if lean == convention else None + + +def sequences( + values: list[Value], + key_texts: list[KeyText], + candidates: list[Candidate], + null_cost: float, + h: float, +) -> dict[int, Side]: + """The side each value of a decided alternating run reads its key from. + + A run is a line of like fields with exactly one text between each two: the + text after one field is the text before the next. It is a key line + only when every field's best key lies along it: the run settles which + text of the line each field reads, never which axis (an option key + beside a checkbox in a column, or a line key far to the left of an + amount, pairs the field across the run). The ends fix the pairing: a key before + the first field and none after the last means keys before (left, + above); the reverse means keys after. With text at both ends one text + is left over (see _leftover). + """ + reach = reaches(candidates, key_texts, values, null_cost) + best: dict[int, float] = {} + best_on: dict[tuple[int, Side], float] = {} + for c in candidates: + if c.side is not None and len(c.members) == 1 and c.cost < null_cost: + i = c.members[0] + best[i] = min(best.get(i, c.cost), c.cost) + best_on[i, c.side] = min(best_on.get((i, c.side), c.cost), c.cost) + decided: dict[int, Side] = {} + conflicts: set[int] = set() + for axis in ("row", "column"): + after, before = FACING[axis] + pairs = neighbours(values, axis) + links = {} + for i, j in pairs.items(): + forward, backward = reach.get((i, after)), reach.get((j, before)) + if forward is None or backward is None: + continue + shared = forward.atoms & backward.atoms + between = _texts_between(values[i], values[j], axis, key_texts) + if shared and between <= shared: + links[i] = j + for first in sorted(set(links) - set(links.values())): + run = [first] + while run[-1] in links: + run.append(links[run[-1]]) + if any( + min(best_on.get((i, side), math.inf) for side in (after, before)) + > best[i] + for i in run + ): + continue + lead, trail = (run[0], before) in reach, (run[-1], after) in reach + if lead and not trail: + side = before + elif trail and not lead: + side = after + elif lead and trail and _leftover(run, reach, axis, values, h): + side = _leftover(run, reach, axis, values, h) + else: + continue + for i in run: + if decided.get(i, side) != side: + conflicts.add(i) + decided[i] = side + return {i: side for i, side in decided.items() if i not in conflicts} + + +def mark_sequences(candidates: list[Candidate], sides: dict[int, Side]) -> None: + """Charge every association of a run's value but its key on the run's side.""" + for c in candidates: + if c.kind == "choice_group": + continue + off = sum( + 1 + for i in c.members + if i in sides and not (len(c.members) == 1 and c.side == sides[i]) + ) + if off: + c.features["sequence"] = SEQUENCE_COST * off + + +def structures( + values: list[Value], + key_texts: list[KeyText], + candidates: list[Candidate], + null_cost: float, +) -> list[list[int]]: + """Groups of at least three like fields repeated along rows and columns. + + Two neighbouring like fields belong together when the only text between + them is what they themselves see (their own keys): a third text in + between starts another part of the form. + """ + reach = reaches(candidates, key_texts, values, null_cost) + uf = UnionFind(range(len(values))) + + for axis in ("row", "column"): + after, before = FACING[axis] + for i, j in neighbours(values, axis).items(): + own = frozenset() + if (i, after) in reach: + own |= reach[i, after].atoms + if (j, before) in reach: + own |= reach[j, before].atoms + if _texts_between(values[i], values[j], axis, key_texts) <= own: + uf.union(i, j) + # neighbours() only pairs values of one scope, so a group shares its + # first member's scope. + return [ + group + for group in uf.get_groups().values() + if len(group) >= 3 and values[group[0]].scope.eligible + ] diff --git a/docling/models/stages/form_field/keying/tables.py b/docling/models/stages/form_field/keying/tables.py new file mode 100644 index 0000000000..0e0aeb157c --- /dev/null +++ b/docling/models/stages/form_field/keying/tables.py @@ -0,0 +1,250 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Field keys from detected table cells, and hints for value grids.""" + +from __future__ import annotations + +from collections import defaultdict + +from docling_core.types.doc import BoundingBox, TableCell + +from docling.datamodel.base_models import Cluster +from docling.models.stages.form_field.keying.candidates import siblings +from docling.models.stages.form_field.keying.geometry import ( + band_index, + gap, + lettered, + span_overlap, +) +from docling.models.stages.form_field.keying.rules import printed_cell, rules_of +from docling.models.stages.form_field.keying.types import ( + Candidate, + KeyText, + Scope, + TableSlot, + Value, +) +from docling.utils.form_fields import without_whitespace + + +def table_fields( + found: list[Cluster], + table_cells: dict[int, list[TableCell]], + rule_boxes: list[BoundingBox], + page_height: float, + values: list[Value], + key_texts: list[KeyText], +) -> tuple[list[Candidate], dict[int, TableSlot]]: + """The grid cell of each value inside a detected table, and its key. + + A value is keyed only by lettered text of its own cell. A row or + column header elsewhere in the table never keys a value: the table's + headers already carry that association. Cell text is used whole; codes + and units without letters never key a value, and neither does cell text + that repeats the value itself (a filled field rendered into the page). + Appends the cell key texts it uses to ``key_texts``. + + A value inside one detected cell sits in that cell. Otherwise it sits in + the row and column whose bands it overlaps most; a band is the extent of + the single-span cells of that row or column, because detected cell boxes + cover their text rather than the printed cell. A value between bands + (text at the top of its printed cell, the box below) takes the nearest + band only when every row, or every column, of the grid has one: a missing + band would hand its values to a neighbour, so such a value gets no cell, + and no key. + + Printed rules correct the bands where the detected grid differs from the + printed table. When rules close a printed cell around the value and that + cell holds detected text, the value belongs with that text: if the bands + point elsewhere, the value moves to the cell of the text in its printed + cell (a key before a code), and its key follows. When the printed cell + holds no text and misses the band of the value's column, the detected grid + lost that column: the value gets no cell. Without a closed printed cell the + bands decide. + """ + fields: list[Candidate] = [] + slots: dict[int, TableSlot] = {} + rules = rules_of(rule_boxes) + frames = {region.id: region.bbox for region in found} + for table_id, detected in sorted(table_cells.items()): + for i, v in enumerate(values): + if v.scope.table == table_id and v.scope.cell is not None: + cell = detected[v.scope.cell] + slots[i] = TableSlot( + table_id, + (cell.start_row_offset_idx, cell.end_row_offset_idx), + (cell.start_col_offset_idx, cell.end_col_offset_idx), + ) + members = [ + i + for i, v in enumerate(values) + if v.scope.table == table_id and not v.scope.eligible + ] + cells = [ + (cell, cell.bbox.to_top_left_origin(page_height)) + for cell in detected + if cell.bbox is not None + ] + rows: dict[int, list[BoundingBox]] = defaultdict(list) + columns: dict[int, list[BoundingBox]] = defaultdict(list) + for cell, box in cells: + if cell.end_row_offset_idx - cell.start_row_offset_idx == 1: + rows[cell.start_row_offset_idx].append(box) + if cell.end_col_offset_idx - cell.start_col_offset_idx == 1: + columns[cell.start_col_offset_idx].append(box) + if not members or not rows or not columns: + continue + row_bands = { + r: (min(b.t for b in bs), max(b.b for b in bs)) for r, bs in rows.items() + } + column_bands = { + c: (min(b.l for b in bs), max(b.r for b in bs)) for c, bs in columns.items() + } + grid_rows = max(cell.end_row_offset_idx for cell in detected) + grid_columns = max(cell.end_col_offset_idx for cell in detected) + + def at(row: int, column: int) -> list[int]: + return [ + k + for k, (cell, _) in enumerate(cells) + if cell.start_row_offset_idx <= row < cell.end_row_offset_idx + and cell.start_col_offset_idx <= column < cell.end_col_offset_idx + ] + + indices: dict[int, int] = {} + + def key_of(k: int) -> int: + if k not in indices: + cell, box = cells[k] + indices[k] = len(key_texts) + key_texts.append( + KeyText( + " ".join(cell.text.split()), + box, + frozenset(), + Scope(table_id), + None, + ) + ) + return indices[k] + + for i in members: + box = values[i].bbox + row = band_index(box.t, box.b, row_bands) + column = band_index(box.l, box.r, column_bands) + printed = ( + None + if not rules or table_id not in frames + else printed_cell(box, rules, frames[table_id]) + ) + by_print = lost_column = False + if printed is not None: + held = [ + k + for k, (cell, cell_box) in enumerate(cells) + if cell.text.strip() + and cell_box.intersection_over_self(printed) >= 0.5 + ] + if held and not set(held) & set(at(row, column)): + k = min( + held, + key=lambda k: ( + not lettered(cells[k][0].text), + gap(cells[k][1], box), + k, + ), + ) + row = cells[k][0].start_row_offset_idx + column = cells[k][0].start_col_offset_idx + by_print = True + elif not held: + lost_column = not any( + printed.x_overlap_with(b) > 0 for b in columns[column] + ) + home = at(row, column) + own = next( + ( + k + for k in home + if lettered(cells[k][0].text) + and without_whitespace(cells[k][0].text) + != without_whitespace(values[i].native.widget_text or "") + ), + None, + ) + # A cell found through its printed text needs no band check; the + # text's column may be one no single-span cell lies in. + placed = by_print or ( + ( + span_overlap(box.t, box.b, *row_bands[row]) > 0 + or len(row_bands) == grid_rows + ) + and ( + span_overlap(box.l, box.r, *column_bands[column]) > 0 + or len(column_bands) == grid_columns + ) + ) + if placed and not lost_column: + spans = cells[own if own is not None else home[0]][0] if home else None + slots[i] = TableSlot( + table_id, + (row, row + 1) + if spans is None + else (spans.start_row_offset_idx, spans.end_row_offset_idx), + (column, column + 1) + if spans is None + else (spans.start_col_offset_idx, spans.end_col_offset_idx), + None if own is None else key_of(own), + ) + if own is not None: + fields.append(Candidate((i,), key_of(own), "table_cell", {})) + return fields, slots + + +def add_hints( + chosen: list[Candidate], + proposed: list[Candidate], + values: list[Value], + null_cost: float, +) -> None: + """Keep an aligned header across the other axis as a field hint in value grids. + + A value with like-sized siblings in both its row and its column sits in a + grid: its field key comes from one axis, and the cheapest aligned text along + the other axis (a column header over a row-keyed amount) is kept as + a hint. A header heads a line of values, so a sibling along that line + must see it too; the key of the previous option in a list does not + qualify, nor does text too costly to key a value on its own (page + headers). A hint is informative only; it never changes the key. + """ + grid = { + i + for i, v in enumerate(values) + if any(siblings(v, u, "left") for u in values if u is not v) + and any(siblings(v, u, "up") for u in values if u is not v) + } + across = {"left": ("up", "down"), "right": ("up", "down")} + for c in chosen: + if c.side is None or c.members[0] not in grid: + continue + sides = across.get(c.side, ("left", "right")) + value = values[c.members[0]] + options = [ + o + for o in proposed + if o.members == c.members + and o.side in sides + and not o.features.get("misaligned") + and o.cost < null_cost + and o.key != c.key + and any( + other.key == o.key + and other.side == o.side + and other.members != c.members + and siblings(values[other.members[0]], value, o.side) + for other in proposed + ) + ] + if options: + c.hint = min(options, key=lambda o: (o.cost, o.key)).key diff --git a/docling/models/stages/form_field/keying/types.py b/docling/models/stages/form_field/keying/types.py new file mode 100644 index 0000000000..ea76d70a6b --- /dev/null +++ b/docling/models/stages/form_field/keying/types.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Internal data of the AcroForm keying.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + +from docling_core.types.doc import BoundingBox, DocItemLabel +from docling_core.types.doc.page import PdfWidget + +# How much of a widget a text container or FORM region must cover to hold it. +# This tolerates imperfect text bounds; detected table/cell ownership still +# requires complete containment in scope_of(). +WIDGET_COVERAGE = 0.8 +# Field flag bit 17 of a /Btn widget (PDF 32000-1, table 226): a push button. +PUSHBUTTON_FLAG = 1 << 16 + +Side = Literal["inside", "up", "down", "left", "right"] + + +def is_skipped(widget: PdfWidget, bbox: BoundingBox) -> bool: + """Widgets that carry no field value for the document. + + A widget of zero height or width is an artifact (Well-Tagged PDF 1.0, + 8.9.2.4.13). Push buttons trigger actions and hold no value. + """ + if bbox.width <= 0 or bbox.height <= 0: + return True + return widget.widget_field_type == "/Btn" and bool( + widget.widget_field_flags & PUSHBUTTON_FLAG + ) + + +@dataclass(frozen=True) +class Scope: + table: int | None = None + cell: int | None = None + + @property + def eligible(self) -> bool: + return self.table is None or self.cell is not None + + +@dataclass +class Value: + native: PdfWidget + bbox: BoundingBox + scope: Scope + + @property + def checkbox(self) -> bool: + return self.native.widget_field_type == "/Btn" + + +@dataclass +class KeyText: + """Printed text considered as a field key or hint.""" + + text: str + bbox: BoundingBox + atoms: frozenset[int] + scope: Scope + layout_label: ( + DocItemLabel | None + ) # Layout label of the source block; None for table cells. + fragment: bool = False + stack: bool = False # Joined from vertically adjacent layout blocks. + + +@dataclass +class Candidate: + members: tuple[int, ...] # Positions in the supplied value list, not y order. + key: int # Index into Assignment.key_texts. + kind: Literal[ + "field_key", + "option_key", + "composite_field", + "inline_clause", + "choice_group", + "table_cell", + ] + features: dict[str, float] + side: Side | None = None # For single-value keys: where the key faces it. + hint: int | None = None # Index of extra context, e.g. a column header. + + @property + def cost(self) -> float: + return sum(self.features.values()) + + +@dataclass(frozen=True) +class TableSlot: + """The grid cell of a detected table that a value sits in.""" + + table: int # Layout id of the table region. + rows: tuple[int, int] # Start and end row offsets of the cell. + columns: tuple[int, int] # Start and end column offsets of the cell. + key: int | None = None # Index of lettered key text in the same cell, if any. + + +@dataclass +class Inputs: + """One page's keying inputs, as keying.inputs reads them.""" + + values: list[Value] + key_texts: list[KeyText] + h: float # Median height of the page's text lines. + # Text atom -> the (cluster id, cell index) pairs it was read from. + sources: dict[int, set[tuple[int, int]]] + # Source cells recognized as painted widget values, separate from key atoms. + painted_cells: set[tuple[int, int]] + + +@dataclass +class Assignment: + values: list[Value] + key_texts: list[KeyText] + candidates: list[Candidate] + selected: list[int] + solver_status: str + # Positions in the value list -> their cell, for values in detected tables. + slots: dict[int, TableSlot] = field(default_factory=dict) + # Text atom -> the (cluster id, cell index) pairs it was read from; more + # than one cluster when the layout put the cell in two clusters. + sources: dict[int, set[tuple[int, int]]] = field(default_factory=dict) + # Source cells recognized as painted widget values, separate from key atoms. + painted_cells: set[tuple[int, int]] = field(default_factory=set) diff --git a/docling/models/stages/page_assemble/page_assemble_model.py b/docling/models/stages/page_assemble/page_assemble_model.py index a7bd223f63..b139d68c47 100644 --- a/docling/models/stages/page_assemble/page_assemble_model.py +++ b/docling/models/stages/page_assemble/page_assemble_model.py @@ -8,13 +8,16 @@ from typing import Dict, List, Optional, Union import numpy as np -from docling_core.types.doc import BoundingBox +from docling_core.types.doc import BoundingBox, DocItemLabel from pydantic import AnyUrl, BaseModel, ValidationError from docling.datamodel.base_models import ( AssembledUnit, + Cluster, ContainerElement, + FieldRegionElement, FigureElement, + FormFieldPrediction, Page, PageElement, Table, @@ -193,6 +196,7 @@ def __call__( elements: List[PageElement] = [] headers: List[PageElement] = [] body: List[PageElement] = [] + form_fields = page.predictions.form_fields or FormFieldPrediction() for cluster in page.predictions.layout.clusters: # _log.info("Cluster label seen:", cluster.label) @@ -204,6 +208,11 @@ def __call__( ] text = self.sanitize_text(textlines) hyperlink = self._match_hyperlink(cluster.bbox, page) + # A paragraph that inlines AcroForm widgets carries a + # single keyed field item; attach it so it materializes + # as a field_item in place (see TextElement.field_item) + # instead of being pulled into a separate field_region. + field_item = form_fields.inline_items.get(cluster.id) text_el = TextElement( label=cluster.label, id=cluster.id, @@ -211,6 +220,7 @@ def __call__( hyperlink=hyperlink, page_no=page.page_no, cluster=cluster, + field_item=field_item, ) elements.append(text_el) @@ -234,7 +244,6 @@ def __call__( cluster=cluster, page_no=page.page_no, ) - elements.append(tbl) body.append(tbl) elif cluster.label == FIGURE_LABEL: @@ -255,12 +264,22 @@ def __call__( elements.append(fig) body.append(fig) elif cluster.label in CONTAINER_LABELS: - container_el = ContainerElement( - label=cluster.label, - id=cluster.id, - page_no=page.page_no, - cluster=cluster, - ) + field_items = form_fields.form_items.get(cluster.id) + if field_items is None: + container_el = ContainerElement( + label=cluster.label, + id=cluster.id, + page_no=page.page_no, + cluster=cluster, + ) + else: + container_el = FieldRegionElement( + label=DocItemLabel.FIELD_REGION, + id=cluster.id, + page_no=page.page_no, + cluster=cluster, + items=field_items, + ) elements.append(container_el) body.append(container_el) diff --git a/docling/models/stages/reading_order/readingorder_model.py b/docling/models/stages/reading_order/readingorder_model.py index 84dcb4c447..51479b1631 100644 --- a/docling/models/stages/reading_order/readingorder_model.py +++ b/docling/models/stages/reading_order/readingorder_model.py @@ -3,15 +3,19 @@ import logging import time +from collections import defaultdict from collections.abc import Iterator +from itertools import takewhile from pathlib import Path from statistics import median from docling_core.types.doc import ( + BoundingBox, CodeItem, DocItemLabel, DoclingDocument, DocumentOrigin, + FieldRegionItem, GroupItem, GroupLabel, NodeItem, @@ -33,9 +37,13 @@ DoclingComponentType, ErrorItem, FailureCategory, + FieldItemPrediction, + FieldRegionElement, + FieldValuePrediction, FigureElement, PageElement, Table, + TableFieldPrediction, TextElement, ) from docling.datamodel.document import ConversionResult @@ -48,6 +56,7 @@ SeparatorElement, build_page_separators, ) +from docling.utils.form_fields import text_slots, without_whitespace from docling.utils.profiling import ProfilingScope, TimeRecorder _log = logging.getLogger(__name__) @@ -174,6 +183,14 @@ def _add_table_children( doc=out_doc, page_height=out_doc.pages[element.page_no].size.height, ) + if element.field_cells: + self._add_rich_table_fields( + table_item=table_item, + field_cells=element.field_cells, + doc=out_doc, + page_no=element.page_no, + page_height=out_doc.pages[element.page_no].size.height, + ) if element.num_rows == 0 and element.num_cols == 0 and element.cluster.children: rich_cell_ref = self._create_rich_cell_group(element, out_doc, table_item) rich_cell = RichTableCell( @@ -283,6 +300,78 @@ def _table_data_from_table(element: Table) -> TableData: def _element_ref(element: BasePageElement) -> str: return f"#/{element.page_no}/{element.cluster.id}" + @staticmethod + def _interleave_fields( + siblings: list[ReadingOrderPageElement], + fields: dict[int, list[FieldItemPrediction]], + id_to_elem: dict[str, PageElement], + ) -> list[ReadingOrderPageElement | tuple[int, list[FieldItemPrediction]]]: + """The siblings with each page's fields placed among them by geometry. + + The siblings keep the model's reading order and the fields their native + widget order; text_slots only chooses how the two interleave, from the + boxes of the siblings and of each field's key and values. A run of + fields comes as one (page number, items) entry. Fields after a page's + last sibling come before the next page's siblings. + """ + + def box(item: FieldItemPrediction) -> BoundingBox: + return BoundingBox.enclosing_bbox( + [ + bbox + for bbox in (item.key_bbox, *(v.bbox for v in item.values)) + if bbox is not None + ] + ) + + before: dict[int, tuple[int, list[FieldItemPrediction]]] = {} + trailing: dict[int, list[FieldItemPrediction]] = {} + for page_no, items in fields.items(): + on_page = [ + rel for rel in siblings if id_to_elem[rel.ref.cref].page_no == page_no + ] + slots = text_slots( + [id_to_elem[rel.ref.cref].cluster.bbox for rel in on_page], + [box(item) for item in items], + ) + placed = 0 + for rel, slot in zip(on_page, slots): + if slot > placed: + before[rel.cid] = (page_no, items[placed:slot]) + placed = slot + trailing[page_no] = items[placed:] + pending = sorted(page_no for page_no, items in trailing.items() if items) + plan: list[ReadingOrderPageElement | tuple[int, list[FieldItemPrediction]]] = [] + for rel in siblings: + while pending and pending[0] < id_to_elem[rel.ref.cref].page_no: + page_no = pending.pop(0) + plan.append((page_no, trailing[page_no])) + if rel.cid in before: + plan.append(before[rel.cid]) + plan.append(rel) + plan.extend((page_no, trailing[page_no]) for page_no in pending) + return plan + + @staticmethod + def _merges_without_hosts( + merges: dict[int, list[int]], host_cids: set[int] + ) -> dict[int, list[int]]: + """Drop the merges that join a paragraph hosting form widgets. + + A host becomes a field item, which takes no merged text, and a merged + element is not emitted on its own: a host source would lose its + continuation, a host target its widgets. A chain stops before its first + host, so the host and what follows it stay separate paragraphs. + """ + kept: dict[int, list[int]] = {} + for source, targets in merges.items(): + if source in host_cids: + continue + chain = list(takewhile(lambda cid: cid not in host_cids, targets)) + if chain: + kept[source] = chain + return kept + @classmethod def _match_table_pictures( cls, @@ -426,6 +515,103 @@ def _add_rich_table_pictures( ref=group.get_ref(), ) + def _add_rich_table_fields( + self, + *, + table_item: TableItem, + field_cells: list[TableFieldPrediction], + doc: DoclingDocument, + page_no: int, + page_height: float, + ) -> None: + """Put AcroForm field items inside the table cells that hold their widgets. + + Each cell becomes a rich cell pointing to a field region with the cell's + items. TableFormer emits no cell for an empty grid position, so such a + cell is added. Printed cell text that is neither a key nor a filled value + of those items stays, as text ahead of them; a cell already made rich by + a picture takes the field region into its group instead. + """ + + for cell_fields in field_cells: + index = next( + ( + i + for i, cell in enumerate(table_item.data.table_cells) + if cell.start_row_offset_idx == cell_fields.start_row_offset_idx + and cell.start_col_offset_idx == cell_fields.start_col_offset_idx + ), + None, + ) + existing = None if index is None else table_item.data.table_cells[index] + values_bbox = BoundingBox.enclosing_bbox( + [value.bbox for item in cell_fields.items for value in item.values] + ) + parent: NodeItem = table_item + if isinstance(existing, RichTableCell): + parent = existing.ref.resolve(doc) + region = doc.add_field_region( + prov=ProvenanceItem( + page_no=page_no, + charspan=(0, 0), + bbox=values_bbox.to_bottom_left_origin(page_height), + ), + parent=parent, + ) + shown = { + without_whitespace(text) + for item in cell_fields.items + for text in (item.key_text, *(value.text for value in item.values)) + } + if ( + existing is not None + and not isinstance(existing, RichTableCell) + and existing.text.strip() + and without_whitespace(existing.text) not in shown + ): + doc.add_text( + label=DocItemLabel.TEXT, + text=existing.text, + prov=None + if existing.bbox is None + else ProvenanceItem( + page_no=page_no, + charspan=(0, len(existing.text)), + bbox=existing.bbox.to_bottom_left_origin(page_height), + ), + parent=region, + ) + for item in cell_fields.items: + self._add_field_item( + item, + out_doc=doc, + parent=region, + page_no=page_no, + page_height=page_height, + ) + if isinstance(existing, RichTableCell): + continue + if existing is None: + rich = RichTableCell( + text="", + bbox=values_bbox, + row_span=cell_fields.end_row_offset_idx + - cell_fields.start_row_offset_idx, + col_span=cell_fields.end_col_offset_idx + - cell_fields.start_col_offset_idx, + start_row_offset_idx=cell_fields.start_row_offset_idx, + end_row_offset_idx=cell_fields.end_row_offset_idx, + start_col_offset_idx=cell_fields.start_col_offset_idx, + end_col_offset_idx=cell_fields.end_col_offset_idx, + ref=region.get_ref(), + ) + doc.add_table_cell(table_item=table_item, cell=rich) + else: + assert index is not None + table_item.data.table_cells[index] = RichTableCell( + **existing.model_dump(exclude={"ref"}), ref=region.get_ref() + ) + def _readingorder_elements_to_docling_doc( self, conv_res: ConversionResult, @@ -487,17 +673,77 @@ def _readingorder_elements_to_docling_doc( rel.cid for rel in ro_elements if rel.ref.cref in rich_picture_refs ) page_no_to_pages = {p.page_no: p for p in conv_res.pages} + # Fields that no assembled block carries (no FORM region, no host + # paragraph) are not in the model's input: they come by page and are + # placed between the blocks by geometry. + loose: dict[int, list[FieldItemPrediction]] = defaultdict(list) + for page in conv_res.pages: + form_fields = page.predictions.form_fields + if form_fields is None: + continue + placed = [ + *((cid, [item]) for cid, item in form_fields.inline_items.items()), + *form_fields.form_items.items(), + ] + for cluster_id, items in placed: + source = id_to_elem.get(f"#/{page.page_no}/{cluster_id}") + if not ( + isinstance(source, FieldRegionElement) + or ( + isinstance(source, TextElement) + and source.field_item is not None + ) + ): + loose[page.page_no].extend(items) + if form_fields.loose_items: + loose[page.page_no].extend(form_fields.loose_items) + + def add_fields( + page_no: int, items: list[FieldItemPrediction], parent: NodeItem | None + ) -> None: + """A run of fields: under a field region as they are, else in one.""" + page_height = page_no_to_pages[page_no].size.height # type: ignore + if not isinstance(parent, FieldRegionItem): + parent = out_doc.add_field_region( + prov=ProvenanceItem( + page_no=page_no, + charspan=(0, 0), + bbox=BoundingBox.enclosing_bbox( + [value.bbox for item in items for value in item.values] + ).to_bottom_left_origin(page_height), + ), + parent=parent, + ) + for item in items: + self._add_field_item( + item, + out_doc=out_doc, + parent=parent, + page_no=page_no, + page_height=page_height, + ) def materialize_siblings( - parent_ref: str | None, parent: NodeItem | None + parent_ref: str | None, + parent: NodeItem | None, + fields: dict[int, list[FieldItemPrediction]] | None = None, ) -> None: + """The siblings in reading order, with each page's fields among them.""" split_lists_on_non_list = ( isinstance(parent, GroupItem) and parent.label == GroupLabel.FORM_AREA - ) + ) or isinstance(parent, FieldRegionItem) current_list: GroupItem | None = None - for rel in ordered_siblings[parent_ref]: - if rel.cid in skippable_cids: + siblings = [ + rel + for rel in ordered_siblings[parent_ref] + if rel.cid not in skippable_cids + ] + for entry in self._interleave_fields(siblings, fields or {}, id_to_elem): + if isinstance(entry, tuple): + add_fields(*entry, parent) + current_list = None # a field ends a list continue + rel = entry element = id_to_elem[rel.ref.cref] page_height = page_no_to_pages[element.page_no].size.height # type: ignore if split_lists_on_non_list and not ( @@ -528,11 +774,12 @@ def materialize_siblings( page_height=page_height, ) - for merged_cid in el_merges_mapping.get(rel.cid, []): - merged_elem = id_to_elem[cid_to_rels[merged_cid].ref.cref] - self._merge_elements( - element, merged_elem, new_item, page_height - ) + if element.field_item is None: + for merged_cid in el_merges_mapping.get(rel.cid, []): + merged_elem = id_to_elem[cid_to_rels[merged_cid].ref.cref] + self._merge_elements( + element, merged_elem, new_item, page_height + ) elif isinstance(element, Table): table_item = self._add_table_element(element, out_doc, parent) @@ -565,6 +812,21 @@ def materialize_siblings( ) self._add_child_elements(element, picture_item, out_doc) + elif isinstance(element, FieldRegionElement): + field_region = out_doc.add_field_region( + prov=ProvenanceItem( + page_no=element.page_no, + charspan=(0, 0), + bbox=element.cluster.bbox.to_bottom_left_origin( + page_height + ), + ), + parent=parent, + ) + materialize_siblings( + rel.ref.cref, field_region, {element.page_no: element.items} + ) + elif isinstance(element, ContainerElement): group_label = ( GroupLabel.FORM_AREA @@ -574,7 +836,7 @@ def materialize_siblings( container_item = out_doc.add_group(label=group_label, parent=parent) materialize_siblings(rel.ref.cref, container_item) - materialize_siblings(None, None) + materialize_siblings(None, None, loose) return out_doc @@ -595,6 +857,84 @@ def _add_caption_or_footnote(self, elem, out_doc, parent, page_height): ) return new_item + def _add_field_item( + self, + item: FieldItemPrediction, + *, + out_doc: DoclingDocument, + parent: NodeItem, + page_no: int, + page_height: float, + ) -> None: + field_item = out_doc.add_field_item(parent=parent) + if item.key_text and item.key_bbox is not None: + out_doc.add_field_key( + text=item.key_text, + prov=ProvenanceItem( + page_no=page_no, + charspan=(0, len(item.key_text)), + bbox=item.key_bbox.to_bottom_left_origin(page_height), + ), + parent=field_item, + ) + self._add_field_values( + item, + out_doc=out_doc, + parent=field_item, + page_no=page_no, + page_height=page_height, + ) + + def _add_field_values( + self, + item: FieldItemPrediction, + *, + out_doc: DoclingDocument, + parent: NodeItem, + page_no: int, + page_height: float, + ) -> None: + """The item's values, then its field hint.""" + for value in item.values: + self._add_field_value( + value, + out_doc=out_doc, + parent=parent, + page_no=page_no, + page_height=page_height, + ) + if item.hint_text: + out_doc.add_field_hint(text=item.hint_text, parent=parent) + + def _add_field_value( + self, + value: FieldValuePrediction, + *, + out_doc: DoclingDocument, + parent: NodeItem, + page_no: int, + page_height: float, + ) -> NodeItem: + prov = ProvenanceItem( + page_no=page_no, + charspan=(0, len(value.text)), + bbox=value.bbox.to_bottom_left_origin(page_height), + ) + field_value = out_doc.add_field_value( + text=value.text, + orig=value.orig, + prov=prov, + parent=parent, + kind="fillable", + ) + if value.checkbox is not None: + # State rides on a nested checkbox child; the value text is empty so + # the token inlines into . No prov on the child: it shares the + # value's rect, and the serializer flattens child locations into + # , so a duplicate prov would emit the same location twice. + out_doc.add_text(label=value.checkbox, text="", parent=field_value) + return field_value + def _handle_text_element( self, element: TextElement, @@ -603,7 +943,7 @@ def _handle_text_element( current_list: GroupItem | None, parent: NodeItem | None, page_height: float, - ) -> tuple[TextItem, GroupItem | None]: + ) -> tuple[NodeItem, GroupItem | None]: cap_text = element.text prov = ProvenanceItem( @@ -612,6 +952,52 @@ def _handle_text_element( bbox=element.cluster.bbox.to_bottom_left_origin(page_height), ) label = element.label + if element.field_item is not None: + # The paragraph inlines widgets: emit a field_item in this paragraph's + # own place -- in a list item of its list when it is a list item, else + # under the current parent -- with its text as the key and the widgets + # as values. No separate field_region: the item stays where the + # paragraph was. The list item keeps the list made of list items; it + # carries no text or location of its own, the key does. + if label == DocItemLabel.LIST_ITEM: + if current_list is None: + current_list = out_doc.add_group( + label=GroupLabel.LIST, name="list", parent=parent + ) + item_parent: NodeItem | None = out_doc.add_list_item( + text="", enumerated=False, parent=current_list + ) + else: + current_list = None + item_parent = parent + field_item = out_doc.add_field_item(parent=item_parent) + # A checkbox block also boxes the drawn checkbox, which is where the + # value is: its key is located by its text alone. + key_bbox = element.cluster.bbox + if ( + label + in {DocItemLabel.CHECKBOX_SELECTED, DocItemLabel.CHECKBOX_UNSELECTED} + and element.field_item.key_bbox is not None + ): + key_bbox = element.field_item.key_bbox + out_doc.add_field_key( + text=cap_text, + prov=ProvenanceItem( + page_no=element.page_no, + charspan=(0, len(cap_text)), + bbox=key_bbox.to_bottom_left_origin(page_height), + ), + parent=field_item, + hyperlink=element.hyperlink, + ) + self._add_field_values( + element.field_item, + out_doc=out_doc, + parent=field_item, + page_no=element.page_no, + page_height=page_height, + ) + return field_item, current_list if label == DocItemLabel.LIST_ITEM: if current_list is None: current_list = out_doc.add_group( @@ -822,6 +1208,15 @@ def iterate_leaves( el_merges_mapping.update( self.ro_model.predict_merges(sorted_elements=siblings) ) + host_cids = { + element.cid + for element in page_elements + if isinstance( + assembled := assembled_by_ref.get(element.ref.cref), TextElement + ) + and assembled.field_item is not None + } + el_merges_mapping = self._merges_without_hosts(el_merges_mapping, host_cids) docling_doc: DoclingDocument = self._readingorder_elements_to_docling_doc( conv_res, diff --git a/docling/pipeline/standard_pdf_pipeline.py b/docling/pipeline/standard_pdf_pipeline.py index 6ab682fe42..f5480a3b2a 100644 --- a/docling/pipeline/standard_pdf_pipeline.py +++ b/docling/pipeline/standard_pdf_pipeline.py @@ -68,6 +68,7 @@ from docling.models.stages.code_formula.code_formula_vlm_model import ( CodeFormulaVlmModel, ) +from docling.models.stages.form_field.form_field_model import PdfFormFieldModel from docling.models.stages.heading_hierarchy.heading_hierarchy_model import ( HeadingHierarchyModel, ) @@ -102,6 +103,7 @@ "ocr": FailureCategory.INFERENCE_FAILURE, "layout": FailureCategory.INFERENCE_FAILURE, "table": FailureCategory.INFERENCE_FAILURE, + "form_field": FailureCategory.INFERENCE_FAILURE, "assemble": FailureCategory.INFERENCE_FAILURE, } @@ -667,6 +669,9 @@ def _init_models(self) -> None: run_postprocessor=self.layout_model.requires_layout_postprocessing, ) ) + self.form_field_model = PdfFormFieldModel( + enabled=self.pipeline_options.extract_form_fields + ) table_factory = get_table_structure_factory( allow_external_plugins=self.pipeline_options.allow_external_plugins @@ -848,11 +853,28 @@ def _create_run_ctx(self) -> RunContext: preprocess.add_output_queue(layout.input_queue) # PDF parsing layout.add_output_queue(ocr.input_queue) # Layout prediction ocr.add_output_queue(layout_postprocess.input_queue) # OCR - layout_postprocess.add_output_queue(table.input_queue) # Layout post-processing - table.add_output_queue(assemble.input_queue) # Table model + layout_postprocess.add_output_queue(table.input_queue) # Layout post-proc + stages = [preprocess, ocr, layout, layout_postprocess, table] + if self.pipeline_options.extract_form_fields: + # Form fields key widgets from table cells, so they follow the table + # model. When disabled, the stage is left out: tables feed assembly. + form_field = ThreadedPipelineStage( + name="form_field", + model=self.form_field_model, + batch_size=1, + batch_timeout=opts.batch_polling_interval_seconds, + queue_max_size=opts.queue_max_size, + shutdown_timeout=opts.stage_shutdown_timeout_seconds, + timed_out_run_ids=timed_out_run_ids, + ) + table.add_output_queue(form_field.input_queue) # Table model + form_field.add_output_queue(assemble.input_queue) # Form fields + stages.append(form_field) + else: + table.add_output_queue(assemble.input_queue) # Table model assemble.add_output_queue(output_q) # Assembly + stages.append(assemble) - stages = [preprocess, ocr, layout, layout_postprocess, table, assemble] return RunContext( stages=stages, first_stage=preprocess, diff --git a/docling/utils/form_fields.py b/docling/utils/form_fields.py new file mode 100644 index 0000000000..19b57967cf --- /dev/null +++ b/docling/utils/form_fields.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Helpers shared by the form-field stage and the reading order.""" + +from collections.abc import Sequence + +from docling_core.types.doc import BoundingBox, CoordOrigin + + +def without_whitespace(text: str) -> str: + """The text with all whitespace removed. + + Printed text equals a widget's painted value when both agree without + whitespace: the layout may split or join the painted characters. + """ + return "".join(text.split()) + + +def reading_order(a: BoundingBox, b: BoundingBox) -> int: + """-1 when ``a`` reads before ``b``, 1 when after, 0 when geometry can't tell. + + Boxes on one line (sharing half the smaller height) read left to right; + boxes in one column (sharing any width) read top to bottom. Boxes that + share neither, as across two columns, set no order. + """ + if a.y_overlap_with(b) >= 0.5 * min(a.height, b.height): + first, second = a.l + a.r, b.l + b.r + elif a.x_overlap_with(b) > 0: + first, second = a.t + a.b, b.t + b.b + if a.coord_origin == CoordOrigin.BOTTOMLEFT: + first, second = -first, -second # up is larger there + else: + return 0 + return -1 if first < second else 1 if first > second else 0 + + +def text_slots( + texts: Sequence[BoundingBox], fields: Sequence[BoundingBox] +) -> list[int]: + """For each text, how many fields precede it once both are interleaved. + + The texts keep their order, as the reading order gives it, and so do the + fields, in their native widget order; only how they interleave is chosen. + The slots minimize the text-field pairs read against reading_order(). On a + tie a text goes as late as it can: a field that geometry leaves free stays + with the fields before it. + """ + count = len(fields) + previous: list[int] = [] + back: list[list[int]] = [] + for text in texts: + order = [reading_order(text, field) for field in fields] + # Placed after k fields: the first k must not read after the text, + # the others must not read before it. + cost = [0] * (count + 1) + late = sum(o > 0 for o in order) + early = 0 + for k in range(count + 1): + cost[k] = early + late + if k < count: + early += order[k] < 0 + late -= order[k] > 0 + if not previous: + back.append(list(range(count + 1))) + previous = cost + continue + best, arg = previous[0], 0 + total, links = [], [] + for k in range(count + 1): + if previous[k] <= best: + best, arg = previous[k], k + total.append(cost[k] + best) + links.append(arg) + back.append(links) + previous = total + if not previous: + return [] + slot = max(range(count + 1), key=lambda k: (-previous[k], k)) + slots = [slot] + for links in reversed(back[1:]): + slots.append(links[slots[-1]]) + return slots[::-1] diff --git a/docling/utils/layout_postprocessor.py b/docling/utils/layout_postprocessor.py index 1735885a40..26bf427337 100644 --- a/docling/utils/layout_postprocessor.py +++ b/docling/utils/layout_postprocessor.py @@ -710,8 +710,13 @@ def _adjust_cluster_bboxes(self, clusters: list[Cluster]) -> list[Cluster]: b=max(cell.rect.to_bounding_box().b for cell in cluster.cells), ) - if cluster.label == DocItemLabel.TABLE: - # For tables, take union of current bbox and cells bbox + if cluster.label in { + DocItemLabel.TABLE, + DocItemLabel.CHECKBOX_SELECTED, + DocItemLabel.CHECKBOX_UNSELECTED, + }: + # For tables, take union of current bbox and cells bbox; a + # checkbox keeps its drawn box, which holds no text cell cluster.bbox = BoundingBox( l=min(cluster.bbox.l, cells_bbox.l), t=min(cluster.bbox.t, cells_bbox.t), diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 24518869d9..278da31751 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -53,6 +53,7 @@ docling convert [OPTIONS] source | `--vlm-model` | `text` | `granite_docling` | Choose the VLM preset to use with PDF or image files. Available presets: smoldocling, granite_docling, deepseek_ocr, granite_vision, pixtral, got_ocr, phi4, qwen, nanonets_ocr2, nemotron_parse_v2, mineru2_pro, gemma_12b, gemma_27b, dolphin, glm_ocr, lightonocr, falcon_ocr, chandra_ocr2, unlimited_ocr, dots_ocr, dots_mocr | | `--vlm-max-new-tokens` | `integer` | | Override max_new_tokens for VLM conversion generation. | | `--vlm-write-native-output` | `boolean` | `false` | Write each page's unparsed VLM response under `/.vlm-native/`. | +| `--extract-form-fields` / `--no-extract-form-fields` | flag | `false` | If enabled, native PDF form widgets (AcroForm) become fillable field values with field keys from printed text. Standard and legacy pipelines with the docling-parse backend only. | | `--asr-model` | `whisper_tiny`, `whisper_small`, `whisper_medium`, `whisper_base`, `whisper_large`, `whisper_turbo`, `whisper_tiny_mlx`, `whisper_small_mlx`, `whisper_medium_mlx`, `whisper_base_mlx`, `whisper_large_mlx`, `whisper_turbo_mlx`, `whisper_tiny_native`, `whisper_small_native`, `whisper_medium_native`, `whisper_base_native`, `whisper_large_native`, `whisper_turbo_native`, `whisper_tiny_en_native`, `whisper_base_en_native`, `whisper_small_en_native`, `whisper_medium_en_native`, `whisper_distil_small_en_native`, `whisper_distil_medium_en_native`, `whisper_distil_large_v3_native`, `whisper_distil_large_v3_5_native`, `whisper_tiny_s2t`, `whisper_tiny_en_s2t`, `whisper_base_s2t`, `whisper_base_en_s2t`, `whisper_small_s2t`, `whisper_small_en_s2t`, `whisper_distil_small_en_s2t`, `whisper_medium_s2t`, `whisper_medium_en_s2t`, `whisper_distil_medium_en_s2t`, `whisper_large_v3_s2t`, `whisper_distil_large_v3_s2t`, `whisper_distil_large_v3_5_s2t`, `whisper_large_v3_turbo_s2t` | `whisper_tiny` | Choose the ASR model to use with audio/video files. | | `--video-sampling-mode` | `fixed`, `scene` | `fixed` | frame sampling mode. | | `--video-frame-interval` | `float` | `10.0` | Seconds between frames in fixed interval mode. | diff --git a/docs/usage/advanced_options.md b/docs/usage/advanced_options.md index b144660249..83e59dd0b2 100644 --- a/docs/usage/advanced_options.md +++ b/docs/usage/advanced_options.md @@ -136,6 +136,40 @@ The options in this list require the explicit `enable_remote_services=True` when The example file [custom_convert.py](../examples/custom_convert.py) contains multiple ways one can adjust the conversion pipeline and features. +### Extract native PDF form fields + +Set `PdfPipelineOptions(extract_form_fields=True)` (CLI: `--extract-form-fields`) +to convert native PDF widgets (AcroForm) into format-neutral `FieldItem` objects +with fillable `FieldValueItem` children. Each field key comes from printed text +associated with its widget by page geometry. The key can be text inside or +beside the widget's cell, or a paragraph that contains the widget. Checkbox and +radio option text becomes an option key. A shared prompt introduces several +options and stays as ordinary text. Additional context becomes a field hint. +A widget inside a detected table goes into its table cell instead (a rich table +cell holding the field). Its key comes only from text printed in that same +cell; the table's row and column headers already carry the association. +Text used as a field key leaves the body text; a section header or formula +used as a key stays inside the key, as a field heading or a formula. PDF field +names are not used as keys. + +Fields keep the native order of their widgets. Page geometry places them +among the text printed around them: the reading-order model orders the text +only, never the widgets. Fields outside a detected form region go in field +regions of their own between the page's other elements, one region per run +of consecutive fields. + +Here, **key** means text that names one or more field values, and **hint** means +extra context. **Layout label** means an element class (`DocItemLabel`). +**Caption** means a document caption associated with a picture, table, code +block, or form; it does not mean a field key or hint. + +The docling-parse backend supplies the widgets. Scanned or flattened forms, +backends without page widgets and full-page OCR produce no field items; table +cells need `do_table_structure=True`. Raw widget metadata is not stored in the +`DoclingDocument`; keep parsed pages with `generate_parsed_pages=True` when +that PDF-specific data is needed. Field items are not supported by the DocTags +export. + ### Image resolution and scale Page coordinates use 72 points per inch. For image inputs, embedded DPI metadata diff --git a/pyproject.toml b/pyproject.toml index 1f7e34c0ca..8459e5726b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ requires-python = '>=3.10,<4.0' # MINIMAL BASE (8 packages) - ~50MB dependencies = [ 'pydantic>=2.0.0,<3.0.0', - 'docling-core>=2.98.0,<3.0.0', + 'docling-core>=2.99.0,<3.0.0', 'pydantic-settings>=2.3.0,<3.0.0', 'filetype>=1.2.0,<2.0.0', 'requests>=2.32.2,<3.0.0', @@ -80,7 +80,7 @@ convert-core = [ 'numpy>=1.24.0,<3.0.0', 'pillow>=10.0.0,<13.0.0', 'rtree>=1.3.0,<2.0.0', - 'scipy>=1.6.0,<2.0.0', + 'scipy>=1.9.0,<2.0.0', ] extract-core = [ @@ -97,7 +97,7 @@ format-pdf-pypdfium2 = [ 'pypdfium2>=4.30.0,!=4.30.1,<6.0.0', ] format-pdf-docling = [ - 'docling-parse>=7.22.1,<8.0.0', + 'docling-parse>=7.22.2,<8.0.0', ] format-pdf = [ @@ -282,7 +282,7 @@ models-vlm-inline = [ feat-chunking = [ # >=2.94.1 relaxed its transformers pin; older releases cap transformers<5.9.0 # on macOS, which conflicts with the mlx-vlm floor in models-vlm-inline. - 'docling-core[chunking]>=2.98.0,<3.0.0', + 'docling-core[chunking]>=2.99.0,<3.0.0', ] service-client = [ @@ -334,7 +334,7 @@ typecheck = [ ] pr-fast-checks = [ { include-group = "typecheck" }, - "docling-core[chunking] (>=2.98.0,<3.0.0)", + "docling-core[chunking] (>=2.99.0,<3.0.0)", "pydantic (>=2.0.0,<3.0.0)", "pydantic-settings (>=2.3.0,<3.0.0)", "ruff==0.15.12", diff --git a/tach.toml b/tach.toml index ac3fb75e02..adcbc8ebed 100644 --- a/tach.toml +++ b/tach.toml @@ -320,6 +320,11 @@ depends_on = [ "docling.utils.ocr_language", ] +[[modules]] +path = "docling.models.stages.form_field" +layer = "models" +depends_on = ["docling.datamodel", "docling.models", "docling.utils"] + [[modules]] path = "docling.models.stages.page_assemble" layer = "models" @@ -526,6 +531,7 @@ depends_on = [ "docling.datamodel", "docling.models.factories", "docling.models.stages.code_formula", + "docling.models.stages.form_field", "docling.models.stages.heading_hierarchy", "docling.models.stages.page_assemble", "docling.models.stages.page_preprocessing", diff --git a/tests/data/pdf/groundtruth/acroform_sample.json b/tests/data/pdf/groundtruth/acroform_sample.json new file mode 100644 index 0000000000..5b293c2f4c --- /dev/null +++ b/tests/data/pdf/groundtruth/acroform_sample.json @@ -0,0 +1,453 @@ +{ + "schema_name": "DoclingDocument", + "version": "1.10.0", + "name": "acroform_sample", + "origin": { + "mimetype": "application/pdf", + "binary_hash": 17759113462838234588, + "filename": "acroform_sample.pdf" + }, + "furniture": { + "self_ref": "#/furniture", + "children": [], + "content_layer": "furniture", + "name": "_root_", + "label": "unspecified" + }, + "body": { + "self_ref": "#/body", + "children": [ + { + "$ref": "#/field_regions/0" + }, + { + "$ref": "#/texts/11" + } + ], + "content_layer": "body", + "name": "_root_", + "label": "unspecified" + }, + "groups": [], + "texts": [ + { + "self_ref": "#/texts/0", + "parent": { + "$ref": "#/field_items/0" + }, + "children": [], + "content_layer": "body", + "label": "field_key", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 72.0, + "t": 703.62, + "r": 128.02, + "b": 692.52, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 10 + ] + } + ], + "orig": "Full name:", + "text": "Full name:" + }, + { + "self_ref": "#/texts/1", + "parent": { + "$ref": "#/field_items/0" + }, + "children": [], + "content_layer": "body", + "label": "field_value", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 150.0, + "t": 710.0, + "r": 400.0, + "b": 690.0, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 12 + ] + } + ], + "orig": "Ada Lovelace", + "text": "Ada Lovelace", + "kind": "fillable" + }, + { + "self_ref": "#/texts/2", + "parent": { + "$ref": "#/field_items/1" + }, + "children": [], + "content_layer": "body", + "label": "field_key", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 95.0, + "t": 651.62, + "r": 199.05, + "b": 640.52, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 20 + ] + } + ], + "orig": "I agree to the terms", + "text": "I agree to the terms" + }, + { + "self_ref": "#/texts/3", + "parent": { + "$ref": "#/field_items/1" + }, + "children": [ + { + "$ref": "#/texts/4" + } + ], + "content_layer": "body", + "label": "field_value", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 72.0, + "t": 652.0, + "r": 86.0, + "b": 638.0, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ], + "orig": "/Yes", + "text": "", + "kind": "fillable" + }, + { + "self_ref": "#/texts/4", + "parent": { + "$ref": "#/texts/3" + }, + "children": [], + "content_layer": "body", + "label": "checkbox_selected", + "prov": [], + "orig": "", + "text": "" + }, + { + "self_ref": "#/texts/5", + "parent": { + "$ref": "#/field_items/2" + }, + "children": [], + "content_layer": "body", + "label": "field_key", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 95.0, + "t": 621.62, + "r": 219.72, + "b": 610.52, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 23 + ] + } + ], + "orig": "Subscribe to newsletter", + "text": "Subscribe to newsletter" + }, + { + "self_ref": "#/texts/6", + "parent": { + "$ref": "#/field_items/2" + }, + "children": [ + { + "$ref": "#/texts/7" + } + ], + "content_layer": "body", + "label": "field_value", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 72.0, + "t": 622.0, + "r": 86.0, + "b": 608.0, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ], + "orig": "/Off", + "text": "", + "kind": "fillable" + }, + { + "self_ref": "#/texts/7", + "parent": { + "$ref": "#/texts/6" + }, + "children": [], + "content_layer": "body", + "label": "checkbox_unselected", + "prov": [], + "orig": "", + "text": "" + }, + { + "self_ref": "#/texts/8", + "parent": { + "$ref": "#/field_items/3" + }, + "children": [], + "content_layer": "body", + "label": "field_key", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 95.0, + "t": 591.62, + "r": 197.7, + "b": 580.52, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 19 + ] + } + ], + "orig": "Monthly mail digest", + "text": "Monthly mail digest" + }, + { + "self_ref": "#/texts/9", + "parent": { + "$ref": "#/field_items/3" + }, + "children": [ + { + "$ref": "#/texts/10" + } + ], + "content_layer": "body", + "label": "field_value", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 72.0, + "t": 592.0, + "r": 86.0, + "b": 578.0, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ], + "orig": "/On", + "text": "", + "kind": "fillable" + }, + { + "self_ref": "#/texts/10", + "parent": { + "$ref": "#/texts/9" + }, + "children": [], + "content_layer": "body", + "label": "checkbox_selected", + "prov": [], + "orig": "", + "text": "" + }, + { + "self_ref": "#/texts/11", + "parent": { + "$ref": "#/body" + }, + "children": [], + "content_layer": "body", + "label": "text", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 72.0, + "t": 571.62, + "r": 218.06, + "b": 560.52, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 27 + ] + } + ], + "orig": "See example.org for details", + "text": "See example.org for details", + "hyperlink": "https://example.org/" + } + ], + "pictures": [], + "tables": [], + "key_value_items": [], + "form_items": [], + "field_regions": [ + { + "self_ref": "#/field_regions/0", + "parent": { + "$ref": "#/body" + }, + "children": [ + { + "$ref": "#/field_items/0" + }, + { + "$ref": "#/field_items/1" + }, + { + "$ref": "#/field_items/2" + }, + { + "$ref": "#/field_items/3" + } + ], + "content_layer": "body", + "label": "field_region", + "prov": [ + { + "page_no": 1, + "bbox": { + "l": 72.0, + "t": 710.0, + "r": 400.0, + "b": 578.0, + "coord_origin": "BOTTOMLEFT" + }, + "charspan": [ + 0, + 0 + ] + } + ] + } + ], + "field_items": [ + { + "self_ref": "#/field_items/0", + "parent": { + "$ref": "#/field_regions/0" + }, + "children": [ + { + "$ref": "#/texts/0" + }, + { + "$ref": "#/texts/1" + } + ], + "content_layer": "body", + "label": "field_item", + "prov": [] + }, + { + "self_ref": "#/field_items/1", + "parent": { + "$ref": "#/field_regions/0" + }, + "children": [ + { + "$ref": "#/texts/2" + }, + { + "$ref": "#/texts/3" + } + ], + "content_layer": "body", + "label": "field_item", + "prov": [] + }, + { + "self_ref": "#/field_items/2", + "parent": { + "$ref": "#/field_regions/0" + }, + "children": [ + { + "$ref": "#/texts/5" + }, + { + "$ref": "#/texts/6" + } + ], + "content_layer": "body", + "label": "field_item", + "prov": [] + }, + { + "self_ref": "#/field_items/3", + "parent": { + "$ref": "#/field_regions/0" + }, + "children": [ + { + "$ref": "#/texts/8" + }, + { + "$ref": "#/texts/9" + } + ], + "content_layer": "body", + "label": "field_item", + "prov": [] + } + ], + "pages": { + "1": { + "size": { + "width": 612.0, + "height": 792.0 + }, + "page_no": 1 + } + } +} \ No newline at end of file diff --git a/tests/data/pdf/groundtruth/acroform_sample.md b/tests/data/pdf/groundtruth/acroform_sample.md new file mode 100644 index 0000000000..2b5dacb337 --- /dev/null +++ b/tests/data/pdf/groundtruth/acroform_sample.md @@ -0,0 +1,17 @@ +Full name: + +Ada Lovelace + +I agree to the terms + +- [x] + +Subscribe to newsletter + +- [ ] + +Monthly mail digest + +- [x] + +[See example.org for details](https://example.org/) \ No newline at end of file diff --git a/tests/data/pdf/groundtruth/acroform_sample.pages.meta.json b/tests/data/pdf/groundtruth/acroform_sample.pages.meta.json new file mode 100644 index 0000000000..6a656b12ad --- /dev/null +++ b/tests/data/pdf/groundtruth/acroform_sample.pages.meta.json @@ -0,0 +1,5 @@ +[ + { + "num_cells": 5 + } +] \ No newline at end of file diff --git a/tests/data/pdf/groundtruth/right_to_left_03.doctags.txt b/tests/data/pdf/groundtruth/right_to_left_03.doctags.txt index aeca92630e..8a4f276c7f 100644 --- a/tests/data/pdf/groundtruth/right_to_left_03.doctags.txt +++ b/tests/data/pdf/groundtruth/right_to_left_03.doctags.txt @@ -3,10 +3,10 @@ 2-5 - استاندارد ک الا شمشه و شمشال توليد شده به روش ريخته گری پيوسته مورد مصرف در فولادهای سازه ای - مطابق آناليز پيوست 20300 -بلی -خير +بلی +خير سازمان ملی استاندارد ايران -بلی خير +بلی خير نام استاندارد شماره استاندارد ملی diff --git a/tests/data/pdf/groundtruth/right_to_left_03.json b/tests/data/pdf/groundtruth/right_to_left_03.json index 1d9b2af72f..87a3f95917 100644 --- a/tests/data/pdf/groundtruth/right_to_left_03.json +++ b/tests/data/pdf/groundtruth/right_to_left_03.json @@ -255,8 +255,8 @@ "page_no": 1, "bbox": { "l": 235.13, - "t": 604.58, - "r": 247.41, + "t": 606.18, + "r": 267.81, "b": 590.59, "coord_origin": "BOTTOMLEFT" }, @@ -282,8 +282,8 @@ "page_no": 1, "bbox": { "l": 197.33, - "t": 604.58, - "r": 211.66, + "t": 605.36, + "r": 228.06, "b": 590.59, "coord_origin": "BOTTOMLEFT" }, @@ -336,7 +336,7 @@ "page_no": 1, "bbox": { "l": 197.33, - "t": 559.22, + "t": 560.13, "r": 247.35, "b": 545.23, "coord_origin": "BOTTOMLEFT" diff --git a/tests/data/pdf/sources/acroform_sample.pdf b/tests/data/pdf/sources/acroform_sample.pdf new file mode 100644 index 0000000000..1c1e5d1c7a Binary files /dev/null and b/tests/data/pdf/sources/acroform_sample.pdf differ diff --git a/tests/test_acroform_keying.py b/tests/test_acroform_keying.py new file mode 100644 index 0000000000..52f74f9a49 --- /dev/null +++ b/tests/test_acroform_keying.py @@ -0,0 +1,832 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Behavioral checks of the AcroForm keying on small synthetic pages.""" + +import random +from dataclasses import dataclass, field, replace +from itertools import combinations + +import pytest +from docling_core.types.doc import BoundingBox, DocItemLabel, TableCell +from docling_core.types.doc.page import BoundingRectangle, PdfWidget, TextCell + +from docling.datamodel.base_models import Cluster +from docling.models.stages.form_field.keying import ( + Assignment, + Scope, + assign, + regions, + scope_of, + solver, +) +from docling.models.stages.form_field.keying.inputs import inputs + + +@dataclass +class KeyingPage: + """The arguments of ``assign`` for one synthetic page.""" + + widgets: list[PdfWidget] + layout: list[Cluster] + tables: dict[int, list[TableCell]] = field(default_factory=dict) + height: float = 400 + rules: list[BoundingBox] = field(default_factory=list) + + +def box(left: float, top: float, right: float, bottom: float) -> BoundingBox: + return BoundingBox(l=left, t=top, r=right, b=bottom) + + +def widget( + index: int, bbox: BoundingBox, *, checkbox: bool = False, name: str | None = None +) -> PdfWidget: + return PdfWidget( + index=index, + rect=BoundingRectangle.from_bounding_box(bbox), + widget_field_type="/Btn" if checkbox else "/Tx", + widget_field_name=name, + widget_field_flags=0, + widget_appearance_state="/Off" if checkbox else None, + widget_text="/Off" if checkbox else "retained value", + ) + + +def key_text(index: int, text: str, bbox: BoundingBox) -> Cluster: + return Cluster( + id=index, + label=DocItemLabel.TEXT, + bbox=bbox, + cells=[ + TextCell( + index=index, + text=text, + orig=text, + from_ocr=False, + rect=BoundingRectangle.from_bounding_box(bbox), + ) + ], + ) + + +def table(bbox: BoundingBox) -> Cluster: + return Cluster(id=100, label=DocItemLabel.TABLE, bbox=bbox) + + +def snapshot( + widgets: list[PdfWidget], + key_texts: list[Cluster], + tables: dict[int, list[TableCell]] | None = None, +) -> KeyingPage: + return KeyingPage(widgets, key_texts, tables or {}) + + +def keyed(page: KeyingPage) -> Assignment: + return assign(page.widgets, page.layout, page.tables, page.height, page.rules) + + +def chosen(page: KeyingPage) -> tuple[list[int], list[tuple[str, list[int], str]]]: + result = keyed(page) + assert result.solver_status == "optimal" + fields = [ + ( + result.candidates[c].kind, + [result.values[i].native.index for i in result.candidates[c].members], + result.key_texts[result.candidates[c].key].text, + ) + for c in result.selected + ] + return [v.native.index for v in result.values], fields + + +def test_table_exception_requires_one_cell_and_never_uses_other_cell_header(): + table_region = table(box(0, 40, 200, 200)) + cells = [ + TableCell( + bbox=box(0, 40, 100, 200), + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=0, + end_col_offset_idx=1, + text="", + ), + TableCell( + bbox=box(100, 40, 200, 200), + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=1, + end_col_offset_idx=2, + text="", + ), + ] + page = snapshot( + [widget(0, box(82, 80, 98, 95)), widget(1, box(110, 150, 170, 170))], + [ + table_region, + key_text(1, "Local name", box(10, 82, 65, 92)), + key_text(2, "Other cell", box(101, 81, 165, 91)), + key_text(3, "Outside table", box(80, 25, 180, 35)), + ], + {100: cells}, + ) + order, fields = chosen(page) + assert order == [0, 1] + assert ("field_key", [0], "Local name") in fields + assert all(text != "Outside table" for _, _, text in fields) + assert scope_of(box(95, 80, 105, 90), regions(page.layout), page.tables) == Scope( + 100 + ) + # Even near-total coverage must not move a cross-cell widget into one cell. + assert scope_of(box(82, 80, 100.1, 95), regions(page.layout), page.tables) == Scope( + 100 + ) + assert scope_of(box(82, 80, 98, 95), regions(page.layout), page.tables) == Scope( + 100, 0 + ) + + # A detected table with no usable cells stays excluded, even with an + # attractive nearby key. Removing detection restores ordinary pairing. + page.tables = {} + assert chosen(page)[1] == [] + page.layout.remove(table_region) + assert chosen(page)[1] + + +def test_column_reset_and_candidate_enumeration_preserve_native_order(): + page = snapshot( + [ + widget(7, box(10, 40, 20, 50), checkbox=True), + widget(2, box(10, 80, 20, 90), checkbox=True), + widget(9, box(200, 40, 210, 50), checkbox=True), + widget(3, box(200, 80, 210, 90), checkbox=True), + ], + [ + key_text(1, "English", box(25, 40, 90, 50)), + key_text(2, "Spanish", box(25, 80, 90, 90)), + key_text(3, "French", box(215, 40, 280, 50)), + key_text(4, "Arabic", box(215, 80, 280, 90)), + ], + ) + order, fields = chosen(page) + assert order == [7, 2, 9, 3] + assert {(tuple(indices), text) for _, indices, text in fields} == { + ((7,), "English"), + ((2,), "Spanish"), + ((9,), "French"), + ((3,), "Arabic"), + } + page.layout.reverse() + assert chosen(page) == (order, fields) + + +def test_shared_prompt_keeps_keys_and_interleaved_native_values(): + page = snapshot( + [ + widget(0, box(10, 50, 20, 60), checkbox=True, name="same native field"), + widget(1, box(220, 120, 300, 140)), + widget(2, box(10, 90, 20, 100), checkbox=True, name="same native field"), + ], + [ + key_text(1, "Preferred language", box(10, 15, 140, 25)), + key_text(2, "English", box(25, 50, 90, 60)), + key_text(3, "French", box(25, 90, 90, 100)), + key_text(4, "Your name", box(220, 103, 290, 113)), + ], + ) + order, fields = chosen(page) + assert order == [0, 1, 2] + assert ("choice_group", [0, 2], "Preferred language") in fields + assert ("option_key", [0], "English") in fields + assert ("option_key", [2], "French") in fields + assert ("field_key", [1], "Your name") in fields + + +@pytest.mark.parametrize("scale, offset", [(1, 0), (3.5, 23)]) +@pytest.mark.parametrize("text", ["Extension requested until", "期限延長"]) +def test_inline_checkbox_and_date_share_clause_without_duplicate_ownership( + scale: float, offset: float, text: str +) -> None: + def positioned(left: float, top: float, right: float, bottom: float) -> BoundingBox: + return box(*(v * scale + offset for v in (left, top, right, bottom))) + + page = snapshot( + [ + widget(7, positioned(10, 60, 20, 70), checkbox=True), + widget(2, positioned(160, 60, 230, 70)), + widget(9, positioned(160, 67, 230, 77)), + ], + # Both clause controls have 80% coverage despite protruding below the + # text bounds. The neighboring control has only 10% and must not join. + [key_text(1, text, positioned(5, 55, 250, 68))], + ) + page.height = 400 * scale + offset + order, fields = chosen(page) + assert order == [7, 2, 9] + assert fields == [("inline_clause", [7, 2], text)] + + +def test_option_checkboxes_inside_a_sentence_keep_their_own_keys(): + # "(a) Business income [ ] Yes [ ] No": the layout returns the line as one + # block, but each checkbox has its own option key printed right after + # it. Options answer the sentence; they are not blanks within it, so each + # takes its key instead of the whole line. + cells = [ + ("(a) Business income", box(10, 50, 110, 58)), + ("Yes", box(132, 50, 150, 58)), + ("No", box(172, 50, 186, 58)), + ] + row = Cluster( + id=1, + label=DocItemLabel.TEXT, + bbox=box(10, 50, 186, 58), + cells=[ + TextCell( + index=k, + text=text, + orig=text, + from_ocr=False, + rect=BoundingRectangle.from_bounding_box(bbox), + ) + for k, (text, bbox) in enumerate(cells) + ], + ) + page = snapshot( + [ + widget(0, box(120, 50, 128, 58), checkbox=True), + widget(1, box(160, 50, 168, 58), checkbox=True), + ], + [row], + ) + _, fields = chosen(page) + assert ("option_key", [0], "Yes") in fields + assert ("option_key", [1], "No") in fields + + +def test_shared_business_number_is_not_split_at_printed_component(): + page = snapshot( + [ + widget(0, box(40, 70, 150, 85)), + widget(1, box(168, 70, 230, 85)), + ], + [ + key_text(1, "Business Number", box(40, 40, 140, 50)), + key_text(2, "RT", box(152, 72, 166, 82)), + ], + ) + _, fields = chosen(page) + assert fields == [("composite_field", [0, 1], "Business Number")] + + +def test_row_and_column_headers_never_key_a_value_in_a_detected_table(): + def cell(row: int, column: int, text: str, bbox: BoundingBox, header=False): + return TableCell( + bbox=bbox, + start_row_offset_idx=row, + end_row_offset_idx=row + 1, + start_col_offset_idx=column, + end_col_offset_idx=column + 1, + text=text, + column_header=header, + ) + + # Detected cell boxes cover only their text; the values sit beside the + # line codes, not inside any cell box. + cells = [ + cell(0, 1, "From head office", box(120, 40, 190, 50), header=True), + cell(0, 2, "From third parties", box(220, 40, 290, 50), header=True), + cell(1, 0, "Financial services", box(10, 62, 90, 72)), + cell(1, 1, "250", box(120, 62, 135, 72)), + cell(1, 2, "251", box(220, 62, 235, 72)), + cell(2, 0, "Taxable goods", box(10, 82, 80, 92)), + cell(2, 1, "260", box(120, 82, 135, 92)), + cell(2, 2, "261", box(220, 82, 235, 92)), + ] + page = snapshot( + [ + widget(0, box(140, 60, 200, 74)), + widget(1, box(240, 60, 300, 74)), + widget(2, box(140, 80, 200, 94)), + widget(3, box(240, 80, 300, 94)), + ], + [table(box(5, 35, 305, 100))], + {100: cells}, + ) + result = keyed(page) + # "Financial services" (row header) and "From head office" (column header) + # sit in other cells: the table already carries that association. + assert result.selected == [] + # Each value still gets the cell of its line code: a code has no letters, + # so the cell has no key of its own. + cells = { + result.values[i].native.index: (slot.table, slot.rows, slot.columns, slot.key) + for i, slot in result.slots.items() + } + assert cells == { + 0: (100, (1, 2), (1, 2), None), + 1: (100, (1, 2), (2, 3), None), + 2: (100, (2, 3), (1, 2), None), + 3: (100, (2, 3), (2, 3), None), + } + + +def grid_cell( + row: int, column: int, text: str, bbox: BoundingBox, *, columns: int = 1 +) -> TableCell: + return TableCell( + bbox=bbox, + start_row_offset_idx=row, + end_row_offset_idx=row + 1, + start_col_offset_idx=column, + end_col_offset_idx=column + columns, + text=text, + ) + + +def ruled(page: KeyingPage, *lines: tuple[str, float, float, float]) -> KeyingPage: + """The page with printed rules: ("h", y, x0, x1) or ("v", x, y0, y1), 0.5 pt thick.""" + boxes = [ + box(start, at - 0.25, end, at + 0.25) + if kind == "h" + else box(at - 0.25, start, at + 0.25, end) + for kind, at, start, end in lines + ] + return replace(page, rules=boxes) + + +def test_printed_cell_moves_a_box_to_the_key_printed_over_it(): + # "Name:" and its box share one printed cell, but the detected grid puts + # the key in its own row, and a title cell whose box runs down beside + # it takes the box. The printed cell holds the key: the box goes to + # the key's cell, keyed by it. + page = snapshot( + [widget(0, box(5, 29, 95, 39))], + [table(box(0, 0, 200, 100))], + { + 100: [ + grid_cell(0, 0, "Personal data", box(2, 2, 60, 30), columns=2), + grid_cell(1, 0, "Name:", box(5, 22, 40, 28)), + grid_cell(1, 1, "Date:", box(105, 22, 140, 28)), + grid_cell(2, 0, "Total", box(5, 62, 40, 68)), + grid_cell(2, 1, "Sum", box(105, 62, 140, 68)), + ] + }, + ) + + def placed(result): + (slot,) = result.slots.values() + (key,) = ( + result.key_texts[c.key].text + for c in (result.candidates[i] for i in result.selected) + if c.kind == "table_cell" + ) + return slot.rows, slot.columns, key + + assert placed(keyed(page)) == ((0, 1), (0, 2), "Personal data") + printed = ruled( + page, + ("h", 20, 0, 200), + ("h", 40, 0, 200), + ("v", 0, 0, 100), + ("v", 100, 20, 40), + ("v", 200, 0, 100), + ) + assert placed(keyed(printed)) == ((1, 2), (0, 1), "Name:") + + +def test_value_in_a_printed_column_the_grid_lost_gets_no_cell(): + # The printed table has a second column the detected grid missed: the + # value's printed cell holds no text and misses the grid's only column, so + # it gets no cell instead of the row number's. + page = snapshot( + [widget(0, box(120, 20, 190, 30))], + [table(box(0, 0, 200, 60))], + { + 100: [ + grid_cell(0, 0, "Name", box(5, 2, 40, 8)), + grid_cell(1, 0, "1", box(5, 22, 15, 28)), + grid_cell(2, 0, "2", box(5, 42, 15, 48)), + ] + }, + ) + assert [(s.rows, s.columns) for s in keyed(page).slots.values()] == [ + ((1, 2), (0, 1)) + ] + printed = ruled( + page, + ("h", 15, 0, 200), + ("h", 35, 0, 200), + ("v", 0, 0, 60), + ("v", 100, 0, 60), + ("v", 200, 0, 60), + ) + assert keyed(printed).slots == {} + + +def test_row_key_is_shared_by_like_sized_values_but_not_operand_boxes(): + page = snapshot( + [ + widget(0, box(100, 60, 160, 72)), + widget(1, box(170, 60, 190, 72)), + widget(2, box(200, 60, 260, 72)), + ], + [key_text(1, "Line 5 total", box(10, 61, 80, 71))], + ) + _, fields = chosen(page) + assert sorted(fields) == [ + ("field_key", [0], "Line 5 total"), + ("field_key", [2], "Line 5 total"), + ] + + +def test_key_split_per_line_is_joined_but_option_lines_are_not(): + page = snapshot( + [ + widget(0, box(150, 72, 220, 84)), + widget(1, box(12, 221, 20, 229), checkbox=True), + widget(2, box(12, 231, 20, 239), checkbox=True), + ], + [ + key_text(1, "Intangible personal", box(10, 60, 100, 68)), + key_text(2, "property and services", box(10, 69, 110, 77)), + key_text(3, "financial services", box(10, 78, 95, 86)), + key_text(4, "a Parent group", box(10, 220, 90, 230)), + key_text(5, "b Brother group", box(10, 230, 95, 240)), + ], + ) + _, fields = chosen(page) + assert ( + "field_key", + [0], + "Intangible personal property and services financial services", + ) in fields + assert ("option_key", [1], "a Parent group") in fields + assert ("option_key", [2], "b Brother group") in fields + + +def test_line_split_into_pieces_is_joined_but_neighbouring_cells_are_not(): + page = snapshot( + [ + widget(0, box(250, 60, 320, 72)), + widget(1, box(40, 131, 60, 143)), + widget(2, box(66, 131, 86, 143)), + ], + [ + key_text(1, "Manitoba", box(10, 61, 45, 69)), + key_text(2, "tax (line 23)", box(47, 61, 100, 69)), + # Sub-keys over their own boxes: separate cells, never one line. + key_text(3, "GIORNO", box(40, 122, 62, 130)), + key_text(4, "MESE", box(66, 122, 84, 130)), + ], + ) + _, fields = chosen(page) + assert ("field_key", [0], "Manitoba tax (line 23)") in fields + assert ("field_key", [1], "GIORNO") in fields + assert ("field_key", [2], "MESE") in fields + + +def test_layout_child_repeated_at_top_level_is_not_consumed_twice(): + key = key_text(1, "Name", box(10, 60, 45, 70)) + container = Cluster( + id=2, label=DocItemLabel.FORM, bbox=box(0, 40, 200, 100), children=[key] + ) + # The second box is not a like-sized sibling, so it may only take "Name" + # if the repeated child produced a second copy of the key. + page = snapshot( + [widget(0, box(50, 60, 80, 70)), widget(1, box(90, 60, 160, 70))], + [key, container], + ) + _, fields = chosen(page) + assert fields == [("field_key", [0], "Name")] + + +def test_option_run_after_a_prompt_reads_keys_on_the_right(): + # "Sexo: (o) M (o) F": each key sits nearer the option before it, but + # more than a text line away. Texts at both ends of the run leave one + # over; the inner key leans on the option before it, so the first text + # is the shared prompt and every option takes the key after it. + page = snapshot( + [ + widget(0, box(74, 50, 82, 58), checkbox=True), + widget(1, box(112, 50, 120, 58), checkbox=True), + ], + [ + key_text(1, "Sexo:", box(40, 50, 72, 58)), + key_text(2, "M", box(91, 50, 99, 58)), + key_text(3, "F", box(129, 50, 137, 58)), + ], + ) + _, fields = chosen(page) + assert ("option_key", [0], "M") in fields + assert ("option_key", [1], "F") in fields + + +def test_run_starting_with_a_key_reads_keys_on_the_left(): + # "Jua (o) Ii (o)": nothing after the last option, so each option takes the + # key before it, even where the next key is as close. + page = snapshot( + [ + widget(0, box(62, 50, 70, 58), checkbox=True), + widget(1, box(88, 50, 96, 58), checkbox=True), + ], + [ + key_text(1, "Jua", box(40, 50, 60, 58)), + key_text(2, "Ii", box(72, 50, 80, 58)), + ], + ) + _, fields = chosen(page) + assert ("option_key", [0], "Jua") in fields + assert ("option_key", [1], "Ii") in fields + + +def test_text_boxes_between_keys_keep_their_leading_keys(): + # "Name [__] Date [__] Place": the inner key leans on the box after + # it, so the trailing text starts the next field and keys come first. + page = snapshot( + [widget(0, box(42, 50, 100, 58)), widget(1, box(127, 50, 180, 58))], + [ + key_text(1, "Name", box(10, 50, 40, 58)), + key_text(2, "Date", box(105, 50, 125, 58)), + key_text(3, "Place", box(185, 50, 210, 58)), + ], + ) + _, fields = chosen(page) + assert ("field_key", [0], "Name") in fields + assert ("field_key", [1], "Date") in fields + + +def test_stacked_boxes_read_the_key_above_when_the_next_one_touches_below(): + # Key above each box; the next box's key touches the first box + # from below. The column run starts with a key: keys are above. + page = snapshot( + [widget(0, box(20, 24, 120, 38)), widget(1, box(20, 54, 120, 68))], + [ + key_text(1, "Name", box(20, 10, 50, 18)), + key_text(2, "Address", box(20, 40, 60, 48)), + ], + ) + _, fields = chosen(page) + assert ("field_key", [0], "Name") in fields + assert ("field_key", [1], "Address") in fields + + +@pytest.mark.parametrize( + "other", + [ + # A slightly closer key on the first option's left. + key_text(1, "Applicant", box(50, 50, 98, 58)), + # The list's shared prompt right above the first option, in its own cell. + key_text(1, "Types:", box(100, 40, 140, 48)), + ], +) +def test_aligned_options_read_their_keys_from_one_side(other: Cluster) -> None: + # The options below the first all read their keys on the right; the + # structure reads one side, so the first option follows the rest. + page = snapshot( + [ + widget(0, box(100, 50, 108, 58), checkbox=True), + widget(1, box(100, 70, 108, 78), checkbox=True), + widget(2, box(100, 90, 108, 98), checkbox=True), + ], + [ + other, + key_text(2, "Type 1A", box(114, 50, 150, 58)), + key_text(3, "Type 1B", box(114, 70, 150, 78)), + key_text(4, "Type 2", box(114, 90, 150, 98)), + ], + ) + _, fields = chosen(page) + assert ("option_key", [0], "Type 1A") in fields + assert ("option_key", [1], "Type 1B") in fields + assert ("option_key", [2], "Type 2") in fields + + +def test_key_touching_both_boxes_does_not_decide_a_run(): + # Keys below their boxes, each touching the next box too: a text that + # touches both neighbours says nothing about which it keys, so the run + # decides nothing and each box keeps the key right under it. + page = snapshot( + [widget(0, box(20, 16, 120, 30)), widget(1, box(20, 40, 120, 54))], + [ + key_text(1, "Application", box(20, 0, 80, 8)), + key_text(2, "UFID", box(20, 32, 40, 40)), + key_text(3, "Last Name", box(20, 56, 60, 64)), + ], + ) + _, fields = chosen(page) + assert ("field_key", [0], "UFID") in fields + assert ("field_key", [1], "Last Name") in fields + + +def block( + cluster_id: int, + bbox: BoundingBox, + *lines: tuple[int, str, BoundingBox], + label: DocItemLabel = DocItemLabel.CHECKBOX_UNSELECTED, +) -> Cluster: + """A layout block, by default a checkbox block that boxes a square and its label.""" + cells = [ + TextCell( + index=index, + text=text, + orig=text, + from_ocr=False, + rect=BoundingRectangle.from_bounding_box(bbox), + ) + for index, text, bbox in lines + ] + return Cluster(id=cluster_id, label=label, bbox=bbox, cells=cells) + + +def block_keys(page: KeyingPage) -> dict[int, str]: + """Widget index -> its key, for every keyed value.""" + _, fields = chosen(page) + return { + index: text + for kind, indices, text in fields + if kind != "choice_group" + for index in indices + } + + +def test_two_checkboxes_in_one_block_are_left_to_the_search() -> None: + # A Yes/No pair the layout boxed as one checkbox block: the block's text + # keys neither box as a whole; each box reads its own option key. + page = snapshot( + [ + widget(0, box(12, 11, 20, 19), checkbox=True), + widget(1, box(12, 21, 20, 29), checkbox=True), + ], + [ + block( + 1, + box(10, 10, 90, 30), + (0, "Sim", box(24, 11, 40, 19)), + (1, "Não", box(24, 21, 40, 29)), + ) + ], + ) + assert block_keys(page) == {0: "Sim", 1: "Não"} + + +def test_block_with_no_letters_is_left_to_the_search() -> None: + # A column of answer boxes printed after leader dots, which the layout + # boxes as a checkbox block: the dots are no key, the column header is. + header = "Grado igual o superior al 65%" + page = snapshot( + [widget(0, box(139, 26, 148, 34), checkbox=True)], + [ + key_text(1, header, box(60, 10, 150, 18)), + block( + 2, + box(60, 25, 150, 35), + (2, "........................", box(60, 26, 135, 34)), + ), + ], + ) + assert block_keys(page) == {0: header} + + +def test_block_a_little_short_of_its_checkbox_still_keys_it() -> None: + # The layout's block ends 0.1 pt above the drawn square's bottom; the + # square is still the block's checkbox, not left to the caption above. + caption_bbox = box(56.7, 19.5, 115.5, 26.3) + page = snapshot( + [widget(0, box(58.8, 34.8, 66.8, 42.8), checkbox=True)], + [ + block( + 1, + caption_bbox, + (0, "Fotocópias de:", caption_bbox), + label=DocItemLabel.SECTION_HEADER, + ), + block( + 2, + box(55.5, 33.0, 300.0, 42.7), + (1, "Recibos de vencimento.", box(76.0, 36.0, 300.0, 42.9)), + ), + ], + ) + assert block_keys(page) == {0: "Recibos de vencimento."} + + +@pytest.mark.parametrize("second_is_block", [True, False]) +def test_checkbox_blocks_below_a_field_leave_its_label_to_the_field( + second_is_block: bool, +) -> None: + # "Contact name:" labels the field to its right. Two options sit 6 pt + # below; the first is a checkbox block, the second a block or plain text. + # A checkbox keyed by its block is not also a member of a choice group, + # so the label cannot become that group's prompt and leave the field + # unkeyed. + second = ( + block(3, box(8, 35, 80, 45), (2, "Send by post", box(22, 36, 76, 44))) + if second_is_block + else key_text(3, "Send by post", box(24, 36, 76, 44)) + ) + page = snapshot( + [ + widget(0, box(64, 10, 140, 18)), + widget(1, box(10, 24, 18, 32), checkbox=True), + widget(2, box(10, 36, 18, 44), checkbox=True), + ], + [ + key_text(1, "Contact name:", box(10, 10, 60, 18)), + block(2, box(8, 23, 80, 33), (1, "Send by email", box(22, 24, 76, 32))), + second, + ], + ) + assert block_keys(page) == { + 0: "Contact name:", + 1: "Send by email", + 2: "Send by post", + } + + +def test_boxed_checkboxes_in_a_grid_keep_their_column_header_as_hint() -> None: + # Two columns of boxed checkboxes under "Applicant" and "Spouse": each is + # keyed by its block, and its column header is still kept as a hint. + clusters, widgets = [], [] + for column, (header, x) in enumerate([("Applicant", 30), ("Spouse", 130)]): + clusters.append( + block( + 100 + column, + box(x, 10, x + 45, 18), + (len(clusters), header, box(x, 10, x + 45, 18)), + label=DocItemLabel.TEXT, + ) + ) + for row, (label, y) in enumerate([("Employed", 30), ("Retired", 45)]): + clusters.append( + block( + 10 * column + row, + box(x, y, x + 52, y + 11), + (len(clusters), label, box(x + 13, y + 1, x + 50, y + 9)), + ) + ) + widgets.append( + widget(len(widgets), box(x + 2, y + 1, x + 11, y + 10), checkbox=True) + ) + result = keyed(snapshot(widgets, clusters)) + keyed_values = { + result.values[member].native.index: ( + result.key_texts[candidate.key].text, + "" if candidate.hint is None else result.key_texts[candidate.hint].text, + ) + for candidate in (result.candidates[i] for i in result.selected) + if candidate.kind != "choice_group" + for member in candidate.members + } + assert keyed_values == { + 0: ("Employed", "Applicant"), + 1: ("Retired", "Applicant"), + 2: ("Employed", "Spouse"), + 3: ("Retired", "Spouse"), + } + + +def test_penalized_pairs_match_scoring_every_pair(): + # penalized_pairs only scores candidates whose boxes meet or whose values + # follow each other; it must find what scoring every pair finds. + compared = 0 + rng = random.Random(4496) + for _ in range(60): + columns = rng.randint(1, 4) + texts, widgets = [], [] + for row in range(rng.randint(2, 12)): + for column in range(columns): + left = 8 + column * 190 / columns + rng.uniform(-3, 3) + top = 6 + row * 13 + rng.uniform(-3.5, 3.5) + width = rng.uniform(8, 20) + if rng.random() < 0.85: + shift = -7 if rng.random() < 0.25 else 0 # a key printed above + key_box = box(left, top + shift, left + width, top + shift + 7) + texts.append( + key_text(len(texts) + 1, f"key {len(texts)}:", key_box) + ) + start = left if rng.random() < 0.25 else left + width + 2 + checkbox = rng.random() < 0.5 + # A flat checkbox: its box may meet no other box on its line. + size = (5, 2) if checkbox else (15, 7) + value_box = box(start, top, start + size[0], top + size[1]) + widgets.append(widget(len(widgets), value_box, checkbox=checkbox)) + if rng.random() < 0.5: # native order unlike the printed order + rng.shuffle(widgets) + widgets = [w.model_copy(update={"index": i}) for i, w in enumerate(widgets)] + page = snapshot(widgets, texts) + result = keyed(page) + h = inputs(widgets, regions(texts), {}, page.height).h + proposed, values, key_texts = result.candidates, result.values, result.key_texts + active = list(range(len(proposed))) + every = [] + for a, b in combinations(active, 2): + first, second = proposed[a], proposed[b] + if key_texts[first.key].scope != key_texts[second.key].scope: + continue + penalty = 3.0 * solver.crossing(first, second, values, key_texts) + ( + solver.transition_penalty(first, second, values, key_texts, h) + ) + if penalty: + every.append((a, b, penalty)) + assert solver.penalized_pairs(active, proposed, values, key_texts, h) == every + compared += len(every) + assert compared > 100 diff --git a/tests/test_backend_docling_parse.py b/tests/test_backend_docling_parse.py index 044fdc5332..c81738c6b1 100644 --- a/tests/test_backend_docling_parse.py +++ b/tests/test_backend_docling_parse.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any +import pypdfium2 as pdfium import pytest from docling_core.types.doc import CoordOrigin from docling_core.types.doc.page import PdfCellRenderingMode @@ -681,6 +682,7 @@ class _FakeSegmentedPage: textline_cells = [_FakeCell()] char_cells = [_FakeCell()] word_cells = [_FakeCell()] + widgets: list = [] bitmap_resources: list[Any] = [] result = _FakeThreadedResult(page_number=4) @@ -795,6 +797,56 @@ def test_threaded_backend_reports_shape_geometry_in_top_left_origin(ruled_table_ doc_backend.unload() +def _pdf_with_filled_rectangles() -> bytes: + """One 200 x 200 pt page filling a 0.5 pt tall rule and a 20 pt tall bar.""" + content = b"0 0 0 rg 20 150 160 0.5 re f 20 50 160 20 re f" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>", + b"<< /Length %d >>\nstream\n%s\nendstream" % (len(content), content), + ] + pdf = b"%PDF-1.4\n" + offsets = [] + for number, body in enumerate(objects, start=1): + offsets.append(len(pdf)) + pdf += b"%d 0 obj\n%s\nendobj\n" % (number, body) + xref = len(pdf) + pdf += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) + pdf += b"".join(b"%010d 00000 n \n" % offset for offset in offsets) + pdf += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % ( + len(objects) + 1, + xref, + ) + return pdf + + +def test_threaded_backend_reports_thin_filled_shapes(tmp_path): + """A rule drawn as a thin filled rectangle surfaces; a thicker fill does not. + + Stroked segments alone miss such rules, which many forms use for borders. + """ + path = tmp_path / "filled_rules.pdf" + path.write_bytes(_pdf_with_filled_rectangles()) + in_doc = InputDocument( + path_or_stream=path, + format=InputFormat.PDF, + backend=ThreadedDoclingParseDocumentBackend, + ) + doc_backend = in_doc._backend + assert isinstance(doc_backend, ThreadedDoclingParseDocumentBackend) + + try: + page_backend = next(iter(doc_backend.iter_pages())) + assert page_backend.get_shape_lines() == [] + + (rule,) = page_backend.get_thin_shape_boxes(max_thickness=3.5) + assert rule.coord_origin == CoordOrigin.TOPLEFT + assert (rule.l, rule.t, rule.r, rule.b) == pytest.approx((20, 49.5, 180, 50)) + finally: + doc_backend.unload() + + def test_threaded_backend_intersects_only_where_content_is(ruled_table_path): """`has_content_in` must discriminate between the ruled table and a blank margin.""" in_doc = InputDocument( @@ -873,3 +925,74 @@ def test_threaded_backend_filters_invisible_text_cells(): } finally: doc_backend.unload() + + +def _widget_offsets_from_text(backend_cls: Any, pdf_path: Path) -> list[tuple]: + """Each widget's top-left corner relative to the page's first word.""" + in_doc = InputDocument( + path_or_stream=pdf_path, format=InputFormat.PDF, backend=backend_cls + ) + doc_backend = in_doc._backend + page_backend = next(doc_backend.iter_pages()) + try: + page = page_backend.get_segmented_page() + word = next(c for c in page.word_cells if c.text.strip()).rect.to_bounding_box() + widgets = [ + w.rect.to_bounding_box().to_top_left_origin(page.dimension.height) + for w in page.widgets + ] + finally: + page_backend.unload() + doc_backend.unload() + return [(round(b.l - word.l, 2), round(b.t - word.t, 2)) for b in widgets] + + +def test_widgets_move_with_the_text_when_the_crop_box_is_offset( + tmp_path: Path, +) -> None: + backend_cls = ThreadedDoclingParseDocumentBackend + # Text cells are relative to the visible page (crop box); widget rectangles + # must be too, or they sit off their field keys when the crop box does not + # start at the PDF origin. + source = Path("tests/data/pdf/sources/acroform_sample.pdf") + shifted = tmp_path / "shifted_crop.pdf" + pdf = pdfium.PdfDocument(source) + left, bottom, right, top = pdf[0].get_mediabox() + pdf[0].set_cropbox(left + 20, bottom + 30, right, top) + pdf.save(shifted) + pdf.close() + + expected = _widget_offsets_from_text(backend_cls, source) + assert len(expected) == 4 + assert _widget_offsets_from_text(backend_cls, shifted) == expected + + +def test_docling_parse_exposes_complete_widget_contract_in_native_order() -> None: + input_doc = InputDocument( + path_or_stream=Path("tests/data/pdf/sources/acroform_sample.pdf"), + format=InputFormat.PDF, + backend=ThreadedDoclingParseDocumentBackend, + ) + backend = input_doc._backend + page = next(backend.iter_pages()) + try: + segmented_page = page.get_segmented_page() + assert segmented_page is not None + assert [widget.widget_field_name for widget in segmented_page.widgets] == [ + "applicant_name", + "agree_terms", + "newsletter", + "mail_optin", + ] + assert [widget.index for widget in segmented_page.widgets] == [0, 1, 2, 3] + assert segmented_page.widgets[0].widget_text == "Ada Lovelace" + assert segmented_page.widgets[0].widget_description == "Full legal name" + assert segmented_page.widgets[0].widget_field_type == "/Tx" + assert segmented_page.widgets[0].widget_field_flags == 2 + assert segmented_page.widgets[1].widget_appearance_state == "/Yes" + assert segmented_page.widgets[2].widget_appearance_state == "/Off" + assert segmented_page.widgets[3].widget_text in {None, ""} + assert segmented_page.widgets[3].widget_appearance_state == "/On" + finally: + page.unload() + backend.unload() diff --git a/tests/test_cli.py b/tests/test_cli.py index 8f3fbe6f36..69a1e7f432 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1895,3 +1895,23 @@ def convert_all( assert captured["pipeline_options"].parser_threads == 3 assert captured["backend_options"].parser_threads == 3 assert captured["pipeline_options"].accelerator_options.num_threads == 7 + + +def test_form_fields_with_doctags_are_rejected_before_reading_inputs( + tmp_path: Path, +) -> None: + # DocTags has no tokens for form fields and its export raises on them. The + # source does not exist: the options alone are refused, nothing is read. + result = runner.invoke( + app, + [ + str(tmp_path / "missing.pdf"), + "--extract-form-fields", + "--to", + "doctags", + "--output", + str(tmp_path), + ], + ) + assert result.exit_code == 2 # usage error, not the missing-input abort + assert "--extract-form-fields cannot be combined" in strip_ansi(result.output) diff --git a/tests/test_e2e_conversion.py b/tests/test_e2e_conversion.py index 8dd31c95d6..f364ad26f6 100644 --- a/tests/test_e2e_conversion.py +++ b/tests/test_e2e_conversion.py @@ -18,12 +18,15 @@ GENERATE_V2 = GEN_TEST_DATA pytestmark = pytest.mark.ml_pdf_model +# acroform_sample.pdf: the DocTags export does not support field items. SKIP_DOCTAGS_COMPARISON = [ "2203.01017v2.pdf", "table_mislabeled_as_picture.pdf", + "acroform_sample.pdf", ] -# PDFs that are tested separately in test_failed_pages.py (intentionally failing pages) +# PDFs that are tested separately: test_failed_pages.py (intentionally failing +# pages). SKIP_E2E_TEST = ["skipped_1page.pdf", "skipped_2pages.pdf"] @@ -46,6 +49,7 @@ def get_converter(): pipeline_options.table_structure_options.do_cell_matching = True pipeline_options.accelerator_options.device = AcceleratorDevice.CPU pipeline_options.generate_parsed_pages = True + pipeline_options.extract_form_fields = True converter = DocumentConverter( format_options={ diff --git a/tests/test_form_checkbox_blocks.py b/tests/test_form_checkbox_blocks.py new file mode 100644 index 0000000000..07e720e1b5 --- /dev/null +++ b/tests/test_form_checkbox_blocks.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""A checkbox the layout boxed together with its label is keyed by that label.""" + +from unittest.mock import MagicMock + +import pytest +from docling_core.types.doc import BoundingBox, DocItemLabel, Size +from docling_core.types.doc.items.form import FieldItem +from docling_core.types.doc.page import BoundingRectangle, PdfWidget, TextCell + +from docling.datamodel.base_models import Cluster, LayoutPrediction, Page + +from .test_form_extraction import _materialize, _regions + +# Tall enough for a note, a caption and three rows of PJ-1 page 4. +PAGE_HEIGHT = 120 + + +def _block( + cluster_id: int, + label: DocItemLabel, + bbox: BoundingBox, + *lines: tuple[int, str, BoundingBox], +) -> Cluster: + cells = [ + TextCell( + index=index, + rect=BoundingRectangle.from_bounding_box(box), + text=text, + orig=text, + from_ocr=False, + ) + for index, text, box in lines + ] + return Cluster(id=cluster_id, label=label, bbox=bbox, cells=cells) + + +def _checkbox(index: int, bbox: BoundingBox) -> PdfWidget: + return PdfWidget( + index=index, + rect=BoundingRectangle.from_bounding_box( + bbox.to_bottom_left_origin(page_height=PAGE_HEIGHT) + ), + widget_text="/Off", + widget_field_type="/Btn", + widget_appearance_state="/Off", + ) + + +def _page(clusters: list[Cluster], widgets) -> Page: + page = Page(page_no=1, size=Size(width=595, height=PAGE_HEIGHT)) + page.parsed_page = MagicMock(widgets=widgets, hyperlinks=[]) + page.predictions.layout = LayoutPrediction(clusters=clusters) + return page + + +def test_checkbox_in_its_label_block_is_keyed_by_the_whole_block() -> None: + # PJ-1 page 4: a "Fotocópias de:" caption under a note, over rows of a + # drawn square and a paragraph. The layout boxes each row as one + # unselected checkbox, square included. The search would key every box + # by the caption, which sits closer than the paragraph on the box's + # right; each box is keyed by its own paragraph instead, which hosts the + # item in place and is not emitted a second time. + note = "Devem ser apresentados os documentos do requerente." + note_bbox = BoundingBox(l=56.7, t=2.4, r=495.8, b=9.2) + caption_bbox = BoundingBox(l=56.7, t=19.5, r=115.5, b=26.3) + above = [ + _block(1, DocItemLabel.TEXT, note_bbox, (0, note, note_bbox)), + _block( + 2, + DocItemLabel.SECTION_HEADER, + caption_bbox, + (1, "Fotocópias de:", caption_bbox), + ), + ] + rows = [ + _block( + 3, + DocItemLabel.CHECKBOX_UNSELECTED, + BoundingBox(l=55.5, t=32.4, r=538.7, b=64.5), + (2, "Bilhete de identidade", BoundingBox(l=73.7, t=37.9, r=538.6, b=44.8)), + (3, "do requerente e demais", BoundingBox(l=73.7, t=47.5, r=538.6, b=54.4)), + (4, "pessoas.", BoundingBox(l=73.7, t=57.1, r=165.3, b=64.0)), + ), + _block( + 4, + DocItemLabel.CHECKBOX_UNSELECTED, + BoundingBox(l=55.2, t=67.9, r=513.3, b=80.7), + (5, "Recibos de vencimento.", BoundingBox(l=73.7, t=73.2, r=513.3, b=80.1)), + ), + _block( + 5, + DocItemLabel.CHECKBOX_UNSELECTED, + BoundingBox(l=55.4, t=84.7, r=538.9, b=106.7), + (6, "Declarações de IVA", BoundingBox(l=73.7, t=90.2, r=538.6, b=97.1)), + (7, "dos dois trimestres.", BoundingBox(l=73.7, t=99.8, r=292.1, b=106.7)), + ), + ] + widgets = [ + _checkbox(0, BoundingBox(l=58.8, t=34.8, r=66.8, b=42.8)), + _checkbox(1, BoundingBox(l=58.8, t=70.2, r=66.8, b=78.2)), + _checkbox(2, BoundingBox(l=58.8, t=87.2, r=66.8, b=95.2)), + ] + page = _page([*above, *rows], widgets) + paragraphs = [" ".join(cell.text for cell in row.cells) for row in rows] + + doc = _materialize(page) + + assert [ + (source, [item.key_text for item in items]) for source, items in _regions(page) + ] == [(row.id, [text]) for row, text in zip(rows, paragraphs)] + order = [] + for item, _ in doc.iterate_items(with_groups=False): + if isinstance(item, FieldItem): + order.append(item.children[0].resolve(doc).text) + elif item.label not in {DocItemLabel.FIELD_KEY, DocItemLabel.FIELD_VALUE}: + order.append(item.text) # the boxes' own checkbox items are empty + assert [text for text in order if text] == [note, "Fotocópias de:", *paragraphs] + + +def _text_field(index: int, bbox: BoundingBox) -> PdfWidget: + return PdfWidget( + index=index, + rect=BoundingRectangle.from_bounding_box( + bbox.to_bottom_left_origin(page_height=PAGE_HEIGHT) + ), + widget_text="", + widget_field_type="/Tx", + ) + + +def _keys(page: Page) -> list[str]: + return [item.key_text for _, items in _regions(page) for item in items] + + +def test_hosted_checkbox_key_is_located_by_its_text() -> None: + # The block boxes the drawn square with its label, but the square is the + # value's place: the key's location covers the label's text only. + text_bbox = BoundingBox(l=73.7, t=37.9, r=300.0, b=44.8) + block = _block( + 1, + DocItemLabel.CHECKBOX_UNSELECTED, + BoundingBox(l=55.5, t=32.4, r=300.0, b=46.0), + (0, "Recibos de vencimento.", text_bbox), + ) + page = _page([block], [_checkbox(0, BoundingBox(l=58.8, t=34.8, r=66.8, b=42.8))]) + + doc = _materialize(page) + + (item,) = doc.field_items + key = item.children[0].resolve(doc) + assert key.text == "Recibos de vencimento." + box = key.prov[0].bbox.to_top_left_origin(PAGE_HEIGHT) + assert (box.l, box.t, box.r, box.b) == pytest.approx( + (text_bbox.l, text_bbox.t, text_bbox.r, text_bbox.b) + ) + + +def test_block_label_also_keys_a_field_of_its_row() -> None: + # "VISA Office" labels the checkbox in its block and the box on the same + # row: the block's text keys the checkbox without taking the label from + # the row's other field. + label_bbox = BoundingBox(l=24, t=21, r=68, b=29) + page = _page( + [ + _block( + 1, + DocItemLabel.CHECKBOX_UNSELECTED, + BoundingBox(l=10, t=20, r=70, b=30), + (0, "VISA Office", label_bbox), + ) + ], + [ + _checkbox(0, BoundingBox(l=12, t=21, r=20, b=29)), + _text_field(1, BoundingBox(l=80, t=20, r=200, b=30)), + ], + ) + + doc = _materialize(page) + + assert _keys(page) == ["VISA Office", "VISA Office"] + assert all(item.label == DocItemLabel.FIELD_KEY for item in doc.texts if item.text) + + +def _keyed_at(page: Page) -> dict[tuple[int, int], str]: + return { + (round(item.values[0].bbox.l), round(item.values[0].bbox.t)): item.key_text + for _, items in _regions(page) + for item in items + } + + +def _option( + cluster_id: int, index: int, left: float, top: float, label: str, right: float +) -> tuple[Cluster, PdfWidget]: + # A checkbox block, 8 pt tall, with the drawn square at its left. + box = BoundingBox(l=left - 2, t=top - 1, r=right, b=top + 9) + text_box = BoundingBox(l=left + 12, t=top, r=right - 4, b=top + 8) + return ( + _block( + cluster_id, DocItemLabel.CHECKBOX_UNSELECTED, box, (index, label, text_box) + ), + _checkbox(index, BoundingBox(l=left, t=top, r=left + 8, b=top + 8)), + ) + + +def test_checkbox_keyed_by_its_block_is_not_printed_a_second_time() -> None: + # A heading, then three options. The first option's row also holds a + # labelled field; the options are indented by different amounts. The + # first option is keyed by its own block, not by the heading, so its + # paragraph is hosted by the field and not left in the body as a second + # checkbox. + heading_bbox = BoundingBox(l=10, t=10, r=70, b=18) + note_bbox = BoundingBox(l=72, t=20, r=98, b=28) + note2_bbox = BoundingBox(l=72, t=44, r=98, b=52) + zero, zero_box = _option(2, 1, 10, 20, "Choice 0 text", 68) + one, one_box = _option(4, 3, 14, 32, "Choice 1 text", 102) + two, two_box = _option(5, 4, 10, 44, "Choice 2 text", 68) + page = _page( + [ + _block(1, DocItemLabel.TEXT, heading_bbox, (0, "Documents", heading_bbox)), + zero, + _block(3, DocItemLabel.TEXT, note_bbox, (2, "Note 0:", note_bbox)), + one, + two, + _block(6, DocItemLabel.TEXT, note2_bbox, (5, "Note 2:", note2_bbox)), + ], + [ + zero_box, + _text_field(5, BoundingBox(l=102, t=20, r=158, b=28)), + one_box, + two_box, + _text_field(6, BoundingBox(l=102, t=44, r=158, b=52)), + ], + ) + + doc = _materialize(page) + + keyed = _keyed_at(page) + assert keyed[(10, 20)] == "Choice 0 text" + assert keyed[(14, 32)] == "Choice 1 text" + assert keyed[(10, 44)] == "Choice 2 text" + assert not [ + item + for item in doc.texts + if item.label + in {DocItemLabel.CHECKBOX_SELECTED, DocItemLabel.CHECKBOX_UNSELECTED} + and item.text + ] diff --git a/tests/test_form_extraction.py b/tests/test_form_extraction.py new file mode 100644 index 0000000000..2df3a8424c --- /dev/null +++ b/tests/test_form_extraction.py @@ -0,0 +1,1223 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +import json +from pathlib import Path, PurePath +from unittest.mock import MagicMock + +import pytest +from docling_core.transforms.serializer.doclang import DocLangDocSerializer +from docling_core.types.doc import ( + BoundingBox, + DocItemLabel, + DoclingDocument, + ListItem, + NodeItem, + RichTableCell, + Size, + TableCell, +) +from docling_core.types.doc.items.form import ( + FieldItem, + FieldRegionItem, + FieldValueItem, +) +from docling_core.types.doc.page import ( + BoundingRectangle, + PdfHyperlink, + PdfWidget, + TextCell, +) + +from docling.datamodel.base_models import ( + AssembledUnit, + Cluster, + FieldItemPrediction, + FieldRegionElement, + InputFormat, + LayoutPrediction, + Page, + Table, + TableStructurePrediction, +) +from docling.datamodel.document import ConversionResult, InputDocument +from docling.datamodel.pipeline_options import PdfPipelineOptions +from docling.document_converter import DocumentConverter, PdfFormatOption +from docling.models.postprocessing.reading_order_rb import ReadingOrderPredictor +from docling.models.stages.form_field import form_field_model +from docling.models.stages.form_field.form_field_model import PdfFormFieldModel +from docling.models.stages.form_field.keying import PUSHBUTTON_FLAG +from docling.models.stages.page_assemble.page_assemble_model import ( + PageAssembleModel, + PageAssembleOptions, +) +from docling.models.stages.reading_order.readingorder_model import ( + ReadingOrderModel, + ReadingOrderOptions, +) + +FORM_PDF = Path("tests/data/pdf/sources/acroform_sample.pdf") +EXPECTED_NAMES = [ + "applicant_name", + "agree_terms", + "newsletter", + "mail_optin", +] + + +def _convert(*, extract_form_fields: bool) -> ConversionResult: + options = PdfPipelineOptions( + do_ocr=False, + do_table_structure=False, + extract_form_fields=extract_form_fields, + ) + return DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption(pipeline_options=options), + } + ).convert(FORM_PDF) + + +def _regions(page: Page) -> list[tuple[int | None, list[FieldItemPrediction]]]: + """The page's field items by route: host paragraph or FORM id, then page-wide.""" + fields = page.predictions.form_fields + if fields is None: + return [] + return [ + *((host_id, [item]) for host_id, item in fields.inline_items.items()), + *fields.form_items.items(), + *([(None, fields.loose_items)] if fields.loose_items else []), + ] + + +def _conversion_result(page: Page) -> ConversionResult: + input_doc = InputDocument.model_construct( + file=PurePath("input.pdf"), + document_hash="0" * 64, + valid=True, + format=InputFormat.PDF, + ) + return ConversionResult(input=input_doc, pages=[page]) + + +def _widget( + index: int, + bbox: BoundingBox, + text: str, + *, + field_type: str = "/Tx", + appearance_state: str | None = None, + field_flags: int = 0, +) -> PdfWidget: + return PdfWidget( + index=index, + rect=BoundingRectangle.from_bounding_box( + bbox.to_bottom_left_origin(page_height=100) + ), + widget_text=text, + widget_field_type=field_type, + widget_appearance_state=appearance_state, + widget_field_flags=field_flags, + ) + + +def test_field_mapping_assembly_and_materialization_preserve_regions_and_order() -> ( + None +): + form_1 = Cluster( + id=1, + label=DocItemLabel.FORM, + bbox=BoundingBox(l=0, t=0, r=60, b=60), + ) + form_2 = Cluster( + id=2, + label=DocItemLabel.FORM, + bbox=BoundingBox(l=40, t=0, r=100, b=60), + ) + form_3 = Cluster( + id=3, + label=DocItemLabel.FORM, + bbox=BoundingBox(l=45, t=30, r=55, b=45), + ) + widgets = [ + _widget(0, BoundingBox(l=10, t=10, r=20, b=20), "first"), + _widget(1, BoundingBox(l=80, t=10, r=90, b=20), "second"), + _widget(2, BoundingBox(l=20, t=30, r=30, b=40), "third"), + _widget(3, BoundingBox(l=45, t=10, r=55, b=20), "tie"), + _widget( + 4, + BoundingBox(l=47, t=32, r=53, b=40), + "/Off", + field_type="/Btn", + appearance_state="/On", + ), + _widget( + 5, + BoundingBox(l=10, t=80, r=20, b=90), + "/Yes", + field_type="/Btn", + appearance_state="/Off", + ), + # Zero-size widgets are artifacts (Well-Tagged PDF 1.0, 8.9.2.4.13). + _widget(6, BoundingBox(l=30, t=30, r=30, b=30), "ghost"), + # Push buttons carry no value. + _widget( + 7, + BoundingBox(l=60, t=80, r=90, b=90), + "Submit", + field_type="/Btn", + field_flags=PUSHBUTTON_FLAG, + ), + ] + # A FORM keeps its child blocks, as layout postprocessing leaves them. + visible_child = Cluster( + id=4, + label=DocItemLabel.TEXT, + bbox=BoundingBox(l=5, t=5, r=35, b=9), + ) + form_1.children = [visible_child] + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=widgets) + page.predictions.layout = LayoutPrediction( + clusters=[form_1, form_2, form_3, visible_child] + ) + conv_res = _conversion_result(page) + + assert list(PdfFormFieldModel(enabled=True)(conv_res, [page])) == [page] + assert [source for source, _ in _regions(page)] == [ + 1, + 2, + 3, + None, + ] + + # Keyless items: one value each, one item per widget -- flat shape preserved. + def _region_values(items): + return [value for item in items for value in item.values] + + regions = _regions(page) + assert [ + [value.text for value in _region_values(items)] for _, items in regions + ] == [["first", "third", "tie"], ["second"], [""], [""]] + assert [ + [value.checkbox for value in _region_values(items)] for _, items in regions + ] == [ + [None, None, None], + [None], + [DocItemLabel.CHECKBOX_SELECTED], + [DocItemLabel.CHECKBOX_UNSELECTED], + ] + assert all(item.key_text == "" for _, items in regions for item in items) + assert regions[2][1][0].values[0].orig == "/On" + assert regions[3][1][0].values[0].orig == "/Off" + # The zero-size and push-button widgets are dropped: no extra region and + # neither their text nor orig reaches any value. + assert not any( + value.orig in {"ghost", "Submit"} + for _, items in regions + for value in _region_values(items) + ) + + layout_before_assembly = page.predictions.layout.model_copy(deep=True) + page._backend = MagicMock() + page._backend.is_valid.return_value = True + + assert list(PageAssembleModel(PageAssembleOptions())(conv_res, [page])) == [page] + assert page.predictions.layout == layout_before_assembly + assert page.assembled is not None + region_elements = [ + element + for element in page.assembled.elements + if isinstance(element, FieldRegionElement) + ] + # Each FORM block with fields becomes a region element; the page-wide + # field is placed by the reading order. + assert [element.id for element in region_elements] == [1, 2, 3] + assert region_elements[0].cluster.children == [visible_child] + assert len({element.id for element in page.assembled.elements}) == len( + page.assembled.elements + ) + + conv_res.assembled = AssembledUnit( + elements=page.assembled.elements, + body=page.assembled.body, + headers=page.assembled.headers, + ) + doc = ReadingOrderModel(ReadingOrderOptions())(conv_res) + # One field region per FORM, plus one for the page-wide field, which the + # reading order places between the blocks by geometry. + assert len(doc.field_regions) == 4 + assert len(doc.field_items) == 6 + assert not any(item.label == DocItemLabel.FIELD_KEY for item in doc.texts) + + def _value_repr(value: FieldValueItem) -> str: + # Text fields carry their text; a checkbox value is empty and nests a + # CHECKBOX_* child that carries the selection state. + checkbox_children = [ + child.resolve(doc) + for child in value.children + if child.resolve(doc).label + in {DocItemLabel.CHECKBOX_SELECTED, DocItemLabel.CHECKBOX_UNSELECTED} + ] + if checkbox_children: + assert value.text == "" + return checkbox_children[0].label.value + return value.text + + values_by_region = [] + for region in doc.field_regions: + field_items = [ + child.resolve(doc) + for child in region.children + if isinstance(child.resolve(doc), FieldItem) + ] + values_by_region.append( + [_value_repr(field.children[0].resolve(doc)) for field in field_items] + ) + assert values_by_region == [ + [DocItemLabel.CHECKBOX_SELECTED.value], + ["first", "third", "tie"], + [DocItemLabel.CHECKBOX_UNSELECTED.value], + ["second"], + ] + doc.validate_document() + + +def _text_cluster(cluster_id: int, bbox: BoundingBox, text: str) -> Cluster: + cell = TextCell( + index=0, + rect=BoundingRectangle.from_bounding_box(bbox), + text=text, + orig=text, + from_ocr=False, + ) + return Cluster(id=cluster_id, label=DocItemLabel.TEXT, bbox=bbox, cells=[cell]) + + +def test_suppresses_rendered_duplicate_of_filled_widget() -> None: + # A filled widget paints its /V into the raster; the layout model re-detects + # it as a plain text cluster inside the widget rect -> suppress it. + widget_bbox = BoundingBox(l=10, t=10, r=90, b=20) + form = Cluster( + id=1, label=DocItemLabel.FORM, bbox=BoundingBox(l=0, t=0, r=100, b=60) + ) + inside = _text_cluster(2, BoundingBox(l=12, t=11, r=30, b=19), "111.00") + # same string but well outside any widget rect -> ordinary printed text, keep + outside = _text_cluster(3, BoundingBox(l=10, t=80, r=30, b=90), "111.00") + form.children = [inside] + + widgets = [_widget(0, widget_bbox, "111.00")] + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=widgets) + page.predictions.layout = LayoutPrediction(clusters=[form, inside, outside]) + conv_res = _conversion_result(page) + + list(PdfFormFieldModel(enabled=True)(conv_res, [page])) + + remaining = page.predictions.layout.clusters + assert inside not in remaining # contained duplicate dropped + assert outside in remaining # coincidental match kept + assert form.children == [] # child ref pruned so assembly stays consistent + # A printed number is never a key: the value stays keyless. + assert _regions(page)[0][1][0].key_text == "" + + +def _checkbox_cluster(cluster_id: int, bbox: BoundingBox) -> Cluster: + # The layout model's checkbox clusters carry the detected mark, no text. + return Cluster(id=cluster_id, label=DocItemLabel.CHECKBOX_SELECTED, bbox=bbox) + + +def test_checkbox_widget_keys_its_option_key_and_lets_as_override_classifier() -> None: + # A /Btn widget with its printed option key to its right. The layout + # model also detected the box as CHECKBOX_SELECTED (a mark glyph), but /AS + # ("/Off") -- not the classifier -- decides the state. The key becomes + # the key and leaves the body; the value keeps the widget's own rect. + widget_bbox = BoundingBox(l=10, t=10, r=18, b=18) + key_bbox = BoundingBox(l=22, t=10, r=80, b=18) + key = _text_cluster(2, key_bbox, "Married filing jointly") + glyph = _checkbox_cluster(3, BoundingBox(l=9, t=9, r=19, b=19)) + widget = _widget(0, widget_bbox, "/Off", field_type="/Btn", appearance_state="/Off") + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=[widget]) + page.predictions.layout = LayoutPrediction(clusters=[key, glyph]) + conv_res = _conversion_result(page) + + list(PdfFormFieldModel(enabled=True)(conv_res, [page])) + + ((_, (item,)),) = _regions(page) + (value,) = item.values + assert item.key_text == "Married filing jointly" + assert item.key_bbox == key_bbox + assert value.text == "" + assert ( + value.checkbox == DocItemLabel.CHECKBOX_UNSELECTED + ) # from /AS, overriding the classifier + assert value.bbox == widget_bbox + assert key not in page.predictions.layout.clusters # promoted to a key + assert glyph in page.predictions.layout.clusters # the mark is no key + + +def test_pipeline_materializes_format_neutral_fields() -> None: + result = _convert(extract_form_fields=True) + + assert result.pages[0].assembled is not None + # The layout detector merges the three checkbox lines and the note below + # them into one text cluster, but each checkbox has its own key printed + # beside it: they are options, + # not blanks of one sentence, so each keeps its own key. The text field + # binds its detached "Full name:" key. All four land in the page-wide + # region; the keys leave the merged paragraph, and its last line, which + # keys nothing, stays in the body. + assert [source for source, _ in _regions(result.pages[0])] == [None] + assert len(result.document.field_regions) == 1 + items = result.document.field_items + keys = [ + next( + ( + child.resolve(result.document).text + for child in item.children + if child.resolve(result.document).label == DocItemLabel.FIELD_KEY + ), + "", + ) + for item in items + ] + assert keys == [ + "Full name:", + "I agree to the terms", + "Subscribe to newsletter", + "Monthly mail digest", + ] + + values = [ + item for item in result.document.texts if isinstance(item, FieldValueItem) + ] + # Checkbox values carry no text; their state rides on a nested CHECKBOX_* child. + assert [value.text for value in values] == ["Ada Lovelace", "", "", ""] + checkbox_labels = [ + [ + child.resolve(result.document).label + for child in value.children + if child.resolve(result.document).label + in {DocItemLabel.CHECKBOX_SELECTED, DocItemLabel.CHECKBOX_UNSELECTED} + ] + for value in values + ] + assert checkbox_labels == [ + [], + [DocItemLabel.CHECKBOX_SELECTED], + [DocItemLabel.CHECKBOX_UNSELECTED], + [DocItemLabel.CHECKBOX_SELECTED], + ] + assert [value.orig for value in values] == [ + "Ada Lovelace", + "/Yes", + "/Off", + "/On", + ] + assert all(value.kind == "fillable" for value in values) + assert not any( + name in {item.text for item in result.document.texts} for name in EXPECTED_NAMES + ) + body = [ + item.text + for item in result.document.texts + if item.label in {DocItemLabel.TEXT, DocItemLabel.PARAGRAPH} + ] + assert body == ["See example.org for details"] + serialized = json.dumps(result.document.export_to_dict()) + assert "widget_field_name" not in serialized + assert "widget_appearance_state" not in serialized + + disabled = _convert(extract_form_fields=False) + assert _regions(disabled.pages[0]) == [] + assert disabled.document.field_regions == [] + assert disabled.document.field_items == [] + # The hosting paragraph is not dropped -- it is reinterpreted as a field_item + # at materialization. The only cluster extraction removes is the "Full name:" + # key promoted out of the body to become the text field's bound key. + enabled_clusters = result.pages[0].predictions.layout.clusters + disabled_clusters = disabled.pages[0].predictions.layout.clusters + enabled_ids = {cluster.id for cluster in enabled_clusters} + dropped = [ + PdfFormFieldModel._cluster_text(cluster) + for cluster in disabled_clusters + if cluster.id not in enabled_ids + ] + assert dropped == ["Fullname:"] + + +def test_widgets_inlined_in_paragraph_group_into_one_keyed_item() -> None: + # IRS Sch.1 line 7 shape: a sentence inlines a checkbox and fillable + # amounts. The layout detects the whole line as one text cluster enclosing + # the widgets; a FORM cluster also wraps the page. The paragraph keys all + # of them as one clause, hosts the item in place (its text is the key) and + # stays in the layout -- not three keyless FORM fields. + paragraph = _text_cluster( + 1, + BoundingBox(l=10, t=10, r=90, b=20), + "check here and enter amount repaid and interest:", + ) + page_form = Cluster( + id=2, label=DocItemLabel.FORM, bbox=BoundingBox(l=0, t=0, r=100, b=100) + ) + widgets = [ + _widget( + 0, + BoundingBox(l=30, t=12, r=34, b=18), + "", + field_type="/Btn", + appearance_state="/Off", + ), + _widget(1, BoundingBox(l=55, t=12, r=68, b=18), "1221.00"), + _widget(2, BoundingBox(l=74, t=12, r=87, b=18), "12.50"), + ] + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=widgets) + page.predictions.layout = LayoutPrediction(clusters=[page_form, paragraph]) + conv_res = _conversion_result(page) + + list(PdfFormFieldModel(enabled=True)(conv_res, [page])) + + keyed = [ + (source, item) + for source, items in _regions(page) + for item in items + if item.key_text + ] + assert len(keyed) == 1 + source, item = keyed[0] + assert source == paragraph.id + assert item.key_text == "check here and enter amount repaid and interest:" + assert item.key_bbox == paragraph.bbox + assert [(v.text, v.checkbox) for v in item.values] == [ + ("", DocItemLabel.CHECKBOX_UNSELECTED), + ("1221.00", None), + ("12.50", None), + ] + # The paragraph stays in the layout -- it is reinterpreted as a field_item in + # place at materialization, keeping its position in its list/container. + assert paragraph in page.predictions.layout.clusters + + +def _table_page( + *, with_structure: bool, amount_cell: str | None = None +) -> tuple[Page, Cluster]: + """A two-by-two table whose amount cell holds a widget. + + TableFormer cell boxes cover their text, so the widget sits in the row and + column bands of the table without being inside any detected cell. Like + TableFormer, the table lists no cell for the empty amount position unless + ``amount_cell`` gives that cell a text. + """ + + def cell(text, bbox, row, col, *, header=False): + return TableCell( + text=text, + bbox=bbox, + start_row_offset_idx=row, + end_row_offset_idx=row + 1, + start_col_offset_idx=col, + end_col_offset_idx=col + 1, + column_header=header, + ) + + cells = [ + cell("Item", BoundingBox(l=2, t=2, r=20, b=8), 0, 0, header=True), + cell("Amount", BoundingBox(l=52, t=2, r=70, b=8), 0, 1, header=True), + cell("Total income", BoundingBox(l=2, t=22, r=40, b=28), 1, 0), + ] + if amount_cell is not None: + cells.append(cell(amount_cell, BoundingBox(l=52, t=22, r=60, b=28), 1, 1)) + table = Cluster( + id=1, label=DocItemLabel.TABLE, bbox=BoundingBox(l=0, t=0, r=100, b=40) + ) + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock( + widgets=[_widget(0, BoundingBox(l=62, t=20, r=82, b=30), "1,000")] + ) + page.predictions.layout = LayoutPrediction(clusters=[table]) + if with_structure: + page.predictions.tablestructure = TableStructurePrediction( + table_map={ + table.id: Table( + label=table.label, + id=table.id, + page_no=page.page_no, + cluster=table, + otsl_seq=[], + num_rows=2, + num_cols=2, + table_cells=cells, + ) + } + ) + return page, table + + +def _materialize(page: Page) -> DoclingDocument: + """Run the form stage, page assembly and reading order on one page.""" + conv_res = _conversion_result(page) + list(PdfFormFieldModel(enabled=True)(conv_res, [page])) + page._backend = MagicMock() + page._backend.is_valid.return_value = True + list(PageAssembleModel(PageAssembleOptions())(conv_res, [page])) + assert page.assembled is not None + conv_res.assembled = AssembledUnit( + elements=page.assembled.elements, + body=page.assembled.body, + headers=page.assembled.headers, + ) + return ReadingOrderModel(ReadingOrderOptions())(conv_res) + + +def test_value_grid_materializes_column_headers_as_field_hints() -> None: + page = Page(page_no=1, size=Size(width=300, height=100)) + boxes = [ + BoundingBox(l=x, t=y, r=x + 60, b=y + 12) for y in (30, 65) for x in (100, 200) + ] + page.parsed_page = MagicMock( + widgets=[_widget(i, bbox, str(i + 1)) for i, bbox in enumerate(boxes)] + ) + page.predictions.layout = LayoutPrediction( + clusters=[ + _text_cluster(1, BoundingBox(l=10, t=31, r=80, b=41), "Revenue"), + _text_cluster(2, BoundingBox(l=10, t=66, r=80, b=76), "Expenses"), + _text_cluster(3, BoundingBox(l=100, t=5, r=160, b=15), "This year"), + _text_cluster(4, BoundingBox(l=200, t=5, r=260, b=15), "Last year"), + ] + ) + + doc = _materialize(page) + + assert [ + [(child.resolve(doc).label, child.resolve(doc).text) for child in item.children] + for item in doc.field_items + ] == [ + [ + (DocItemLabel.FIELD_KEY, key), + (DocItemLabel.FIELD_VALUE, str(i + 1)), + (DocItemLabel.FIELD_HINT, hint), + ] + for i, (key, hint) in enumerate( + [ + ("Revenue", "This year"), + ("Revenue", "Last year"), + ("Expenses", "This year"), + ("Expenses", "Last year"), + ] + ) + ] + + +def _cell_region(doc: DoclingDocument, row: int, col: int) -> NodeItem: + """The item a rich cell of the document's only table points to.""" + (table,) = doc.tables + (cell,) = [ + c + for c in table.data.table_cells + if c.start_row_offset_idx == row and c.start_col_offset_idx == col + ] + assert isinstance(cell, RichTableCell) + return cell.ref.resolve(doc) + + +def test_widget_in_table_goes_into_its_cell_with_value_only(caplog) -> None: + # The row header names the value in another column and the header names + # the column: the table already says both, so the empty cell TableFormer + # did not list becomes a rich cell holding only the value. + page, table = _table_page(with_structure=True) + + with caplog.at_level("WARNING"): + doc = _materialize(page) + assert "Form field" not in caplog.text # the keying ran, not the fallback + + assert _regions(page) == [] + assert page.predictions.tablestructure is not None + (cell_fields,) = page.predictions.tablestructure.table_map[table.id].field_cells + assert [(i.key_text, i.hint_text) for i in cell_fields.items] == [("", "")] + # The table keeps its cells: nothing is promoted out of the layout. + assert [cluster.id for cluster in page.predictions.layout.clusters] == [1] + + region = _cell_region(doc, 1, 1) + assert region.label == DocItemLabel.FIELD_REGION + (field_item,) = doc.field_items + assert field_item.parent is not None + assert field_item.parent.cref == region.self_ref + children = [child.resolve(doc) for child in field_item.children] + assert [(child.label, child.text) for child in children] == [ + (DocItemLabel.FIELD_VALUE, "1,000") + ] + doc.validate_document() + doclang = doc.export_to_doclang() + assert doclang.index("") < doclang.index("") + + +@pytest.mark.parametrize( + ("amount_cell", "key", "kept_text"), + [ + # Text in the widget's own cell is its key, not repeated as text. + ("Paid", "Paid", []), + # A code without letters never keys a value, but stays in its cell. + ("250", "", ["250"]), + ], +) +def test_text_in_the_widgets_own_cell_stays_in_the_rich_cell( + amount_cell: str, key: str, kept_text: list[str] +) -> None: + page, _ = _table_page(with_structure=True, amount_cell=amount_cell) + + doc = _materialize(page) + + region = _cell_region(doc, 1, 1) + children = [child.resolve(doc) for child in region.children] + assert [c.text for c in children if c.label == DocItemLabel.TEXT] == kept_text + (field_item,) = [c for c in children if c.label == DocItemLabel.FIELD_ITEM] + keys = [ + child.resolve(doc).text + for child in field_item.children + if child.resolve(doc).label == DocItemLabel.FIELD_KEY + ] + assert keys == ([key] if key else []) + doc.validate_document() + + +def test_widget_between_columns_of_an_incomplete_grid_stays_out_of_the_table() -> None: + # A three-column grid whose middle column has no text has no band there: + # a widget in that column, overlapping neither neighbour's band, has no + # reliable cell and stays after the table, without a key: the row header + # sits in another cell and never keys it. + def cell(text, bbox, row, col): + return TableCell( + text=text, + bbox=bbox, + start_row_offset_idx=row, + end_row_offset_idx=row + 1, + start_col_offset_idx=col, + end_col_offset_idx=col + 1, + ) + + table = Cluster( + id=1, label=DocItemLabel.TABLE, bbox=BoundingBox(l=0, t=0, r=100, b=40) + ) + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock( + widgets=[_widget(0, BoundingBox(l=44, t=20, r=56, b=30), "1,000")] + ) + page.predictions.layout = LayoutPrediction(clusters=[table]) + page.predictions.tablestructure = TableStructurePrediction( + table_map={ + table.id: Table( + label=table.label, + id=table.id, + page_no=page.page_no, + cluster=table, + otsl_seq=[], + num_rows=2, + num_cols=3, + table_cells=[ + cell("Item", BoundingBox(l=2, t=2, r=20, b=8), 0, 0), + cell("Total", BoundingBox(l=62, t=2, r=80, b=8), 0, 2), + cell("Total income", BoundingBox(l=2, t=22, r=40, b=28), 1, 0), + ], + ) + } + ) + + list(PdfFormFieldModel(enabled=True)(_conversion_result(page), [page])) + + assert page.predictions.tablestructure is not None + assert page.predictions.tablestructure.table_map[table.id].field_cells == [] + ((_, (item,)),) = _regions(page) + assert item.key_text == "" + assert [value.text for value in item.values] == ["1,000"] + + +def test_widget_in_table_without_structure_stays_keyless() -> None: + # Table keys come from TableFormer cells: without table structure the + # widget keeps its value and no free-form key crosses into the table. + page, _ = _table_page(with_structure=False) + + list(PdfFormFieldModel(enabled=True)(_conversion_result(page), [page])) + + ((_, (item,)),) = _regions(page) + assert item.key_text == "" + assert item.hint_text == "" + assert [value.text for value in item.values] == ["1,000"] + + +def test_keying_failure_keeps_values_without_keys(caplog, monkeypatch) -> None: + # The page must not fail when the keying raises: its values come through + # unkeyed and the key stays in the body. + def failing(*args, **kwargs): + raise RuntimeError("keying failed") + + monkeypatch.setattr(form_field_model, "assign", failing) + key = _text_cluster(2, BoundingBox(l=22, t=10, r=80, b=18), "Full name") + form = Cluster( + id=1, label=DocItemLabel.FORM, bbox=BoundingBox(l=0, t=0, r=100, b=60) + ) + widgets = [ + _widget(0, BoundingBox(l=10, t=10, r=18, b=18), "Ada"), + _widget(1, BoundingBox(l=10, t=30, r=18, b=38), "Lovelace"), + ] + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=widgets) + page.predictions.layout = LayoutPrediction(clusters=[form, key]) + + with caplog.at_level("WARNING"): + list(PdfFormFieldModel(enabled=True)(_conversion_result(page), [page])) + + assert "Form field keying failed on page 1" in caplog.text + ((source, items),) = _regions(page) + assert source == form.id + assert [(item.key_text, item.values[0].text) for item in items] == [ + ("", "Ada"), + ("", "Lovelace"), + ] + assert key in page.predictions.layout.clusters + + +def _options_page() -> tuple[Page, Cluster]: + """Three checkboxes with their option keys, then a note, in one cluster. + + acroform_sample.pdf at a quarter of its size: the layout model merges the + four text lines into one text cluster. + """ + lines = [ + ("I agree to the terms", 35, 49.75), + ("Subscribe to newsletter", 42.5, 55), + ("Monthly mail digest", 50, 49.5), + ] + note = ("See example.org for details", BoundingBox(l=18, t=55, r=54.5, b=57.75)) + boxes = [ + (text, BoundingBox(l=23.75, t=top, r=right, b=top + 2.75)) + for text, top, right in lines + ] + cells = [ + TextCell( + index=i, + rect=BoundingRectangle.from_bounding_box(box), + text=text, + orig=text, + from_ocr=False, + ) + for i, (text, box) in enumerate([*boxes, note]) + ] + paragraph = Cluster( + id=1, + label=DocItemLabel.TEXT, + bbox=BoundingBox(l=18, t=35, r=55, b=57.75), + cells=cells, + ) + widgets = [ + _widget( + i, + BoundingBox(l=18, t=top, r=21.5, b=top + 3.5), + "", + field_type="/Btn", + appearance_state="/Off", + ) + for i, (_, top, _) in enumerate(lines) + ] + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=widgets) + page.predictions.layout = LayoutPrediction(clusters=[paragraph]) + return page, paragraph + + +def test_option_keys_leave_a_partly_used_paragraph() -> None: + # Each checkbox uses its own option key, but the note is no key, so + # the paragraph is only partly used: the keys leave it, and the note + # stays in the body with a box fitted to it, instead of the whole paragraph + # repeating the keys after the fields. + page, paragraph = _options_page() + + list(PdfFormFieldModel(enabled=True)(_conversion_result(page), [page])) + + assert [item.key_text for _, items in _regions(page) for item in items] == [ + "I agree to the terms", + "Subscribe to newsletter", + "Monthly mail digest", + ] + assert page.predictions.layout.clusters == [paragraph] + assert [cell.text for cell in paragraph.cells] == ["See example.org for details"] + assert paragraph.bbox == BoundingBox(l=18, t=55, r=54.5, b=57.75) + + +def test_filled_values_do_not_remain_as_a_paragraph_after_their_keys() -> None: + # A filled form: the page paints each widget's value, and the layout model + # merges two key-and-value lines into one text cluster. Both keys + # become keys and the painted values are the fields' own values, so + # nothing of the cluster is left for the body. + def cell(index: int, text: str, bbox: BoundingBox) -> TextCell: + return TextCell( + index=index, + rect=BoundingRectangle.from_bounding_box(bbox), + text=text, + orig=text, + from_ocr=False, + ) + + paragraph = Cluster( + id=1, + label=DocItemLabel.TEXT, + bbox=BoundingBox(l=10, t=10, r=50, b=20), + cells=[ + cell(0, "Full name:", BoundingBox(l=10, t=10, r=28, b=14)), + cell(1, "John Smith", BoundingBox(l=31, t=10, r=50, b=14)), + cell(2, "City:", BoundingBox(l=10, t=16, r=20, b=20)), + cell(3, "Paris", BoundingBox(l=31, t=16, r=40, b=20)), + ], + ) + widgets = [ + _widget(0, BoundingBox(l=30, t=9.5, r=70, b=14.5), "John Smith"), + _widget(1, BoundingBox(l=30, t=15.5, r=70, b=20.5), "Paris"), + ] + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=widgets) + page.predictions.layout = LayoutPrediction(clusters=[paragraph]) + + list(PdfFormFieldModel(enabled=True)(_conversion_result(page), [page])) + + assert [ + (item.key_text, [value.text for value in item.values]) + for _, items in _regions(page) + for item in items + ] == [("Full name:", ["John Smith"]), ("City:", ["Paris"])] + assert page.predictions.layout.clusters == [] + + +def test_failed_commit_restores_a_trimmed_paragraph(caplog, monkeypatch) -> None: + # Writing the plan fails after the keys left the paragraph: the page + # is left unchanged, so the paragraph gets its cells and box back. + page, paragraph = _options_page() + cells, bbox = list(paragraph.cells), paragraph.bbox + + def failing(*args, **kwargs): + raise RuntimeError("commit failed") + + monkeypatch.setattr(PdfFormFieldModel, "_suppress_duplicate_text", failing) + with caplog.at_level("WARNING"): + list(PdfFormFieldModel(enabled=True)(_conversion_result(page), [page])) + + assert "Form field extraction failed on page 1" in caplog.text + assert page.predictions.form_fields is None + assert page.predictions.layout.clusters == [paragraph] + assert paragraph.cells == cells + assert paragraph.bbox == bbox + + +def test_inline_host_is_not_dropped_as_rendered_duplicate() -> None: + # A filled widget whose painted value the layout absorbed into a paragraph + # slightly larger than the widget: the paragraph hosts the item in place, + # so dropping it as a duplicate of the value would lose the value. + host = _text_cluster(1, BoundingBox(l=10, t=10, r=90, b=20), "Amount 12") + widgets = [_widget(0, BoundingBox(l=12, t=11, r=88, b=19), "Amount 12")] + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=widgets) + page.predictions.layout = LayoutPrediction(clusters=[host]) + + list(PdfFormFieldModel(enabled=True)(_conversion_result(page), [page])) + + ((source, items),) = _regions(page) + assert source == host.id + assert [value.text for value in items[0].values] == ["Amount 12"] + assert host in page.predictions.layout.clusters + + +def _field_region_page(page_wide: bool) -> Page: + """Name: [box] / a note / Date: [box], in a FORM region or on the page.""" + clusters = [ + _text_cluster(1, BoundingBox(l=10, t=10, r=30, b=20), "Name:"), + _text_cluster(2, BoundingBox(l=10, t=30, r=90, b=40), "Write in capitals."), + _text_cluster(3, BoundingBox(l=10, t=50, r=30, b=60), "Date:"), + ] + if not page_wide: + form = Cluster( + id=4, + label=DocItemLabel.FORM, + bbox=BoundingBox(l=5, t=5, r=95, b=65), + children=clusters, + ) + clusters = [form, *clusters] + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock( + widgets=[ + _widget(0, BoundingBox(l=35, t=10, r=90, b=20), "Ada"), + _widget(1, BoundingBox(l=35, t=50, r=90, b=60), "2026-10-12"), + ], + hyperlinks=[], + ) + page.predictions.layout = LayoutPrediction(clusters=clusters) + return page + + +@pytest.mark.parametrize("page_wide", [False, True]) +def test_fields_go_between_the_text_they_are_printed_between(page_wide: bool) -> None: + # The keys leave the text, so the reading order has only the note left. + # The note is printed between the two fields, so it is emitted there, + # in a FORM region as on the page. + doc = _materialize(_field_region_page(page_wide)) + + order = [] + for item, _ in doc.iterate_items(with_groups=False): + if isinstance(item, FieldItem): + order.append(item.children[0].resolve(doc).text) + elif item.label == DocItemLabel.TEXT: + order.append(item.text) + assert order == ["Name:", "Write in capitals.", "Date:"] + # On the page each run of fields has a field region of its own. + assert len(doc.field_regions) == (2 if page_wide else 1) + + +def _host_between_paragraphs( + link: str | None = None, +) -> tuple[Page, Cluster, Cluster, Cluster]: + """Three stacked paragraphs; the middle one hosts a checkbox widget. + + ``link`` puts a hyperlink annotation over the hosting paragraph. + """ + host_bbox = BoundingBox(l=10, t=30, r=90, b=40) + previous = _text_cluster( + 1, BoundingBox(l=10, t=10, r=90, b=20), "Applicants who live in" + ) + host = _text_cluster(2, host_bbox, "check here if you rent") + following = _text_cluster( + 3, BoundingBox(l=10, t=50, r=90, b=60), "and attach the lease" + ) + hyperlinks = ( + [] + if link is None + else [ + PdfHyperlink( + index=0, + rect=BoundingRectangle.from_bounding_box( + host_bbox.to_bottom_left_origin(page_height=100) + ), + uri=link, + ) + ] + ) + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock( + widgets=[ + _widget( + 0, + BoundingBox(l=80, t=32, r=86, b=38), + "", + field_type="/Btn", + appearance_state="/Off", + ) + ], + hyperlinks=hyperlinks, + ) + page.predictions.layout = LayoutPrediction(clusters=[previous, host, following]) + return page, previous, host, following + + +@pytest.mark.parametrize("host_is_source", [True, False]) +def test_reading_order_never_merges_an_inline_host(host_is_source: bool) -> None: + # predict_merges joins a paragraph to its continuation at the top of the + # next column, and the merged paragraph is not emitted on its own. A host + # becomes a field item and takes no merged text: merged from, its + # continuation would vanish; merged into, its widgets would. + left, right = (10, 90), (110, 190) + host_column, host_top = (left, 30) if host_is_source else (right, 10) + host_bbox = BoundingBox( + l=host_column[0], t=host_top, r=host_column[1], b=host_top + 10 + ) + previous = _text_cluster( + 1, + BoundingBox( + l=10, t=10 if host_is_source else 30, r=90, b=20 if host_is_source else 40 + ), + "Applicants who live in", + ) + host = _text_cluster(2, host_bbox, "check here if you rent") + following = _text_cluster( + 3, + BoundingBox( + l=110, t=10 if host_is_source else 30, r=190, b=20 if host_is_source else 40 + ), + "and attach the lease", + ) + page = Page(page_no=1, size=Size(width=200, height=100)) + page.parsed_page = MagicMock( + widgets=[ + _widget( + 0, + BoundingBox( + l=host_bbox.r - 10, + t=host_top + 2, + r=host_bbox.r - 4, + b=host_top + 8, + ), + "", + field_type="/Btn", + appearance_state="/Off", + ) + ], + hyperlinks=[], + ) + page.predictions.layout = LayoutPrediction(clusters=[previous, host, following]) + + doc = _materialize(page) + + texts = [(text.label, text.text) for text in doc.texts] + assert (DocItemLabel.TEXT, "Applicants who live in") in texts + assert (DocItemLabel.FIELD_KEY, "check here if you rent") in texts + assert (DocItemLabel.TEXT, "and attach the lease") in texts + (item,) = doc.field_items + assert [child.resolve(doc).label for child in item.children] == [ + DocItemLabel.FIELD_KEY, + DocItemLabel.FIELD_VALUE, + ] + + +def test_inline_host_keeps_its_hyperlink() -> None: + page, _, _, _ = _host_between_paragraphs(link="https://example.org/rent") + + doc = _materialize(page) + + (key,) = [text for text in doc.texts if text.label == DocItemLabel.FIELD_KEY] + assert str(key.hyperlink) == "https://example.org/rent" + + +def test_list_item_host_keeps_the_list_made_of_list_items() -> None: + # A list item that hosts a checkbox becomes a field item in place. List + # serializers expect list items under a list, so the field item sits in an + # empty list item of its own. + texts = ["Read the terms first", "I agree to the terms", "Sign below"] + clusters = [ + _text_cluster(n + 1, BoundingBox(l=10, t=10 + 15 * n, r=90, b=20 + 15 * n), t) + for n, t in enumerate(texts) + ] + for cluster in clusters: + cluster.label = DocItemLabel.LIST_ITEM + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock( + widgets=[ + _widget( + 0, + BoundingBox(l=82, t=27, r=88, b=33), + "", + field_type="/Btn", + appearance_state="/Off", + ) + ], + hyperlinks=[], + ) + page.predictions.layout = LayoutPrediction(clusters=clusters) + + doc = _materialize(page) + + (group,) = doc.groups + children = [child.resolve(doc) for child in group.children] + assert all(isinstance(child, ListItem) for child in children) + assert [child.text for child in children] == [texts[0], "", texts[2]] + (item,) = [grandchild.resolve(doc) for grandchild in children[1].children] + assert isinstance(item, FieldItem) + key = item.children[0].resolve(doc) + assert (key.label, key.text) == (DocItemLabel.FIELD_KEY, texts[1]) + + +@pytest.mark.parametrize( + ("label", "widget"), + [ + # A text box beside a heading, and a checkbox inside one: a heading + # never hosts a field in place, so both go to a field region. + (DocItemLabel.SECTION_HEADER, "beside"), + (DocItemLabel.SECTION_HEADER, "inside"), + (DocItemLabel.FORMULA, "beside"), + ], +) +def test_heading_or_formula_key_is_a_plain_key_in_a_field_region( + label: DocItemLabel, widget: str +) -> None: + key = _text_cluster(1, BoundingBox(l=10, t=10, r=60, b=20), "Applicant name") + key.label = label + widgets = ( + [_widget(0, BoundingBox(l=65, t=10, r=95, b=20), "Jane Doe")] + if widget == "beside" + else [ + _widget( + 0, + BoundingBox(l=52, t=12, r=58, b=18), + "", + field_type="/Btn", + appearance_state="/Off", + ) + ] + ) + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock(widgets=widgets, hyperlinks=[]) + page.predictions.layout = LayoutPrediction(clusters=[key]) + + doc = _materialize(page) + + (item,) = doc.field_items + assert isinstance(item.parent.resolve(doc), FieldRegionItem) + field_key = item.children[0].resolve(doc) + assert field_key.label == DocItemLabel.FIELD_KEY + assert field_key.text == "Applicant name" + assert field_key.prov + assert not field_key.children + assert "Applicant name" in doc.export_to_markdown() + + +@pytest.mark.parametrize("boundary", ["ordinary", "zero_area", "table_scope"]) +@pytest.mark.parametrize("abstain", [False, True]) +def test_painted_cells_survive_candidate_filters_and_abstention( + boundary: str, abstain: bool, monkeypatch +) -> None: + # Recognition must cover cells cleanup can read even when candidate filters + # reject them. Painted cells must not become atoms shared with keys. + key = _text_cluster(1, BoundingBox(l=10, t=10, r=28, b=14), "Full name:") + painted = _text_cluster( + 2, + BoundingBox(l=31, t=31, r=31 if boundary == "zero_area" else 50, b=34), + " Paris ", + ) + painted.cells[0].index = 2 + key.cells += painted.cells + key.bbox = BoundingBox.enclosing_bbox([key.bbox, painted.bbox]) + clusters = [key] + if boundary == "table_scope": + clusters.append( + Cluster( + id=3, + label=DocItemLabel.TABLE, + bbox=BoundingBox(l=30, t=30, r=70, b=35), + ) + ) + page = Page(page_no=1, size=Size(width=100, height=100)) + page.parsed_page = MagicMock( + widgets=[ + _widget(0, BoundingBox(l=30, t=9.5, r=70, b=14.5), "John Smith"), + _widget(1, BoundingBox(l=30, t=30, r=70, b=35), "Paris"), + ] + ) + page.predictions.layout = LayoutPrediction(clusters=clusters) + if abstain: + monkeypatch.setattr( + "docling.models.stages.form_field.keying.inputs.MAX_KEY_TEXTS", 0 + ) + monkeypatch.setattr( + "docling.models.stages.form_field.keying.solver.MAX_KEY_TEXTS", 0 + ) + + assignment = form_field_model._assign(page) + + assert assignment.painted_cells == {(key.id, 2)} + assert all(key_text.text == "Full name:" for key_text in assignment.key_texts) + assert set().union(*assignment.sources.values()) == {(key.id, 0)} + assert [value.native.index for value in assignment.values] == [0, 1] + assert (assignment.solver_status == "optimal") is not abstain + list(PdfFormFieldModel(enabled=True)(_conversion_result(page), [page])) + assert (key in page.predictions.layout.clusters) is abstain + assert [ + (item.key_text, [value.text for value in item.values]) + for _, items in _regions(page) + for item in items + ] == [("" if abstain else "Full name:", ["John Smith"]), ("", ["Paris"])] diff --git a/tests/test_form_fields_order.py b/tests/test_form_fields_order.py new file mode 100644 index 0000000000..014ec7b9eb --- /dev/null +++ b/tests/test_form_fields_order.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +import random +from itertools import combinations_with_replacement + +from docling_core.types.doc import BoundingBox + +from docling.utils.form_fields import reading_order, text_slots + + +def _box(left: float, top: float, right: float, bottom: float) -> BoundingBox: + return BoundingBox(l=left, t=top, r=right, b=bottom) + + +def test_reading_order_reads_lines_left_to_right_and_columns_top_down() -> None: + name, date = _box(10, 10, 40, 20), _box(50, 10, 90, 20) + below = _box(10, 30, 40, 40) + other_column = _box(60, 30, 90, 40) + assert reading_order(name, date) == -1 + assert reading_order(date, name) == 1 + assert reading_order(name, below) == -1 + assert reading_order(name, other_column) == 0 + + +def test_text_between_two_fields_goes_between_them() -> None: + # Name: [___] / Please write in capitals. / Date: [___] + name, note, date = _box(10, 10, 90, 20), _box(10, 30, 90, 40), _box(10, 50, 90, 60) + assert text_slots([note], [name, date]) == [1] + + +def test_text_slots_match_exhaustive_search() -> None: + rng = random.Random(4496) + + def box() -> BoundingBox: + left, top = rng.uniform(0, 80), rng.uniform(0, 80) + return _box(left, top, left + rng.uniform(2, 40), top + rng.uniform(2, 15)) + + for _ in range(400): + texts = [box() for _ in range(rng.randint(0, 5))] + fields = [box() for _ in range(rng.randint(0, 5))] + + def cost(slots: tuple[int, ...]) -> int: + return sum( + (order < 0 and k < slot) or (order > 0 and k >= slot) + for text, slot in zip(texts, slots) + for k, field in enumerate(fields) + for order in [reading_order(text, field)] + ) + + options = list( + combinations_with_replacement(range(len(fields) + 1), len(texts)) + ) + least = min(cost(o) for o in options) + # Ties: the latest slots, compared from the last text backwards. + expected = max((o for o in options if cost(o) == least), key=lambda o: o[::-1]) + assert tuple(text_slots(texts, fields)) == expected diff --git a/tests/test_layout_postprocessor.py b/tests/test_layout_postprocessor.py index cfb216ca04..ca4ad5a269 100644 --- a/tests/test_layout_postprocessor.py +++ b/tests/test_layout_postprocessor.py @@ -94,6 +94,22 @@ def test_sort_cells_uses_native_cell_index_order() -> None: assert [cell.index for cell in cells] == [3, 1, 2] +def test_checkbox_keeps_its_drawn_box_left_of_its_text() -> None: + # The model boxes a checkbox together with its label; the drawn square + # holds no text cell, so shrinking to the cells would cut it off. + checkbox = _cluster( + 1, BoundingBox(l=55, t=10, r=540, b=30), DocItemLabel.CHECKBOX_UNSELECTED + ) + text = _cluster(2, BoundingBox(l=55, t=40, r=540, b=50)) + checkbox.cells = [_text_cell(0, BoundingBox(l=73, t=12, r=530, b=28))] + text.cells = [_text_cell(1, BoundingBox(l=73, t=41, r=530, b=49))] + + _postprocessor()._adjust_cluster_bboxes([checkbox, text]) + + assert checkbox.bbox == BoundingBox(l=55, t=10, r=540, b=30) + assert text.bbox == BoundingBox(l=73, t=41, r=530, b=49) + + def test_assign_cells_to_clusters_matches_exhaustive_selection() -> None: clusters = [ _cluster(0, BoundingBox(l=0, t=0, r=100, b=100)), diff --git a/tests/verify_utils.py b/tests/verify_utils.py index d80299ed5d..9e4a33afac 100644 --- a/tests/verify_utils.py +++ b/tests/verify_utils.py @@ -494,7 +494,11 @@ def run_check(check: str, verify: Callable[[], bool], mismatch: str) -> None: doc_pred_md = _normalize_newlines( doc_result.document.export_to_markdown(compact_tables=True) ) - doc_pred_dt = _normalize_newlines(doc_result.document.export_to_doctags()) + doc_pred_dt = ( + _normalize_newlines(doc_result.document.export_to_doctags()) + if verify_doctags + else "" + ) pages_path = gt.pages_meta json_path = gt.doc_json diff --git a/uv.lock b/uv.lock index f2b4d8cceb..81c3092aae 100644 --- a/uv.lock +++ b/uv.lock @@ -2422,10 +2422,10 @@ requires-dist = [ { name = "defusedxml", marker = "extra == 'format-iwork'", specifier = ">=0.7.1,<0.8.0" }, { name = "defusedxml", marker = "extra == 'format-xml-uspto'", specifier = ">=0.7.1,<0.8.0" }, { name = "defusedxml", marker = "extra == 'models-local'", specifier = ">=0.7.1,<0.8.0" }, - { name = "docling-core", specifier = ">=2.98.0,<3.0.0" }, - { name = "docling-core", extras = ["chunking"], marker = "extra == 'feat-chunking'", specifier = ">=2.98.0,<3.0.0" }, + { name = "docling-core", specifier = ">=2.99.0,<3.0.0" }, + { name = "docling-core", extras = ["chunking"], marker = "extra == 'feat-chunking'", specifier = ">=2.99.0,<3.0.0" }, { name = "docling-ibm-models", marker = "extra == 'models-local'", specifier = ">=4.0.2,<5" }, - { name = "docling-parse", marker = "extra == 'format-pdf-docling'", specifier = ">=7.22.1,<8.0.0" }, + { name = "docling-parse", marker = "extra == 'format-pdf-docling'", specifier = ">=7.22.2,<8.0.0" }, { name = "docling-slim", extras = ["convert-core"], marker = "extra == 'extract-core'" }, { name = "docling-slim", extras = ["format-audio"], marker = "extra == 'format-video'" }, { name = "docling-slim", extras = ["format-docx", "format-pptx", "format-xlsx"], marker = "extra == 'format-office'" }, @@ -2491,7 +2491,7 @@ requires-dist = [ { name = "rtree", marker = "extra == 'convert-core'", specifier = ">=1.3.0,<2.0.0" }, { name = "scikit-image", marker = "extra == 'feat-ocr-easyocr'", specifier = ">=0.19" }, { name = "scikit-learn", marker = "extra == 'format-video'", specifier = ">=1.3.0,<2.0.0" }, - { name = "scipy", marker = "extra == 'convert-core'", specifier = ">=1.6.0,<2.0.0" }, + { name = "scipy", marker = "extra == 'convert-core'", specifier = ">=1.9.0,<2.0.0" }, { name = "soundfile", marker = "extra == 'format-video'", specifier = ">=0.12.1,<1.0.0" }, { name = "tesserocr", marker = "extra == 'feat-ocr-tesserocr'", specifier = ">=2.7.1,<3.0.0" }, { name = "timm", marker = "extra == 'models-vlm-inline'", specifier = ">=1.0.22,<2.0.0" }, @@ -2562,7 +2562,7 @@ examples = [ ] pr-fast-checks = [ { name = "boto3-stubs", specifier = "~=1.37" }, - { name = "docling-core", extras = ["chunking"], specifier = ">=2.98.0,<3.0.0" }, + { name = "docling-core", extras = ["chunking"], specifier = ">=2.99.0,<3.0.0" }, { name = "pandas-stubs", specifier = "~=2.1" }, { name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.3.0,<3.0.0" },