diff --git a/docling_core/compat.py b/docling_core/compat.py index c01776f00..925580702 100644 --- a/docling_core/compat.py +++ b/docling_core/compat.py @@ -179,7 +179,9 @@ def project_to(doc: DoclingDocument, target_version: str) -> DoclingDocument: # The current SDK's version validator normalises the version field to # CURRENT_VERSION on success. We restore the target version afterwards so # callers and downstream serialisation correctly reflect the projected schema. - result = DoclingDocument.model_validate(data) + # Validate under the source stamp: the projected stamp would trigger load-time upgrades + # (the inline separator migration) on data that is already current. + result = DoclingDocument.model_validate({**data, "version": doc.version}) object.__setattr__(result, "version", data["version"]) return result @@ -198,6 +200,49 @@ def list_projectors() -> list[tuple[int, int]]: # the serialised dict without error. Full semantic fidelity is not always # possible; the goal is a document that validates rather than crashes. +# --- 1.11 → 1.10 ---------------------------------------------------------- +# Schema 1.11 (docling-core v2.105.0): InlineGroup runs carry their own +# significant whitespace and serializers concatenate them faithfully. +# +# The 1.11 and 1.10 schemas are structurally identical -- no new fields, +# labels, or lists. The only difference is *semantic*: a 1.11 run's text +# contains its boundary whitespace, whereas a 1.10 client re-inserts a +# single " " between runs at serialization time. Left as-is, a 1.10 client +# renders every boundary twice and keeps the space inside the markup +# (``**bold **``, which CommonMark does not render as bold). The projector +# therefore strips inline runs to the shape old producers emitted; the old +# join puts one space back. 1.10 cannot represent a tight run boundary, so +# ``H2O`` comes back as the old ``H 2 O``. +# --------------------------------------------------------------------------- + + +@register_projector(from_minor=11, to_minor=10) +def _project_1_11_to_1_10(data: dict) -> dict: + """Downgrade a schema 1.11 document dict to schema 1.10. + + 1.11 and 1.10 share an identical structure; the difference is only in how + inline-run whitespace is represented. A 1.10 client joins runs with ``" "``, + so the text of each inline-group child is stripped. A whitespace-only run + becomes ``""``, which old serializers skip, so no refs need renumbering. + ``orig`` is left alone. + """ + data = dict(data) + data["version"] = "1.10.0" + inline_runs = { + child.get("$ref", child.get("cref")) + for group in data.get("groups", []) + if group.get("label") == "inline" + for child in group.get("children", []) + } + data["texts"] = [ + {**item, "text": item["text"].strip()} + if item.get("self_ref") in inline_runs and isinstance(item.get("text"), str) + else item + for item in data.get("texts", []) + ] + return data + + # --- 1.10 → 1.9 ----------------------------------------------------------- # Schema 1.10 (docling-core v2.69.0, PR #519): # • Added FieldRegionItem, FieldHeadingItem, FieldItem, FieldValueItem diff --git a/docling_core/transforms/deserializer/doclang.py b/docling_core/transforms/deserializer/doclang.py index 611e8c69a..3ebe09900 100644 --- a/docling_core/transforms/deserializer/doclang.py +++ b/docling_core/transforms/deserializer/doclang.py @@ -147,6 +147,22 @@ def _repair_unescaped_xml_text(text: str) -> str: return "".join(parts) +# A whitespace run touching a newline is pretty-print indentation; one without is content. +_PRETTY_INDENT = re.compile(r"\A\s*\n\s*|\s*\n\s*\Z") + + +def _normalize_bare_text(data: str) -> str: + """Drop pretty-print indentation from a bare XML text node, keep significant whitespace. + + Standard XML heuristic: whitespace containing a newline is layout noise, whitespace + without one is meaningful. Serialized DocLang uses ```` for whitespace-bearing + runs, so this only governs hand-written and VLM-emitted input. + """ + if not data.strip(): + return "" if "\n" in data else data + return _PRETTY_INDENT.sub("", data) + + def _enforce_doclang_dom_budgets( root: Element, *, @@ -521,7 +537,8 @@ def _parse_text_like(self, *, doc: DoclingDocument, el: Element, parent: NodeIte prov_list = self._extract_provenance(doc=doc, el=el) content_layer = self._extract_layer(el=el) - text, formatting = self._extract_text_with_formatting(el) + # A run inside an inline group owns its edge whitespace; a block host does not. + text, formatting = self._extract_text_with_formatting(el, trim=not isinstance(parent, InlineGroup)) if not text and not is_empty: if ( thread_id @@ -676,17 +693,12 @@ def _extract_code_content_and_language(self, el: Element) -> tuple[str, CodeLang parts.append(node.data) elif isinstance(node, Element): nm_child = node.tagName - if nm_child in { - DocLangToken.LOCATION.value, - DocLangToken.LAYER.value, - DocLangToken.LABEL.value, - DocLangToken.CAPTION.value, - }: + if self._is_element_head_tag(node): continue elif nm_child == DocLangToken.BR.value: parts.append("\n") else: - parts.append(self._get_text(node)) + parts.append(self._collect_text(node)) return "".join(parts), lang_label @@ -1524,7 +1536,20 @@ def _parse_inline_runs( """Add text runs for ``nodes``, carrying ``formatting`` through nested formatting tags.""" for node in nodes: if isinstance(node, Element): - if node.tagName in _FORMATTING_TAG_VALUES: + if node.tagName == DocLangToken.CONTENT.value: + # is the explicit whitespace-preserving text wrapper: it is a run, + # not a nested item, so it inherits the formatting accumulated on the way in. + # Dispatching it would drop that formatting (x + # would come back unformatted). + text_content = self._collect_text(node) + if text_content: + doc.add_text( + label=DocItemLabel.TEXT, + text=text_content, + parent=parent, + formatting=formatting, + ) + elif node.tagName in _FORMATTING_TAG_VALUES: # Accumulate this tag's style and keep descending, so e.g. a # b gives "a" bold and "b" bold+italic. merged = self._merge_formatting_tag(node.tagName, formatting) @@ -1532,7 +1557,8 @@ def _parse_inline_runs( else: self._dispatch_element(doc=doc, el=node, parent=parent) elif isinstance(node, Text): - text_content = node.data.strip() + # A run owns its boundary whitespace; only pretty-print indentation is dropped. + text_content = _normalize_bare_text(node.data) if text_content: doc.add_text( label=DocItemLabel.TEXT, @@ -2377,20 +2403,23 @@ def _parse_otsl_table_content( table_cells=table_cells, ) - def _extract_text_with_formatting(self, el: Element) -> tuple[str, Formatting | None]: + def _extract_text_with_formatting(self, el: Element, *, trim: bool = True) -> tuple[str, Formatting | None]: """Extract text content and formatting from an element. If the element contains a single formatting child (bold, italic, etc.), recursively extract the text and build up the Formatting object. + Args: + el: the element to extract from. + trim: trim the outer boundary. True for a block-level host, False when ``el`` is a + run inside an inline group, whose edge whitespace is significant. + Returns: Tuple of (text_content, formatting_object or None) """ - # Get non-whitespace, non-location child elements + # Body child elements only -- element-head tags carry metadata, not text. child_elements = [ - node - for node in el.childNodes - if isinstance(node, Element) and node.tagName not in {DocLangToken.LOCATION.value} + node for node in el.childNodes if isinstance(node, Element) and not self._is_element_head_tag(node) ] # Check if we have a single child that is a formatting tag @@ -2411,7 +2440,7 @@ def _extract_text_with_formatting(self, el: Element) -> tuple[str, Formatting | if tag_name in format_tags: # Recursively extract text and formatting from the child - text, child_formatting = self._extract_text_with_formatting(child) + text, child_formatting = self._extract_text_with_formatting(child, trim=trim) # Build up the formatting object if child_formatting is None: @@ -2433,25 +2462,57 @@ def _extract_text_with_formatting(self, el: Element) -> tuple[str, Formatting | return text, child_formatting - # No formatting found, just extract plain text - return self._get_text(el), None + # No formatting found, just extract plain text. + return (self._trimmed_text(el) if trim else self._collect_text(el)), None def _get_text(self, el: Element) -> str: - out: list[str] = [] + """Extract the text of a block-level element, trimming the block boundary only.""" + return self._trimmed_text(el) + + def _collect_text(self, el: Element) -> str: + """Concatenate descendant text verbatim, per the inline whitespace contract. + + Fragments are joined faithfully -- stripping each one is what collapsed + `Advanced Topics` into `AdvancedTopics`. + """ + return "".join(text for text, _ in self._collect_fragments(el)) + + def _trimmed_text(self, el: Element) -> str: + """Concatenate descendant text, trimming the outer block boundary only. + + Whitespace delivered through ```` is protected: that tag is the format's + explicit "this whitespace is significant" marker, so a block edge never strips it. + Same shape as docling's ``_normalize_odf_text_runs``. + """ + frags = self._collect_fragments(el) + for idx, strip in ((0, str.lstrip), (-1, str.rstrip)): + while frags and not frags[idx][1]: + if trimmed := strip(frags[idx][0]): + frags[idx] = (trimmed, False) + break + frags.pop(idx) + return "".join(text for text, _ in frags) + + def _collect_fragments(self, el: Element) -> list[tuple[str, bool]]: + """Return ``(text, protected)`` fragments; protected text came from ````.""" + out: list[tuple[str, bool]] = [] for node in el.childNodes: if isinstance(node, Text): - # Skip pure indentation/pretty-print whitespace - if node.data.strip(): - out.append(node.data if el.tagName == DocLangToken.CONTENT.value else node.data.strip()) + if el.tagName == DocLangToken.CONTENT.value: + # `` is the lossless whitespace channel: take it verbatim. + out.append((node.data, True)) + else: + out.append((_normalize_bare_text(node.data), False)) elif isinstance(node, Element): - nm = node.tagName - if nm in {DocLangToken.LOCATION.value, DocLangToken.CUSTOM.value}: + # Element-head children (, , , , ...) are + # metadata; they are modelled in item.meta and must not bleed into body text. + if self._is_element_head_tag(node): continue - if nm == DocLangToken.BR.value: - out.append("\n") + if node.tagName == DocLangToken.BR.value: + out.append(("\n", False)) else: - out.append(self._get_text(node)) - return "".join(out) + out.extend(self._collect_fragments(node)) + return out # --------- Location helpers --------- def _ensure_page_exists(self, *, doc: DoclingDocument, page_no: int, resolution: int | None = None) -> None: diff --git a/docling_core/transforms/serializer/common.py b/docling_core/transforms/serializer/common.py index deac69cbd..e476d4c63 100644 --- a/docling_core/transforms/serializer/common.py +++ b/docling_core/transforms/serializer/common.py @@ -8,7 +8,7 @@ from collections.abc import Iterable from functools import cached_property from pathlib import Path, PurePath -from typing import Annotated, Any, Final, Optional, Union +from typing import Annotated, Any, ClassVar, Final, Optional, Union from urllib.parse import quote, urlsplit, urlunsplit from pydantic import ( @@ -408,6 +408,11 @@ class DocSerializer(BaseModel, BaseDocSerializer): params: CommonParams = CommonParams() + # Move whitespace at the edges of a run outside the formatting/hyperlink markers. + # Off for formats whose own whitespace channel must keep it inside the markup + # (DocLang wraps it in ``), on where adjacency breaks the markup (Markdown). + hoist_decoration_whitespace: ClassVar[bool] = False + _excluded_refs_cache: dict[str, set[str]] = {} @computed_field # type: ignore[misc] @@ -755,7 +760,23 @@ def post_process( ) -> str: """Apply some text post-processing steps.""" params = self.params.merge_with_patch(patch=kwargs) + + lead = trail = "" res = text + if self.hoist_decoration_whitespace: + # Inline runs carry their own significant whitespace, which may sit *inside* a + # formatted run (common in DOCX/HTML). Decorating it verbatim produces + # `**bold ** tail`, which CommonMark renders as literal asterisks. Hoist the edge + # whitespace around the *complete* decoration stack -- hoisting inside an + # individual formatter would let the hyperlink capture it again. + core = text.strip() + if not core: + # Whitespace-only (or empty) run: no markers, no empty hyperlink. + return text + lead = text[: len(text) - len(text.lstrip())] + trail = text[len(text.rstrip()) :] + res = core + if params.include_formatting and formatting: if formatting.bold: res = self.serialize_bold(text=res, **kwargs) @@ -774,7 +795,7 @@ def post_process( res = self.serialize_hyperlink(text=res, hyperlink=hyperlink, **kwargs) else: _logger.warning("Dropping hyperlink with unsafe scheme; keeping the link text only.") - return res + return f"{lead}{res}{trail}" @override def serialize_bold(self, text: str, **kwargs: Any) -> str: diff --git a/docling_core/transforms/serializer/doclang.py b/docling_core/transforms/serializer/doclang.py index b5d22b9ed..c74d79f63 100644 --- a/docling_core/transforms/serializer/doclang.py +++ b/docling_core/transforms/serializer/doclang.py @@ -1865,17 +1865,23 @@ def serialize( yres=params.ysize, ) parts.append(create_ser_result(text=loc_str)) - parts.extend( - doc_serializer.get_parts( - item=item, - list_level=list_level, - is_inline_scope=True, - visited=my_visited, - **{**kwargs, **params.model_dump()}, - ) + head_parts = list(parts) + inline_parts = doc_serializer.get_parts( + item=item, + list_level=list_level, + is_inline_scope=True, + visited=my_visited, + **{**kwargs, **params.model_dump()}, ) + parts.extend(inline_parts) delim = _get_delim(params=params) - text_res = delim.join([p.text for p in parts if p.text]) + # Inline runs carry their own significant whitespace; concatenate them faithfully. + # The delimiter separates the element head from the body, it is not a whitespace + # channel — `` is (see `_escape_text`). + segments = [p.text for p in head_parts if p.text] + if inline_text := "".join([p.text for p in inline_parts if p.text]): + segments.append(inline_text) + text_res = delim.join(segments) if text_res: text_res = f"{text_res}{delim}" diff --git a/docling_core/transforms/serializer/html.py b/docling_core/transforms/serializer/html.py index 923c4054e..7c8e4f62f 100644 --- a/docling_core/transforms/serializer/html.py +++ b/docling_core/transforms/serializer/html.py @@ -8,7 +8,7 @@ from html.parser import HTMLParser from io import BytesIO from pathlib import Path, PurePath -from typing import Any, Optional, Union +from typing import Any, ClassVar, Optional, Union from xml.etree.ElementTree import tostring import latex2mathml.converter @@ -926,8 +926,8 @@ def serialize( **kwargs, ) - # Join all parts without separators - inline_html = " ".join([p.text for p in parts if p.text]) + # Inline runs carry their own significant whitespace; concatenate faithfully. + inline_html = "".join([p.text for p in parts if p.text]) # Wrap in span if needed if inline_html: @@ -1114,6 +1114,8 @@ class HTMLDocSerializer(DocSerializer): params: HTMLParams = HTMLParams() + hoist_decoration_whitespace: ClassVar[bool] = True + @override def _item_wraps_meta(self, item: NodeItem) -> bool: """PictureItem meta is wrapped inside the figure element.""" diff --git a/docling_core/transforms/serializer/latex.py b/docling_core/transforms/serializer/latex.py index 973d6987d..e02da4c2d 100644 --- a/docling_core/transforms/serializer/latex.py +++ b/docling_core/transforms/serializer/latex.py @@ -2,7 +2,7 @@ import re from pathlib import PurePath -from typing import Annotated, Any +from typing import Annotated, Any, ClassVar from pydantic import AnyUrl, BaseModel, Field from typing_extensions import override @@ -663,14 +663,15 @@ def serialize( list_level: int = 0, **kwargs: Any, ) -> SerializationResult: - """Serialize inline children joining them with spaces for LaTeX output.""" + """Serialize inline children concatenating them faithfully for LaTeX output.""" parts = doc_serializer.get_parts( item=item, list_level=list_level, is_inline_scope=True, **kwargs, ) - text_res = " ".join([p.text for p in parts if p.text]) + # Inline runs carry their own significant whitespace; concatenate faithfully. + text_res = "".join([p.text for p in parts if p.text]) return create_ser_result(text=text_res, span_source=parts) @@ -716,6 +717,10 @@ class LaTeXDocSerializer(DocSerializer): params: LaTeXParams = LaTeXParams() + # `\textbf{bold }tail` renders the same as `\textbf{bold} tail`, but keeping the space + # outside the group matches the other formats and keeps escaping predictable. + hoist_decoration_whitespace: ClassVar[bool] = True + @override def _item_wraps_meta(self, item: NodeItem) -> bool: """PictureItem and TableItem meta is wrapped inside the figure / table environment.""" diff --git a/docling_core/transforms/serializer/markdown.py b/docling_core/transforms/serializer/markdown.py index 3906cf8da..630667b45 100644 --- a/docling_core/transforms/serializer/markdown.py +++ b/docling_core/transforms/serializer/markdown.py @@ -6,7 +6,7 @@ import textwrap from enum import Enum from pathlib import Path, PurePath -from typing import Annotated, Any, Final, Optional, Union +from typing import Annotated, Any, ClassVar, Final, Optional, Union from urllib.parse import quote, urlsplit, urlunsplit from pydantic import AnyUrl, BaseModel, Field, PositiveInt @@ -173,8 +173,9 @@ def _mark_subtree_visited( def _collect_subtree_text(item: NodeItem, doc: DoclingDocument, excluded_refs: set[str]) -> str: """Collect all text from *item*'s subtree, flattening nested tables. - Returns a space-joined string of every piece of text found so that the - content of a nested table is preserved in a flat, readable form. + Returns every piece of text found, joined with spaces (the runs of an inline + group are concatenated as they are), so that the content of a nested table is + preserved in a flat, readable form. For TableItems the text is pulled from ``data.grid`` cells directly; RichTableCells are resolved through their references, without visiting the @@ -213,7 +214,8 @@ def _collect_subtree_text(item: NodeItem, doc: DoclingDocument, excluded_refs: s if child_text: parts.append(child_text) - return " ".join(parts) + # Inline runs carry their own boundary whitespace; everything else is joined as blocks. + return ("" if isinstance(item, InlineGroup) else " ").join(parts) def _longest_backtick_run(text: str) -> int: @@ -1081,7 +1083,11 @@ def serialize( and p.spans[0].item.parent and isinstance(p.spans[0].item.parent.resolve(doc), InlineGroup) ): - my_parts[-1].text = f"{my_parts[-1].text}{p.text}" # append to last + # append to last; a group that is the child of a text item keeps the host separator + group = p.spans[0].item.parent.resolve(doc) + host = group.parent.resolve(doc) if group.parent else None + sep_host = " " if isinstance(host, TextItem) and host.text else "" + my_parts[-1].text = f"{my_parts[-1].text}{sep_host}{p.text}" my_parts[-1].spans.extend(p.spans) else: my_parts.append(p) @@ -1129,7 +1135,8 @@ def serialize( visited=my_visited, **kwargs, ) - text_res = " ".join([p.text for p in parts if p.text]) + # Inline runs carry their own significant whitespace; concatenate faithfully. + text_res = "".join([p.text for p in parts if p.text]) return create_ser_result(text=text_res, span_source=parts) @@ -1177,6 +1184,9 @@ class MarkdownDocSerializer(DocSerializer): params: MarkdownParams = MarkdownParams() + # `**bold ** tail` renders as literal asterisks in CommonMark. + hoist_decoration_whitespace: ClassVar[bool] = True + @override def serialize_bold(self, text: str, **kwargs: Any): """Apply Markdown-specific bold serialization.""" diff --git a/docling_core/types/doc/common/constants.py b/docling_core/types/doc/common/constants.py index ff68fdba7..67eb75770 100644 --- a/docling_core/types/doc/common/constants.py +++ b/docling_core/types/doc/common/constants.py @@ -5,7 +5,7 @@ from docling_core.types.doc.labels import DocItemLabel -CURRENT_VERSION: Final = "1.10.0" +CURRENT_VERSION: Final = "1.11.0" """Current DoclingDocument schema version. Bump this (minor) whenever a change to the serialised format would cause an @@ -20,6 +20,11 @@ {"schema": "1.8.0", "library_version": "v2.49.0", "note": "BasePrediction, BaseMeta, meta field"}, {"schema": "1.9.0", "library_version": "v2.57.0", "note": "FineRef, DocItem.comments"}, {"schema": "1.10.0", "library_version": "v2.69.0", "note": "FieldItem types, 7 new DocItemLabel values"}, + { + "schema": "1.11.0", + "library_version": "v2.105.0", + "note": "InlineGroup runs carry their own significant whitespace", + }, ] """Ordered history of DoclingDocument schema versions in docling-core v2.x. @@ -47,6 +52,11 @@ every step from ``_CURRENT_MINOR`` down to ``FIRST_SUPPORTED_MINOR + 1``. """ +# First schema version in which InlineGroup runs carry their own significant whitespace and +# serializers concatenate them faithfully. Documents stamped below this are normalized on load +# (see `DoclingDocument._migrate_legacy_inline_separators`). +INLINE_WHITESPACE_CONTRACT_VERSION: Final = "1.11.0" + DEFAULT_EXPORT_LABELS = { DocItemLabel.TITLE, diff --git a/docling_core/types/doc/document.py b/docling_core/types/doc/document.py index 40219539f..af5d3fa36 100644 --- a/docling_core/types/doc/document.py +++ b/docling_core/types/doc/document.py @@ -3,6 +3,7 @@ import base64 import copy import hashlib +import itertools import json import logging import mimetypes @@ -69,6 +70,7 @@ CURRENT_VERSION, DEFAULT_EXPORT_LABELS, DOCUMENT_TOKENS_EXPORT_LABELS, + INLINE_WHITESPACE_CONTRACT_VERSION, ) from docling_core.types.doc.common.content_layer import DEFAULT_CONTENT_LAYERS, ContentLayer from docling_core.types.doc.common.formatting import Formatting, Script @@ -172,6 +174,34 @@ _logger = logging.getLogger(__name__) +_warned_legacy_inline_separators = False + + +def _predates_inline_whitespace_contract(version: str) -> bool: + """Return True when ``version`` is older than the inline whitespace contract.""" + doc = re.match(VERSION_PATTERN, version) + contract = re.match(VERSION_PATTERN, INLINE_WHITESPACE_CONTRACT_VERSION) + if doc is None or contract is None: + return False + return (int(doc["major"]), int(doc["minor"])) < (int(contract["major"]), int(contract["minor"])) + + +def _warn_legacy_inline_separators(version: str) -> None: + """Warn once per process that legacy inline separators were injected on load.""" + global _warned_legacy_inline_separators + if _warned_legacy_inline_separators: + return + _warned_legacy_inline_separators = True + warnings.warn( + f"Document uses schema version {version}, predating the inline whitespace contract " + f"({INLINE_WHITESPACE_CONTRACT_VERSION}). Legacy inline separators were injected to " + "preserve the previous rendering; re-convert the source for faithful whitespace. " + "Note that inline text differs from a fresh conversion, which affects chunking and " + "any persisted embeddings.", + UserWarning, + stacklevel=2, + ) + class DoclingDocument(BaseModel): """DoclingDocument.""" @@ -1530,7 +1560,19 @@ def add_inline_group( parent: NodeItem | None = None, content_layer: ContentLayer | None = None, ) -> InlineGroup: - """add_inline_group.""" + """Add an inline group, whose children are runs of one visual line of text. + + Whitespace contract: each run carries its own significant whitespace and serializers + concatenate the runs faithfully, with no separator of their own. A producer that omits + the boundary whitespace gets merged words (``["H", "2", "O"]`` renders as ``H2O``), so + put the boundary in the run text: ``["Advanced Topics", " in ", "Machine Learning"]``. + Whitespace at the edge of a formatted run is fine -- the serializers move it outside the + emphasis markers and hyperlink. + + Children of a text item are block structure: serializers keep a parent text item's own + text apart from this group (by a space or a block break). To put text flush against the + runs, make it a run of the group. + """ _parent = parent or self.body cref = f"#/groups/{len(self.groups)}" group = InlineGroup(self_ref=cref, parent=_parent.get_ref()) @@ -4479,7 +4521,12 @@ def save_as_vtt( def load_from_doctags( # noqa: C901 doctag_document: DocTagsDocument, document_name: str = "Document" ) -> "DoclingDocument": - r"""Load Docling document from lists of DocTags and Images.""" + r"""Load Docling document from lists of DocTags and Images. + + Limitation: DocTags stores each ```` run trimmed, so the boundary whitespace that + the inline whitespace contract expects in the runs (see ``add_inline_group``) is not in the + source and is not restored. Serializing such an inline group joins its runs without spaces. + """ # Maps the recognized tag to a Docling label. # Code items will be given DocItemLabel.CODE tag_to_doclabel = { @@ -4858,6 +4905,8 @@ def _add_text( doc.add_table(data=table_data, caption=caption) elif tag_name == GroupLabel.INLINE: + # DocTags runs are trimmed (by the serializer and by extract_inner_text), so the + # source has lost their boundary whitespace; it is not guessed back here. inline_group = doc.add_inline_group() content = match.group("content") common_bbox = extract_bounding_box(content) @@ -5340,6 +5389,11 @@ def load_from_doclang_archive( ``document.xml`` is additionally capped by ``settings.max_doclang_xml_bytes`` before parsing. Archive ``assets/`` and ``pages/`` images are capped by ``settings.max_image_decoded_size`` before Pillow opens them. + + Unlike JSON, DocLang carries no schema version, so archives written by docling-core + before schema 1.11.0 (the inline whitespace contract) are loaded as current: their + inline runs were stored without boundary whitespace and now join without a separator + (``Normal**bold**tail``). Re-convert the source, or keep JSON for long-term storage. """ from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer @@ -5582,6 +5636,141 @@ def get_visualization( images = visualizer_obj.get_visualization(doc=self) return images + @model_validator(mode="before") + @classmethod + def _migrate_legacy_inline_separators(cls, data: Any) -> Any: + """Reproduce the legacy inline separator for documents written before the contract. + + Serializers used to join inline runs with a hard ``" "``, so producers of that era did + not put boundary whitespace in the run text. Now that the join is faithful, those saved + documents would render ``Normalitalicbold``. Re-inject one separator at each boundary + the old serializer inserted, preserving the previous *visible* output -- including its + extra-space bugs (``["H", "2", "O"]`` stays ``H 2 O``). + + This is compatibility normalization, not semantic repair: the old document does not + contain enough information to recover the real whitespace. It must run in ``mode="before"`` + because ``check_version_is_compatible`` rewrites ``version`` to ``CURRENT_VERSION`` and + destroys the evidence. Versionless input is left alone rather than guessed at. + """ + if not isinstance(data, dict): + return data + raw_version = data.get("version") + if not isinstance(raw_version, str) or not _predates_inline_whitespace_contract(raw_version): + return data + + # Work on a copy: the caller owns the dict it passed in, and mutating it in place would + # make a second `model_validate` of the same value migrate again, compounding separators. + data = copy.deepcopy(data) + texts = data.get("texts") + groups = data.get("groups") + if not isinstance(texts, list) or not isinstance(groups, list): + return data + + def resolve(child: Any) -> dict | None: + """Return the text or group dict an inline child reference points to, else None.""" + ref = (child.get("$ref") or child.get("cref")) if isinstance(child, dict) else None + for prefix, items in (("#/texts/", texts), ("#/groups/", groups)): + if isinstance(ref, str) and ref.startswith(prefix): + try: + item = items[int(ref.removeprefix(prefix))] + except (ValueError, IndexError): + return None + return item if isinstance(item, dict) else None + return None + + def renders_text(item: dict, seen: set[int]) -> bool: + """Whether the item has text, or is an empty host or nested group whose subtree has text.""" + if id(item) in seen: # a reference cycle in malformed input + return False + seen.add(id(item)) + if isinstance(item.get("text"), str) and item["text"]: + return True + children = item.get("children") + return isinstance(children, list) and any( + (sub := resolve(child)) is not None and renders_text(sub, seen) for child in children + ) + + # Code and formula runs are wrapped in delimiters (`` ` ``, ``$``) by the serializers, so a + # separator appended to their text would land *inside* the delimiter -- the same defect as + # `**bold **`. Put it on the neighbouring plain run instead. + delimited = {DocItemLabel.CODE.value, DocItemLabel.FORMULA.value} + + def carries_separator(run: dict) -> bool: + """Only a plain run with its own text: an empty host's text must stay empty to render its group.""" + return isinstance(run.get("text"), str) and bool(run["text"]) and run.get("label") not in delimited + + migrated = False + for group in groups: + if not isinstance(group, dict) or group.get("label") != GroupLabel.INLINE.value: + continue + children = group.get("children") + if not isinstance(children, list): + continue + # shortcut: a child is a run when it or its subtree has text (an empty formula host + # holding an inline group, a nested inline group). A picture, table or empty formula also + # got the old separator, and HTML renders the empty formula, but whether such a child + # rendered text depends on the serializer. Widen this if such a document shows up. + runs = [ + (idx, run) for idx, child in enumerate(children) if (run := resolve(child)) and renders_text(run, set()) + ] + # Boundaries where the separator fits in neither neighbour; filled in afterwards so + # the indices collected here stay valid. + standalone: list[tuple[int, list]] = [] + for (idx, run), (nxt_idx, nxt) in itertools.pairwise(runs): + # An empty plain run between them is the separator the 1.11 -> 1.10 projector + # stripped; refilling it keeps repeated projection and migration from adding runs. + empty = next( + ( + between + for child in children[idx + 1 : nxt_idx] + if (between := resolve(child)) is not None + and between.get("text") == "" + and between.get("label") == DocItemLabel.TEXT.value + and not between.get("children") + ), + None, + ) + if carries_separator(run): + run["text"] += " " + elif carries_separator(nxt): + nxt["text"] = " " + nxt["text"] + elif empty is not None: + empty["text"] = " " + else: + # Neither side can carry it: the separator would corrupt `` `a``b` `` or + # ``$a$$b$``, so it becomes a plain run of its own between them. Page filters + # keep only located items, so it takes the left run's location when both + # runs are on one page: it then survives exactly when both of them do. + left = (run.get("prov") or [None])[-1] + right = (nxt.get("prov") or [None])[0] + same_page = ( + isinstance(left, dict) + and isinstance(right, dict) + and left.get("page_no") == right.get("page_no") + ) + standalone.append((nxt_idx, [{**left, "charspan": [0, 1]}] if same_page else [])) + migrated = True + + for shift, (insert_at, prov) in enumerate(standalone): + ref = f"#/texts/{len(texts)}" + texts.append( + { + "self_ref": ref, + "parent": {"$ref": group.get("self_ref")}, + "label": DocItemLabel.TEXT.value, + "prov": prov, + "orig": " ", + "text": " ", + "children": [], + "content_layer": group.get("content_layer", ContentLayer.BODY.value), + } + ) + children.insert(insert_at + shift, {"$ref": ref}) + + if migrated: + _warn_legacy_inline_separators(raw_version) + return data + @field_validator("version") @classmethod def check_version_is_compatible(cls, v: str) -> str: diff --git a/docs/DoclingDocument.json b/docs/DoclingDocument.json index 74f4d839d..88833795b 100644 --- a/docs/DoclingDocument.json +++ b/docs/DoclingDocument.json @@ -4494,7 +4494,7 @@ "type": "string" }, "version": { - "default": "1.10.0", + "default": "1.11.0", "pattern": "^(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)(?:-(?P(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", "title": "Version", "type": "string" diff --git a/docs/schemas/DoclingDocument_1_11.json b/docs/schemas/DoclingDocument_1_11.json new file mode 100644 index 000000000..88833795b --- /dev/null +++ b/docs/schemas/DoclingDocument_1_11.json @@ -0,0 +1,4655 @@ +{ + "$defs": { + "BaseMeta": { + "additionalProperties": true, + "description": "Base class for metadata.", + "properties": { + "summary": { + "anyOf": [ + { + "$ref": "#/$defs/SummaryMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A condensed natural-language summary of the content rooted at this node.", + "examples": [ + { + "text": "A short company/location statement." + } + ] + }, + "language": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The detected human language of the node content, expressed as a BCP 47 code.", + "examples": [ + { + "code": "en" + } + ] + }, + "entities": { + "anyOf": [ + { + "$ref": "#/$defs/EntitiesMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Named entities extracted from the node text (persons, organisations, locations, etc.). Each mention carries the entity text, an optional type label, and an optional character span.", + "examples": [ + { + "mentions": [ + { + "charspan": [ + 0, + 3 + ], + "label": "ORG", + "text": "IBM" + } + ] + } + ] + }, + "keywords": { + "anyOf": [ + { + "$ref": "#/$defs/KeywordsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Salient terms or short keyphrases that characterise the node content. Keywords are more specific than topics and typically correspond to individual words or short multi-word expressions found in or closely related to the text. Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "transformer", + "attention mechanism", + "BERT" + ] + } + ] + }, + "topics": { + "anyOf": [ + { + "$ref": "#/$defs/TopicsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Higher-level subject categories or thematic labels inferred for the node content. Topics are broader than keywords and describe the domain or theme rather than specific terms (e.g., 'machine learning' rather than 'gradient descent'). Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "natural language processing", + "computer vision" + ] + } + ] + } + }, + "title": "BaseMeta", + "type": "object" + }, + "BoundingBox": { + "description": "BoundingBox.", + "properties": { + "l": { + "title": "L", + "type": "number" + }, + "t": { + "title": "T", + "type": "number" + }, + "r": { + "title": "R", + "type": "number" + }, + "b": { + "title": "B", + "type": "number" + }, + "coord_origin": { + "$ref": "#/$defs/CoordOrigin", + "default": "TOPLEFT" + } + }, + "required": [ + "l", + "t", + "r", + "b" + ], + "title": "BoundingBox", + "type": "object" + }, + "ChartBar": { + "description": "Represents a bar in a bar chart.\n\nAttributes:\n label (str): The label for the bar.\n values (float): The value associated with the bar.", + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "values": { + "title": "Values", + "type": "number" + } + }, + "required": [ + "label", + "values" + ], + "title": "ChartBar", + "type": "object" + }, + "ChartLine": { + "description": "Represents a line in a line chart.\n\nAttributes:\n label (str): The label for the line.\n values (list[tuple[float, float]]): A list of (x, y) coordinate pairs\n representing the line's data points.", + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "values": { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "type": "array" + }, + "title": "Values", + "type": "array" + } + }, + "required": [ + "label", + "values" + ], + "title": "ChartLine", + "type": "object" + }, + "ChartPoint": { + "description": "Represents a point in a scatter chart.\n\nAttributes:\n value (Tuple[float, float]): A (x, y) coordinate pair representing a point in a\n chart.", + "properties": { + "value": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "title": "Value", + "type": "array" + } + }, + "required": [ + "value" + ], + "title": "ChartPoint", + "type": "object" + }, + "ChartSlice": { + "description": "Represents a slice in a pie chart.\n\nAttributes:\n label (str): The label for the slice.\n value (float): The value represented by the slice.", + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "value": { + "title": "Value", + "type": "number" + } + }, + "required": [ + "label", + "value" + ], + "title": "ChartSlice", + "type": "object" + }, + "ChartStackedBar": { + "description": "Represents a stacked bar in a stacked bar chart.\n\nAttributes:\n label (list[str]): The labels for the stacked bars. Multiple values are stored\n in cases where the chart is \"double stacked,\" meaning bars are stacked both\n horizontally and vertically.\n values (list[tuple[str, int]]): A list of values representing different segments\n of the stacked bar along with their label.", + "properties": { + "label": { + "items": { + "type": "string" + }, + "title": "Label", + "type": "array" + }, + "values": { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "type": "array" + }, + "title": "Values", + "type": "array" + } + }, + "required": [ + "label", + "values" + ], + "title": "ChartStackedBar", + "type": "object" + }, + "CodeItem": { + "additionalProperties": false, + "description": "CodeItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/FloatingMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "code", + "default": "code", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "code_language": { + "$ref": "#/$defs/CodeLanguageLabel", + "default": "unknown" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "CodeItem", + "type": "object" + }, + "CodeLanguageLabel": { + "description": "CodeLanguageLabel.", + "enum": [ + "Ada", + "Awk", + "Bash", + "bc", + "C", + "C#", + "C++", + "CMake", + "COBOL", + "CSS", + "Ceylon", + "Clojure", + "Crystal", + "Cuda", + "Cython", + "D", + "Dart", + "dc", + "Dockerfile", + "DocLang", + "Elixir", + "Erlang", + "FORTRAN", + "Forth", + "Go", + "HTML", + "Haskell", + "Haxe", + "Java", + "JavaScript", + "JSON", + "Julia", + "Kotlin", + "Latex", + "Lisp", + "Lua", + "Matlab", + "MoonScript", + "Nim", + "OCaml", + "ObjectiveC", + "Octave", + "PHP", + "Pascal", + "Perl", + "Prolog", + "Python", + "Racket", + "Ruby", + "Rust", + "SML", + "SQL", + "Scala", + "Scheme", + "Swift", + "Tikz", + "TypeScript", + "unknown", + "VisualBasic", + "XML", + "YAML" + ], + "title": "CodeLanguageLabel", + "type": "string" + }, + "CodeMetaField": { + "additionalProperties": true, + "description": "Code representation for the respective item.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "title": "Text", + "type": "string" + }, + "language": { + "anyOf": [ + { + "$ref": "#/$defs/CodeLanguageLabel" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "text" + ], + "title": "CodeMetaField", + "type": "object" + }, + "ContentLayer": { + "description": "ContentLayer.", + "enum": [ + "body", + "furniture", + "background", + "invisible", + "notes" + ], + "title": "ContentLayer", + "type": "string" + }, + "CoordOrigin": { + "description": "CoordOrigin.", + "enum": [ + "TOPLEFT", + "BOTTOMLEFT" + ], + "title": "CoordOrigin", + "type": "string" + }, + "DescriptionAnnotation": { + "description": "DescriptionAnnotation.", + "properties": { + "kind": { + "const": "description", + "default": "description", + "title": "Kind", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "provenance": { + "title": "Provenance", + "type": "string" + } + }, + "required": [ + "text", + "provenance" + ], + "title": "DescriptionAnnotation", + "type": "object" + }, + "DescriptionMetaField": { + "additionalProperties": true, + "description": "Description metadata field.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "DescriptionMetaField", + "type": "object" + }, + "DocumentOrigin": { + "description": "FileSource.", + "properties": { + "mimetype": { + "title": "Mimetype", + "type": "string" + }, + "binary_hash": { + "maximum": 18446744073709551615, + "minimum": 0, + "title": "Binary Hash", + "type": "integer" + }, + "filename": { + "title": "Filename", + "type": "string" + }, + "uri": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Uri" + } + }, + "required": [ + "mimetype", + "binary_hash", + "filename" + ], + "title": "DocumentOrigin", + "type": "object" + }, + "EntitiesMetaField": { + "additionalProperties": true, + "description": "Container for extracted entity mentions.", + "properties": { + "mentions": { + "items": { + "$ref": "#/$defs/EntityMention" + }, + "minItems": 1, + "title": "Mentions", + "type": "array" + } + }, + "required": [ + "mentions" + ], + "title": "EntitiesMetaField", + "type": "object" + }, + "EntityMention": { + "additionalProperties": true, + "description": "Entity mention extracted from text.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "description": "Normalized text of the entity mention.", + "title": "Text", + "type": "string" + }, + "orig": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact source text extracted from the original charspan, analogous to TextItem.orig. This may differ from 'text' when the mention has been normalized.", + "title": "Orig" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Entity type or category.", + "title": "Label" + }, + "charspan": { + "anyOf": [ + { + "description": "Character span (0-indexed)", + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Character span (0-indexed) of the entity mention in the source text.", + "title": "Charspan" + } + }, + "required": [ + "text" + ], + "title": "EntityMention", + "type": "object" + }, + "FieldHeadingItem": { + "additionalProperties": false, + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "field_heading", + "default": "field_heading", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "level": { + "default": 1, + "maximum": 100, + "minimum": 1, + "title": "Level", + "type": "integer" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "FieldHeadingItem", + "type": "object" + }, + "FieldItem": { + "additionalProperties": false, + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "field_item", + "default": "field_item", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + } + }, + "required": [ + "self_ref" + ], + "title": "FieldItem", + "type": "object" + }, + "FieldRegionItem": { + "additionalProperties": false, + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "field_region", + "default": "field_region", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + } + }, + "required": [ + "self_ref" + ], + "title": "FieldRegionItem", + "type": "object" + }, + "FieldValueItem": { + "additionalProperties": false, + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "field_value", + "default": "field_value", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "kind": { + "default": "read_only", + "enum": [ + "read_only", + "fillable" + ], + "title": "Kind", + "type": "string" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "FieldValueItem", + "type": "object" + }, + "FineRef": { + "description": "Fine-granular reference item that can capture span range info.", + "properties": { + "$ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "$Ref", + "type": "string" + }, + "range": { + "anyOf": [ + { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Range" + } + }, + "required": [ + "$ref" + ], + "title": "FineRef", + "type": "object" + }, + "FloatingMeta": { + "additionalProperties": true, + "description": "Metadata model for floating.", + "properties": { + "summary": { + "anyOf": [ + { + "$ref": "#/$defs/SummaryMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A condensed natural-language summary of the content rooted at this node.", + "examples": [ + { + "text": "A short company/location statement." + } + ] + }, + "language": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The detected human language of the node content, expressed as a BCP 47 code.", + "examples": [ + { + "code": "en" + } + ] + }, + "entities": { + "anyOf": [ + { + "$ref": "#/$defs/EntitiesMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Named entities extracted from the node text (persons, organisations, locations, etc.). Each mention carries the entity text, an optional type label, and an optional character span.", + "examples": [ + { + "mentions": [ + { + "charspan": [ + 0, + 3 + ], + "label": "ORG", + "text": "IBM" + } + ] + } + ] + }, + "keywords": { + "anyOf": [ + { + "$ref": "#/$defs/KeywordsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Salient terms or short keyphrases that characterise the node content. Keywords are more specific than topics and typically correspond to individual words or short multi-word expressions found in or closely related to the text. Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "transformer", + "attention mechanism", + "BERT" + ] + } + ] + }, + "topics": { + "anyOf": [ + { + "$ref": "#/$defs/TopicsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Higher-level subject categories or thematic labels inferred for the node content. Topics are broader than keywords and describe the domain or theme rather than specific terms (e.g., 'machine learning' rather than 'gradient descent'). Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "natural language processing", + "computer vision" + ] + } + ] + }, + "description": { + "anyOf": [ + { + "$ref": "#/$defs/DescriptionMetaField" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "FloatingMeta", + "type": "object" + }, + "FormItem": { + "additionalProperties": false, + "description": "FormItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/FloatingMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "form", + "default": "form", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "graph": { + "$ref": "#/$defs/GraphData" + } + }, + "required": [ + "self_ref", + "graph" + ], + "title": "FormItem", + "type": "object" + }, + "Formatting": { + "description": "Formatting.", + "properties": { + "bold": { + "default": false, + "title": "Bold", + "type": "boolean" + }, + "italic": { + "default": false, + "title": "Italic", + "type": "boolean" + }, + "underline": { + "default": false, + "title": "Underline", + "type": "boolean" + }, + "strikethrough": { + "default": false, + "title": "Strikethrough", + "type": "boolean" + }, + "script": { + "$ref": "#/$defs/Script", + "default": "baseline" + } + }, + "title": "Formatting", + "type": "object" + }, + "FormulaItem": { + "additionalProperties": false, + "description": "FormulaItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "formula", + "default": "formula", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "FormulaItem", + "type": "object" + }, + "GraphCell": { + "description": "GraphCell.", + "properties": { + "label": { + "$ref": "#/$defs/GraphCellLabel" + }, + "cell_id": { + "title": "Cell Id", + "type": "integer" + }, + "text": { + "title": "Text", + "type": "string" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "prov": { + "anyOf": [ + { + "$ref": "#/$defs/ProvenanceItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "item_ref": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "label", + "cell_id", + "text", + "orig" + ], + "title": "GraphCell", + "type": "object" + }, + "GraphCellLabel": { + "description": "GraphCellLabel.", + "enum": [ + "unspecified", + "key", + "value", + "checkbox" + ], + "title": "GraphCellLabel", + "type": "string" + }, + "GraphData": { + "description": "GraphData.", + "properties": { + "cells": { + "items": { + "$ref": "#/$defs/GraphCell" + }, + "title": "Cells", + "type": "array" + }, + "links": { + "items": { + "$ref": "#/$defs/GraphLink" + }, + "title": "Links", + "type": "array" + } + }, + "title": "GraphData", + "type": "object" + }, + "GraphLink": { + "description": "GraphLink.", + "properties": { + "label": { + "$ref": "#/$defs/GraphLinkLabel" + }, + "source_cell_id": { + "title": "Source Cell Id", + "type": "integer" + }, + "target_cell_id": { + "title": "Target Cell Id", + "type": "integer" + } + }, + "required": [ + "label", + "source_cell_id", + "target_cell_id" + ], + "title": "GraphLink", + "type": "object" + }, + "GraphLinkLabel": { + "description": "GraphLinkLabel.", + "enum": [ + "unspecified", + "to_value", + "to_key", + "to_parent", + "to_child" + ], + "title": "GraphLinkLabel", + "type": "string" + }, + "GroupItem": { + "additionalProperties": false, + "description": "GroupItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "name": { + "default": "group", + "title": "Name", + "type": "string" + }, + "label": { + "$ref": "#/$defs/GroupLabel", + "default": "unspecified" + } + }, + "required": [ + "self_ref" + ], + "title": "GroupItem", + "type": "object" + }, + "GroupLabel": { + "description": "GroupLabel.", + "enum": [ + "unspecified", + "list", + "ordered_list", + "chapter", + "section", + "sheet", + "slide", + "form_area", + "key_value_area", + "comment_section", + "inline", + "picture_area" + ], + "title": "GroupLabel", + "type": "string" + }, + "HumanLanguageLabel": { + "description": "Two-letter human language primary subtags using BCP-47 values.", + "enum": [ + "aa", + "ab", + "ae", + "af", + "ak", + "am", + "an", + "ar", + "as", + "av", + "ay", + "az", + "ba", + "be", + "bg", + "bh", + "bi", + "bm", + "bn", + "bo", + "br", + "bs", + "ca", + "ce", + "ch", + "co", + "cr", + "cs", + "cu", + "cv", + "cy", + "da", + "de", + "dv", + "dz", + "ee", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "ff", + "fi", + "fj", + "fo", + "fr", + "fy", + "ga", + "gd", + "gl", + "gn", + "gu", + "gv", + "ha", + "he", + "hi", + "ho", + "hr", + "ht", + "hu", + "hy", + "hz", + "ia", + "id", + "ie", + "ig", + "ii", + "ik", + "io", + "is", + "it", + "iu", + "ja", + "jv", + "ka", + "kg", + "ki", + "kj", + "kk", + "kl", + "km", + "kn", + "ko", + "kr", + "ks", + "ku", + "kv", + "kw", + "ky", + "la", + "lb", + "lg", + "li", + "ln", + "lo", + "lt", + "lu", + "lv", + "mg", + "mh", + "mi", + "mk", + "ml", + "mn", + "mr", + "ms", + "mt", + "my", + "na", + "nb", + "nd", + "ne", + "ng", + "nl", + "nn", + "no", + "nr", + "nv", + "ny", + "oc", + "oj", + "om", + "or", + "os", + "pa", + "pi", + "pl", + "ps", + "pt", + "qu", + "rm", + "rn", + "ro", + "ru", + "rw", + "sa", + "sc", + "sd", + "se", + "sg", + "sh", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr", + "ss", + "st", + "su", + "sv", + "sw", + "ta", + "te", + "tg", + "th", + "ti", + "tk", + "tl", + "tn", + "to", + "tr", + "ts", + "tt", + "tw", + "ty", + "ug", + "uk", + "ur", + "uz", + "ve", + "vi", + "vo", + "wa", + "wo", + "xh", + "yi", + "yo", + "za", + "zh", + "zu" + ], + "title": "HumanLanguageLabel", + "type": "string" + }, + "ImageRef": { + "description": "ImageRef.", + "properties": { + "mimetype": { + "title": "Mimetype", + "type": "string" + }, + "dpi": { + "title": "Dpi", + "type": "integer" + }, + "size": { + "$ref": "#/$defs/Size" + }, + "uri": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + } + ], + "title": "Uri" + } + }, + "required": [ + "mimetype", + "dpi", + "size", + "uri" + ], + "title": "ImageRef", + "type": "object" + }, + "InlineGroup": { + "additionalProperties": false, + "description": "InlineGroup.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "name": { + "default": "group", + "title": "Name", + "type": "string" + }, + "label": { + "const": "inline", + "default": "inline", + "title": "Label", + "type": "string" + } + }, + "required": [ + "self_ref" + ], + "title": "InlineGroup", + "type": "object" + }, + "KeyValueItem": { + "additionalProperties": false, + "description": "KeyValueItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/FloatingMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "key_value_region", + "default": "key_value_region", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "graph": { + "$ref": "#/$defs/GraphData" + } + }, + "required": [ + "self_ref", + "graph" + ], + "title": "KeyValueItem", + "type": "object" + }, + "KeywordsMetaField": { + "additionalProperties": true, + "description": "Container for a list of unique keywords / keyphrases.", + "properties": { + "values": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Values", + "type": "array", + "uniqueItems": true + } + }, + "required": [ + "values" + ], + "title": "KeywordsMetaField", + "type": "object" + }, + "LanguageMetaField": { + "additionalProperties": true, + "description": "Detected human language.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "code": { + "$ref": "#/$defs/HumanLanguageLabel" + } + }, + "required": [ + "code" + ], + "title": "LanguageMetaField", + "type": "object" + }, + "ListGroup": { + "additionalProperties": false, + "description": "ListGroup.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "name": { + "default": "group", + "title": "Name", + "type": "string" + }, + "label": { + "const": "list", + "default": "list", + "title": "Label", + "type": "string" + } + }, + "required": [ + "self_ref" + ], + "title": "ListGroup", + "type": "object" + }, + "ListItem": { + "additionalProperties": false, + "description": "SectionItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "list_item", + "default": "list_item", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "enumerated": { + "default": false, + "title": "Enumerated", + "type": "boolean" + }, + "marker": { + "default": "-", + "title": "Marker", + "type": "string" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "ListItem", + "type": "object" + }, + "MiscAnnotation": { + "description": "MiscAnnotation.", + "properties": { + "kind": { + "const": "misc", + "default": "misc", + "title": "Kind", + "type": "string" + }, + "content": { + "additionalProperties": true, + "title": "Content", + "type": "object" + } + }, + "required": [ + "content" + ], + "title": "MiscAnnotation", + "type": "object" + }, + "MoleculeMetaField": { + "additionalProperties": true, + "description": "Molecule metadata field.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "smi": { + "description": "The SMILES representation of the molecule.", + "title": "Smi", + "type": "string" + } + }, + "required": [ + "smi" + ], + "title": "MoleculeMetaField", + "type": "object" + }, + "Orientation": { + "description": "Counter-clockwise rotation of a table on the page, in degrees.\n\nFollows the convention used by PIL/Pillow's ``Image.rotate``: positive\nangles rotate the table counter-clockwise. ``ROT_0`` / ``ROT_180`` keep\nrows running horizontally on the page; ``ROT_90`` / ``ROT_270`` turn\nrows into vertical stripes.", + "enum": [ + "rot_0", + "rot_90", + "rot_180", + "rot_270" + ], + "title": "Orientation", + "type": "string" + }, + "PageItem": { + "description": "PageItem.", + "properties": { + "size": { + "$ref": "#/$defs/Size" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "page_no": { + "title": "Page No", + "type": "integer" + } + }, + "required": [ + "size", + "page_no" + ], + "title": "PageItem", + "type": "object" + }, + "PictureBarChartData": { + "description": "Represents data of a bar chart.\n\nAttributes:\n kind (Literal[\"bar_chart_data\"]): The type of the chart.\n x_axis_label (str): The label for the x-axis.\n y_axis_label (str): The label for the y-axis.\n bars (list[ChartBar]): A list of bars in the chart.", + "properties": { + "kind": { + "const": "bar_chart_data", + "default": "bar_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "x_axis_label": { + "title": "X Axis Label", + "type": "string" + }, + "y_axis_label": { + "title": "Y Axis Label", + "type": "string" + }, + "bars": { + "items": { + "$ref": "#/$defs/ChartBar" + }, + "title": "Bars", + "type": "array" + } + }, + "required": [ + "title", + "x_axis_label", + "y_axis_label", + "bars" + ], + "title": "PictureBarChartData", + "type": "object" + }, + "PictureClassificationClass": { + "description": "PictureClassificationData.", + "properties": { + "class_name": { + "title": "Class Name", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + } + }, + "required": [ + "class_name", + "confidence" + ], + "title": "PictureClassificationClass", + "type": "object" + }, + "PictureClassificationData": { + "description": "PictureClassificationData.", + "properties": { + "kind": { + "const": "classification", + "default": "classification", + "title": "Kind", + "type": "string" + }, + "provenance": { + "title": "Provenance", + "type": "string" + }, + "predicted_classes": { + "items": { + "$ref": "#/$defs/PictureClassificationClass" + }, + "title": "Predicted Classes", + "type": "array" + } + }, + "required": [ + "provenance", + "predicted_classes" + ], + "title": "PictureClassificationData", + "type": "object" + }, + "PictureClassificationMetaField": { + "additionalProperties": true, + "description": "Picture classification metadata field.", + "properties": { + "predictions": { + "items": { + "$ref": "#/$defs/PictureClassificationPrediction" + }, + "minItems": 1, + "title": "Predictions", + "type": "array" + } + }, + "title": "PictureClassificationMetaField", + "type": "object" + }, + "PictureClassificationPrediction": { + "additionalProperties": true, + "description": "Picture classification instance.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "class_name": { + "title": "Class Name", + "type": "string" + } + }, + "required": [ + "class_name" + ], + "title": "PictureClassificationPrediction", + "type": "object" + }, + "PictureItem": { + "additionalProperties": false, + "description": "PictureItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/PictureMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "default": "picture", + "enum": [ + "picture", + "chart" + ], + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "annotations": { + "default": [], + "deprecated": true, + "items": { + "discriminator": { + "mapping": { + "bar_chart_data": "#/$defs/PictureBarChartData", + "classification": "#/$defs/PictureClassificationData", + "description": "#/$defs/DescriptionAnnotation", + "line_chart_data": "#/$defs/PictureLineChartData", + "misc": "#/$defs/MiscAnnotation", + "molecule_data": "#/$defs/PictureMoleculeData", + "pie_chart_data": "#/$defs/PicturePieChartData", + "scatter_chart_data": "#/$defs/PictureScatterChartData", + "stacked_bar_chart_data": "#/$defs/PictureStackedBarChartData", + "tabular_chart_data": "#/$defs/PictureTabularChartData" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/DescriptionAnnotation" + }, + { + "$ref": "#/$defs/MiscAnnotation" + }, + { + "$ref": "#/$defs/PictureClassificationData" + }, + { + "$ref": "#/$defs/PictureMoleculeData" + }, + { + "$ref": "#/$defs/PictureTabularChartData" + }, + { + "$ref": "#/$defs/PictureLineChartData" + }, + { + "$ref": "#/$defs/PictureBarChartData" + }, + { + "$ref": "#/$defs/PictureStackedBarChartData" + }, + { + "$ref": "#/$defs/PicturePieChartData" + }, + { + "$ref": "#/$defs/PictureScatterChartData" + } + ] + }, + "title": "Annotations", + "type": "array" + } + }, + "required": [ + "self_ref" + ], + "title": "PictureItem", + "type": "object" + }, + "PictureLineChartData": { + "description": "Represents data of a line chart.\n\nAttributes:\n kind (Literal[\"line_chart_data\"]): The type of the chart.\n x_axis_label (str): The label for the x-axis.\n y_axis_label (str): The label for the y-axis.\n lines (list[ChartLine]): A list of lines in the chart.", + "properties": { + "kind": { + "const": "line_chart_data", + "default": "line_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "x_axis_label": { + "title": "X Axis Label", + "type": "string" + }, + "y_axis_label": { + "title": "Y Axis Label", + "type": "string" + }, + "lines": { + "items": { + "$ref": "#/$defs/ChartLine" + }, + "title": "Lines", + "type": "array" + } + }, + "required": [ + "title", + "x_axis_label", + "y_axis_label", + "lines" + ], + "title": "PictureLineChartData", + "type": "object" + }, + "PictureMeta": { + "additionalProperties": true, + "description": "Metadata model for pictures.", + "properties": { + "summary": { + "anyOf": [ + { + "$ref": "#/$defs/SummaryMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A condensed natural-language summary of the content rooted at this node.", + "examples": [ + { + "text": "A short company/location statement." + } + ] + }, + "language": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The detected human language of the node content, expressed as a BCP 47 code.", + "examples": [ + { + "code": "en" + } + ] + }, + "entities": { + "anyOf": [ + { + "$ref": "#/$defs/EntitiesMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Named entities extracted from the node text (persons, organisations, locations, etc.). Each mention carries the entity text, an optional type label, and an optional character span.", + "examples": [ + { + "mentions": [ + { + "charspan": [ + 0, + 3 + ], + "label": "ORG", + "text": "IBM" + } + ] + } + ] + }, + "keywords": { + "anyOf": [ + { + "$ref": "#/$defs/KeywordsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Salient terms or short keyphrases that characterise the node content. Keywords are more specific than topics and typically correspond to individual words or short multi-word expressions found in or closely related to the text. Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "transformer", + "attention mechanism", + "BERT" + ] + } + ] + }, + "topics": { + "anyOf": [ + { + "$ref": "#/$defs/TopicsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Higher-level subject categories or thematic labels inferred for the node content. Topics are broader than keywords and describe the domain or theme rather than specific terms (e.g., 'machine learning' rather than 'gradient descent'). Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "natural language processing", + "computer vision" + ] + } + ] + }, + "description": { + "anyOf": [ + { + "$ref": "#/$defs/DescriptionMetaField" + }, + { + "type": "null" + } + ], + "default": null + }, + "classification": { + "anyOf": [ + { + "$ref": "#/$defs/PictureClassificationMetaField" + }, + { + "type": "null" + } + ], + "default": null + }, + "molecule": { + "anyOf": [ + { + "$ref": "#/$defs/MoleculeMetaField" + }, + { + "type": "null" + } + ], + "default": null + }, + "tabular_chart": { + "anyOf": [ + { + "$ref": "#/$defs/TabularChartMetaField" + }, + { + "type": "null" + } + ], + "default": null + }, + "code": { + "anyOf": [ + { + "$ref": "#/$defs/CodeMetaField" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "PictureMeta", + "type": "object" + }, + "PictureMoleculeData": { + "description": "PictureMoleculeData.", + "properties": { + "kind": { + "const": "molecule_data", + "default": "molecule_data", + "title": "Kind", + "type": "string" + }, + "smi": { + "title": "Smi", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "class_name": { + "title": "Class Name", + "type": "string" + }, + "segmentation": { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "type": "array" + }, + "title": "Segmentation", + "type": "array" + }, + "provenance": { + "title": "Provenance", + "type": "string" + } + }, + "required": [ + "smi", + "confidence", + "class_name", + "segmentation", + "provenance" + ], + "title": "PictureMoleculeData", + "type": "object" + }, + "PicturePieChartData": { + "description": "Represents data of a pie chart.\n\nAttributes:\n kind (Literal[\"pie_chart_data\"]): The type of the chart.\n slices (list[ChartSlice]): A list of slices in the pie chart.", + "properties": { + "kind": { + "const": "pie_chart_data", + "default": "pie_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "slices": { + "items": { + "$ref": "#/$defs/ChartSlice" + }, + "title": "Slices", + "type": "array" + } + }, + "required": [ + "title", + "slices" + ], + "title": "PicturePieChartData", + "type": "object" + }, + "PictureScatterChartData": { + "description": "Represents data of a scatter chart.\n\nAttributes:\n kind (Literal[\"scatter_chart_data\"]): The type of the chart.\n x_axis_label (str): The label for the x-axis.\n y_axis_label (str): The label for the y-axis.\n points (list[ChartPoint]): A list of points in the scatter chart.", + "properties": { + "kind": { + "const": "scatter_chart_data", + "default": "scatter_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "x_axis_label": { + "title": "X Axis Label", + "type": "string" + }, + "y_axis_label": { + "title": "Y Axis Label", + "type": "string" + }, + "points": { + "items": { + "$ref": "#/$defs/ChartPoint" + }, + "title": "Points", + "type": "array" + } + }, + "required": [ + "title", + "x_axis_label", + "y_axis_label", + "points" + ], + "title": "PictureScatterChartData", + "type": "object" + }, + "PictureStackedBarChartData": { + "description": "Represents data of a stacked bar chart.\n\nAttributes:\n kind (Literal[\"stacked_bar_chart_data\"]): The type of the chart.\n x_axis_label (str): The label for the x-axis.\n y_axis_label (str): The label for the y-axis.\n stacked_bars (list[ChartStackedBar]): A list of stacked bars in the chart.", + "properties": { + "kind": { + "const": "stacked_bar_chart_data", + "default": "stacked_bar_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "x_axis_label": { + "title": "X Axis Label", + "type": "string" + }, + "y_axis_label": { + "title": "Y Axis Label", + "type": "string" + }, + "stacked_bars": { + "items": { + "$ref": "#/$defs/ChartStackedBar" + }, + "title": "Stacked Bars", + "type": "array" + } + }, + "required": [ + "title", + "x_axis_label", + "y_axis_label", + "stacked_bars" + ], + "title": "PictureStackedBarChartData", + "type": "object" + }, + "PictureTabularChartData": { + "description": "Base class for picture chart data.\n\nAttributes:\n title (str): The title of the chart.\n chart_data (TableData): Chart data in the table format.", + "properties": { + "kind": { + "const": "tabular_chart_data", + "default": "tabular_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "chart_data": { + "$ref": "#/$defs/TableData" + } + }, + "required": [ + "title", + "chart_data" + ], + "title": "PictureTabularChartData", + "type": "object" + }, + "ProvenanceItem": { + "description": "Provenance information for elements extracted from a textual document.\n\nA `ProvenanceItem` object acts as a lightweight pointer back into the original\ndocument for an extracted element. It applies to documents with an explicit\nor implicit layout, such as PDF, HTML, docx, or pptx.", + "properties": { + "page_no": { + "description": "Page number", + "title": "Page No", + "type": "integer" + }, + "bbox": { + "$ref": "#/$defs/BoundingBox", + "description": "Bounding box" + }, + "charspan": { + "description": "Character span (0-indexed)", + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "title": "Charspan", + "type": "array" + } + }, + "required": [ + "page_no", + "bbox", + "charspan" + ], + "title": "ProvenanceItem", + "type": "object" + }, + "RefItem": { + "description": "RefItem.", + "properties": { + "$ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "$Ref", + "type": "string" + } + }, + "required": [ + "$ref" + ], + "title": "RefItem", + "type": "object" + }, + "RichTableCell": { + "description": "RichTableCell.", + "properties": { + "bbox": { + "anyOf": [ + { + "$ref": "#/$defs/BoundingBox" + }, + { + "type": "null" + } + ], + "default": null + }, + "row_span": { + "default": 1, + "title": "Row Span", + "type": "integer" + }, + "col_span": { + "default": 1, + "title": "Col Span", + "type": "integer" + }, + "start_row_offset_idx": { + "title": "Start Row Offset Idx", + "type": "integer" + }, + "end_row_offset_idx": { + "title": "End Row Offset Idx", + "type": "integer" + }, + "start_col_offset_idx": { + "title": "Start Col Offset Idx", + "type": "integer" + }, + "end_col_offset_idx": { + "title": "End Col Offset Idx", + "type": "integer" + }, + "text": { + "title": "Text", + "type": "string" + }, + "column_header": { + "default": false, + "title": "Column Header", + "type": "boolean" + }, + "row_header": { + "default": false, + "title": "Row Header", + "type": "boolean" + }, + "row_section": { + "default": false, + "title": "Row Section", + "type": "boolean" + }, + "fillable": { + "default": false, + "title": "Fillable", + "type": "boolean" + }, + "ref": { + "$ref": "#/$defs/RefItem" + } + }, + "required": [ + "start_row_offset_idx", + "end_row_offset_idx", + "start_col_offset_idx", + "end_col_offset_idx", + "text", + "ref" + ], + "title": "RichTableCell", + "type": "object" + }, + "Script": { + "description": "Text script position.", + "enum": [ + "baseline", + "sub", + "super" + ], + "title": "Script", + "type": "string" + }, + "SectionHeaderItem": { + "additionalProperties": false, + "description": "SectionItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "section_header", + "default": "section_header", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "level": { + "default": 1, + "maximum": 100, + "minimum": 1, + "title": "Level", + "type": "integer" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "SectionHeaderItem", + "type": "object" + }, + "Size": { + "description": "Size.", + "properties": { + "width": { + "default": 0.0, + "title": "Width", + "type": "number" + }, + "height": { + "default": 0.0, + "title": "Height", + "type": "number" + } + }, + "title": "Size", + "type": "object" + }, + "SummaryMetaField": { + "additionalProperties": true, + "description": "Summary data.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "SummaryMetaField", + "type": "object" + }, + "TableCell": { + "description": "TableCell.", + "properties": { + "bbox": { + "anyOf": [ + { + "$ref": "#/$defs/BoundingBox" + }, + { + "type": "null" + } + ], + "default": null + }, + "row_span": { + "default": 1, + "title": "Row Span", + "type": "integer" + }, + "col_span": { + "default": 1, + "title": "Col Span", + "type": "integer" + }, + "start_row_offset_idx": { + "title": "Start Row Offset Idx", + "type": "integer" + }, + "end_row_offset_idx": { + "title": "End Row Offset Idx", + "type": "integer" + }, + "start_col_offset_idx": { + "title": "Start Col Offset Idx", + "type": "integer" + }, + "end_col_offset_idx": { + "title": "End Col Offset Idx", + "type": "integer" + }, + "text": { + "title": "Text", + "type": "string" + }, + "column_header": { + "default": false, + "title": "Column Header", + "type": "boolean" + }, + "row_header": { + "default": false, + "title": "Row Header", + "type": "boolean" + }, + "row_section": { + "default": false, + "title": "Row Section", + "type": "boolean" + }, + "fillable": { + "default": false, + "title": "Fillable", + "type": "boolean" + } + }, + "required": [ + "start_row_offset_idx", + "end_row_offset_idx", + "start_col_offset_idx", + "end_col_offset_idx", + "text" + ], + "title": "TableCell", + "type": "object" + }, + "TableData": { + "description": "BaseTableData.", + "properties": { + "table_cells": { + "default": [], + "items": { + "anyOf": [ + { + "$ref": "#/$defs/RichTableCell" + }, + { + "$ref": "#/$defs/TableCell" + } + ] + }, + "title": "Table Cells", + "type": "array" + }, + "num_rows": { + "default": 0, + "title": "Num Rows", + "type": "integer" + }, + "num_cols": { + "default": 0, + "title": "Num Cols", + "type": "integer" + }, + "orientation": { + "$ref": "#/$defs/Orientation", + "default": "rot_0" + } + }, + "title": "TableData", + "type": "object" + }, + "TableItem": { + "additionalProperties": false, + "description": "TableItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/FloatingMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "default": "table", + "enum": [ + "document_index", + "table" + ], + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "data": { + "$ref": "#/$defs/TableData" + }, + "annotations": { + "default": [], + "deprecated": true, + "items": { + "discriminator": { + "mapping": { + "description": "#/$defs/DescriptionAnnotation", + "misc": "#/$defs/MiscAnnotation" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/DescriptionAnnotation" + }, + { + "$ref": "#/$defs/MiscAnnotation" + } + ] + }, + "title": "Annotations", + "type": "array" + } + }, + "required": [ + "self_ref", + "data" + ], + "title": "TableItem", + "type": "object" + }, + "TabularChartMetaField": { + "additionalProperties": true, + "description": "Tabular chart metadata field.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Title" + }, + "chart_data": { + "$ref": "#/$defs/TableData" + } + }, + "required": [ + "chart_data" + ], + "title": "TabularChartMetaField", + "type": "object" + }, + "TextItem": { + "additionalProperties": false, + "description": "TextItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "enum": [ + "caption", + "checkbox_selected", + "checkbox_unselected", + "footnote", + "page_footer", + "page_header", + "paragraph", + "reference", + "text", + "empty_value", + "field_key", + "field_hint", + "marker", + "handwritten_text" + ], + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + } + }, + "required": [ + "self_ref", + "label", + "orig", + "text" + ], + "title": "TextItem", + "type": "object" + }, + "TitleItem": { + "additionalProperties": false, + "description": "TitleItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "title", + "default": "title", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "TitleItem", + "type": "object" + }, + "TopicsMetaField": { + "additionalProperties": true, + "description": "Container for a list of unique topics / subjects.", + "properties": { + "values": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Values", + "type": "array", + "uniqueItems": true + } + }, + "required": [ + "values" + ], + "title": "TopicsMetaField", + "type": "object" + }, + "TrackSource": { + "description": "Source metadata for a cue extracted from a media track.\n\nA `TrackSource` instance identifies a cue in a media track (audio, video, subtitles, screen-recording captions,\netc.). A *cue* here refers to any discrete segment that was pulled out of the original asset, e.g., a subtitle\nblock, an audio clip, or a timed marker in a screen-recording.", + "properties": { + "kind": { + "const": "track", + "default": "track", + "description": "Identifies this type of source.", + "title": "Kind", + "type": "string" + }, + "start_time": { + "description": "Start time offset of the track cue in seconds", + "examples": [ + 11.0, + 6.5, + 5370.0 + ], + "title": "Start Time", + "type": "number" + }, + "end_time": { + "description": "End time offset of the track cue in seconds", + "examples": [ + 12.0, + 8.2, + 5370.1 + ], + "title": "End Time", + "type": "number" + }, + "identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An identifier of the cue", + "examples": [ + "test", + "123", + "b72d946" + ], + "title": "Identifier" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The name of the voice in this track (the speaker)", + "examples": [ + "John", + "Mary", + "Speaker 1" + ], + "title": "Voice" + } + }, + "required": [ + "start_time", + "end_time" + ], + "title": "TrackSource", + "type": "object" + } + }, + "description": "DoclingDocument.", + "properties": { + "schema_name": { + "const": "DoclingDocument", + "default": "DoclingDocument", + "title": "Schema Name", + "type": "string" + }, + "version": { + "default": "1.11.0", + "pattern": "^(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)(?:-(?P(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "title": "Version", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "origin": { + "anyOf": [ + { + "$ref": "#/$defs/DocumentOrigin" + }, + { + "type": "null" + } + ], + "default": null + }, + "furniture": { + "$ref": "#/$defs/GroupItem", + "default": { + "self_ref": "#/furniture", + "parent": null, + "children": [], + "content_layer": "furniture", + "meta": null, + "name": "_root_", + "label": "unspecified" + }, + "deprecated": true + }, + "body": { + "$ref": "#/$defs/GroupItem", + "default": { + "self_ref": "#/body", + "parent": null, + "children": [], + "content_layer": "body", + "meta": null, + "name": "_root_", + "label": "unspecified" + } + }, + "groups": { + "default": [], + "items": { + "anyOf": [ + { + "$ref": "#/$defs/ListGroup" + }, + { + "$ref": "#/$defs/InlineGroup" + }, + { + "$ref": "#/$defs/GroupItem" + } + ] + }, + "title": "Groups", + "type": "array" + }, + "texts": { + "default": [], + "items": { + "anyOf": [ + { + "$ref": "#/$defs/TitleItem" + }, + { + "$ref": "#/$defs/SectionHeaderItem" + }, + { + "$ref": "#/$defs/ListItem" + }, + { + "$ref": "#/$defs/CodeItem" + }, + { + "$ref": "#/$defs/FormulaItem" + }, + { + "$ref": "#/$defs/FieldHeadingItem" + }, + { + "$ref": "#/$defs/FieldValueItem" + }, + { + "$ref": "#/$defs/TextItem" + } + ] + }, + "title": "Texts", + "type": "array" + }, + "pictures": { + "default": [], + "items": { + "$ref": "#/$defs/PictureItem" + }, + "title": "Pictures", + "type": "array" + }, + "tables": { + "default": [], + "items": { + "$ref": "#/$defs/TableItem" + }, + "title": "Tables", + "type": "array" + }, + "key_value_items": { + "default": [], + "items": { + "$ref": "#/$defs/KeyValueItem" + }, + "title": "Key Value Items", + "type": "array" + }, + "form_items": { + "default": [], + "items": { + "$ref": "#/$defs/FormItem" + }, + "title": "Form Items", + "type": "array" + }, + "field_regions": { + "default": [], + "items": { + "$ref": "#/$defs/FieldRegionItem" + }, + "title": "Field Regions", + "type": "array" + }, + "field_items": { + "default": [], + "items": { + "$ref": "#/$defs/FieldItem" + }, + "title": "Field Items", + "type": "array" + }, + "pages": { + "additionalProperties": { + "$ref": "#/$defs/PageItem" + }, + "default": {}, + "title": "Pages", + "type": "object" + } + }, + "required": [ + "name" + ], + "title": "DoclingDocument", + "type": "object" +} \ No newline at end of file diff --git a/packages/dclq/tests/test_cli.py b/packages/dclq/tests/test_cli.py index 61b7bef7b..cf5a41808 100644 --- a/packages/dclq/tests/test_cli.py +++ b/packages/dclq/tests/test_cli.py @@ -259,7 +259,7 @@ def test_text_output_collapses_formatting_and_marks_truncation(tmp_path: Path, c assert main(["show", str(path), "/formula[1]", "-B", "1", "-n"]) == 0 output = capsys.readouterr().out - assert "/code[1]-public class Main {}" in output + assert "/code[1]-public class Main {}" in output assert "/formula[1]:$$E = mc^2$$" in output assert "/d:" not in output assert "\n--\n" not in output diff --git a/tests/conftest.py b/tests/conftest.py index f7f1615a0..a4ab6ebc8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -208,7 +208,7 @@ def _construct_doc_impl() -> DoclingDocument: inline1 = doc.add_inline_group(parent=g2_subgroup_li_1) doc.add_text( label=DocItemLabel.TEXT, - text="Here a code snippet:", + text="Here a code snippet: ", parent=inline1, ) doc.add_code( @@ -216,17 +216,17 @@ def _construct_doc_impl() -> DoclingDocument: parent=inline1, code_language=CodeLanguageLabel.PYTHON, ) - doc.add_text(label=DocItemLabel.TEXT, text="(to be displayed inline)", parent=inline1) + doc.add_text(label=DocItemLabel.TEXT, text=" (to be displayed inline)", parent=inline1) g2_subgroup_li_2 = doc.add_list_item(text="", parent=g2_subgroup, marker="□") inline2 = doc.add_inline_group(parent=g2_subgroup_li_2) doc.add_text( label=DocItemLabel.TEXT, - text="Here a formula:", + text="Here a formula: ", parent=inline2, ) doc.add_text(label=DocItemLabel.FORMULA, text="E=mc^2", parent=inline2) - doc.add_text(label=DocItemLabel.TEXT, text="(to be displayed inline)", parent=inline2) + doc.add_text(label=DocItemLabel.TEXT, text=" (to be displayed inline)", parent=inline2) doc.add_text(label=DocItemLabel.TEXT, text="Here a code block:", parent=None) doc.add_code(text='print("Hello world")', parent=None, code_language=CodeLanguageLabel.PYTHON) @@ -264,52 +264,52 @@ def _construct_doc_impl() -> DoclingDocument: doc.add_form(graph=graph) inline_fmt = doc.add_inline_group() - doc.add_text(label=DocItemLabel.TEXT, text="Some formatting chops:", parent=inline_fmt) + doc.add_text(label=DocItemLabel.TEXT, text="Some formatting chops: ", parent=inline_fmt) doc.add_text( label=DocItemLabel.TEXT, - text="bold", + text="bold ", parent=inline_fmt, formatting=Formatting(bold=True), ) doc.add_text( label=DocItemLabel.TEXT, - text="italic", + text="italic ", parent=inline_fmt, formatting=Formatting(italic=True), ) doc.add_text( label=DocItemLabel.TEXT, - text="underline", + text="underline ", parent=inline_fmt, formatting=Formatting(underline=True), ) doc.add_text( label=DocItemLabel.TEXT, - text="strikethrough", + text="strikethrough ", parent=inline_fmt, formatting=Formatting(strikethrough=True), ) doc.add_text( label=DocItemLabel.TEXT, - text="subscript", - orig="subscript", + text="subscript ", + orig="subscript ", formatting=Formatting(script=Script.SUB), parent=inline_fmt, ) doc.add_text( label=DocItemLabel.TEXT, - text="superscript", - orig="superscript", + text="superscript ", + orig="superscript ", formatting=Formatting(script=Script.SUPER), parent=inline_fmt, ) doc.add_text( label=DocItemLabel.TEXT, - text="hyperlink", + text="hyperlink ", parent=inline_fmt, hyperlink=Path("."), ) - doc.add_text(label=DocItemLabel.TEXT, text="&", parent=inline_fmt) + doc.add_text(label=DocItemLabel.TEXT, text="& ", parent=inline_fmt) doc.add_text( label=DocItemLabel.TEXT, text="everything at the same time.", diff --git a/tests/data/doc/2206.01062.yaml.dt.json b/tests/data/doc/2206.01062.yaml.dt.json index f41500ac7..89733e008 100644 --- a/tests/data/doc/2206.01062.yaml.dt.json +++ b/tests/data/doc/2206.01062.yaml.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/2408.09869v3_enriched.out.dt.json b/tests/data/doc/2408.09869v3_enriched.out.dt.json index 0e03e45f1..d3ef38582 100644 --- a/tests/data/doc/2408.09869v3_enriched.out.dt.json +++ b/tests/data/doc/2408.09869v3_enriched.out.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.json b/tests/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.json index b93becd88..149fc5618 100644 --- a/tests/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.json +++ b/tests/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "2408.09869v3", "furniture": { "self_ref": "#/furniture", @@ -497,7 +497,7 @@ } ], "orig": "To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at ", - "text": "To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at " + "text": "To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at " }, { "self_ref": "#/texts/9", @@ -524,7 +524,7 @@ } ], "orig": "github.com/DS4SD/docling", - "text": "github.com/DS4SD/docling", + "text": "github.com/DS4SD/docling ", "hyperlink": "https://github.com/DS4SD/docling" }, { diff --git a/tests/data/doc/barchart.dt.out.json b/tests/data/doc/barchart.dt.out.json index 655c9a2ed..8859b75f9 100644 --- a/tests/data/doc/barchart.dt.out.json +++ b/tests/data/doc/barchart.dt.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/cdata_always.gt.dclg.xml b/tests/data/doc/cdata_always.gt.dclg.xml index 5dc8f35e3..d41be3106 100644 --- a/tests/data/doc/cdata_always.gt.dclg.xml +++ b/tests/data/doc/cdata_always.gt.dclg.xml @@ -99,17 +99,17 @@ Affiliation 2]]> - + - + - + - + @@ -119,15 +119,27 @@ Affiliation 2]]> - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/doc/cdata_when_needed.gt.dclg.xml b/tests/data/doc/cdata_when_needed.gt.dclg.xml index f2b8efe4c..b6b268e93 100644 --- a/tests/data/doc/cdata_when_needed.gt.dclg.xml +++ b/tests/data/doc/cdata_when_needed.gt.dclg.xml @@ -106,17 +106,17 @@ Affiliation 2 □ - Here a code snippet: + Here a code snippet: -(to be displayed inline) + (to be displayed inline) □ - Here a formula: + Here a formula: E=mc^2 -(to be displayed inline) + (to be displayed inline) Here a code block: @@ -126,15 +126,27 @@ Affiliation 2 Here a formula block: E=mc^2 - Some formatting chops: - bold - italic - underline - strikethrough - subscript - superscript -hyperlink - + Some formatting chops: + + bold + + + italic + + + underline + + + strikethrough + + + subscript + + + superscript + + hyperlink + diff --git a/tests/data/doc/concatenated.json b/tests/data/doc/concatenated.json index 23c0b1833..e0e8f89fb 100644 --- a/tests/data/doc/concatenated.json +++ b/tests/data/doc/concatenated.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "2501.17887v1 + Untitled 1 + 2311.18481v1", "furniture": { "self_ref": "#/furniture", @@ -9335,8 +9335,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/313", @@ -9363,8 +9363,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/315", @@ -9393,8 +9393,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/317", @@ -9417,8 +9417,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/319", @@ -9481,8 +9481,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/324", @@ -9493,8 +9493,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -9512,8 +9512,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -9531,8 +9531,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -9550,8 +9550,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -9569,8 +9569,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -9588,8 +9588,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -9607,8 +9607,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -9620,8 +9620,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/332", diff --git a/tests/data/doc/constr_doc_reserialized.dclg.xml b/tests/data/doc/constr_doc_reserialized.dclg.xml index 109d2f82c..36afbcdef 100644 --- a/tests/data/doc/constr_doc_reserialized.dclg.xml +++ b/tests/data/doc/constr_doc_reserialized.dclg.xml @@ -105,17 +105,17 @@ Affiliation 2 □ - Here a code snippet: + Here a code snippet: -(to be displayed inline) + (to be displayed inline) □ - Here a formula: + Here a formula: E=mc^2 -(to be displayed inline) + (to be displayed inline) Here a code block: @@ -125,15 +125,27 @@ Affiliation 2 Here a formula block: E=mc^2 - Some formatting chops: - bold - italic - underline - strikethrough - subscript - superscript -hyperlink - + Some formatting chops: + + bold + + + italic + + + underline + + + strikethrough + + + subscript + + + superscript + + hyperlink + diff --git a/tests/data/doc/constructed_doc.added_extracted_doc.json.gt b/tests/data/doc/constructed_doc.added_extracted_doc.json.gt index 2b8057f01..baac1a322 100644 --- a/tests/data/doc/constructed_doc.added_extracted_doc.json.gt +++ b/tests/data/doc/constructed_doc.added_extracted_doc.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -715,8 +715,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -743,8 +743,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -773,8 +773,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -797,8 +797,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -861,8 +861,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -873,8 +873,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -892,8 +892,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -911,8 +911,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -930,8 +930,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -949,8 +949,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -968,8 +968,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -987,8 +987,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1000,8 +1000,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.appended_child.json.gt b/tests/data/doc/constructed_doc.appended_child.json.gt index 8c273f2bb..9b961131a 100644 --- a/tests/data/doc/constructed_doc.appended_child.json.gt +++ b/tests/data/doc/constructed_doc.appended_child.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -544,8 +544,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -572,8 +572,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -602,8 +602,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -626,8 +626,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -690,8 +690,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -702,8 +702,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -721,8 +721,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -740,8 +740,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -759,8 +759,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -778,8 +778,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -797,8 +797,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -816,8 +816,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -829,8 +829,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.bulk_item_addition.json.gt b/tests/data/doc/constructed_doc.bulk_item_addition.json.gt index c15f774f7..4a98c1641 100644 --- a/tests/data/doc/constructed_doc.bulk_item_addition.json.gt +++ b/tests/data/doc/constructed_doc.bulk_item_addition.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -708,8 +708,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -736,8 +736,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -766,8 +766,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -790,8 +790,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -854,8 +854,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -866,8 +866,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -885,8 +885,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -904,8 +904,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -923,8 +923,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -942,8 +942,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -961,8 +961,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -980,8 +980,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -993,8 +993,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.bulk_item_insertion.json.gt b/tests/data/doc/constructed_doc.bulk_item_insertion.json.gt index 57214fd39..c40452f36 100644 --- a/tests/data/doc/constructed_doc.bulk_item_insertion.json.gt +++ b/tests/data/doc/constructed_doc.bulk_item_insertion.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -714,8 +714,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -742,8 +742,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -772,8 +772,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -796,8 +796,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -860,8 +860,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -872,8 +872,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -891,8 +891,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -910,8 +910,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -929,8 +929,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -948,8 +948,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -967,8 +967,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -986,8 +986,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -999,8 +999,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.deleted_group.json.gt b/tests/data/doc/constructed_doc.deleted_group.json.gt index e6499979c..0545b58fe 100644 --- a/tests/data/doc/constructed_doc.deleted_group.json.gt +++ b/tests/data/doc/constructed_doc.deleted_group.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -541,8 +541,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -569,8 +569,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -599,8 +599,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -623,8 +623,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -687,8 +687,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -699,8 +699,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -718,8 +718,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -737,8 +737,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -756,8 +756,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -775,8 +775,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -794,8 +794,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -813,8 +813,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -826,8 +826,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.deleted_items_range.json.gt b/tests/data/doc/constructed_doc.deleted_items_range.json.gt index 1340b897d..c55abcb4a 100644 --- a/tests/data/doc/constructed_doc.deleted_items_range.json.gt +++ b/tests/data/doc/constructed_doc.deleted_items_range.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -544,8 +544,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -572,8 +572,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -602,8 +602,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -626,8 +626,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -690,8 +690,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -702,8 +702,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -721,8 +721,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -740,8 +740,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -759,8 +759,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -778,8 +778,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -797,8 +797,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -816,8 +816,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -829,8 +829,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.deleted_picture.json.gt b/tests/data/doc/constructed_doc.deleted_picture.json.gt index 3ffb7b326..c23a5c0fb 100644 --- a/tests/data/doc/constructed_doc.deleted_picture.json.gt +++ b/tests/data/doc/constructed_doc.deleted_picture.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -538,8 +538,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -566,8 +566,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -596,8 +596,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -620,8 +620,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -684,8 +684,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -696,8 +696,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -715,8 +715,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -734,8 +734,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -753,8 +753,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -772,8 +772,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -791,8 +791,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -810,8 +810,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -823,8 +823,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.deleted_text.json.gt b/tests/data/doc/constructed_doc.deleted_text.json.gt index 363090f98..9d2b6e0a9 100644 --- a/tests/data/doc/constructed_doc.deleted_text.json.gt +++ b/tests/data/doc/constructed_doc.deleted_text.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -755,8 +755,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/24", @@ -783,8 +783,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/26", @@ -813,8 +813,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/28", @@ -837,8 +837,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/30", @@ -901,8 +901,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/35", @@ -913,8 +913,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -932,8 +932,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -951,8 +951,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -970,8 +970,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -989,8 +989,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1008,8 +1008,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1027,8 +1027,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1040,8 +1040,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/43", diff --git a/tests/data/doc/constructed_doc.embedded.json.gt b/tests/data/doc/constructed_doc.embedded.json.gt index 73a327570..49b12ad56 100644 --- a/tests/data/doc/constructed_doc.embedded.json.gt +++ b/tests/data/doc/constructed_doc.embedded.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -766,8 +766,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/25", @@ -794,8 +794,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/27", @@ -824,8 +824,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/29", @@ -848,8 +848,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/31", @@ -912,8 +912,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/36", @@ -924,8 +924,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -943,8 +943,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -962,8 +962,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -981,8 +981,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -1000,8 +1000,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1019,8 +1019,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1038,8 +1038,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1051,8 +1051,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/44", diff --git a/tests/data/doc/constructed_doc.embedded.yaml.gt b/tests/data/doc/constructed_doc.embedded.yaml.gt index 27a201bcc..92b2fa5b1 100644 --- a/tests/data/doc/constructed_doc.embedded.yaml.gt +++ b/tests/data/doc/constructed_doc.embedded.yaml.gt @@ -752,12 +752,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Here a code snippet:' + orig: 'Here a code snippet: ' parent: $ref: '#/groups/10' prov: [] self_ref: '#/texts/24' - text: 'Here a code snippet:' + text: 'Here a code snippet: ' - captions: [] children: [] code_language: Python @@ -774,12 +774,12 @@ texts: - children: [] content_layer: body label: text - orig: (to be displayed inline) + orig: ' (to be displayed inline)' parent: $ref: '#/groups/10' prov: [] self_ref: '#/texts/26' - text: (to be displayed inline) + text: ' (to be displayed inline)' - children: - $ref: '#/groups/11' content_layer: body @@ -795,12 +795,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Here a formula:' + orig: 'Here a formula: ' parent: $ref: '#/groups/11' prov: [] self_ref: '#/texts/28' - text: 'Here a formula:' + text: 'Here a formula: ' - children: [] content_layer: body label: formula @@ -813,12 +813,12 @@ texts: - children: [] content_layer: body label: text - orig: (to be displayed inline) + orig: ' (to be displayed inline)' parent: $ref: '#/groups/11' prov: [] self_ref: '#/texts/30' - text: (to be displayed inline) + text: ' (to be displayed inline)' - children: [] content_layer: body label: text @@ -862,12 +862,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Some formatting chops:' + orig: 'Some formatting chops: ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/35' - text: 'Some formatting chops:' + text: 'Some formatting chops: ' - children: [] content_layer: body formatting: @@ -877,12 +877,12 @@ texts: strikethrough: false underline: false label: text - orig: bold + orig: 'bold ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/36' - text: bold + text: 'bold ' - children: [] content_layer: body formatting: @@ -892,12 +892,12 @@ texts: strikethrough: false underline: false label: text - orig: italic + orig: 'italic ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/37' - text: italic + text: 'italic ' - children: [] content_layer: body formatting: @@ -907,12 +907,12 @@ texts: strikethrough: false underline: true label: text - orig: underline + orig: 'underline ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/38' - text: underline + text: 'underline ' - children: [] content_layer: body formatting: @@ -922,12 +922,12 @@ texts: strikethrough: true underline: false label: text - orig: strikethrough + orig: 'strikethrough ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/39' - text: strikethrough + text: 'strikethrough ' - children: [] content_layer: body formatting: @@ -937,12 +937,12 @@ texts: strikethrough: false underline: false label: text - orig: subscript + orig: 'subscript ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/40' - text: subscript + text: 'subscript ' - children: [] content_layer: body formatting: @@ -952,31 +952,31 @@ texts: strikethrough: false underline: false label: text - orig: superscript + orig: 'superscript ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/41' - text: superscript + text: 'superscript ' - children: [] content_layer: body hyperlink: . label: text - orig: hyperlink + orig: 'hyperlink ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/42' - text: hyperlink + text: 'hyperlink ' - children: [] content_layer: body label: text - orig: '&' + orig: '& ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/43' - text: '&' + text: '& ' - children: [] content_layer: body formatting: @@ -1114,4 +1114,4 @@ texts: prov: [] self_ref: '#/texts/55' text: The end. -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/constructed_doc.extracted_with_deletion.json.gt b/tests/data/doc/constructed_doc.extracted_with_deletion.json.gt index 1a8865d8c..733018a11 100644 --- a/tests/data/doc/constructed_doc.extracted_with_deletion.json.gt +++ b/tests/data/doc/constructed_doc.extracted_with_deletion.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -621,8 +621,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -649,8 +649,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -679,8 +679,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -703,8 +703,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -767,8 +767,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -779,8 +779,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -798,8 +798,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -817,8 +817,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -836,8 +836,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -855,8 +855,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -874,8 +874,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -893,8 +893,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -906,8 +906,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.inserted_extracted_doc.json.gt b/tests/data/doc/constructed_doc.inserted_extracted_doc.json.gt index e526f3be1..18450a97a 100644 --- a/tests/data/doc/constructed_doc.inserted_extracted_doc.json.gt +++ b/tests/data/doc/constructed_doc.inserted_extracted_doc.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -808,8 +808,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -836,8 +836,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -866,8 +866,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -890,8 +890,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -954,8 +954,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -966,8 +966,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -985,8 +985,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -1004,8 +1004,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -1023,8 +1023,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -1042,8 +1042,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1061,8 +1061,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1080,8 +1080,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1093,8 +1093,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.inserted_items.json.gt b/tests/data/doc/constructed_doc.inserted_items.json.gt index be83e7229..9eef19967 100644 --- a/tests/data/doc/constructed_doc.inserted_items.json.gt +++ b/tests/data/doc/constructed_doc.inserted_items.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -661,8 +661,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -689,8 +689,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -719,8 +719,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -743,8 +743,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -807,8 +807,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -819,8 +819,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -838,8 +838,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -857,8 +857,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -876,8 +876,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -895,8 +895,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -914,8 +914,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -933,8 +933,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -946,8 +946,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.inserted_list_items.json.gt b/tests/data/doc/constructed_doc.inserted_list_items.json.gt index 1bfc2e5cb..fba326109 100644 --- a/tests/data/doc/constructed_doc.inserted_list_items.json.gt +++ b/tests/data/doc/constructed_doc.inserted_list_items.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -701,8 +701,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -729,8 +729,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -759,8 +759,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -783,8 +783,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -847,8 +847,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -859,8 +859,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -878,8 +878,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -897,8 +897,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -916,8 +916,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -935,8 +935,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -954,8 +954,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -973,8 +973,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -986,8 +986,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.inserted_text.json.gt b/tests/data/doc/constructed_doc.inserted_text.json.gt index 4303eb248..33ae714bf 100644 --- a/tests/data/doc/constructed_doc.inserted_text.json.gt +++ b/tests/data/doc/constructed_doc.inserted_text.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -772,8 +772,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/25", @@ -800,8 +800,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/27", @@ -830,8 +830,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/29", @@ -854,8 +854,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/31", @@ -918,8 +918,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/36", @@ -930,8 +930,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -949,8 +949,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -968,8 +968,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -987,8 +987,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -1006,8 +1006,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1025,8 +1025,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1044,8 +1044,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1057,8 +1057,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/44", diff --git a/tests/data/doc/constructed_doc.manipulated_table.json.gt b/tests/data/doc/constructed_doc.manipulated_table.json.gt index 8ff89be35..9bf334a61 100644 --- a/tests/data/doc/constructed_doc.manipulated_table.json.gt +++ b/tests/data/doc/constructed_doc.manipulated_table.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -714,8 +714,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -742,8 +742,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -772,8 +772,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -796,8 +796,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -860,8 +860,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -872,8 +872,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -891,8 +891,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -910,8 +910,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -929,8 +929,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -948,8 +948,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -967,8 +967,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -986,8 +986,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -999,8 +999,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_doc.referenced.json.gt b/tests/data/doc/constructed_doc.referenced.json.gt index 4ad56ae05..45441b440 100644 --- a/tests/data/doc/constructed_doc.referenced.json.gt +++ b/tests/data/doc/constructed_doc.referenced.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -766,8 +766,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/25", @@ -794,8 +794,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/27", @@ -824,8 +824,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/29", @@ -848,8 +848,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/31", @@ -912,8 +912,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/36", @@ -924,8 +924,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -943,8 +943,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -962,8 +962,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -981,8 +981,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -1000,8 +1000,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1019,8 +1019,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1038,8 +1038,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1051,8 +1051,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/44", diff --git a/tests/data/doc/constructed_doc.referenced.yaml.gt b/tests/data/doc/constructed_doc.referenced.yaml.gt index 5dab83356..a16651f6f 100644 --- a/tests/data/doc/constructed_doc.referenced.yaml.gt +++ b/tests/data/doc/constructed_doc.referenced.yaml.gt @@ -752,12 +752,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Here a code snippet:' + orig: 'Here a code snippet: ' parent: $ref: '#/groups/10' prov: [] self_ref: '#/texts/24' - text: 'Here a code snippet:' + text: 'Here a code snippet: ' - captions: [] children: [] code_language: Python @@ -774,12 +774,12 @@ texts: - children: [] content_layer: body label: text - orig: (to be displayed inline) + orig: ' (to be displayed inline)' parent: $ref: '#/groups/10' prov: [] self_ref: '#/texts/26' - text: (to be displayed inline) + text: ' (to be displayed inline)' - children: - $ref: '#/groups/11' content_layer: body @@ -795,12 +795,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Here a formula:' + orig: 'Here a formula: ' parent: $ref: '#/groups/11' prov: [] self_ref: '#/texts/28' - text: 'Here a formula:' + text: 'Here a formula: ' - children: [] content_layer: body label: formula @@ -813,12 +813,12 @@ texts: - children: [] content_layer: body label: text - orig: (to be displayed inline) + orig: ' (to be displayed inline)' parent: $ref: '#/groups/11' prov: [] self_ref: '#/texts/30' - text: (to be displayed inline) + text: ' (to be displayed inline)' - children: [] content_layer: body label: text @@ -862,12 +862,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Some formatting chops:' + orig: 'Some formatting chops: ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/35' - text: 'Some formatting chops:' + text: 'Some formatting chops: ' - children: [] content_layer: body formatting: @@ -877,12 +877,12 @@ texts: strikethrough: false underline: false label: text - orig: bold + orig: 'bold ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/36' - text: bold + text: 'bold ' - children: [] content_layer: body formatting: @@ -892,12 +892,12 @@ texts: strikethrough: false underline: false label: text - orig: italic + orig: 'italic ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/37' - text: italic + text: 'italic ' - children: [] content_layer: body formatting: @@ -907,12 +907,12 @@ texts: strikethrough: false underline: true label: text - orig: underline + orig: 'underline ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/38' - text: underline + text: 'underline ' - children: [] content_layer: body formatting: @@ -922,12 +922,12 @@ texts: strikethrough: true underline: false label: text - orig: strikethrough + orig: 'strikethrough ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/39' - text: strikethrough + text: 'strikethrough ' - children: [] content_layer: body formatting: @@ -937,12 +937,12 @@ texts: strikethrough: false underline: false label: text - orig: subscript + orig: 'subscript ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/40' - text: subscript + text: 'subscript ' - children: [] content_layer: body formatting: @@ -952,31 +952,31 @@ texts: strikethrough: false underline: false label: text - orig: superscript + orig: 'superscript ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/41' - text: superscript + text: 'superscript ' - children: [] content_layer: body hyperlink: . label: text - orig: hyperlink + orig: 'hyperlink ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/42' - text: hyperlink + text: 'hyperlink ' - children: [] content_layer: body label: text - orig: '&' + orig: '& ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/43' - text: '&' + text: '& ' - children: [] content_layer: body formatting: @@ -1114,4 +1114,4 @@ texts: prov: [] self_ref: '#/texts/55' text: The end. -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/constructed_doc.replaced_item.json.gt b/tests/data/doc/constructed_doc.replaced_item.json.gt index 1340b897d..c55abcb4a 100644 --- a/tests/data/doc/constructed_doc.replaced_item.json.gt +++ b/tests/data/doc/constructed_doc.replaced_item.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -544,8 +544,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -572,8 +572,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -602,8 +602,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -626,8 +626,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -690,8 +690,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -702,8 +702,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -721,8 +721,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -740,8 +740,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -759,8 +759,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -778,8 +778,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -797,8 +797,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -816,8 +816,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -829,8 +829,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/tests/data/doc/constructed_document.yaml.et b/tests/data/doc/constructed_document.yaml.et index 8bc0b024e..2b6b64c49 100644 --- a/tests/data/doc/constructed_document.yaml.et +++ b/tests/data/doc/constructed_document.yaml.et @@ -39,14 +39,14 @@ Affiliation 2 36: list_item: item 1 of sub list 37: list_item: 38: inline with name=group - 39: text: Here a code snippet: + 39: text: Here a code snippet: 40: code: print("Hello world") - 41: text: (to be displayed inline) + 41: text: (to be displayed inline) 42: list_item: 43: inline with name=group - 44: text: Here a formula: + 44: text: Here a formula: 45: formula: E=mc^2 - 46: text: (to be displayed inline) + 46: text: (to be displayed inline) 47: text: Here a code block: 48: code: print("Hello world") 49: text: Here a formula block: @@ -54,15 +54,15 @@ Affiliation 2 51: key_value_region 52: form 53: inline with name=group - 54: text: Some formatting chops: - 55: text: bold - 56: text: italic - 57: text: underline - 58: text: strikethrough - 59: text: subscript - 60: text: superscript - 61: text: hyperlink - 62: text: & + 54: text: Some formatting chops: + 55: text: bold + 56: text: italic + 57: text: underline + 58: text: strikethrough + 59: text: subscript + 60: text: superscript + 61: text: hyperlink + 62: text: & 63: text: everything at the same time. 64: list with name=list A 65: list_item: Item 1 in A diff --git a/tests/data/doc/content_all.gt.dclg.xml b/tests/data/doc/content_all.gt.dclg.xml index fc893e73a..914144eef 100644 --- a/tests/data/doc/content_all.gt.dclg.xml +++ b/tests/data/doc/content_all.gt.dclg.xml @@ -106,17 +106,17 @@ Affiliation 2 □ - Here a code snippet: + Here a code snippet: -(to be displayed inline) + (to be displayed inline) □ - Here a formula: + Here a formula: E=mc^2 -(to be displayed inline) + (to be displayed inline) Here a code block: @@ -126,15 +126,27 @@ Affiliation 2 Here a formula block: E=mc^2 - Some formatting chops: - bold - italic - underline - strikethrough - subscript - superscript -hyperlink - + Some formatting chops: + + bold + + + italic + + + underline + + + strikethrough + + + subscript + + + superscript + + hyperlink + diff --git a/tests/data/doc/content_block_specific.gt.dclg.xml b/tests/data/doc/content_block_specific.gt.dclg.xml index 18e3c918a..78e68eb5e 100644 --- a/tests/data/doc/content_block_specific.gt.dclg.xml +++ b/tests/data/doc/content_block_specific.gt.dclg.xml @@ -87,17 +87,17 @@ Affiliation 2 □ - Here a code snippet: + Here a code snippet: -(to be displayed inline) + (to be displayed inline) □ - Here a formula: + Here a formula: E=mc^2 -(to be displayed inline) + (to be displayed inline) Here a code block: @@ -107,15 +107,27 @@ Affiliation 2 Here a formula block: E=mc^2 - Some formatting chops: - bold - italic - underline - strikethrough - subscript - superscript -hyperlink - + Some formatting chops: + + bold + + + italic + + + underline + + + strikethrough + + + subscript + + + superscript + + hyperlink + diff --git a/tests/data/doc/cross_column_list/deserialized.json b/tests/data/doc/cross_column_list/deserialized.json index 8baa629f8..9638da6ff 100644 --- a/tests/data/doc/cross_column_list/deserialized.json +++ b/tests/data/doc/cross_column_list/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/cross_column_list/input.json b/tests/data/doc/cross_column_list/input.json index 74acc8371..565c1288a 100644 --- a/tests/data/doc/cross_column_list/input.json +++ b/tests/data/doc/cross_column_list/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "cross_column_list", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/cross_page_list/deserialized.json b/tests/data/doc/cross_page_list/deserialized.json index b9befd7c9..0defaf972 100644 --- a/tests/data/doc/cross_page_list/deserialized.json +++ b/tests/data/doc/cross_page_list/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/cross_page_list/input.json b/tests/data/doc/cross_page_list/input.json index 1252c18d8..390ce97af 100644 --- a/tests/data/doc/cross_page_list/input.json +++ b/tests/data/doc/cross_page_list/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "cross_page_list", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/cross_page_paragraph/deserialized.json b/tests/data/doc/cross_page_paragraph/deserialized.json index 0ba545230..9a5bf544b 100644 --- a/tests/data/doc/cross_page_paragraph/deserialized.json +++ b/tests/data/doc/cross_page_paragraph/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/cross_page_paragraph/input.json b/tests/data/doc/cross_page_paragraph/input.json index 6e4a478c2..50acf39e7 100644 --- a/tests/data/doc/cross_page_paragraph/input.json +++ b/tests/data/doc/cross_page_paragraph/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "cross_page_paragraph", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/cross_page_table/deserialized.json b/tests/data/doc/cross_page_table/deserialized.json index 69a4d7ac4..9921ef477 100644 --- a/tests/data/doc/cross_page_table/deserialized.json +++ b/tests/data/doc/cross_page_table/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/cross_page_table/input.json b/tests/data/doc/cross_page_table/input.json index 47c8c57a2..5dd2a2497 100644 --- a/tests/data/doc/cross_page_table/input.json +++ b/tests/data/doc/cross_page_table/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "cross_page_table", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/default_mode.gt.dclg.xml b/tests/data/doc/default_mode.gt.dclg.xml index ba9b52828..6de298fc7 100644 --- a/tests/data/doc/default_mode.gt.dclg.xml +++ b/tests/data/doc/default_mode.gt.dclg.xml @@ -18,7 +18,7 @@ E=mc^2 -in line + in line Here a bold diff --git a/tests/data/doc/doc_with_kv.dt.json b/tests/data/doc/doc_with_kv.dt.json index 352900710..03e4944d2 100644 --- a/tests/data/doc/doc_with_kv.dt.json +++ b/tests/data/doc/doc_with_kv.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/docitem_comments_field.out.yaml b/tests/data/doc/docitem_comments_field.out.yaml index 2c7e88c4d..cf418a3b6 100644 --- a/tests/data/doc/docitem_comments_field.out.yaml +++ b/tests/data/doc/docitem_comments_field.out.yaml @@ -51,4 +51,4 @@ texts: prov: [] self_ref: '#/texts/2' text: '[John Reviewer]: This is a reviewer comment.' -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/docitem_comments_multiple.out.yaml b/tests/data/doc/docitem_comments_multiple.out.yaml index 24c6f1d1e..1458032a1 100644 --- a/tests/data/doc/docitem_comments_multiple.out.yaml +++ b/tests/data/doc/docitem_comments_multiple.out.yaml @@ -79,4 +79,4 @@ texts: prov: [] self_ref: '#/texts/4' text: '[Reviewer B]: This is a comment on texts 2 (range [0,6)) and 3.' -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/doclang_archive/load/two_pages.gt.json b/tests/data/doc/doclang_archive/load/two_pages.gt.json index a19789fd6..50647c241 100644 --- a/tests/data/doc/doclang_archive/load/two_pages.gt.json +++ b/tests/data/doc/doclang_archive/load/two_pages.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "two_pages", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/doclang_archive/save/two_pages.json b/tests/data/doc/doclang_archive/save/two_pages.json index 5bc6551e8..7efc7a00c 100644 --- a/tests/data/doc/doclang_archive/save/two_pages.json +++ b/tests/data/doc/doclang_archive/save/two_pages.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "2408.09869v3", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/doclang_ref/002a_table_order/output.json b/tests/data/doc/doclang_ref/002a_table_order/output.json index d280e0b32..673648029 100644 --- a/tests/data/doc/doclang_ref/002a_table_order/output.json +++ b/tests/data/doc/doclang_ref/002a_table_order/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "input", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/dummy_doc_2_prec.yaml b/tests/data/doc/dummy_doc_2_prec.yaml index 1d2800bd2..b445d4f5f 100644 --- a/tests/data/doc/dummy_doc_2_prec.yaml +++ b/tests/data/doc/dummy_doc_2_prec.yaml @@ -272,4 +272,4 @@ texts: self_ref: '#/texts/3' text: 'Figure 1: Four examples of complex page layouts across different document categories' -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/dummy_doc_with_meta_modified.yaml b/tests/data/doc/dummy_doc_with_meta_modified.yaml index 1a4a0d33c..a24b30376 100644 --- a/tests/data/doc/dummy_doc_with_meta_modified.yaml +++ b/tests/data/doc/dummy_doc_with_meta_modified.yaml @@ -290,4 +290,4 @@ texts: self_ref: '#/texts/3' text: 'Figure 1: Four examples of complex page layouts across different document categories' -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/field_region_kv/deserialized.json b/tests/data/doc/field_region_kv/deserialized.json index b6984b6ed..ff680b22d 100644 --- a/tests/data/doc/field_region_kv/deserialized.json +++ b/tests/data/doc/field_region_kv/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", @@ -283,8 +283,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Clark ", - "text": "Clark " + "orig": "Clark ", + "text": "Clark " }, { "self_ref": "#/texts/12", @@ -295,8 +295,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Kent", - "text": "Kent", + "orig": "Kent ", + "text": "Kent ", "formatting": { "bold": true, "italic": false, @@ -339,8 +339,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "My first input ", - "text": "My first input " + "orig": "My first input ", + "text": "My first input " }, { "self_ref": "#/texts/16", @@ -364,8 +364,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": " and my second input ", - "text": " and my second input " + "orig": " and my second input ", + "text": " and my second input " }, { "self_ref": "#/texts/18", diff --git a/tests/data/doc/field_region_kv/input.json b/tests/data/doc/field_region_kv/input.json index 33cf6bd47..3d1cc2fb1 100644 --- a/tests/data/doc/field_region_kv/input.json +++ b/tests/data/doc/field_region_kv/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", @@ -284,7 +284,7 @@ "label": "text", "prov": [], "orig": "Clark ", - "text": "Clark " + "text": "Clark " }, { "self_ref": "#/texts/12", @@ -296,7 +296,7 @@ "label": "text", "prov": [], "orig": "Kent", - "text": "Kent", + "text": "Kent ", "formatting": { "bold": true, "italic": false, @@ -340,7 +340,7 @@ "label": "text", "prov": [], "orig": "My first input ", - "text": "My first input " + "text": "My first input " }, { "self_ref": "#/texts/16", @@ -365,7 +365,7 @@ "label": "text", "prov": [], "orig": " and my second input ", - "text": " and my second input " + "text": " and my second input " }, { "self_ref": "#/texts/18", diff --git a/tests/data/doc/field_region_kv/reserialized.dclg.xml b/tests/data/doc/field_region_kv/reserialized.dclg.xml index 6959ba029..768728dff 100644 --- a/tests/data/doc/field_region_kv/reserialized.dclg.xml +++ b/tests/data/doc/field_region_kv/reserialized.dclg.xml @@ -20,15 +20,17 @@ Max Mustermann - Clark - Kent + Clark + + Kent + Select this if you are a Superman fan - My first input + My first input - and my second input + and my second input m diff --git a/tests/data/doc/field_region_kv/serialized.dclg.xml b/tests/data/doc/field_region_kv/serialized.dclg.xml index 6959ba029..768728dff 100644 --- a/tests/data/doc/field_region_kv/serialized.dclg.xml +++ b/tests/data/doc/field_region_kv/serialized.dclg.xml @@ -20,15 +20,17 @@ Max Mustermann - Clark - Kent + Clark + + Kent + Select this if you are a Superman fan - My first input + My first input - and my second input + and my second input m diff --git a/tests/data/doc/field_region_kv_invoice/deserialized.json b/tests/data/doc/field_region_kv_invoice/deserialized.json index 55d9381ec..6763fc6c4 100644 --- a/tests/data/doc/field_region_kv_invoice/deserialized.json +++ b/tests/data/doc/field_region_kv_invoice/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/field_region_kv_invoice/input.json b/tests/data/doc/field_region_kv_invoice/input.json index 7fd2a45eb..0e52cb4af 100644 --- a/tests/data/doc/field_region_kv_invoice/input.json +++ b/tests/data/doc/field_region_kv_invoice/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/field_region_kv_migration/deserialized.json b/tests/data/doc/field_region_kv_migration/deserialized.json index 8e8fe92b8..c71e651c9 100644 --- a/tests/data/doc/field_region_kv_migration/deserialized.json +++ b/tests/data/doc/field_region_kv_migration/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/field_region_kv_migration/input.json b/tests/data/doc/field_region_kv_migration/input.json index 2a9ebee65..9e6c13bb4 100644 --- a/tests/data/doc/field_region_kv_migration/input.json +++ b/tests/data/doc/field_region_kv_migration/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/flattened.dclg.xml b/tests/data/doc/flattened.dclg.xml index 440d557e0..7149867c3 100644 --- a/tests/data/doc/flattened.dclg.xml +++ b/tests/data/doc/flattened.dclg.xml @@ -9,7 +9,7 @@ Rich heading without -other children besides the inline + other children besides the inline Section content as sibling of the heading. Subheading @@ -17,7 +17,7 @@ other children besides the inline Rich heading with -other children besides the inline + other children besides the inline Section content as child of the heading. Section content as sibling of the heading. diff --git a/tests/data/doc/flattened.json b/tests/data/doc/flattened.json index 7a9161650..bae77d18d 100644 --- a/tests/data/doc/flattened.json +++ b/tests/data/doc/flattened.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/group_with_metadata.yaml b/tests/data/doc/group_with_metadata.yaml index 37b00a434..548eb06d7 100644 --- a/tests/data/doc/group_with_metadata.yaml +++ b/tests/data/doc/group_with_metadata.yaml @@ -125,4 +125,4 @@ texts: prov: [] self_ref: '#/texts/4' text: Regarding bar... -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/hierarchized.dclg.xml b/tests/data/doc/hierarchized.dclg.xml index 440d557e0..7149867c3 100644 --- a/tests/data/doc/hierarchized.dclg.xml +++ b/tests/data/doc/hierarchized.dclg.xml @@ -9,7 +9,7 @@ Rich heading without -other children besides the inline + other children besides the inline Section content as sibling of the heading. Subheading @@ -17,7 +17,7 @@ other children besides the inline Rich heading with -other children besides the inline + other children besides the inline Section content as child of the heading. Section content as sibling of the heading. diff --git a/tests/data/doc/hierarchized.json b/tests/data/doc/hierarchized.json index 09dd4bd3a..30f477814 100644 --- a/tests/data/doc/hierarchized.json +++ b/tests/data/doc/hierarchized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/inline_and_formatting.gt.dt b/tests/data/doc/inline_and_formatting.gt.dt index f19624326..755a08e15 100644 --- a/tests/data/doc/inline_and_formatting.gt.dt +++ b/tests/data/doc/inline_and_formatting.gt.dt @@ -40,6 +40,7 @@ Partially formatted heading to_escape <_unknown_>not_to_escape + E=mc^2 & ampersand diff --git a/tests/data/doc/inline_group.gt.dclg.xml b/tests/data/doc/inline_group.gt.dclg.xml index 6ce42cdf5..700021335 100644 --- a/tests/data/doc/inline_group.gt.dclg.xml +++ b/tests/data/doc/inline_group.gt.dclg.xml @@ -6,7 +6,7 @@ One Two -Three + Three @@ -14,6 +14,6 @@ Three Four Five -Six + Six diff --git a/tests/data/doc/kv.out.json b/tests/data/doc/kv.out.json index 33cf6bd47..8b020e883 100644 --- a/tests/data/doc/kv.out.json +++ b/tests/data/doc/kv.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/deserialized.json b/tests/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/deserialized.json index 422e0beac..9f3986254 100644 --- a/tests/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/deserialized.json +++ b/tests/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/output.json b/tests/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/output.json index 011421cdd..9e8eb6fca 100644 --- a/tests/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/output.json +++ b/tests/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/deserialized.json b/tests/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/deserialized.json index 29cc4845c..887711460 100644 --- a/tests/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/deserialized.json +++ b/tests/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/output.json b/tests/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/output.json index 9555b87fc..ff2f76a07 100644 --- a/tests/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/output.json +++ b/tests/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/deserialized.json b/tests/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/deserialized.json index f0ab7e3be..95a6cc8ff 100644 --- a/tests/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/deserialized.json +++ b/tests/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/output.json b/tests/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/output.json index 2e4f1da4f..515dea899 100644 --- a/tests/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/output.json +++ b/tests/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/deserialized.json b/tests/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/deserialized.json index 6b2823fe5..5bd3da79f 100644 --- a/tests/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/deserialized.json +++ b/tests/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/output.json b/tests/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/output.json index 914ba3f8c..ddb51c277 100644 --- a/tests/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/output.json +++ b/tests/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/deserialized.json b/tests/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/deserialized.json index 03d87bc9a..13197af7c 100644 --- a/tests/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/deserialized.json +++ b/tests/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/output.json b/tests/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/output.json index 5be82748a..d112259a9 100644 --- a/tests/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/output.json +++ b/tests/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json b/tests/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json index 552b2079d..967790bde 100644 --- a/tests/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json +++ b/tests/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json b/tests/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json index ec07558af..8cb59e98c 100644 --- a/tests/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json +++ b/tests/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/deserialized.json b/tests/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/deserialized.json index 1e585c67b..461babe11 100644 --- a/tests/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/deserialized.json +++ b/tests/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/output.json b/tests/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/output.json index ccf09e939..4295134fc 100644 --- a/tests/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/output.json +++ b/tests/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/deserialized.json b/tests/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/deserialized.json index 3f64f48dd..42658ce9a 100644 --- a/tests/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/deserialized.json +++ b/tests/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/output.json b/tests/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/output.json index 345d40552..21d34a758 100644 --- a/tests/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/output.json +++ b/tests/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json b/tests/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json index 552b2079d..967790bde 100644 --- a/tests/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json +++ b/tests/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json b/tests/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json index ec07558af..8cb59e98c 100644 --- a/tests/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json +++ b/tests/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv_advanced_inline.out.dclg.xml b/tests/data/doc/kv_advanced_inline.out.dclg.xml index 29fb0a9cf..88264412a 100644 --- a/tests/data/doc/kv_advanced_inline.out.dclg.xml +++ b/tests/data/doc/kv_advanced_inline.out.dclg.xml @@ -13,7 +13,7 @@ -. + . diff --git a/tests/data/doc/kv_advanced_inline.out.json b/tests/data/doc/kv_advanced_inline.out.json index b25b59f7d..03954c4d1 100644 --- a/tests/data/doc/kv_advanced_inline.out.json +++ b/tests/data/doc/kv_advanced_inline.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv_form_with_table.out.json b/tests/data/doc/kv_form_with_table.out.json index 5d33e97eb..8577433b2 100644 --- a/tests/data/doc/kv_form_with_table.out.json +++ b/tests/data/doc/kv_form_with_table.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv_invoice.out.json b/tests/data/doc/kv_invoice.out.json index 7fd2a45eb..0e52cb4af 100644 --- a/tests/data/doc/kv_invoice.out.json +++ b/tests/data/doc/kv_invoice.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv_nested.out.json b/tests/data/doc/kv_nested.out.json index b7605a45a..ca19c8ad8 100644 --- a/tests/data/doc/kv_nested.out.json +++ b/tests/data/doc/kv_nested.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv_post_migration.out.json b/tests/data/doc/kv_post_migration.out.json index 2a9ebee65..9e6c13bb4 100644 --- a/tests/data/doc/kv_post_migration.out.json +++ b/tests/data/doc/kv_post_migration.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/kv_pre_migration.out.json b/tests/data/doc/kv_pre_migration.out.json index 3dfd68225..4514cd055 100644 --- a/tests/data/doc/kv_pre_migration.out.json +++ b/tests/data/doc/kv_pre_migration.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/misplaced_list_items.norm.out.yaml b/tests/data/doc/misplaced_list_items.norm.out.yaml index 3d0586d3f..f75b7778f 100644 --- a/tests/data/doc/misplaced_list_items.norm.out.yaml +++ b/tests/data/doc/misplaced_list_items.norm.out.yaml @@ -81,4 +81,4 @@ texts: prov: [] self_ref: '#/texts/3' text: there -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/misplaced_list_items.out.yaml b/tests/data/doc/misplaced_list_items.out.yaml index 3daa2763e..0771ba3bd 100644 --- a/tests/data/doc/misplaced_list_items.out.yaml +++ b/tests/data/doc/misplaced_list_items.out.yaml @@ -81,4 +81,4 @@ texts: prov: [] self_ref: '#/texts/3' text: there -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/multi_captions_footnotes.deserialized.gt.json b/tests/data/doc/multi_captions_footnotes.deserialized.gt.json index 48fc4c6a6..0128145fe 100644 --- a/tests/data/doc/multi_captions_footnotes.deserialized.gt.json +++ b/tests/data/doc/multi_captions_footnotes.deserialized.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/multi_captions_footnotes_top_level.deserialized.gt.json b/tests/data/doc/multi_captions_footnotes_top_level.deserialized.gt.json index 48fc4c6a6..0128145fe 100644 --- a/tests/data/doc/multi_captions_footnotes_top_level.deserialized.gt.json +++ b/tests/data/doc/multi_captions_footnotes_top_level.deserialized.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/multi_page_roundtrip/deserialized.json b/tests/data/doc/multi_page_roundtrip/deserialized.json index 2e75ef58a..d941ef9cd 100644 --- a/tests/data/doc/multi_page_roundtrip/deserialized.json +++ b/tests/data/doc/multi_page_roundtrip/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/multi_page_roundtrip/input.json b/tests/data/doc/multi_page_roundtrip/input.json index acc8b2f12..c081cf1a9 100644 --- a/tests/data/doc/multi_page_roundtrip/input.json +++ b/tests/data/doc/multi_page_roundtrip/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "multi_page_roundtrip", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/multi_prov_thread/deserialized.json b/tests/data/doc/multi_prov_thread/deserialized.json index 1c38c5f70..ed8646dc4 100644 --- a/tests/data/doc/multi_prov_thread/deserialized.json +++ b/tests/data/doc/multi_prov_thread/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/multi_prov_thread/input.json b/tests/data/doc/multi_prov_thread/input.json index b44bc279e..27c61bdc0 100644 --- a/tests/data/doc/multi_prov_thread/input.json +++ b/tests/data/doc/multi_prov_thread/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "multi_prov_thread", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/page_with_pic.dt.json b/tests/data/doc/page_with_pic.dt.json index d53c9c46a..9e548f2b4 100644 --- a/tests/data/doc/page_with_pic.dt.json +++ b/tests/data/doc/page_with_pic.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/page_with_pic_from_files.dt.json b/tests/data/doc/page_with_pic_from_files.dt.json index d53c9c46a..9e548f2b4 100644 --- a/tests/data/doc/page_with_pic_from_files.dt.json +++ b/tests/data/doc/page_with_pic_from_files.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/page_without_pic.dt.json b/tests/data/doc/page_without_pic.dt.json index b4a0b3c5e..c9a7dca3a 100644 --- a/tests/data/doc/page_without_pic.dt.json +++ b/tests/data/doc/page_without_pic.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/picture_molecule_meta/deserialized.json b/tests/data/doc/picture_molecule_meta/deserialized.json index 0813f708a..e6d36bc08 100644 --- a/tests/data/doc/picture_molecule_meta/deserialized.json +++ b/tests/data/doc/picture_molecule_meta/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/picture_molecule_meta/input.json b/tests/data/doc/picture_molecule_meta/input.json index bb292c990..8d5ec6891 100644 --- a/tests/data/doc/picture_molecule_meta/input.json +++ b/tests/data/doc/picture_molecule_meta/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "picture_molecule_meta", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/referenced_caption/deserialized.json b/tests/data/doc/referenced_caption/deserialized.json index b0ecb2039..f2230a78d 100644 --- a/tests/data/doc/referenced_caption/deserialized.json +++ b/tests/data/doc/referenced_caption/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/referenced_caption/input.json b/tests/data/doc/referenced_caption/input.json index e2017077c..50c9332d6 100644 --- a/tests/data/doc/referenced_caption/input.json +++ b/tests/data/doc/referenced_caption/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "referenced_caption", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/rich_table.out.yaml b/tests/data/doc/rich_table.out.yaml index 992c08a8c..d486bbde6 100644 --- a/tests/data/doc/rich_table.out.yaml +++ b/tests/data/doc/rich_table.out.yaml @@ -501,4 +501,4 @@ texts: prov: [] self_ref: '#/texts/5' text: More text in the group. -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/rich_table_item_ins_norm_1.out.yaml b/tests/data/doc/rich_table_item_ins_norm_1.out.yaml index 008cd1523..05597efa9 100644 --- a/tests/data/doc/rich_table_item_ins_norm_1.out.yaml +++ b/tests/data/doc/rich_table_item_ins_norm_1.out.yaml @@ -241,4 +241,4 @@ texts: prov: [] self_ref: '#/texts/1' text: text in italic -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/rich_table_item_ins_norm_2.out.yaml b/tests/data/doc/rich_table_item_ins_norm_2.out.yaml index 4762a2ba1..0bc332819 100644 --- a/tests/data/doc/rich_table_item_ins_norm_2.out.yaml +++ b/tests/data/doc/rich_table_item_ins_norm_2.out.yaml @@ -251,4 +251,4 @@ texts: prov: [] self_ref: '#/texts/2' text: text before -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/rich_table_item_ins_norm_3.out.yaml b/tests/data/doc/rich_table_item_ins_norm_3.out.yaml index 16c93668e..eaf069aa3 100644 --- a/tests/data/doc/rich_table_item_ins_norm_3.out.yaml +++ b/tests/data/doc/rich_table_item_ins_norm_3.out.yaml @@ -251,4 +251,4 @@ texts: prov: [] self_ref: '#/texts/2' text: text in italic -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/rich_table_post_text_del.out.yaml b/tests/data/doc/rich_table_post_text_del.out.yaml index 06752fc27..30f0bc97c 100644 --- a/tests/data/doc/rich_table_post_text_del.out.yaml +++ b/tests/data/doc/rich_table_post_text_del.out.yaml @@ -491,4 +491,4 @@ texts: prov: [] self_ref: '#/texts/4' text: More text in the group. -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/roundtrip_list_item_with_inline_deserialized.yaml b/tests/data/doc/roundtrip_list_item_with_inline_deserialized.yaml index 5bb6c0975..2aa0d9945 100644 --- a/tests/data/doc/roundtrip_list_item_with_inline_deserialized.yaml +++ b/tests/data/doc/roundtrip_list_item_with_inline_deserialized.yaml @@ -378,4 +378,4 @@ texts: prov: [] self_ref: '#/texts/24' text: final element -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/roundtrip_list_item_with_inline_reserialized.dclg.xml b/tests/data/doc/roundtrip_list_item_with_inline_reserialized.dclg.xml index f26cbb160..16098eaf1 100644 --- a/tests/data/doc/roundtrip_list_item_with_inline_reserialized.dclg.xml +++ b/tests/data/doc/roundtrip_list_item_with_inline_reserialized.dclg.xml @@ -20,7 +20,7 @@ E=mc^2 -in line + in line Here a bold diff --git a/tests/data/doc/roundtrip_list_item_with_inline_serialized.dclg.xml b/tests/data/doc/roundtrip_list_item_with_inline_serialized.dclg.xml index ba9b52828..6de298fc7 100644 --- a/tests/data/doc/roundtrip_list_item_with_inline_serialized.dclg.xml +++ b/tests/data/doc/roundtrip_list_item_with_inline_serialized.dclg.xml @@ -18,7 +18,7 @@ E=mc^2 -in line + in line Here a bold diff --git a/tests/data/doc/virtual_text_index/deserialized.json b/tests/data/doc/virtual_text_index/deserialized.json index 3838f06a8..25ed806c0 100644 --- a/tests/data/doc/virtual_text_index/deserialized.json +++ b/tests/data/doc/virtual_text_index/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/virtual_text_index/input.json b/tests/data/doc/virtual_text_index/input.json index b8c3b9a89..d45d62321 100644 --- a/tests/data/doc/virtual_text_index/input.json +++ b/tests/data/doc/virtual_text_index/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "virtual_text_index", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/virtual_text_list/deserialized.json b/tests/data/doc/virtual_text_list/deserialized.json index c70380951..b39c0511b 100644 --- a/tests/data/doc/virtual_text_list/deserialized.json +++ b/tests/data/doc/virtual_text_list/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/virtual_text_list/input.json b/tests/data/doc/virtual_text_list/input.json index 25c80c987..b6bf5f1ef 100644 --- a/tests/data/doc/virtual_text_list/input.json +++ b/tests/data/doc/virtual_text_list/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "virtual_text_list", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/virtual_text_table/deserialized.json b/tests/data/doc/virtual_text_table/deserialized.json index a878507ce..98fb97e9e 100644 --- a/tests/data/doc/virtual_text_table/deserialized.json +++ b/tests/data/doc/virtual_text_table/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/virtual_text_table/input.json b/tests/data/doc/virtual_text_table/input.json index a5732fba5..322fcc5e7 100644 --- a/tests/data/doc/virtual_text_table/input.json +++ b/tests/data/doc/virtual_text_table/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "virtual_text_table", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/webvtt_example_02.json b/tests/data/doc/webvtt_example_02.json index c418ac506..07e3cfbde 100644 --- a/tests/data/doc/webvtt_example_02.json +++ b/tests/data/doc/webvtt_example_02.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.8.0", + "version": "1.11.0", "name": "webvtt_example_02", "origin": { "mimetype": "text/vtt", diff --git a/tests/data/doc/webvtt_example_04.json b/tests/data/doc/webvtt_example_04.json index 6fe7d0ea9..5fd5df4f4 100644 --- a/tests/data/doc/webvtt_example_04.json +++ b/tests/data/doc/webvtt_example_04.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.8.0", + "version": "1.11.0", "name": "webvtt_example_04", "origin": { "mimetype": "text/vtt", diff --git a/tests/data/doc/webvtt_example_05.json b/tests/data/doc/webvtt_example_05.json index 76803cf6f..088766d8d 100644 --- a/tests/data/doc/webvtt_example_05.json +++ b/tests/data/doc/webvtt_example_05.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.8.0", + "version": "1.11.0", "name": "webvtt_example_04", "origin": { "mimetype": "text/vtt", diff --git a/tests/test_compat.py b/tests/test_compat.py index 070064594..662aa2054 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -44,7 +44,7 @@ FIRST_SUPPORTED_MINOR, SCHEMA_VERSION_HISTORY, ) -from docling_core.types.doc.document import DoclingDocument +from docling_core.types.doc.document import DocItemLabel, DoclingDocument, Formatting # Root of the repository, used to locate docs/schemas/. _REPO_ROOT = Path(__file__).parent.parent @@ -284,6 +284,120 @@ def test_rendered_schema_describes_the_current_models(self): assert set(schema["$defs"]["FieldItem"]["properties"]) == set(FieldItem.model_fields) +class TestProjector_1_11_to_1_10: + """Tests for the 1.11 → 1.10 downgrade projector.""" + + def _apply(self, data: dict) -> dict: + from docling_core.compat import _project_1_11_to_1_10 + + return _project_1_11_to_1_10(data) + + def test_version_set_to_1_10(self): + result = self._apply(_minimal_doc_dict("1.11.0")) + assert result["version"] == "1.10.0" + + def test_plain_text_keeps_its_whitespace(self): + """Only inline-group runs are stripped; other text items stay as they are.""" + data = _minimal_doc_dict("1.11.0") + data["texts"] = [ + { + "self_ref": "#/texts/0", + "parent": None, + "children": [], + "content_layer": "body", + "label": "text", + "orig": "bold ", + "text": "bold ", + "prov": [], + } + ] + result = self._apply(data) + assert result["texts"][0]["text"] == "bold " + + def test_inline_runs_are_stripped_to_the_old_shape(self): + """A 1.10 client joins runs with " ", so runs must not carry boundary whitespace.""" + doc = DoclingDocument(name="runs") + caption = doc.add_text(label=DocItemLabel.CAPTION, text=" Caption ") + heading = doc.add_heading(text="Heading ") + group = doc.add_inline_group() + doc.add_text(label=DocItemLabel.TEXT, text="Some ", parent=group) + doc.add_text(label=DocItemLabel.TEXT, text="bold ", parent=group, formatting=Formatting(bold=True)) + doc.add_text(label=DocItemLabel.TEXT, text="text with a ", parent=group) + doc.add_text(label=DocItemLabel.TEXT, text="link ", parent=group, hyperlink="https://example.com") + doc.add_text(label=DocItemLabel.TEXT, text=" ", parent=group) + doc.add_text(label=DocItemLabel.TEXT, text="core", parent=group, formatting=Formatting(bold=True)) + data = json.loads(doc.model_dump_json()) + + result = self._apply(data) + + assert len(result["texts"]) == len(data["texts"]) + assert [t["text"] for t in result["texts"]] == [ + " Caption ", + "Heading ", + "Some", + "bold", + "text with a", + "link", + "", + "core", + ] + assert [t["orig"] for t in result["texts"]] == [t["orig"] for t in data["texts"]] + assert data["texts"][2]["text"] == "Some " + assert caption.text == " Caption " and heading.text == "Heading " + + def test_does_not_mutate_caller_input(self): + data = _minimal_doc_dict("1.11.0") + self._apply(data) + assert data["version"] == "1.11.0" + + @pytest.mark.parametrize( + "runs", + [ + [("code", "a"), ("code", "b")], + [("formula", "x"), ("formula", "y")], + [("code", "a"), ("formula", "x"), ("code", "b")], + [("text", "Install with "), ("code", "pip"), ("text", ", then")], + [("text", "a"), ("text", " "), ("text", "b")], + ], + ) + def test_repeated_projection_and_migration_do_not_drift(self, runs): + """Down-projecting and reloading again must not add runs or change the output.""" + doc = DoclingDocument(name="drift") + group = doc.add_inline_group() + for kind, text in runs: + if kind == "code": + doc.add_code(text=text, parent=group) + elif kind == "formula": + doc.add_formula(text=text, parent=group) + else: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=group) + + def round_trip(d: DoclingDocument) -> DoclingDocument: + return DoclingDocument.model_validate(json.loads(project_to(d, "1.10.0").model_dump_json())) + + first = round_trip(doc) + again = first + for _ in range(3): + again = round_trip(again) + assert again.export_to_dict() == first.export_to_dict() + + def test_project_to_does_not_run_the_legacy_migration(self): + """project_to keeps the old-shape runs; reloading them on this core restores the spacing.""" + doc = DoclingDocument(name="projected") + group = doc.add_inline_group() + doc.add_text(label=DocItemLabel.TEXT, text="Normal ", parent=group) + doc.add_text(label=DocItemLabel.TEXT, text="bold", parent=group, formatting=Formatting(bold=True)) + doc.add_text(label=DocItemLabel.TEXT, text=" tail", parent=group) + + result = project_to(doc, target_version="1.10.0") + dumped = json.loads(result.model_dump_json()) + + assert dumped["version"] == "1.10.0" + assert [t["text"] for t in dumped["texts"]] == ["Normal", "bold", "tail"] + reloaded = DoclingDocument.model_validate(dumped) + assert reloaded.export_to_markdown() == "Normal **bold** tail" + + class TestProjector_1_10_to_1_9: """Tests for the 1.10 → 1.9 downgrade projector.""" diff --git a/tests/test_deserializer_doclang.py b/tests/test_deserializer_doclang.py index 176947173..dda37e549 100644 --- a/tests/test_deserializer_doclang.py +++ b/tests/test_deserializer_doclang.py @@ -1897,7 +1897,9 @@ def test_otsl_xml_sensitive_virtual_and_explicit_text_cells(): doc = _deserialize(doclang) assert doc.tables[0].data.grid[0][0].text == "virtual & \"quoted\" 'apostrophe'" - assert doc.tables[0].data.grid[0][1].text == "nested & bold & styled" + # The source writes ` bold & styled`; that leading space is a significant + # run boundary under the inline whitespace contract, so it survives. + assert doc.tables[0].data.grid[0][1].text == "nested & bold & styled" assert isinstance(doc.tables[0].data.grid[0][1], RichTableCell) @@ -2608,7 +2610,7 @@ def test_mixed_text_like_archive_roundtrip( for node, _ in doc.iterate_items(root=root, with_groups=True, included_content_layers=set(ContentLayer)) ) runs = [ref.resolve(doc) for ref in inline.children] - assert [run.text for run in runs] == ["Before", "x" if "formula" in body else "bold", "after"] + assert [run.text for run in runs] == ["Before ", "x" if "formula" in body else "bold", " after"] if "formula" in body: assert runs[1].label == DocItemLabel.FORMULA else: @@ -2637,16 +2639,22 @@ def test_plain_caption_layer_roundtrip(kind: str, layer: ContentLayer, tmp_path: @pytest.mark.parametrize("kind", ["table", "picture"]) @pytest.mark.parametrize( - ("caption", "expected", "markdown"), + ("caption", "expected", "markdown", "doctags"), [ - ("Table 1: results", "Table 1: results", "Table 1: results"), - ("Table 1: results", "Table 1 : results", "Table **1** : results"), - ("Table 1: results", "Table 1 : results", "**Table** ***1*** : results"), + ("Table 1: results", "Table 1: results", "Table 1: results", "Table 1: results"), + # DocTags predates the whitespace contract: it strips each run and puts them on separate lines. + ("Table 1: results", "Table 1: results", "Table **1**: results", "Table 1 : results"), + ( + "Table 1: results", + "Table 1: results", + "**Table** ***1***: results", + "Table 1 : results", + ), ], ) @pytest.mark.parametrize("layer", [ContentLayer.BODY, ContentLayer.FURNITURE]) def test_caption_readers_after_archive_roundtrip( - kind: str, caption: str, expected: str, markdown: str, layer: ContentLayer, tmp_path: Path + kind: str, caption: str, expected: str, markdown: str, doctags: str, layer: ContentLayer, tmp_path: Path ): table_body = "AB12" if kind == "table" else "" doc = _deserialize( @@ -2663,14 +2671,15 @@ def test_caption_readers_after_archive_roundtrip( DocTagsDocSerializer(doc=doc), ]: result = serializer.serialize_captions(item=owner, layers=set(ContentLayer)).text - if isinstance(serializer, HTMLDocSerializer | DocTagsDocSerializer): + if isinstance(serializer, DocTagsDocSerializer): content = " ".join(text.strip() for text in ET.fromstring(result).itertext() if text.strip()) - assert content == expected - if isinstance(serializer, HTMLDocSerializer): - if "" in caption: - assert "1" in result - if "" in caption: - assert "" in result + assert content == doctags + elif isinstance(serializer, HTMLDocSerializer): + assert "".join(ET.fromstring(result).itertext()) == expected + if "" in caption: + assert "1" in result + if "" in caption: + assert "" in result elif isinstance(serializer, PlainTextDocSerializer): assert expected in result else: @@ -2692,10 +2701,10 @@ def test_native_caption_children_and_filters(kind: str, host_text: str): caption = doc.add_text(label=DocItemLabel.CAPTION, text=host_text) owner.captions.append(caption.get_ref()) inline = doc.add_inline_group(parent=caption) - doc.add_text(label=DocItemLabel.TEXT, text="CHILD", formatting=Formatting(bold=True), parent=inline) + doc.add_text(label=DocItemLabel.TEXT, text="CHILD ", formatting=Formatting(bold=True), parent=inline) doc.add_text(label=DocItemLabel.FORMULA, text="FORMULA", parent=inline) DoclingDocument.model_validate(doc.model_dump()) - assert owner.caption_text(doc) == " ".join(text for text in [host_text, "CHILD", "FORMULA"] if text) + assert owner.caption_text(doc) == " ".join(text for text in [host_text, "CHILD FORMULA"] if text) labels = {DocItemLabel.CAPTION, DocItemLabel.TEXT, DocItemLabel.TABLE, DocItemLabel.PICTURE} for serializer in [ MarkdownDocSerializer(doc=doc), @@ -2717,12 +2726,15 @@ def test_native_caption_children_and_filters(kind: str, host_text: str): if isinstance(serializer, HTMLDocSerializer): assert "CHILD" in result if host_text: - assert "HOST &" in result + # Host text and its inline group are joined with a space; the runs inside are not. + assert "HOST & CHILD" in result assert "&amp;" not in result elif isinstance(serializer, LaTeXDocSerializer): assert r"\textbf{CHILD}" in result elif isinstance(serializer, MarkdownDocSerializer) and not isinstance(serializer, PlainTextDocSerializer): assert "**CHILD**" in result + if host_text: + assert "HOST & **CHILD**" in result def test_rich_caption_html_direction(): @@ -2759,7 +2771,8 @@ def test_roundtrip_furniture_inline_group(): ("$q < 0$ and p <0.05 and $a<\\ln b$", ["$q < 0$ and p <0.05 and $a<\\ln b$"]), ("x]]> y", ["x]]> y"]), ("a\x18b", ["ab"]), - (" & d", ["a & b < c", "& d"]), + # The run after the CDATA section owns its leading space under the whitespace contract. + (" & d", ["a & b < c", " & d"]), ], ) def test_repair_unescaped_text(body: str, expected: list[str]): diff --git a/tests/test_hierarchical_chunker.py b/tests/test_hierarchical_chunker.py index f5d811f7d..9824dde2c 100644 --- a/tests/test_hierarchical_chunker.py +++ b/tests/test_hierarchical_chunker.py @@ -37,8 +37,8 @@ def test_chunk(): ("caption", "expected"), [ ("Table 1: results", "Table 1: results"), - ("Table 1: results", "Table **1** : results"), - ("Table x: results", "Table $x$ : results"), + ("Table 1: results", "Table **1**: results"), + ("Table x: results", "Table $x$: results"), ], ) def test_chunk_caption_stays_with_owner(kind: str, caption: str, expected: str, tmp_path: Path) -> None: @@ -74,7 +74,7 @@ def test_chunk_satellite_check_keeps_unrelated_inline_text() -> None: "After" ) chunks = [DocChunk.model_validate(chunk) for chunk in HierarchicalChunker().chunk(doc)] - assert [chunk.text for chunk in chunks] == ["Before **body**", "Table **1** : results\n\nA", "After"] + assert [chunk.text for chunk in chunks] == ["Before **body**", "Table **1**: results\n\nA", "After"] assert doc.tables[0].self_ref in {item.self_ref for item in chunks[1].meta.doc_items} diff --git a/tests/test_inline_whitespace_contract.py b/tests/test_inline_whitespace_contract.py new file mode 100644 index 000000000..cbe5ea0e2 --- /dev/null +++ b/tests/test_inline_whitespace_contract.py @@ -0,0 +1,539 @@ +"""The InlineGroup whitespace contract. + +Runs carry their own significant whitespace; serializers concatenate them faithfully and never +insert a separator of their own. This module is the contract table: one set of inline-run +sequences checked against every text format, plus a DocLang round-trip identity check and the +load-time normalization that keeps pre-contract documents rendering as they used to. +""" + +import copy +import json +import re +import warnings + +import pytest +from pydantic import AnyUrl + +from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer +from docling_core.transforms.serializer.doclang import DocLangDocSerializer, DocLangParams +from docling_core.transforms.serializer.html import HTMLDocSerializer +from docling_core.transforms.serializer.latex import LaTeXDocSerializer +from docling_core.transforms.serializer.markdown import MarkdownDocSerializer +from docling_core.types.doc.document import ( + CURRENT_VERSION, + DocItemLabel, + DoclingDocument, + Formatting, + InlineGroup, +) + +# (name, runs) where a run is (text, formatting, hyperlink) +Run = tuple[str, Formatting | None, str | None] + +_BOLD = Formatting(bold=True) +_ITALIC = Formatting(italic=True) +_SUB = Formatting(script="sub") +_SUPER = Formatting(script="super") + +# Expected output per format. `html` is the inline-group span only, `latex`/`doclang` the body. +CONTRACT_TABLE: list[tuple[str, list[Run], dict[str, str]]] = [ + ( + "ordinal", + [("2", None, None), ("nd", _SUPER, None)], + { + "md": "2nd", + "itxt": "2nd", + "html": "2nd", + "latex": "2$^{nd}$", + "doclang": "2nd", + }, + ), + ( + "formula_unit", + [("O", None, None), ("2", _SUB, None), (".", None, None)], + { + "md": "O2.", + "itxt": "O2.", + "html": "O2.", + "latex": "O$_{2}$.", + "doclang": "O2.", + }, + ), + ( + "water", + [("H", None, None), ("2", _SUB, None), ("O", None, None)], + { + "md": "H2O", + "itxt": "H2O", + "html": "H2O", + "latex": "H$_{2}$O", + "doclang": "H2O", + }, + ), + ( + "spaced_emphasis", + [("Advanced Topics", _BOLD, None), (" in ", None, None), ("Machine Learning", _ITALIC, None)], + { + "md": "**Advanced Topics** in *Machine Learning*", + "itxt": "Advanced Topics in Machine Learning", + "html": "Advanced Topics in Machine Learning", + "latex": "\\textbf{Advanced Topics} in \\textit{Machine Learning}", + "doclang": ( + "Advanced Topics in " + "Machine Learning" + ), + }, + ), + ( + "trailing_period", + [("This is ", None, None), ("italic text", _ITALIC, None), (".", None, None)], + { + "md": "This is *italic text*.", + "itxt": "This is italic text.", + "html": "This is italic text.", + "latex": "This is \\textit{italic text}.", + "doclang": "This is italic text.", + }, + ), + ( + # Whitespace *inside* a formatted run: it must end up outside the emphasis markers, + # otherwise CommonMark renders `**bold **` as literal asterisks. + "whitespace_inside_bold", + [("bold ", _BOLD, None), ("tail", None, None)], + { + "md": "**bold** tail", + "itxt": "bold tail", + "html": "bold tail", + "latex": "\\textbf{bold} tail", + "doclang": "bold tail", + }, + ), + ( + # ... and outside the *complete* decoration stack, hyperlink included. + "whitespace_inside_bold_link", + [("bold ", _BOLD, "https://example.com/x"), ("tail", None, None)], + { + "md": "[**bold**](https://example.com/x) tail", + "itxt": "bold tail", + "html": 'bold tail', + "latex": "\\href{https://example.com/x}{\\textbf{bold}} tail", + # DocLang has no place for a hyperlink on an inline run (pre-existing gap, not + # part of this contract): the href is dropped. The whitespace still hoists. + "doclang": "bold tail", + }, + ), + ( + # A whitespace-only run survives as one space, with no empty markers. + "whitespace_only_run", + [("a", None, None), (" ", _BOLD, None), ("b", None, None)], + { + "md": "a b", + "itxt": "a b", + "html": "a b", + "latex": "a b", + "doclang": "a b", + }, + ), +] + + +def _build(runs: list[Run]) -> DoclingDocument: + doc = DoclingDocument(name="contract") + group = doc.add_inline_group() + for text, formatting, hyperlink in runs: + doc.add_text( + label=DocItemLabel.TEXT, + text=text, + parent=group, + formatting=formatting, + hyperlink=AnyUrl(hyperlink) if hyperlink else None, + ) + return doc + + +def _inline_span(doc: DoclingDocument) -> str: + html = doc.export_to_html(split_page_view=False) + match = re.search(r"(.*?)\s*$", html, re.DOTALL | re.MULTILINE) + if match is None: + match = re.search(r"(.*)", html, re.DOTALL) + assert match is not None, html + return match.group(1) + + +def _latex_body(doc: DoclingDocument) -> str: + text = LaTeXDocSerializer(doc=doc).serialize().text + body = text.split("\\begin{document}", 1)[1].split("\\end{document}", 1)[0] + return body.strip() + + +def _doclang_compact(doc: DoclingDocument) -> str: + """Serialize to DocLang without pretty-printing, so whitespace is unambiguous.""" + params = DocLangParams(include_version=False, pretty_indentation=None) + return DocLangDocSerializer(doc=doc, params=params).serialize().text.strip() + + +@pytest.mark.parametrize(("name", "runs", "expected"), CONTRACT_TABLE, ids=[c[0] for c in CONTRACT_TABLE]) +def test_contract_table(name: str, runs: list[Run], expected: dict[str, str]) -> None: + """Inline runs are concatenated faithfully in every text format.""" + doc = _build(runs) + actual = { + "md": doc.export_to_markdown(), + "itxt": doc.export_to_text(), + "html": _inline_span(doc), + "latex": _latex_body(doc), + "doclang": _doclang_compact(doc), + } + assert actual == expected + + +@pytest.mark.parametrize(("name", "runs", "expected"), CONTRACT_TABLE, ids=[c[0] for c in CONTRACT_TABLE]) +def test_doclang_roundtrip_is_lossless(name: str, runs: list[Run], expected: dict[str, str]) -> None: + """DocLang carries the run whitespace losslessly, via ``. + + Checked both pretty-printed (the default, where bare text nodes pick up indentation) and + compact, since the whitespace channel must survive the pretty printer. + """ + doc = _build(runs) + # DocLang drops hyperlinks on inline runs (pre-existing, see the table), so compare the + # whitespace rather than the full markdown for that one case. + expected_md = expected["md"] if not any(link for _, _, link in runs) else "**bold** tail" + for params in (DocLangParams(), DocLangParams(pretty_indentation=None)): + xml = DocLangDocSerializer(doc=doc, params=params).serialize().text + roundtripped = DocLangDocDeserializer().deserialize_str(xml) + assert roundtripped.export_to_markdown() == expected_md, (name, params.pretty_indentation) + assert roundtripped.export_to_text() == doc.export_to_text(), (name, params.pretty_indentation) + + +def test_no_separator_is_invented_between_runs() -> None: + """The serializers add nothing of their own: joined run text == the plain text export.""" + for name, runs, _ in CONTRACT_TABLE: + doc = _build(runs) + assert doc.export_to_text() == "".join(text for text, _, _ in runs), name + + +def test_host_text_stays_apart_from_its_inline_group() -> None: + """Children of a text item are block structure, so the host text keeps its separator. + + Producers put a section's first paragraph under its heading; joining the two with "" would + read ``ResultsWe use bold words.``. Host boundaries match the pre-contract output; only the + runs inside the group are concatenated. + """ + doc = DoclingDocument(name="host") + heading = doc.add_heading(text="Results", level=1) + group = doc.add_inline_group(parent=heading) + for text, formatting in [("We use ", None), ("bold", _BOLD), (" words.", None)]: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=group, formatting=formatting) + caption = doc.add_text(label=DocItemLabel.CAPTION, text="Table") + picture = doc.add_picture() + picture.captions.append(caption.get_ref()) + caption_group = doc.add_inline_group(parent=caption) + doc.add_text(label=DocItemLabel.TEXT, text="1", parent=caption_group, formatting=_BOLD) + doc.add_text(label=DocItemLabel.TEXT, text=": results", parent=caption_group) + + assert heading._get_plain_text(doc) == "Results We use bold words." + assert doc.export_to_markdown().startswith("## Results\n\nWe use **bold** words.") + html = HTMLDocSerializer(doc=doc).serialize().text + assert "

Results

\nWe use bold words." in html + assert _doclang_compact(doc).startswith( + 'Results We use bold' + " words." + ) + assert picture.caption_text(doc) == "Table 1: results" + assert MarkdownDocSerializer(doc=doc).serialize_captions(item=picture).text == "Table **1**: results" + + +def test_list_item_text_stays_apart_from_its_inline_group() -> None: + """A list item's own text and its inline-group child follow the same host rule in Markdown.""" + doc = DoclingDocument(name="host") + lst = doc.add_list_group() + item = doc.add_list_item(text="Intro", parent=lst) + group = doc.add_inline_group(parent=item) + for text, formatting in [("more ", None), ("x", _BOLD), (" y", None)]: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=group, formatting=formatting) + + assert doc.export_to_markdown() == "- Intro more **x** y" + assert item._get_plain_text(doc) == "Intro more x y" + + +# -------------------------------------------------------------------------------------------- +# Load normalization for pre-contract documents +# -------------------------------------------------------------------------------------------- + +_LEGACY_VERSION = "1.8.0" + + +def _legacy_doc_dict(runs: list[str], *, version: str = _LEGACY_VERSION, label: str = "text") -> dict: + return { + "schema_name": "DoclingDocument", + "version": version, + "name": "legacy", + "body": { + "name": "_root_", + "self_ref": "#/body", + "label": "unspecified", + "children": [{"$ref": "#/groups/0"}], + }, + "furniture": {"name": "_root_", "self_ref": "#/furniture", "label": "unspecified", "children": []}, + "groups": [ + { + "self_ref": "#/groups/0", + "parent": {"$ref": "#/body"}, + "label": "inline", + "name": "group", + "content_layer": "body", + "children": [{"$ref": f"#/texts/{i}"} for i in range(len(runs))], + } + ], + "texts": [ + { + "self_ref": f"#/texts/{i}", + "parent": {"$ref": "#/groups/0"}, + "label": label, + "prov": [], + "orig": text, + "text": text, + "children": [], + "content_layer": "body", + } + for i, text in enumerate(runs) + ], + "pictures": [], + "tables": [], + "key_value_items": [], + "form_items": [], + "pages": {}, + } + + +def _load(data: dict) -> DoclingDocument: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + return DoclingDocument.model_validate(json.loads(json.dumps(data))) + + +@pytest.mark.parametrize( + ("runs", "expected_runs", "expected_md"), + [ + # Space-less DOCX-style runs stay spaced, as the old `" "` join rendered them. + (["Normal", "italic"], ["Normal ", "italic"], "Normal italic"), + # The legacy extra-space bug is retained: this used to render `H 2 O`. + (["H", "2", "O"], ["H ", "2 ", "O"], "H 2 O"), + # An existing boundary keeps its legacy double space -- the old serializer added one too. + (["left ", "right"], ["left ", "right"], "left right"), + ], +) +def test_legacy_documents_keep_their_separators(runs: list[str], expected_runs: list[str], expected_md: str) -> None: + doc = _load(_legacy_doc_dict(runs)) + assert [t.text for t in doc.texts] == expected_runs + assert doc.export_to_markdown() == expected_md + + +def test_current_version_documents_are_not_normalized() -> None: + doc = _load(_legacy_doc_dict(["H", "2", "O"], version=CURRENT_VERSION)) + assert [t.text for t in doc.texts] == ["H", "2", "O"] + assert doc.export_to_markdown() == "H2O" + + +def test_versionless_input_is_not_guessed_at() -> None: + data = _legacy_doc_dict(["H", "2", "O"]) + del data["version"] + assert [t.text for t in _load(data).texts] == ["H", "2", "O"] + + +def test_migration_is_one_way() -> None: + """Loading stamps CURRENT_VERSION, so a re-save cannot pick up a second separator.""" + once = _load(_legacy_doc_dict(["H", "2"])) + assert once.version == CURRENT_VERSION + twice = _load(json.loads(once.model_dump_json())) + assert [t.text for t in twice.texts] == [t.text for t in once.texts] == ["H ", "2"] + + +@pytest.mark.parametrize( + ("labels", "expected_md", "expected_txt"), + [ + # The separator goes on whichever neighbour is not delimiter-wrapped ... + (("text", "code"), "a `b`", "a b"), + (("code", "text"), "`a` b", "a b"), + (("text", "formula"), "a $b$", "a $b$"), + (("formula", "text"), "$a$ b", "$a$ b"), + # ... and when both are, it becomes a plain run of its own, because appending to + # either side would produce `a``b` or $a$$b$. + (("code", "code"), "`a` `b`", "a b"), + (("formula", "formula"), "$a$ $b$", "$a$ $b$"), + (("code", "formula"), "`a` $b$", "a $b$"), + (("formula", "code"), "$a$ `b`", "$a$ b"), + ], +) +def test_legacy_separator_never_lands_inside_a_delimiter( + labels: tuple[str, str], expected_md: str, expected_txt: str +) -> None: + data = _legacy_doc_dict(["a", "b"]) + for item, label in zip(data["texts"], labels): + item["label"] = label + doc = _load(data) + assert doc.export_to_markdown() == expected_md + assert doc.export_to_text() == expected_txt + + +def _load_once(data: dict) -> DoclingDocument: + """Load a legacy dict, checking that the caller's value is untouched and a re-load is stable.""" + pristine = copy.deepcopy(data) + doc = _load(data) + assert data == pristine + assert _load(json.loads(doc.model_dump_json())).export_to_markdown() == doc.export_to_markdown() + return doc + + +def _locate(data: dict, pages: list[int]) -> dict: + data["pages"] = {str(p): {"page_no": p, "size": {"width": 100, "height": 100}} for p in set(pages)} + for item, page in zip(data["texts"], pages): + bbox = {"l": 0, "t": 0, "r": 10, "b": 10, "coord_origin": "BOTTOMLEFT"} + item["prov"] = [{"page_no": page, "bbox": bbox, "charspan": [0, len(item["text"])]}] + return data + + +@pytest.mark.parametrize( + ("label", "both", "first", "second"), + [("code", "`a` `b`", "`a`", "`b`"), ("formula", "$a$ $b$", "$a$", "$b$")], +) +def test_legacy_standalone_separator_survives_page_filters_with_its_runs( + label: str, both: str, first: str, second: str +) -> None: + """Page filters keep only located items; the separator must survive exactly when both runs do.""" + same_page = _load_once(_locate(_legacy_doc_dict(["a", "b"], label=label), [1, 1])) + assert same_page.filter(page_nrs={1}).export_to_markdown() == both + assert same_page.export_to_markdown(page_no=1) == both + + split = _load_once(_locate(_legacy_doc_dict(["a", "b"], label=label), [1, 2])) + assert split.filter(page_nrs={1}).export_to_markdown() == first + assert split.filter(page_nrs={2}).export_to_markdown() == second + assert split.export_to_markdown(page_no=1) == first + + +def _host(doc: DoclingDocument, group: InlineGroup, label: DocItemLabel, runs: list[str]) -> None: + """An empty-text item whose visible content is a child inline group.""" + nested = doc.add_inline_group(parent=doc.add_text(label=label, text="", parent=group)) + for text in runs: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=nested) + + +def _nested(doc: DoclingDocument, group: InlineGroup, runs: list[str]) -> None: + nested = doc.add_inline_group(parent=group) + for text in runs: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=nested) + + +@pytest.mark.parametrize( + ("build", "expected"), + [ + pytest.param( + lambda d, g: (d.add_text(DocItemLabel.TEXT, "left", parent=g), _host(d, g, DocItemLabel.FORMULA, ["x"])), + "left $x$", + id="formula-host-last", + ), + pytest.param( + lambda d, g: ( + d.add_text(DocItemLabel.TEXT, "left", parent=g), + _host(d, g, DocItemLabel.FORMULA, ["x"]), + d.add_text(DocItemLabel.TEXT, "right", parent=g), + ), + "left $x$ right", + id="formula-host", + ), + pytest.param( + lambda d, g: ( + d.add_text(DocItemLabel.FORMULA, "l", parent=g), + _host(d, g, DocItemLabel.FORMULA, ["x"]), + d.add_text(DocItemLabel.CODE, "r", parent=g), + ), + "$l$ $x$ `r`", + id="formula-host-between-delimited", + ), + # The DocLang deserializer writes these two for nested mixed-content elements. + pytest.param( + lambda d, g: ( + d.add_text(DocItemLabel.TEXT, "left", parent=g), + _host(d, g, DocItemLabel.FOOTNOTE, ["a", "b"]), + d.add_text(DocItemLabel.TEXT, "right", parent=g), + ), + "left a b right", + id="footnote-host", + ), + pytest.param( + lambda d, g: ( + d.add_text(DocItemLabel.TEXT, "left", parent=g), + _nested(d, g, ["a", "b"]), + d.add_text(DocItemLabel.TEXT, "right", parent=g), + ), + "left a b right", + id="nested-group", + ), + ], +) +def test_legacy_separator_counts_runs_whose_text_is_in_a_child_group(build, expected: str) -> None: + """An empty host renders its child inline group, so the old join spaced it like any other run.""" + doc = DoclingDocument(name="legacy") + build(doc, doc.add_inline_group()) + data = doc.model_dump(mode="json") | {"version": _LEGACY_VERSION} + assert _load_once(data).export_to_markdown() == expected + + +def test_legacy_migration_does_not_touch_caller_input() -> None: + """The caller owns the dict it passes in, and repeated validation must be stable. + + The migration inserts a standalone separator run for adjacent delimited runs, so an + in-place edit would both corrupt the caller's value and compound on every re-validation. + """ + data = _legacy_doc_dict(["a", "b"]) + data["texts"][0]["label"] = "code" + data["texts"][1]["label"] = "code" + pristine = copy.deepcopy(data) + + rendered = [_load(data).export_to_markdown() for _ in range(3)] + + assert rendered == ["`a` `b`"] * 3 + assert data == pristine + + +def test_legacy_load_warns_once() -> None: + import docling_core.types.doc.document as document_module + + original = document_module._warned_legacy_inline_separators + document_module._warned_legacy_inline_separators = False + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + DoclingDocument.model_validate(_legacy_doc_dict(["a", "b"])) + DoclingDocument.model_validate(_legacy_doc_dict(["c", "d"])) + messages = [str(w.message) for w in caught if w.category is UserWarning] + assert len(messages) == 1, messages + assert _LEGACY_VERSION in messages[0] + finally: + document_module._warned_legacy_inline_separators = original + + +def test_chunk_text_follows_the_contract() -> None: + """The chunker has no inline serializer, so it inherits the Markdown join. + + That makes chunk text -- and therefore embeddings -- change with this contract. Pinned + here because the drift is otherwise invisible: nothing in a chunker golden fails, but a + persisted vector index silently stops matching. + """ + from docling_core.transforms.chunker.hierarchical_chunker import HierarchicalChunker + + doc = DoclingDocument(name="chunk") + doc.add_heading(text="Chemistry") + group = doc.add_inline_group() + for text, formatting in [ + ("Water is ", None), + ("H", None), + ("2", _SUB), + ("O", None), + (" and the ", None), + ("melting point", _BOLD), + (" is 0 C.", None), + ]: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=group, formatting=formatting) + + chunks = list(HierarchicalChunker().chunk(doc)) + assert [c.text for c in chunks] == ["Water is H2O and the **melting point** is 0 C."] diff --git a/tests/test_latex_serialization.py b/tests/test_latex_serialization.py index ec0d8a793..709dae51a 100644 --- a/tests/test_latex_serialization.py +++ b/tests/test_latex_serialization.py @@ -180,9 +180,10 @@ def test_inline_group_no_duplication(): # Formatted section heading via InlineGroup heading = doc.add_heading(text="", level=1) ig = doc.add_inline_group(parent=heading) + # Runs carry their own boundary whitespace; the serializer does not invent a separator. doc.add_text( label="text", - text="Partially formatted", + text="Partially formatted ", formatting=Formatting(italic=True), parent=ig, ) @@ -199,11 +200,11 @@ def test_inline_group_no_duplication(): # Each piece of content must appear exactly once in the body assert body.count("\\textit{Partially formatted} heading") == 1 - assert body.count("\\textbf{Term} : definition") == 1 + assert body.count("\\textbf{Term}: definition") == 1 # The content must be inside the LaTeX command, not on a standalone line assert "\\section{\\textit{Partially formatted} heading}" in body - assert "\\item \\textbf{Term} : definition" in body + assert "\\item \\textbf{Term}: definition" in body def test_latex_column_spanning_rich_cell_keeps_text_in_every_column(col_span_rich_table_doc): @@ -435,7 +436,7 @@ def test_latex_inline_group_child_meta_is_suppressed(): """Meta on items inside an InlineGroup must not produce comment blocks mid-sentence.""" doc = DoclingDocument(name="test") ig = doc.add_inline_group() - first = doc.add_text(label=DocItemLabel.TEXT, text="Alpha", parent=ig) + first = doc.add_text(label=DocItemLabel.TEXT, text="Alpha ", parent=ig) first.meta = BaseMeta(summary=SummaryMetaField(text="A summary")) doc.add_text(label=DocItemLabel.TEXT, text="Beta", parent=ig) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 17fa1ba67..37aa16ebc 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1764,7 +1764,7 @@ def test_html_formula_with_inline_group_is_not_double_escaped(inline_scope: bool parent = doc.add_inline_group(parent=doc.add_heading(text="", level=1)) formula = doc.add_formula(text="", parent=parent) group = doc.add_inline_group(parent=formula) - doc.add_text(label=DocItemLabel.TEXT, text="a ProvenanceItem: