From 7600953bc08ca8b33ea758c8a8192897515c4d86 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Sun, 26 Jul 2026 11:04:07 +0200 Subject: [PATCH 01/18] fix(doclang): preserve leading text before a lone formatting tag `_get_children_simple_text_block` guarded against a second *text* fragment but the Element branch just overwrote `result`, so `2nd` returned "nd" and the leading `2` was silently discarded. Guard both branches so the element is routed to `_parse_inline_group`, which already handles mixed children. --- docling_core/transforms/deserializer/doclang.py | 2 ++ test/test_deserializer_doclang.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/docling_core/transforms/deserializer/doclang.py b/docling_core/transforms/deserializer/doclang.py index d718d6b8e..bb79e3545 100644 --- a/docling_core/transforms/deserializer/doclang.py +++ b/docling_core/transforms/deserializer/doclang.py @@ -402,6 +402,8 @@ def _get_children_simple_text_block(self, element: Element) -> Optional[str]: }: return None elif tmp := self._get_children_simple_text_block(el): + if result is not None: + return None result = tmp elif isinstance(el, Text) and el.data.strip(): # TODO should still support whitespace-only if result is None: diff --git a/test/test_deserializer_doclang.py b/test/test_deserializer_doclang.py index 6f970173b..f13e5b9cc 100644 --- a/test/test_deserializer_doclang.py +++ b/test/test_deserializer_doclang.py @@ -2286,3 +2286,18 @@ def test_deserialize_accepts_document_within_budgets() -> None: ) assert len(doc.texts) == 1 assert doc.texts[0].text == "hello" + + +def test_leading_text_before_lone_formatting_tag_is_preserved() -> None: + """Regression: the leading run was silently dropped for text + one formatting child.""" + cases = { + "2nd": ["2", "nd"], + "plain b": ["plain", "b"], + "see ibid": ["see", "ibid"], + # guards for the shapes that already worked + "nd place": ["nd", "place"], + "H2O": ["H", "2", "O"], + } + for frag, expected in cases.items(): + doc = DocLangDocDeserializer().deserialize_str(f'{frag}') + assert [t.text.strip() for t in doc.texts] == expected, frag From f0e26b6722e2c1b8c543a8b68f04641772f3ef5a Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Sun, 26 Jul 2026 19:50:55 +0200 Subject: [PATCH 02/18] feat: inline runs carry their own whitespace; serializers concatenate faithfully The Markdown, HTML, LaTeX and DocLang inline serializers joined runs with a hard `" "`, inventing spaces the model never asked for: `2nd` -> `2 nd`, `O2.` -> `O 2 .`, `italic text.` -> `italic text .`. Those artifacts can never be removed while the separator lives in the serializer. Serializers: - join inline parts with `""`; DocLang keeps its delimiter between the element head and the body only, where `` is the whitespace channel - hoist edge whitespace around the *complete* decoration stack (formatting and hyperlink together) for Markdown, HTML and LaTeX. Decorating it verbatim emits `**bold ** tail`, which CommonMark renders as literal asterisks; hoisting inside one formatter would let the hyperlink recapture it. DocLang opts out: it keeps whitespace inside the markup, in ``. DocLang deserializer: - `_get_text` stripped every fragment and joined with `""`, collapsing `Advanced Topics` into `AdvancedTopics`. It now trims the outer block boundary only, and never trims whitespace delivered by ``. - bare text nodes use the standard XML heuristic: a whitespace run containing a newline is pretty-print indentation, one without it is content. This is what hand-written and VLM-emitted DocLang needs, since it will not use ``. - element-head children (``, ``, ``, ``) no longer bleed into body text, where they were also double-modelled in `item.meta`. Data at rest: schema minor bumped to 1.11.0 and a `mode="before"` validator re-injects the legacy separator for documents stamped below it, preserving their previous *visible* output including its extra-space bugs (`H 2 O` stays `H 2 O`). It must run before `check_version_is_compatible`, which rewrites `version` and destroys the evidence. Versionless input is left alone. The validator works on a copy, so the caller's dict is untouched and re-validating the same raw value cannot compound separators. A separator can never land inside a delimiter. Code and formula runs are wrapped in `` ` ``/`$` by the serializers, so the space goes on the plain neighbour; when both neighbours are delimited it becomes a plain run of its own, because `` `a``b` `` and `$a$$b$` are both corrupt. Chunk text changes too, since ChunkingDocSerializer inherits the Markdown join -- anyone with a persisted vector index gets silent retrieval drift, so that needs its own line in the release notes. DocTags code is untouched. Its golden gains one empty `` for the legacy fixture because that document now carries an extra whitespace run, which DocTags renders empty by its own pre-existing strip-per-run behaviour. Ships as a minor, not a major: `feat!:` would make python-semantic-release (7.34.6, both repos) bump to 3.0, and a 3.0 is not affordable on either package. --- .../transforms/deserializer/doclang.py | 106 +++-- docling_core/transforms/serializer/common.py | 25 +- docling_core/transforms/serializer/doclang.py | 24 +- docling_core/transforms/serializer/html.py | 8 +- docling_core/transforms/serializer/latex.py | 11 +- .../transforms/serializer/markdown.py | 8 +- docling_core/types/doc/common/constants.py | 8 +- docling_core/types/doc/document.py | 133 +++++- docs/DoclingDocument.json | 2 +- test/conftest.py | 30 +- test/data/doc/2206.01062.yaml.dt.json | 2 +- .../doc/2408.09869v3_enriched.out.dt.json | 2 +- .../2408.09869v3_enriched_p2_p3_p5.gt.json | 6 +- test/data/doc/barchart.dt.out.json | 2 +- test/data/doc/cdata_always.gt.dclg.xml | 38 +- test/data/doc/cdata_when_needed.gt.dclg.xml | 38 +- test/data/doc/concatenated.json | 54 +-- .../data/doc/constr_doc_reserialized.dclg.xml | 38 +- ...onstructed_doc.added_extracted_doc.json.gt | 54 +-- .../constructed_doc.appended_child.json.gt | 54 +-- ...constructed_doc.bulk_item_addition.json.gt | 54 +-- ...onstructed_doc.bulk_item_insertion.json.gt | 54 +-- .../doc/constructed_doc.deleted_group.json.gt | 54 +-- ...onstructed_doc.deleted_items_range.json.gt | 54 +-- .../constructed_doc.deleted_picture.json.gt | 54 +-- .../doc/constructed_doc.deleted_text.json.gt | 54 +-- .../data/doc/constructed_doc.embedded.json.gt | 54 +-- .../data/doc/constructed_doc.embedded.yaml.gt | 54 +-- ...ructed_doc.extracted_with_deletion.json.gt | 54 +-- ...tructed_doc.inserted_extracted_doc.json.gt | 54 +-- .../constructed_doc.inserted_items.json.gt | 54 +-- ...onstructed_doc.inserted_list_items.json.gt | 54 +-- .../doc/constructed_doc.inserted_text.json.gt | 54 +-- .../constructed_doc.manipulated_table.json.gt | 54 +-- .../doc/constructed_doc.referenced.json.gt | 54 +-- .../doc/constructed_doc.referenced.yaml.gt | 54 +-- .../doc/constructed_doc.replaced_item.json.gt | 54 +-- test/data/doc/constructed_document.yaml.et | 26 +- test/data/doc/content_all.gt.dclg.xml | 38 +- .../doc/content_block_specific.gt.dclg.xml | 38 +- .../doc/cross_column_list/deserialized.json | 2 +- test/data/doc/cross_column_list/input.json | 2 +- .../doc/cross_page_list/deserialized.json | 2 +- test/data/doc/cross_page_list/input.json | 2 +- .../cross_page_paragraph/deserialized.json | 2 +- test/data/doc/cross_page_paragraph/input.json | 2 +- .../doc/cross_page_table/deserialized.json | 2 +- test/data/doc/cross_page_table/input.json | 2 +- test/data/doc/default_mode.gt.dclg.xml | 2 +- test/data/doc/doc_with_kv.dt.json | 2 +- test/data/doc/docitem_comments_field.out.yaml | 2 +- .../doc/docitem_comments_multiple.out.yaml | 2 +- .../doclang_archive/load/two_pages.gt.json | 2 +- .../doc/doclang_archive/save/two_pages.json | 2 +- .../doclang_ref/002a_table_order/output.json | 2 +- test/data/doc/dummy_doc_2_prec.yaml | 2 +- .../doc/dummy_doc_with_meta_modified.yaml | 2 +- .../doc/field_region_kv/deserialized.json | 18 +- test/data/doc/field_region_kv/input.json | 10 +- .../doc/field_region_kv/reserialized.dclg.xml | 10 +- .../doc/field_region_kv/serialized.dclg.xml | 10 +- .../field_region_kv_invoice/deserialized.json | 2 +- .../doc/field_region_kv_invoice/input.json | 2 +- .../deserialized.json | 2 +- .../doc/field_region_kv_migration/input.json | 2 +- test/data/doc/flattened.dclg.xml | 4 +- test/data/doc/flattened.json | 2 +- test/data/doc/group_with_metadata.yaml | 2 +- test/data/doc/hierarchized.dclg.xml | 4 +- test/data/doc/hierarchized.json | 2 +- test/data/doc/inline_and_formatting.gt.dt | 1 + test/data/doc/inline_group.gt.dclg.xml | 4 +- test/data/doc/kv.out.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- .../deserialized.json | 2 +- .../output.json | 2 +- test/data/doc/kv_advanced_inline.out.dclg.xml | 2 +- test/data/doc/kv_advanced_inline.out.json | 2 +- test/data/doc/kv_form_with_table.out.json | 2 +- test/data/doc/kv_invoice.out.json | 2 +- test/data/doc/kv_nested.out.json | 2 +- test/data/doc/kv_post_migration.out.json | 2 +- test/data/doc/kv_pre_migration.out.json | 2 +- .../doc/misplaced_list_items.norm.out.yaml | 2 +- test/data/doc/misplaced_list_items.out.yaml | 2 +- .../multi_page_roundtrip/deserialized.json | 2 +- test/data/doc/multi_page_roundtrip/input.json | 2 +- .../doc/multi_prov_thread/deserialized.json | 2 +- test/data/doc/multi_prov_thread/input.json | 2 +- test/data/doc/page_with_pic.dt.json | 2 +- .../data/doc/page_with_pic_from_files.dt.json | 2 +- test/data/doc/page_without_pic.dt.json | 2 +- .../picture_molecule_meta/deserialized.json | 2 +- .../data/doc/picture_molecule_meta/input.json | 2 +- .../doc/referenced_caption/deserialized.json | 2 +- test/data/doc/referenced_caption/input.json | 2 +- test/data/doc/rich_table.out.yaml | 2 +- .../doc/rich_table_item_ins_norm_1.out.yaml | 2 +- .../doc/rich_table_item_ins_norm_2.out.yaml | 2 +- .../doc/rich_table_item_ins_norm_3.out.yaml | 2 +- .../doc/rich_table_post_text_del.out.yaml | 2 +- ...ip_list_item_with_inline_deserialized.yaml | 2 +- ...ist_item_with_inline_reserialized.dclg.xml | 2 +- ..._list_item_with_inline_serialized.dclg.xml | 2 +- .../doc/virtual_text_index/deserialized.json | 2 +- test/data/doc/virtual_text_index/input.json | 2 +- .../doc/virtual_text_list/deserialized.json | 2 +- test/data/doc/virtual_text_list/input.json | 2 +- .../doc/virtual_text_table/deserialized.json | 2 +- test/data/doc/virtual_text_table/input.json | 2 +- test/data/doc/webvtt_example_02.json | 2 +- test/data/doc/webvtt_example_04.json | 2 +- test/data/doc/webvtt_example_05.json | 2 +- test/test_deserializer_doclang.py | 4 +- test/test_inline_whitespace_contract.py | 390 ++++++++++++++++++ 130 files changed, 1481 insertions(+), 797 deletions(-) create mode 100644 test/test_inline_whitespace_contract.py diff --git a/docling_core/transforms/deserializer/doclang.py b/docling_core/transforms/deserializer/doclang.py index bb79e3545..031587245 100644 --- a/docling_core/transforms/deserializer/doclang.py +++ b/docling_core/transforms/deserializer/doclang.py @@ -3,6 +3,7 @@ Aligned to the DocLang specification version ``_DOCLANG_VERSION``. """ +import re from collections.abc import Callable, Sequence from itertools import groupby from pathlib import Path @@ -104,6 +105,22 @@ def _utf8_byte_length(text: str) -> int: return len(text.encode("utf-8")) +# A whitespace run touching a newline is pretty-print indentation; one without is content. +_PRETTY_INDENT = re.compile(r"\A\s*\n\s*|\s*\n\s*\Z") + + +def _normalize_bare_text(data: str) -> str: + """Drop pretty-print indentation from a bare XML text node, keep significant whitespace. + + Standard XML heuristic: whitespace containing a newline is layout noise, whitespace + without one is meaningful. Serialized DocLang uses ```` for whitespace-bearing + runs, so this only governs hand-written and VLM-emitted input. + """ + if not data.strip(): + return "" if "\n" in data else data + return _PRETTY_INDENT.sub("", data) + + def _enforce_doclang_dom_budgets( root: Element, *, @@ -426,7 +443,8 @@ def _parse_text_like(self, *, doc: DoclingDocument, el: Element, parent: Optiona prov_list = self._extract_provenance(doc=doc, el=el) content_layer = self._extract_layer(el=el) - text, formatting = self._extract_text_with_formatting(el) + # A run inside an inline group owns its edge whitespace; a block host does not. + text, formatting = self._extract_text_with_formatting(el, trim=not isinstance(parent, InlineGroup)) if not text: if ( thread_id @@ -581,16 +599,12 @@ def _extract_code_content_and_language(self, el: Element) -> tuple[str, CodeLang parts.append(node.data) elif isinstance(node, Element): nm_child = node.tagName - if nm_child in { - DocLangToken.LOCATION.value, - DocLangToken.LAYER.value, - DocLangToken.LABEL.value, - }: + if self._is_element_head_tag(node): continue elif nm_child == DocLangToken.BR.value: parts.append("\n") else: - parts.append(self._get_text(node)) + parts.append(self._collect_text(node)) return "".join(parts), lang_label @@ -1419,8 +1433,11 @@ def _parse_inline_group( # Recursively dispatch child elements with the inline group as parent self._dispatch_element(doc=doc, el=node, parent=inline_group) elif isinstance(node, Text): - # Handle direct text content - text_content = node.data.strip() + # Direct text content: a run of the inline group, carrying its own whitespace. + # ponytail: no outer block-edge trim here -- pretty-printed input has newlines at + # the edges and loses them to the indentation heuristic anyway. Add edge trimming + # if hand-written single-line DocLang turns out to need it. + text_content = _normalize_bare_text(node.data) if text_content: doc.add_text( label=DocItemLabel.TEXT, @@ -2090,20 +2107,23 @@ def _parse_otsl_table_content( table_cells=table_cells, ) - def _extract_text_with_formatting(self, el: Element) -> tuple[str, Optional[Formatting]]: + def _extract_text_with_formatting(self, el: Element, *, trim: bool = True) -> tuple[str, Optional[Formatting]]: """Extract text content and formatting from an element. If the element contains a single formatting child (bold, italic, etc.), recursively extract the text and build up the Formatting object. + Args: + el: the element to extract from. + trim: trim the outer boundary. True for a block-level host, False when ``el`` is a + run inside an inline group, whose edge whitespace is significant. + Returns: Tuple of (text_content, formatting_object or None) """ - # Get non-whitespace, non-location child elements + # Body child elements only -- element-head tags carry metadata, not text. child_elements = [ - node - for node in el.childNodes - if isinstance(node, Element) and node.tagName not in {DocLangToken.LOCATION.value} + node for node in el.childNodes if isinstance(node, Element) and not self._is_element_head_tag(node) ] # Check if we have a single child that is a formatting tag @@ -2124,7 +2144,7 @@ def _extract_text_with_formatting(self, el: Element) -> tuple[str, Optional[Form if tag_name in format_tags: # Recursively extract text and formatting from the child - text, child_formatting = self._extract_text_with_formatting(child) + text, child_formatting = self._extract_text_with_formatting(child, trim=trim) # Build up the formatting object if child_formatting is None: @@ -2146,25 +2166,57 @@ def _extract_text_with_formatting(self, el: Element) -> tuple[str, Optional[Form return text, child_formatting - # No formatting found, just extract plain text - return self._get_text(el), None + # No formatting found, just extract plain text. + return (self._trimmed_text(el) if trim else self._collect_text(el)), None def _get_text(self, el: Element) -> str: - out: list[str] = [] + """Extract the text of a block-level element, trimming the block boundary only.""" + return self._trimmed_text(el) + + def _collect_text(self, el: Element) -> str: + """Concatenate descendant text verbatim, per the inline whitespace contract. + + Fragments are joined faithfully -- stripping each one is what collapsed + `Advanced Topics` into `AdvancedTopics`. + """ + return "".join(text for text, _ in self._collect_fragments(el)) + + def _trimmed_text(self, el: Element) -> str: + """Concatenate descendant text, trimming the outer block boundary only. + + Whitespace delivered through ```` is protected: that tag is the format's + explicit "this whitespace is significant" marker, so a block edge never strips it. + Same shape as docling's ``_normalize_odf_text_runs``. + """ + frags = self._collect_fragments(el) + for idx, strip in ((0, str.lstrip), (-1, str.rstrip)): + while frags and not frags[idx][1]: + if trimmed := strip(frags[idx][0]): + frags[idx] = (trimmed, False) + break + frags.pop(idx) + return "".join(text for text, _ in frags) + + def _collect_fragments(self, el: Element) -> list[tuple[str, bool]]: + """Return ``(text, protected)`` fragments; protected text came from ````.""" + out: list[tuple[str, bool]] = [] for node in el.childNodes: if isinstance(node, Text): - # Skip pure indentation/pretty-print whitespace - if node.data.strip(): - out.append(node.data if el.tagName == DocLangToken.CONTENT.value else node.data.strip()) + if el.tagName == DocLangToken.CONTENT.value: + # `` is the lossless whitespace channel: take it verbatim. + out.append((node.data, True)) + else: + out.append((_normalize_bare_text(node.data), False)) elif isinstance(node, Element): - nm = node.tagName - if nm in {DocLangToken.LOCATION.value}: + # Element-head children (, , , , ...) are + # metadata; they are modelled in item.meta and must not bleed into body text. + if self._is_element_head_tag(node): continue - if nm == DocLangToken.BR.value: - out.append("\n") + if node.tagName == DocLangToken.BR.value: + out.append(("\n", False)) else: - out.append(self._get_text(node)) - return "".join(out) + out.extend(self._collect_fragments(node)) + return out # --------- Location helpers --------- def _ensure_page_exists(self, *, doc: DoclingDocument, page_no: int, resolution: int) -> None: diff --git a/docling_core/transforms/serializer/common.py b/docling_core/transforms/serializer/common.py index d78cf8fa7..9579029ed 100644 --- a/docling_core/transforms/serializer/common.py +++ b/docling_core/transforms/serializer/common.py @@ -8,7 +8,7 @@ from collections.abc import Iterable from functools import cached_property from pathlib import Path -from typing import Annotated, Any, Optional, Union +from typing import Annotated, Any, ClassVar, Optional, Union from pydantic import ( AnyUrl, @@ -250,6 +250,11 @@ class DocSerializer(BaseModel, BaseDocSerializer): params: CommonParams = CommonParams() + # Move whitespace at the edges of a run outside the formatting/hyperlink markers. + # Off for formats whose own whitespace channel must keep it inside the markup + # (DocLang wraps it in ``), on where adjacency breaks the markup (Markdown). + hoist_decoration_whitespace: ClassVar[bool] = False + _excluded_refs_cache: dict[str, set[str]] = {} @computed_field # type: ignore[misc] @@ -555,7 +560,23 @@ def post_process( ) -> str: """Apply some text post-processing steps.""" params = self.params.merge_with_patch(patch=kwargs) + + lead = trail = "" res = text + if self.hoist_decoration_whitespace: + # Inline runs carry their own significant whitespace, which may sit *inside* a + # formatted run (common in DOCX/HTML). Decorating it verbatim produces + # `**bold ** tail`, which CommonMark renders as literal asterisks. Hoist the edge + # whitespace around the *complete* decoration stack -- hoisting inside an + # individual formatter would let the hyperlink capture it again. + core = text.strip() + if not core: + # Whitespace-only (or empty) run: no markers, no empty hyperlink. + return text + lead = text[: len(text) - len(text.lstrip())] + trail = text[len(text.rstrip()) :] + res = core + if params.include_formatting and formatting: if formatting.bold: res = self.serialize_bold(text=res, **kwargs) @@ -571,7 +592,7 @@ def post_process( res = self.serialize_superscript(text=res, **kwargs) if params.include_hyperlinks and hyperlink: res = self.serialize_hyperlink(text=res, hyperlink=hyperlink, **kwargs) - return res + return f"{lead}{res}{trail}" @override def serialize_bold(self, text: str, **kwargs: Any) -> str: diff --git a/docling_core/transforms/serializer/doclang.py b/docling_core/transforms/serializer/doclang.py index 21050e078..bf33cf3ce 100644 --- a/docling_core/transforms/serializer/doclang.py +++ b/docling_core/transforms/serializer/doclang.py @@ -1620,17 +1620,23 @@ def serialize( ) parts.append(create_ser_result(text=loc_str)) params.add_location = False - parts.extend( - doc_serializer.get_parts( - item=item, - list_level=list_level, - is_inline_scope=True, - visited=my_visited, - **{**kwargs, **params.model_dump()}, - ) + head_parts = list(parts) + inline_parts = doc_serializer.get_parts( + item=item, + list_level=list_level, + is_inline_scope=True, + visited=my_visited, + **{**kwargs, **params.model_dump()}, ) + parts.extend(inline_parts) delim = _get_delim(params=params) - text_res = delim.join([p.text for p in parts if p.text]) + # Inline runs carry their own significant whitespace; concatenate them faithfully. + # The delimiter separates the element head from the body, it is not a whitespace + # channel — `` is (see `_escape_text`). + segments = [p.text for p in head_parts if p.text] + if inline_text := "".join([p.text for p in inline_parts if p.text]): + segments.append(inline_text) + text_res = delim.join(segments) if text_res: text_res = f"{text_res}{delim}" diff --git a/docling_core/transforms/serializer/html.py b/docling_core/transforms/serializer/html.py index 862ee184a..2b859897e 100644 --- a/docling_core/transforms/serializer/html.py +++ b/docling_core/transforms/serializer/html.py @@ -7,7 +7,7 @@ from html.parser import HTMLParser from io import BytesIO from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, ClassVar, Optional, Union from urllib.parse import quote from xml.etree.ElementTree import SubElement, tostring from xml.sax.saxutils import unescape @@ -891,8 +891,8 @@ def serialize( **kwargs, ) - # Join all parts without separators - inline_html = " ".join([p.text for p in parts if p.text]) + # Inline runs carry their own significant whitespace; concatenate faithfully. + inline_html = "".join([p.text for p in parts if p.text]) # Wrap in span if needed if inline_html: @@ -1079,6 +1079,8 @@ class HTMLDocSerializer(DocSerializer): params: HTMLParams = HTMLParams() + hoist_decoration_whitespace: ClassVar[bool] = True + @override def _item_wraps_meta(self, item: NodeItem) -> bool: """PictureItem meta is wrapped inside the figure element.""" diff --git a/docling_core/transforms/serializer/latex.py b/docling_core/transforms/serializer/latex.py index 2d95ec361..f0ed546a7 100644 --- a/docling_core/transforms/serializer/latex.py +++ b/docling_core/transforms/serializer/latex.py @@ -2,7 +2,7 @@ import re from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, ClassVar, Optional, Union from pydantic import AnyUrl, BaseModel from typing_extensions import override @@ -514,14 +514,15 @@ def serialize( list_level: int = 0, **kwargs: Any, ) -> SerializationResult: - """Serialize inline children joining them with spaces for LaTeX output.""" + """Serialize inline children concatenating them faithfully for LaTeX output.""" parts = doc_serializer.get_parts( item=item, list_level=list_level, is_inline_scope=True, **kwargs, ) - text_res = " ".join([p.text for p in parts if p.text]) + # Inline runs carry their own significant whitespace; concatenate faithfully. + text_res = "".join([p.text for p in parts if p.text]) return create_ser_result(text=text_res, span_source=parts) @@ -566,6 +567,10 @@ class LaTeXDocSerializer(DocSerializer): params: LaTeXParams = LaTeXParams() + # `\textbf{bold }tail` renders the same as `\textbf{bold} tail`, but keeping the space + # outside the group matches the other formats and keeps escaping predictable. + hoist_decoration_whitespace: ClassVar[bool] = True + @override def serialize_bold(self, text: str, **kwargs: Any) -> str: """Return LaTeX for bold text.""" diff --git a/docling_core/transforms/serializer/markdown.py b/docling_core/transforms/serializer/markdown.py index 7077499e2..e055bbceb 100644 --- a/docling_core/transforms/serializer/markdown.py +++ b/docling_core/transforms/serializer/markdown.py @@ -6,7 +6,7 @@ import textwrap from enum import Enum from pathlib import Path -from typing import Annotated, Any, Optional, Union +from typing import Annotated, Any, ClassVar, Optional, Union from pydantic import AnyUrl, BaseModel, Field, PositiveInt from tabulate import _column_type, tabulate @@ -824,7 +824,8 @@ def serialize( visited=my_visited, **kwargs, ) - text_res = " ".join([p.text for p in parts if p.text]) + # Inline runs carry their own significant whitespace; concatenate faithfully. + text_res = "".join([p.text for p in parts if p.text]) return create_ser_result(text=text_res, span_source=parts) @@ -870,6 +871,9 @@ class MarkdownDocSerializer(DocSerializer): params: MarkdownParams = MarkdownParams() + # `**bold ** tail` renders as literal asterisks in CommonMark. + hoist_decoration_whitespace: ClassVar[bool] = True + @override def serialize_bold(self, text: str, **kwargs: Any): """Apply Markdown-specific bold serialization.""" diff --git a/docling_core/types/doc/common/constants.py b/docling_core/types/doc/common/constants.py index 6ca699126..c8ba85ee2 100644 --- a/docling_core/types/doc/common/constants.py +++ b/docling_core/types/doc/common/constants.py @@ -5,7 +5,7 @@ from docling_core.types.doc.labels import DocItemLabel -CURRENT_VERSION: Final = "1.10.0" +CURRENT_VERSION: Final = "1.11.0" """Current DoclingDocument schema version. Bump this (minor) whenever a change to the serialised format would cause an @@ -20,6 +20,7 @@ {"schema": "1.8.0", "library_version": "v2.49.0", "note": "BasePrediction, BaseMeta, meta field"}, {"schema": "1.9.0", "library_version": "v2.57.0", "note": "FineRef, DocItem.comments"}, {"schema": "1.10.0", "library_version": "v2.69.0", "note": "FieldItem types, 7 new DocItemLabel values"}, + {"schema": "1.11.0", "library_version": "v2.86.0", "note": "InlineGroup runs carry their own significant whitespace"}, ] """Ordered history of DoclingDocument schema versions in docling-core v2.x. @@ -47,6 +48,11 @@ every step from ``_CURRENT_MINOR`` down to ``FIRST_SUPPORTED_MINOR + 1``. """ +# First schema version in which InlineGroup runs carry their own significant whitespace and +# serializers concatenate them faithfully. Documents stamped below this are normalized on load +# (see `DoclingDocument._migrate_legacy_inline_separators`). +INLINE_WHITESPACE_CONTRACT_VERSION: Final = "1.11.0" + DEFAULT_EXPORT_LABELS = { DocItemLabel.TITLE, diff --git a/docling_core/types/doc/document.py b/docling_core/types/doc/document.py index ce795f974..837d841fb 100644 --- a/docling_core/types/doc/document.py +++ b/docling_core/types/doc/document.py @@ -3,6 +3,7 @@ import base64 import copy import hashlib +import itertools import json import logging import mimetypes @@ -68,6 +69,7 @@ CURRENT_VERSION, DEFAULT_EXPORT_LABELS, DOCUMENT_TOKENS_EXPORT_LABELS, + INLINE_WHITESPACE_CONTRACT_VERSION, ) from docling_core.types.doc.common.content_layer import DEFAULT_CONTENT_LAYERS, ContentLayer from docling_core.types.doc.common.formatting import Formatting, Script @@ -170,6 +172,34 @@ _logger = logging.getLogger(__name__) +_warned_legacy_inline_separators = False + + +def _predates_inline_whitespace_contract(version: str) -> bool: + """Return True when ``version`` is older than the inline whitespace contract.""" + doc = re.match(VERSION_PATTERN, version) + contract = re.match(VERSION_PATTERN, INLINE_WHITESPACE_CONTRACT_VERSION) + if doc is None or contract is None: + return False + return (int(doc["major"]), int(doc["minor"])) < (int(contract["major"]), int(contract["minor"])) + + +def _warn_legacy_inline_separators(version: str) -> None: + """Warn once per process that legacy inline separators were injected on load.""" + global _warned_legacy_inline_separators + if _warned_legacy_inline_separators: + return + _warned_legacy_inline_separators = True + warnings.warn( + f"Document uses schema version {version}, predating the inline whitespace contract " + f"({INLINE_WHITESPACE_CONTRACT_VERSION}). Legacy inline separators were injected to " + "preserve the previous rendering; re-convert the source for faithful whitespace. " + "Note that inline text differs from a fresh conversion, which affects chunking and " + "any persisted embeddings.", + UserWarning, + stacklevel=2, + ) + class DoclingDocument(BaseModel): """DoclingDocument.""" @@ -1376,7 +1406,15 @@ def add_inline_group( parent: Optional[NodeItem] = None, content_layer: Optional[ContentLayer] = None, ) -> InlineGroup: - """add_inline_group.""" + """Add an inline group, whose children are runs of one visual line of text. + + Whitespace contract: each run carries its own significant whitespace and serializers + concatenate the runs faithfully, with no separator of their own. A producer that omits + the boundary whitespace gets merged words (``["H", "2", "O"]`` renders as ``H2O``), so + put the boundary in the run text: ``["Advanced Topics", " in ", "Machine Learning"]``. + Whitespace at the edge of a formatted run is fine -- the serializers move it outside the + emphasis markers and hyperlink. + """ _parent = parent or self.body cref = f"#/groups/{len(self.groups)}" group = InlineGroup(self_ref=cref, parent=_parent.get_ref()) @@ -5190,6 +5228,99 @@ def get_visualization( images = visualizer_obj.get_visualization(doc=self) return images + @model_validator(mode="before") + @classmethod + def _migrate_legacy_inline_separators(cls, data: Any) -> Any: + """Reproduce the legacy inline separator for documents written before the contract. + + Serializers used to join inline runs with a hard ``" "``, so producers of that era did + not put boundary whitespace in the run text. Now that the join is faithful, those saved + documents would render ``Normalitalicbold``. Re-inject one separator at each boundary + the old serializer inserted, preserving the previous *visible* output -- including its + extra-space bugs (``["H", "2", "O"]`` stays ``H 2 O``). + + This is compatibility normalization, not semantic repair: the old document does not + contain enough information to recover the real whitespace. It must run in ``mode="before"`` + because ``check_version_is_compatible`` rewrites ``version`` to ``CURRENT_VERSION`` and + destroys the evidence. Versionless input is left alone rather than guessed at. + """ + if not isinstance(data, dict): + return data + raw_version = data.get("version") + if not isinstance(raw_version, str) or not _predates_inline_whitespace_contract(raw_version): + return data + + # Work on a copy: the caller owns the dict it passed in, and mutating it in place would + # make a second `model_validate` of the same value migrate again, compounding separators. + data = copy.deepcopy(data) + texts = data.get("texts") + groups = data.get("groups") + if not isinstance(texts, list) or not isinstance(groups, list): + return data + + def resolve_run(child: Any) -> Optional[dict]: + """Return the text item dict for an inline run reference, else None.""" + if not isinstance(child, dict): + return None + ref = child.get("$ref") or child.get("cref") + if not isinstance(ref, str) or not ref.startswith("#/texts/"): + return None + try: + item = texts[int(ref.removeprefix("#/texts/"))] + except (ValueError, IndexError): + return None + return item if isinstance(item, dict) and isinstance(item.get("text"), str) else None + + # Code and formula runs are wrapped in delimiters (`` ` ``, ``$``) by the serializers, so a + # separator appended to their text would land *inside* the delimiter -- the same defect as + # `**bold **`. Put it on the neighbouring plain run instead. + delimited = {DocItemLabel.CODE.value, DocItemLabel.FORMULA.value} + + migrated = False + for group in groups: + if not isinstance(group, dict) or group.get("label") != GroupLabel.INLINE.value: + continue + children = group.get("children") + if not isinstance(children, list): + continue + # ponytail: only `#/texts/N` children are treated as runs. A picture or table inside + # an inline group also received a separator, but whether it serialized to non-empty + # text is not knowable from the raw dict. Widen this if such a document shows up. + runs = [(idx, run) for idx, child in enumerate(children) if (run := resolve_run(child)) and run["text"]] + # Boundaries where the separator fits in neither neighbour; filled in afterwards so + # the indices collected here stay valid. + standalone: list[int] = [] + for (_, run), (nxt_idx, nxt) in itertools.pairwise(runs): + if run.get("label") not in delimited: + run["text"] += " " + elif nxt.get("label") not in delimited: + nxt["text"] = " " + nxt["text"] + else: + # Both sides are delimited: the separator would corrupt `` `a``b` `` or + # ``$a$$b$``, so it becomes a plain run of its own between them. + standalone.append(nxt_idx) + migrated = True + + for shift, insert_at in enumerate(standalone): + ref = f"#/texts/{len(texts)}" + texts.append( + { + "self_ref": ref, + "parent": {"$ref": group.get("self_ref")}, + "label": DocItemLabel.TEXT.value, + "prov": [], + "orig": " ", + "text": " ", + "children": [], + "content_layer": group.get("content_layer", ContentLayer.BODY.value), + } + ) + children.insert(insert_at + shift, {"$ref": ref}) + + if migrated: + _warn_legacy_inline_separators(raw_version) + return data + @field_validator("version") @classmethod def check_version_is_compatible(cls, v: str) -> str: diff --git a/docs/DoclingDocument.json b/docs/DoclingDocument.json index 74f4d839d..88833795b 100644 --- a/docs/DoclingDocument.json +++ b/docs/DoclingDocument.json @@ -4494,7 +4494,7 @@ "type": "string" }, "version": { - "default": "1.10.0", + "default": "1.11.0", "pattern": "^(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)(?:-(?P(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", "title": "Version", "type": "string" diff --git a/test/conftest.py b/test/conftest.py index 7adf21dd8..43348c4b1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -208,7 +208,7 @@ def _construct_doc_impl() -> DoclingDocument: inline1 = doc.add_inline_group(parent=g2_subgroup_li_1) doc.add_text( label=DocItemLabel.TEXT, - text="Here a code snippet:", + text="Here a code snippet: ", parent=inline1, ) doc.add_code( @@ -216,17 +216,17 @@ def _construct_doc_impl() -> DoclingDocument: parent=inline1, code_language=CodeLanguageLabel.PYTHON, ) - doc.add_text(label=DocItemLabel.TEXT, text="(to be displayed inline)", parent=inline1) + doc.add_text(label=DocItemLabel.TEXT, text=" (to be displayed inline)", parent=inline1) g2_subgroup_li_2 = doc.add_list_item(text="", parent=g2_subgroup, marker="□") inline2 = doc.add_inline_group(parent=g2_subgroup_li_2) doc.add_text( label=DocItemLabel.TEXT, - text="Here a formula:", + text="Here a formula: ", parent=inline2, ) doc.add_text(label=DocItemLabel.FORMULA, text="E=mc^2", parent=inline2) - doc.add_text(label=DocItemLabel.TEXT, text="(to be displayed inline)", parent=inline2) + doc.add_text(label=DocItemLabel.TEXT, text=" (to be displayed inline)", parent=inline2) doc.add_text(label=DocItemLabel.TEXT, text="Here a code block:", parent=None) doc.add_code(text='print("Hello world")', parent=None, code_language=CodeLanguageLabel.PYTHON) @@ -264,52 +264,52 @@ def _construct_doc_impl() -> DoclingDocument: doc.add_form(graph=graph) inline_fmt = doc.add_inline_group() - doc.add_text(label=DocItemLabel.TEXT, text="Some formatting chops:", parent=inline_fmt) + doc.add_text(label=DocItemLabel.TEXT, text="Some formatting chops: ", parent=inline_fmt) doc.add_text( label=DocItemLabel.TEXT, - text="bold", + text="bold ", parent=inline_fmt, formatting=Formatting(bold=True), ) doc.add_text( label=DocItemLabel.TEXT, - text="italic", + text="italic ", parent=inline_fmt, formatting=Formatting(italic=True), ) doc.add_text( label=DocItemLabel.TEXT, - text="underline", + text="underline ", parent=inline_fmt, formatting=Formatting(underline=True), ) doc.add_text( label=DocItemLabel.TEXT, - text="strikethrough", + text="strikethrough ", parent=inline_fmt, formatting=Formatting(strikethrough=True), ) doc.add_text( label=DocItemLabel.TEXT, - text="subscript", - orig="subscript", + text="subscript ", + orig="subscript ", formatting=Formatting(script=Script.SUB), parent=inline_fmt, ) doc.add_text( label=DocItemLabel.TEXT, - text="superscript", - orig="superscript", + text="superscript ", + orig="superscript ", formatting=Formatting(script=Script.SUPER), parent=inline_fmt, ) doc.add_text( label=DocItemLabel.TEXT, - text="hyperlink", + text="hyperlink ", parent=inline_fmt, hyperlink=Path("."), ) - doc.add_text(label=DocItemLabel.TEXT, text="&", parent=inline_fmt) + doc.add_text(label=DocItemLabel.TEXT, text="& ", parent=inline_fmt) doc.add_text( label=DocItemLabel.TEXT, text="everything at the same time.", diff --git a/test/data/doc/2206.01062.yaml.dt.json b/test/data/doc/2206.01062.yaml.dt.json index f41500ac7..89733e008 100644 --- a/test/data/doc/2206.01062.yaml.dt.json +++ b/test/data/doc/2206.01062.yaml.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/2408.09869v3_enriched.out.dt.json b/test/data/doc/2408.09869v3_enriched.out.dt.json index e973e6861..825d0ac02 100644 --- a/test/data/doc/2408.09869v3_enriched.out.dt.json +++ b/test/data/doc/2408.09869v3_enriched.out.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.json b/test/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.json index b93becd88..149fc5618 100644 --- a/test/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.json +++ b/test/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "2408.09869v3", "furniture": { "self_ref": "#/furniture", @@ -497,7 +497,7 @@ } ], "orig": "To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at ", - "text": "To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at " + "text": "To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at " }, { "self_ref": "#/texts/9", @@ -524,7 +524,7 @@ } ], "orig": "github.com/DS4SD/docling", - "text": "github.com/DS4SD/docling", + "text": "github.com/DS4SD/docling ", "hyperlink": "https://github.com/DS4SD/docling" }, { diff --git a/test/data/doc/barchart.dt.out.json b/test/data/doc/barchart.dt.out.json index 8ed2c9a40..fb5e73889 100644 --- a/test/data/doc/barchart.dt.out.json +++ b/test/data/doc/barchart.dt.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/cdata_always.gt.dclg.xml b/test/data/doc/cdata_always.gt.dclg.xml index 49ab38880..636914fb0 100644 --- a/test/data/doc/cdata_always.gt.dclg.xml +++ b/test/data/doc/cdata_always.gt.dclg.xml @@ -99,17 +99,17 @@ Affiliation 2]]> - + - + - + - + @@ -119,15 +119,27 @@ Affiliation 2]]> - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/test/data/doc/cdata_when_needed.gt.dclg.xml b/test/data/doc/cdata_when_needed.gt.dclg.xml index c8367d918..3c356677d 100644 --- a/test/data/doc/cdata_when_needed.gt.dclg.xml +++ b/test/data/doc/cdata_when_needed.gt.dclg.xml @@ -106,17 +106,17 @@ Affiliation 2 □ - Here a code snippet: + Here a code snippet: -(to be displayed inline) + (to be displayed inline) □ - Here a formula: + Here a formula: E=mc^2 -(to be displayed inline) + (to be displayed inline) Here a code block: @@ -126,15 +126,27 @@ Affiliation 2 Here a formula block: E=mc^2 - Some formatting chops: - bold - italic - underline - strikethrough - subscript - superscript -hyperlink - + Some formatting chops: + + bold + + + italic + + + underline + + + strikethrough + + + subscript + + + superscript + + hyperlink + diff --git a/test/data/doc/concatenated.json b/test/data/doc/concatenated.json index fc2c554ff..b069c899b 100644 --- a/test/data/doc/concatenated.json +++ b/test/data/doc/concatenated.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "2501.17887v1 + Untitled 1 + 2311.18481v1", "furniture": { "self_ref": "#/furniture", @@ -9335,8 +9335,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/313", @@ -9363,8 +9363,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/315", @@ -9393,8 +9393,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/317", @@ -9417,8 +9417,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/319", @@ -9481,8 +9481,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/324", @@ -9493,8 +9493,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -9512,8 +9512,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -9531,8 +9531,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -9550,8 +9550,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -9569,8 +9569,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -9588,8 +9588,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -9607,8 +9607,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -9620,8 +9620,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/332", diff --git a/test/data/doc/constr_doc_reserialized.dclg.xml b/test/data/doc/constr_doc_reserialized.dclg.xml index 109d2f82c..36afbcdef 100644 --- a/test/data/doc/constr_doc_reserialized.dclg.xml +++ b/test/data/doc/constr_doc_reserialized.dclg.xml @@ -105,17 +105,17 @@ Affiliation 2 □ - Here a code snippet: + Here a code snippet: -(to be displayed inline) + (to be displayed inline) □ - Here a formula: + Here a formula: E=mc^2 -(to be displayed inline) + (to be displayed inline) Here a code block: @@ -125,15 +125,27 @@ Affiliation 2 Here a formula block: E=mc^2 - Some formatting chops: - bold - italic - underline - strikethrough - subscript - superscript -hyperlink - + Some formatting chops: + + bold + + + italic + + + underline + + + strikethrough + + + subscript + + + superscript + + hyperlink + diff --git a/test/data/doc/constructed_doc.added_extracted_doc.json.gt b/test/data/doc/constructed_doc.added_extracted_doc.json.gt index 010675d11..64764bfee 100644 --- a/test/data/doc/constructed_doc.added_extracted_doc.json.gt +++ b/test/data/doc/constructed_doc.added_extracted_doc.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -715,8 +715,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -743,8 +743,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -773,8 +773,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -797,8 +797,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -861,8 +861,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -873,8 +873,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -892,8 +892,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -911,8 +911,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -930,8 +930,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -949,8 +949,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -968,8 +968,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -987,8 +987,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1000,8 +1000,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.appended_child.json.gt b/test/data/doc/constructed_doc.appended_child.json.gt index 8c273f2bb..9b961131a 100644 --- a/test/data/doc/constructed_doc.appended_child.json.gt +++ b/test/data/doc/constructed_doc.appended_child.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -544,8 +544,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -572,8 +572,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -602,8 +602,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -626,8 +626,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -690,8 +690,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -702,8 +702,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -721,8 +721,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -740,8 +740,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -759,8 +759,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -778,8 +778,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -797,8 +797,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -816,8 +816,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -829,8 +829,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.bulk_item_addition.json.gt b/test/data/doc/constructed_doc.bulk_item_addition.json.gt index 734b4bfa0..8568fb8be 100644 --- a/test/data/doc/constructed_doc.bulk_item_addition.json.gt +++ b/test/data/doc/constructed_doc.bulk_item_addition.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -708,8 +708,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -736,8 +736,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -766,8 +766,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -790,8 +790,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -854,8 +854,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -866,8 +866,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -885,8 +885,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -904,8 +904,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -923,8 +923,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -942,8 +942,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -961,8 +961,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -980,8 +980,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -993,8 +993,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.bulk_item_insertion.json.gt b/test/data/doc/constructed_doc.bulk_item_insertion.json.gt index d6a3e4eed..a13adb9e6 100644 --- a/test/data/doc/constructed_doc.bulk_item_insertion.json.gt +++ b/test/data/doc/constructed_doc.bulk_item_insertion.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -714,8 +714,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -742,8 +742,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -772,8 +772,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -796,8 +796,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -860,8 +860,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -872,8 +872,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -891,8 +891,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -910,8 +910,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -929,8 +929,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -948,8 +948,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -967,8 +967,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -986,8 +986,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -999,8 +999,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.deleted_group.json.gt b/test/data/doc/constructed_doc.deleted_group.json.gt index e1f185961..38ce8ee8f 100644 --- a/test/data/doc/constructed_doc.deleted_group.json.gt +++ b/test/data/doc/constructed_doc.deleted_group.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -541,8 +541,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -569,8 +569,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -599,8 +599,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -623,8 +623,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -687,8 +687,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -699,8 +699,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -718,8 +718,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -737,8 +737,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -756,8 +756,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -775,8 +775,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -794,8 +794,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -813,8 +813,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -826,8 +826,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.deleted_items_range.json.gt b/test/data/doc/constructed_doc.deleted_items_range.json.gt index 1340b897d..c55abcb4a 100644 --- a/test/data/doc/constructed_doc.deleted_items_range.json.gt +++ b/test/data/doc/constructed_doc.deleted_items_range.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -544,8 +544,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -572,8 +572,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -602,8 +602,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -626,8 +626,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -690,8 +690,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -702,8 +702,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -721,8 +721,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -740,8 +740,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -759,8 +759,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -778,8 +778,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -797,8 +797,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -816,8 +816,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -829,8 +829,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.deleted_picture.json.gt b/test/data/doc/constructed_doc.deleted_picture.json.gt index 3ffb7b326..c23a5c0fb 100644 --- a/test/data/doc/constructed_doc.deleted_picture.json.gt +++ b/test/data/doc/constructed_doc.deleted_picture.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -538,8 +538,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -566,8 +566,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -596,8 +596,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -620,8 +620,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -684,8 +684,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -696,8 +696,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -715,8 +715,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -734,8 +734,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -753,8 +753,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -772,8 +772,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -791,8 +791,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -810,8 +810,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -823,8 +823,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.deleted_text.json.gt b/test/data/doc/constructed_doc.deleted_text.json.gt index 75bdedcd7..e9875c2dc 100644 --- a/test/data/doc/constructed_doc.deleted_text.json.gt +++ b/test/data/doc/constructed_doc.deleted_text.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -755,8 +755,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/24", @@ -783,8 +783,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/26", @@ -813,8 +813,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/28", @@ -837,8 +837,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/30", @@ -901,8 +901,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/35", @@ -913,8 +913,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -932,8 +932,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -951,8 +951,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -970,8 +970,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -989,8 +989,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1008,8 +1008,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1027,8 +1027,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1040,8 +1040,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/43", diff --git a/test/data/doc/constructed_doc.embedded.json.gt b/test/data/doc/constructed_doc.embedded.json.gt index 8d61eed39..d66ee7d74 100644 --- a/test/data/doc/constructed_doc.embedded.json.gt +++ b/test/data/doc/constructed_doc.embedded.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -766,8 +766,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/25", @@ -794,8 +794,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/27", @@ -824,8 +824,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/29", @@ -848,8 +848,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/31", @@ -912,8 +912,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/36", @@ -924,8 +924,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -943,8 +943,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -962,8 +962,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -981,8 +981,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -1000,8 +1000,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1019,8 +1019,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1038,8 +1038,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1051,8 +1051,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/44", diff --git a/test/data/doc/constructed_doc.embedded.yaml.gt b/test/data/doc/constructed_doc.embedded.yaml.gt index 767cdb8c5..76ddd9468 100644 --- a/test/data/doc/constructed_doc.embedded.yaml.gt +++ b/test/data/doc/constructed_doc.embedded.yaml.gt @@ -752,12 +752,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Here a code snippet:' + orig: 'Here a code snippet: ' parent: $ref: '#/groups/10' prov: [] self_ref: '#/texts/24' - text: 'Here a code snippet:' + text: 'Here a code snippet: ' - captions: [] children: [] code_language: Python @@ -774,12 +774,12 @@ texts: - children: [] content_layer: body label: text - orig: (to be displayed inline) + orig: ' (to be displayed inline)' parent: $ref: '#/groups/10' prov: [] self_ref: '#/texts/26' - text: (to be displayed inline) + text: ' (to be displayed inline)' - children: - $ref: '#/groups/11' content_layer: body @@ -795,12 +795,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Here a formula:' + orig: 'Here a formula: ' parent: $ref: '#/groups/11' prov: [] self_ref: '#/texts/28' - text: 'Here a formula:' + text: 'Here a formula: ' - children: [] content_layer: body label: formula @@ -813,12 +813,12 @@ texts: - children: [] content_layer: body label: text - orig: (to be displayed inline) + orig: ' (to be displayed inline)' parent: $ref: '#/groups/11' prov: [] self_ref: '#/texts/30' - text: (to be displayed inline) + text: ' (to be displayed inline)' - children: [] content_layer: body label: text @@ -862,12 +862,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Some formatting chops:' + orig: 'Some formatting chops: ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/35' - text: 'Some formatting chops:' + text: 'Some formatting chops: ' - children: [] content_layer: body formatting: @@ -877,12 +877,12 @@ texts: strikethrough: false underline: false label: text - orig: bold + orig: 'bold ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/36' - text: bold + text: 'bold ' - children: [] content_layer: body formatting: @@ -892,12 +892,12 @@ texts: strikethrough: false underline: false label: text - orig: italic + orig: 'italic ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/37' - text: italic + text: 'italic ' - children: [] content_layer: body formatting: @@ -907,12 +907,12 @@ texts: strikethrough: false underline: true label: text - orig: underline + orig: 'underline ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/38' - text: underline + text: 'underline ' - children: [] content_layer: body formatting: @@ -922,12 +922,12 @@ texts: strikethrough: true underline: false label: text - orig: strikethrough + orig: 'strikethrough ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/39' - text: strikethrough + text: 'strikethrough ' - children: [] content_layer: body formatting: @@ -937,12 +937,12 @@ texts: strikethrough: false underline: false label: text - orig: subscript + orig: 'subscript ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/40' - text: subscript + text: 'subscript ' - children: [] content_layer: body formatting: @@ -952,31 +952,31 @@ texts: strikethrough: false underline: false label: text - orig: superscript + orig: 'superscript ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/41' - text: superscript + text: 'superscript ' - children: [] content_layer: body hyperlink: . label: text - orig: hyperlink + orig: 'hyperlink ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/42' - text: hyperlink + text: 'hyperlink ' - children: [] content_layer: body label: text - orig: '&' + orig: '& ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/43' - text: '&' + text: '& ' - children: [] content_layer: body formatting: @@ -1114,4 +1114,4 @@ texts: prov: [] self_ref: '#/texts/55' text: The end. -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/constructed_doc.extracted_with_deletion.json.gt b/test/data/doc/constructed_doc.extracted_with_deletion.json.gt index 19e83a0f9..c5ee37c00 100644 --- a/test/data/doc/constructed_doc.extracted_with_deletion.json.gt +++ b/test/data/doc/constructed_doc.extracted_with_deletion.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -621,8 +621,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -649,8 +649,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -679,8 +679,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -703,8 +703,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -767,8 +767,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -779,8 +779,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -798,8 +798,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -817,8 +817,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -836,8 +836,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -855,8 +855,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -874,8 +874,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -893,8 +893,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -906,8 +906,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.inserted_extracted_doc.json.gt b/test/data/doc/constructed_doc.inserted_extracted_doc.json.gt index 817ad4c34..ced5e6a70 100644 --- a/test/data/doc/constructed_doc.inserted_extracted_doc.json.gt +++ b/test/data/doc/constructed_doc.inserted_extracted_doc.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -808,8 +808,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -836,8 +836,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -866,8 +866,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -890,8 +890,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -954,8 +954,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -966,8 +966,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -985,8 +985,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -1004,8 +1004,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -1023,8 +1023,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -1042,8 +1042,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1061,8 +1061,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1080,8 +1080,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1093,8 +1093,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.inserted_items.json.gt b/test/data/doc/constructed_doc.inserted_items.json.gt index e7f181684..4306fa896 100644 --- a/test/data/doc/constructed_doc.inserted_items.json.gt +++ b/test/data/doc/constructed_doc.inserted_items.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -661,8 +661,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -689,8 +689,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -719,8 +719,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -743,8 +743,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -807,8 +807,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -819,8 +819,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -838,8 +838,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -857,8 +857,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -876,8 +876,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -895,8 +895,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -914,8 +914,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -933,8 +933,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -946,8 +946,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.inserted_list_items.json.gt b/test/data/doc/constructed_doc.inserted_list_items.json.gt index ff7fb5c28..9d8f9d7f2 100644 --- a/test/data/doc/constructed_doc.inserted_list_items.json.gt +++ b/test/data/doc/constructed_doc.inserted_list_items.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -701,8 +701,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -729,8 +729,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -759,8 +759,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -783,8 +783,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -847,8 +847,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -859,8 +859,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -878,8 +878,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -897,8 +897,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -916,8 +916,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -935,8 +935,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -954,8 +954,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -973,8 +973,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -986,8 +986,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.inserted_text.json.gt b/test/data/doc/constructed_doc.inserted_text.json.gt index 2e5ffb471..57925da32 100644 --- a/test/data/doc/constructed_doc.inserted_text.json.gt +++ b/test/data/doc/constructed_doc.inserted_text.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -772,8 +772,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/25", @@ -800,8 +800,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/27", @@ -830,8 +830,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/29", @@ -854,8 +854,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/31", @@ -918,8 +918,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/36", @@ -930,8 +930,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -949,8 +949,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -968,8 +968,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -987,8 +987,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -1006,8 +1006,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1025,8 +1025,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1044,8 +1044,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1057,8 +1057,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/44", diff --git a/test/data/doc/constructed_doc.manipulated_table.json.gt b/test/data/doc/constructed_doc.manipulated_table.json.gt index 2ac09576f..ce995d417 100644 --- a/test/data/doc/constructed_doc.manipulated_table.json.gt +++ b/test/data/doc/constructed_doc.manipulated_table.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -714,8 +714,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -742,8 +742,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -772,8 +772,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -796,8 +796,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -860,8 +860,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -872,8 +872,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -891,8 +891,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -910,8 +910,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -929,8 +929,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -948,8 +948,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -967,8 +967,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -986,8 +986,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -999,8 +999,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_doc.referenced.json.gt b/test/data/doc/constructed_doc.referenced.json.gt index 4ad56ae05..45441b440 100644 --- a/test/data/doc/constructed_doc.referenced.json.gt +++ b/test/data/doc/constructed_doc.referenced.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -766,8 +766,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/25", @@ -794,8 +794,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/27", @@ -824,8 +824,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/29", @@ -848,8 +848,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/31", @@ -912,8 +912,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/36", @@ -924,8 +924,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -943,8 +943,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -962,8 +962,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -981,8 +981,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -1000,8 +1000,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -1019,8 +1019,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -1038,8 +1038,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -1051,8 +1051,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/44", diff --git a/test/data/doc/constructed_doc.referenced.yaml.gt b/test/data/doc/constructed_doc.referenced.yaml.gt index 5dab83356..a16651f6f 100644 --- a/test/data/doc/constructed_doc.referenced.yaml.gt +++ b/test/data/doc/constructed_doc.referenced.yaml.gt @@ -752,12 +752,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Here a code snippet:' + orig: 'Here a code snippet: ' parent: $ref: '#/groups/10' prov: [] self_ref: '#/texts/24' - text: 'Here a code snippet:' + text: 'Here a code snippet: ' - captions: [] children: [] code_language: Python @@ -774,12 +774,12 @@ texts: - children: [] content_layer: body label: text - orig: (to be displayed inline) + orig: ' (to be displayed inline)' parent: $ref: '#/groups/10' prov: [] self_ref: '#/texts/26' - text: (to be displayed inline) + text: ' (to be displayed inline)' - children: - $ref: '#/groups/11' content_layer: body @@ -795,12 +795,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Here a formula:' + orig: 'Here a formula: ' parent: $ref: '#/groups/11' prov: [] self_ref: '#/texts/28' - text: 'Here a formula:' + text: 'Here a formula: ' - children: [] content_layer: body label: formula @@ -813,12 +813,12 @@ texts: - children: [] content_layer: body label: text - orig: (to be displayed inline) + orig: ' (to be displayed inline)' parent: $ref: '#/groups/11' prov: [] self_ref: '#/texts/30' - text: (to be displayed inline) + text: ' (to be displayed inline)' - children: [] content_layer: body label: text @@ -862,12 +862,12 @@ texts: - children: [] content_layer: body label: text - orig: 'Some formatting chops:' + orig: 'Some formatting chops: ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/35' - text: 'Some formatting chops:' + text: 'Some formatting chops: ' - children: [] content_layer: body formatting: @@ -877,12 +877,12 @@ texts: strikethrough: false underline: false label: text - orig: bold + orig: 'bold ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/36' - text: bold + text: 'bold ' - children: [] content_layer: body formatting: @@ -892,12 +892,12 @@ texts: strikethrough: false underline: false label: text - orig: italic + orig: 'italic ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/37' - text: italic + text: 'italic ' - children: [] content_layer: body formatting: @@ -907,12 +907,12 @@ texts: strikethrough: false underline: true label: text - orig: underline + orig: 'underline ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/38' - text: underline + text: 'underline ' - children: [] content_layer: body formatting: @@ -922,12 +922,12 @@ texts: strikethrough: true underline: false label: text - orig: strikethrough + orig: 'strikethrough ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/39' - text: strikethrough + text: 'strikethrough ' - children: [] content_layer: body formatting: @@ -937,12 +937,12 @@ texts: strikethrough: false underline: false label: text - orig: subscript + orig: 'subscript ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/40' - text: subscript + text: 'subscript ' - children: [] content_layer: body formatting: @@ -952,31 +952,31 @@ texts: strikethrough: false underline: false label: text - orig: superscript + orig: 'superscript ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/41' - text: superscript + text: 'superscript ' - children: [] content_layer: body hyperlink: . label: text - orig: hyperlink + orig: 'hyperlink ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/42' - text: hyperlink + text: 'hyperlink ' - children: [] content_layer: body label: text - orig: '&' + orig: '& ' parent: $ref: '#/groups/12' prov: [] self_ref: '#/texts/43' - text: '&' + text: '& ' - children: [] content_layer: body formatting: @@ -1114,4 +1114,4 @@ texts: prov: [] self_ref: '#/texts/55' text: The end. -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/constructed_doc.replaced_item.json.gt b/test/data/doc/constructed_doc.replaced_item.json.gt index 1340b897d..c55abcb4a 100644 --- a/test/data/doc/constructed_doc.replaced_item.json.gt +++ b/test/data/doc/constructed_doc.replaced_item.json.gt @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Untitled 1", "furniture": { "self_ref": "#/furniture", @@ -544,8 +544,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a code snippet:", - "text": "Here a code snippet:" + "orig": "Here a code snippet: ", + "text": "Here a code snippet: " }, { "self_ref": "#/texts/15", @@ -572,8 +572,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/17", @@ -602,8 +602,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Here a formula:", - "text": "Here a formula:" + "orig": "Here a formula: ", + "text": "Here a formula: " }, { "self_ref": "#/texts/19", @@ -626,8 +626,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "(to be displayed inline)", - "text": "(to be displayed inline)" + "orig": " (to be displayed inline)", + "text": " (to be displayed inline)" }, { "self_ref": "#/texts/21", @@ -690,8 +690,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Some formatting chops:", - "text": "Some formatting chops:" + "orig": "Some formatting chops: ", + "text": "Some formatting chops: " }, { "self_ref": "#/texts/26", @@ -702,8 +702,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "bold", - "text": "bold", + "orig": "bold ", + "text": "bold ", "formatting": { "bold": true, "italic": false, @@ -721,8 +721,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "italic", - "text": "italic", + "orig": "italic ", + "text": "italic ", "formatting": { "bold": false, "italic": true, @@ -740,8 +740,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "underline", - "text": "underline", + "orig": "underline ", + "text": "underline ", "formatting": { "bold": false, "italic": false, @@ -759,8 +759,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "strikethrough", - "text": "strikethrough", + "orig": "strikethrough ", + "text": "strikethrough ", "formatting": { "bold": false, "italic": false, @@ -778,8 +778,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "subscript", - "text": "subscript", + "orig": "subscript ", + "text": "subscript ", "formatting": { "bold": false, "italic": false, @@ -797,8 +797,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "superscript", - "text": "superscript", + "orig": "superscript ", + "text": "superscript ", "formatting": { "bold": false, "italic": false, @@ -816,8 +816,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "hyperlink", - "text": "hyperlink", + "orig": "hyperlink ", + "text": "hyperlink ", "hyperlink": "." }, { @@ -829,8 +829,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "&", - "text": "&" + "orig": "& ", + "text": "& " }, { "self_ref": "#/texts/34", diff --git a/test/data/doc/constructed_document.yaml.et b/test/data/doc/constructed_document.yaml.et index 8bc0b024e..2b6b64c49 100644 --- a/test/data/doc/constructed_document.yaml.et +++ b/test/data/doc/constructed_document.yaml.et @@ -39,14 +39,14 @@ Affiliation 2 36: list_item: item 1 of sub list 37: list_item: 38: inline with name=group - 39: text: Here a code snippet: + 39: text: Here a code snippet: 40: code: print("Hello world") - 41: text: (to be displayed inline) + 41: text: (to be displayed inline) 42: list_item: 43: inline with name=group - 44: text: Here a formula: + 44: text: Here a formula: 45: formula: E=mc^2 - 46: text: (to be displayed inline) + 46: text: (to be displayed inline) 47: text: Here a code block: 48: code: print("Hello world") 49: text: Here a formula block: @@ -54,15 +54,15 @@ Affiliation 2 51: key_value_region 52: form 53: inline with name=group - 54: text: Some formatting chops: - 55: text: bold - 56: text: italic - 57: text: underline - 58: text: strikethrough - 59: text: subscript - 60: text: superscript - 61: text: hyperlink - 62: text: & + 54: text: Some formatting chops: + 55: text: bold + 56: text: italic + 57: text: underline + 58: text: strikethrough + 59: text: subscript + 60: text: superscript + 61: text: hyperlink + 62: text: & 63: text: everything at the same time. 64: list with name=list A 65: list_item: Item 1 in A diff --git a/test/data/doc/content_all.gt.dclg.xml b/test/data/doc/content_all.gt.dclg.xml index ed0520590..7c83f14bb 100644 --- a/test/data/doc/content_all.gt.dclg.xml +++ b/test/data/doc/content_all.gt.dclg.xml @@ -106,17 +106,17 @@ Affiliation 2 □ - Here a code snippet: + Here a code snippet: -(to be displayed inline) + (to be displayed inline) □ - Here a formula: + Here a formula: E=mc^2 -(to be displayed inline) + (to be displayed inline) Here a code block: @@ -126,15 +126,27 @@ Affiliation 2 Here a formula block: E=mc^2 - Some formatting chops: - bold - italic - underline - strikethrough - subscript - superscript -hyperlink - + Some formatting chops: + + bold + + + italic + + + underline + + + strikethrough + + + subscript + + + superscript + + hyperlink + diff --git a/test/data/doc/content_block_specific.gt.dclg.xml b/test/data/doc/content_block_specific.gt.dclg.xml index acfce03e8..5c61a2ae9 100644 --- a/test/data/doc/content_block_specific.gt.dclg.xml +++ b/test/data/doc/content_block_specific.gt.dclg.xml @@ -87,17 +87,17 @@ Affiliation 2 □ - Here a code snippet: + Here a code snippet: -(to be displayed inline) + (to be displayed inline) □ - Here a formula: + Here a formula: E=mc^2 -(to be displayed inline) + (to be displayed inline) Here a code block: @@ -107,15 +107,27 @@ Affiliation 2 Here a formula block: E=mc^2 - Some formatting chops: - bold - italic - underline - strikethrough - subscript - superscript -hyperlink - + Some formatting chops: + + bold + + + italic + + + underline + + + strikethrough + + + subscript + + + superscript + + hyperlink + diff --git a/test/data/doc/cross_column_list/deserialized.json b/test/data/doc/cross_column_list/deserialized.json index 8baa629f8..9638da6ff 100644 --- a/test/data/doc/cross_column_list/deserialized.json +++ b/test/data/doc/cross_column_list/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/cross_column_list/input.json b/test/data/doc/cross_column_list/input.json index 74acc8371..565c1288a 100644 --- a/test/data/doc/cross_column_list/input.json +++ b/test/data/doc/cross_column_list/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "cross_column_list", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/cross_page_list/deserialized.json b/test/data/doc/cross_page_list/deserialized.json index b9befd7c9..0defaf972 100644 --- a/test/data/doc/cross_page_list/deserialized.json +++ b/test/data/doc/cross_page_list/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/cross_page_list/input.json b/test/data/doc/cross_page_list/input.json index 1252c18d8..390ce97af 100644 --- a/test/data/doc/cross_page_list/input.json +++ b/test/data/doc/cross_page_list/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "cross_page_list", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/cross_page_paragraph/deserialized.json b/test/data/doc/cross_page_paragraph/deserialized.json index 0ba545230..9a5bf544b 100644 --- a/test/data/doc/cross_page_paragraph/deserialized.json +++ b/test/data/doc/cross_page_paragraph/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/cross_page_paragraph/input.json b/test/data/doc/cross_page_paragraph/input.json index 6e4a478c2..50acf39e7 100644 --- a/test/data/doc/cross_page_paragraph/input.json +++ b/test/data/doc/cross_page_paragraph/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "cross_page_paragraph", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/cross_page_table/deserialized.json b/test/data/doc/cross_page_table/deserialized.json index 69a4d7ac4..9921ef477 100644 --- a/test/data/doc/cross_page_table/deserialized.json +++ b/test/data/doc/cross_page_table/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/cross_page_table/input.json b/test/data/doc/cross_page_table/input.json index 47c8c57a2..5dd2a2497 100644 --- a/test/data/doc/cross_page_table/input.json +++ b/test/data/doc/cross_page_table/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "cross_page_table", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/default_mode.gt.dclg.xml b/test/data/doc/default_mode.gt.dclg.xml index ba9b52828..6de298fc7 100644 --- a/test/data/doc/default_mode.gt.dclg.xml +++ b/test/data/doc/default_mode.gt.dclg.xml @@ -18,7 +18,7 @@ E=mc^2 -in line + in line Here a bold diff --git a/test/data/doc/doc_with_kv.dt.json b/test/data/doc/doc_with_kv.dt.json index 64ea0ea2c..068d61a5d 100644 --- a/test/data/doc/doc_with_kv.dt.json +++ b/test/data/doc/doc_with_kv.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/docitem_comments_field.out.yaml b/test/data/doc/docitem_comments_field.out.yaml index 2c7e88c4d..cf418a3b6 100644 --- a/test/data/doc/docitem_comments_field.out.yaml +++ b/test/data/doc/docitem_comments_field.out.yaml @@ -51,4 +51,4 @@ texts: prov: [] self_ref: '#/texts/2' text: '[John Reviewer]: This is a reviewer comment.' -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/docitem_comments_multiple.out.yaml b/test/data/doc/docitem_comments_multiple.out.yaml index 24c6f1d1e..1458032a1 100644 --- a/test/data/doc/docitem_comments_multiple.out.yaml +++ b/test/data/doc/docitem_comments_multiple.out.yaml @@ -79,4 +79,4 @@ texts: prov: [] self_ref: '#/texts/4' text: '[Reviewer B]: This is a comment on texts 2 (range [0,6)) and 3.' -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/doclang_archive/load/two_pages.gt.json b/test/data/doc/doclang_archive/load/two_pages.gt.json index d6b3ffc02..ad3a479b1 100644 --- a/test/data/doc/doclang_archive/load/two_pages.gt.json +++ b/test/data/doc/doclang_archive/load/two_pages.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "two_pages", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/doclang_archive/save/two_pages.json b/test/data/doc/doclang_archive/save/two_pages.json index 5bc6551e8..7efc7a00c 100644 --- a/test/data/doc/doclang_archive/save/two_pages.json +++ b/test/data/doc/doclang_archive/save/two_pages.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "2408.09869v3", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/doclang_ref/002a_table_order/output.json b/test/data/doc/doclang_ref/002a_table_order/output.json index d280e0b32..673648029 100644 --- a/test/data/doc/doclang_ref/002a_table_order/output.json +++ b/test/data/doc/doclang_ref/002a_table_order/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "input", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/dummy_doc_2_prec.yaml b/test/data/doc/dummy_doc_2_prec.yaml index 1d2800bd2..b445d4f5f 100644 --- a/test/data/doc/dummy_doc_2_prec.yaml +++ b/test/data/doc/dummy_doc_2_prec.yaml @@ -272,4 +272,4 @@ texts: self_ref: '#/texts/3' text: 'Figure 1: Four examples of complex page layouts across different document categories' -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/dummy_doc_with_meta_modified.yaml b/test/data/doc/dummy_doc_with_meta_modified.yaml index 1a4a0d33c..a24b30376 100644 --- a/test/data/doc/dummy_doc_with_meta_modified.yaml +++ b/test/data/doc/dummy_doc_with_meta_modified.yaml @@ -290,4 +290,4 @@ texts: self_ref: '#/texts/3' text: 'Figure 1: Four examples of complex page layouts across different document categories' -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/field_region_kv/deserialized.json b/test/data/doc/field_region_kv/deserialized.json index b6984b6ed..ff680b22d 100644 --- a/test/data/doc/field_region_kv/deserialized.json +++ b/test/data/doc/field_region_kv/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", @@ -283,8 +283,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Clark ", - "text": "Clark " + "orig": "Clark ", + "text": "Clark " }, { "self_ref": "#/texts/12", @@ -295,8 +295,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "Kent", - "text": "Kent", + "orig": "Kent ", + "text": "Kent ", "formatting": { "bold": true, "italic": false, @@ -339,8 +339,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": "My first input ", - "text": "My first input " + "orig": "My first input ", + "text": "My first input " }, { "self_ref": "#/texts/16", @@ -364,8 +364,8 @@ "content_layer": "body", "label": "text", "prov": [], - "orig": " and my second input ", - "text": " and my second input " + "orig": " and my second input ", + "text": " and my second input " }, { "self_ref": "#/texts/18", diff --git a/test/data/doc/field_region_kv/input.json b/test/data/doc/field_region_kv/input.json index 33cf6bd47..3d1cc2fb1 100644 --- a/test/data/doc/field_region_kv/input.json +++ b/test/data/doc/field_region_kv/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", @@ -284,7 +284,7 @@ "label": "text", "prov": [], "orig": "Clark ", - "text": "Clark " + "text": "Clark " }, { "self_ref": "#/texts/12", @@ -296,7 +296,7 @@ "label": "text", "prov": [], "orig": "Kent", - "text": "Kent", + "text": "Kent ", "formatting": { "bold": true, "italic": false, @@ -340,7 +340,7 @@ "label": "text", "prov": [], "orig": "My first input ", - "text": "My first input " + "text": "My first input " }, { "self_ref": "#/texts/16", @@ -365,7 +365,7 @@ "label": "text", "prov": [], "orig": " and my second input ", - "text": " and my second input " + "text": " and my second input " }, { "self_ref": "#/texts/18", diff --git a/test/data/doc/field_region_kv/reserialized.dclg.xml b/test/data/doc/field_region_kv/reserialized.dclg.xml index 6959ba029..768728dff 100644 --- a/test/data/doc/field_region_kv/reserialized.dclg.xml +++ b/test/data/doc/field_region_kv/reserialized.dclg.xml @@ -20,15 +20,17 @@ Max Mustermann - Clark - Kent + Clark + + Kent + Select this if you are a Superman fan - My first input + My first input - and my second input + and my second input m diff --git a/test/data/doc/field_region_kv/serialized.dclg.xml b/test/data/doc/field_region_kv/serialized.dclg.xml index 6959ba029..768728dff 100644 --- a/test/data/doc/field_region_kv/serialized.dclg.xml +++ b/test/data/doc/field_region_kv/serialized.dclg.xml @@ -20,15 +20,17 @@ Max Mustermann - Clark - Kent + Clark + + Kent + Select this if you are a Superman fan - My first input + My first input - and my second input + and my second input m diff --git a/test/data/doc/field_region_kv_invoice/deserialized.json b/test/data/doc/field_region_kv_invoice/deserialized.json index 55d9381ec..6763fc6c4 100644 --- a/test/data/doc/field_region_kv_invoice/deserialized.json +++ b/test/data/doc/field_region_kv_invoice/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/field_region_kv_invoice/input.json b/test/data/doc/field_region_kv_invoice/input.json index 7fd2a45eb..0e52cb4af 100644 --- a/test/data/doc/field_region_kv_invoice/input.json +++ b/test/data/doc/field_region_kv_invoice/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/field_region_kv_migration/deserialized.json b/test/data/doc/field_region_kv_migration/deserialized.json index 8e8fe92b8..c71e651c9 100644 --- a/test/data/doc/field_region_kv_migration/deserialized.json +++ b/test/data/doc/field_region_kv_migration/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/field_region_kv_migration/input.json b/test/data/doc/field_region_kv_migration/input.json index 2a9ebee65..9e6c13bb4 100644 --- a/test/data/doc/field_region_kv_migration/input.json +++ b/test/data/doc/field_region_kv_migration/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/flattened.dclg.xml b/test/data/doc/flattened.dclg.xml index 440d557e0..7149867c3 100644 --- a/test/data/doc/flattened.dclg.xml +++ b/test/data/doc/flattened.dclg.xml @@ -9,7 +9,7 @@ Rich heading without -other children besides the inline + other children besides the inline Section content as sibling of the heading. Subheading @@ -17,7 +17,7 @@ other children besides the inline Rich heading with -other children besides the inline + other children besides the inline Section content as child of the heading. Section content as sibling of the heading. diff --git a/test/data/doc/flattened.json b/test/data/doc/flattened.json index 7a9161650..bae77d18d 100644 --- a/test/data/doc/flattened.json +++ b/test/data/doc/flattened.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/group_with_metadata.yaml b/test/data/doc/group_with_metadata.yaml index 37b00a434..548eb06d7 100644 --- a/test/data/doc/group_with_metadata.yaml +++ b/test/data/doc/group_with_metadata.yaml @@ -125,4 +125,4 @@ texts: prov: [] self_ref: '#/texts/4' text: Regarding bar... -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/hierarchized.dclg.xml b/test/data/doc/hierarchized.dclg.xml index 440d557e0..7149867c3 100644 --- a/test/data/doc/hierarchized.dclg.xml +++ b/test/data/doc/hierarchized.dclg.xml @@ -9,7 +9,7 @@ Rich heading without -other children besides the inline + other children besides the inline Section content as sibling of the heading. Subheading @@ -17,7 +17,7 @@ other children besides the inline Rich heading with -other children besides the inline + other children besides the inline Section content as child of the heading. Section content as sibling of the heading. diff --git a/test/data/doc/hierarchized.json b/test/data/doc/hierarchized.json index 09dd4bd3a..30f477814 100644 --- a/test/data/doc/hierarchized.json +++ b/test/data/doc/hierarchized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/inline_and_formatting.gt.dt b/test/data/doc/inline_and_formatting.gt.dt index f19624326..755a08e15 100644 --- a/test/data/doc/inline_and_formatting.gt.dt +++ b/test/data/doc/inline_and_formatting.gt.dt @@ -40,6 +40,7 @@ Partially formatted heading to_escape <_unknown_>not_to_escape + E=mc^2 & ampersand diff --git a/test/data/doc/inline_group.gt.dclg.xml b/test/data/doc/inline_group.gt.dclg.xml index 6ce42cdf5..700021335 100644 --- a/test/data/doc/inline_group.gt.dclg.xml +++ b/test/data/doc/inline_group.gt.dclg.xml @@ -6,7 +6,7 @@ One Two -Three + Three @@ -14,6 +14,6 @@ Three Four Five -Six + Six diff --git a/test/data/doc/kv.out.json b/test/data/doc/kv.out.json index 33cf6bd47..8b020e883 100644 --- a/test/data/doc/kv.out.json +++ b/test/data/doc/kv.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/deserialized.json b/test/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/deserialized.json index 422e0beac..9f3986254 100644 --- a/test/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/deserialized.json +++ b/test/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/output.json b/test/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/output.json index 011421cdd..9e8eb6fca 100644 --- a/test/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/output.json +++ b/test/data/doc/kv/01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/deserialized.json b/test/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/deserialized.json index 29cc4845c..887711460 100644 --- a/test/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/deserialized.json +++ b/test/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/output.json b/test/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/output.json index 9555b87fc..ff2f76a07 100644 --- a/test/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/output.json +++ b/test/data/doc/kv/08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/deserialized.json b/test/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/deserialized.json index f0ab7e3be..95a6cc8ff 100644 --- a/test/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/deserialized.json +++ b/test/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/output.json b/test/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/output.json index 2e4f1da4f..515dea899 100644 --- a/test/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/output.json +++ b/test/data/doc/kv/0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "0cb12d33b02867dc8708f4877480743533f1248683091188000d25456ba12d73", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/deserialized.json b/test/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/deserialized.json index 6b2823fe5..5bd3da79f 100644 --- a/test/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/deserialized.json +++ b/test/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/output.json b/test/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/output.json index 914ba3f8c..ddb51c277 100644 --- a/test/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/output.json +++ b/test/data/doc/kv/167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "167f6658cd410df8d4d14acc53e8c8f509e94c44b8005e6b76de8d17329363a7", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/deserialized.json b/test/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/deserialized.json index 03d87bc9a..13197af7c 100644 --- a/test/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/deserialized.json +++ b/test/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/output.json b/test/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/output.json index 5be82748a..d112259a9 100644 --- a/test/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/output.json +++ b/test/data/doc/kv/1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json b/test/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json index e81dd9188..3986898dc 100644 --- a/test/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json +++ b/test/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json b/test/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json index ec07558af..8cb59e98c 100644 --- a/test/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json +++ b/test/data/doc/kv/587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/deserialized.json b/test/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/deserialized.json index 1e585c67b..461babe11 100644 --- a/test/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/deserialized.json +++ b/test/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/output.json b/test/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/output.json index ccf09e939..4295134fc 100644 --- a/test/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/output.json +++ b/test/data/doc/kv/ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/deserialized.json b/test/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/deserialized.json index 83c1aa4f8..456607dab 100644 --- a/test/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/deserialized.json +++ b/test/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/output.json b/test/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/output.json index 345d40552..21d34a758 100644 --- a/test/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/output.json +++ b/test/data/doc/kv/fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "fedf68d45c1acd279f7c34fd5b4cbae80677138bdc1d03b2869466e0cf4e89e9", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json b/test/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json index e81dd9188..3986898dc 100644 --- a/test/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json +++ b/test/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json b/test/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json index ec07558af..8cb59e98c 100644 --- a/test/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json +++ b/test/data/doc/kv/non_seq_cell_ids_587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4/output.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "587558a074858891bdf1d625a9e9fce4ea116ac61bd276d86a069697744b62a4", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv_advanced_inline.out.dclg.xml b/test/data/doc/kv_advanced_inline.out.dclg.xml index 29fb0a9cf..88264412a 100644 --- a/test/data/doc/kv_advanced_inline.out.dclg.xml +++ b/test/data/doc/kv_advanced_inline.out.dclg.xml @@ -13,7 +13,7 @@ -. + . diff --git a/test/data/doc/kv_advanced_inline.out.json b/test/data/doc/kv_advanced_inline.out.json index b25b59f7d..03954c4d1 100644 --- a/test/data/doc/kv_advanced_inline.out.json +++ b/test/data/doc/kv_advanced_inline.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv_form_with_table.out.json b/test/data/doc/kv_form_with_table.out.json index 5d33e97eb..8577433b2 100644 --- a/test/data/doc/kv_form_with_table.out.json +++ b/test/data/doc/kv_form_with_table.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv_invoice.out.json b/test/data/doc/kv_invoice.out.json index 7fd2a45eb..0e52cb4af 100644 --- a/test/data/doc/kv_invoice.out.json +++ b/test/data/doc/kv_invoice.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv_nested.out.json b/test/data/doc/kv_nested.out.json index b7605a45a..ca19c8ad8 100644 --- a/test/data/doc/kv_nested.out.json +++ b/test/data/doc/kv_nested.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv_post_migration.out.json b/test/data/doc/kv_post_migration.out.json index 2a9ebee65..9e6c13bb4 100644 --- a/test/data/doc/kv_post_migration.out.json +++ b/test/data/doc/kv_post_migration.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/kv_pre_migration.out.json b/test/data/doc/kv_pre_migration.out.json index 3dfd68225..4514cd055 100644 --- a/test/data/doc/kv_pre_migration.out.json +++ b/test/data/doc/kv_pre_migration.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/misplaced_list_items.norm.out.yaml b/test/data/doc/misplaced_list_items.norm.out.yaml index 3d0586d3f..f75b7778f 100644 --- a/test/data/doc/misplaced_list_items.norm.out.yaml +++ b/test/data/doc/misplaced_list_items.norm.out.yaml @@ -81,4 +81,4 @@ texts: prov: [] self_ref: '#/texts/3' text: there -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/misplaced_list_items.out.yaml b/test/data/doc/misplaced_list_items.out.yaml index c8efd5617..75f022b3e 100644 --- a/test/data/doc/misplaced_list_items.out.yaml +++ b/test/data/doc/misplaced_list_items.out.yaml @@ -81,4 +81,4 @@ texts: prov: [] self_ref: '#/texts/3' text: foo -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/multi_page_roundtrip/deserialized.json b/test/data/doc/multi_page_roundtrip/deserialized.json index 2e75ef58a..d941ef9cd 100644 --- a/test/data/doc/multi_page_roundtrip/deserialized.json +++ b/test/data/doc/multi_page_roundtrip/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/multi_page_roundtrip/input.json b/test/data/doc/multi_page_roundtrip/input.json index acc8b2f12..c081cf1a9 100644 --- a/test/data/doc/multi_page_roundtrip/input.json +++ b/test/data/doc/multi_page_roundtrip/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "multi_page_roundtrip", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/multi_prov_thread/deserialized.json b/test/data/doc/multi_prov_thread/deserialized.json index 1c38c5f70..ed8646dc4 100644 --- a/test/data/doc/multi_prov_thread/deserialized.json +++ b/test/data/doc/multi_prov_thread/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/multi_prov_thread/input.json b/test/data/doc/multi_prov_thread/input.json index b44bc279e..27c61bdc0 100644 --- a/test/data/doc/multi_prov_thread/input.json +++ b/test/data/doc/multi_prov_thread/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "multi_prov_thread", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/page_with_pic.dt.json b/test/data/doc/page_with_pic.dt.json index c0c13673f..266a42c78 100644 --- a/test/data/doc/page_with_pic.dt.json +++ b/test/data/doc/page_with_pic.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/page_with_pic_from_files.dt.json b/test/data/doc/page_with_pic_from_files.dt.json index c0c13673f..266a42c78 100644 --- a/test/data/doc/page_with_pic_from_files.dt.json +++ b/test/data/doc/page_with_pic_from_files.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/page_without_pic.dt.json b/test/data/doc/page_without_pic.dt.json index b4a0b3c5e..c9a7dca3a 100644 --- a/test/data/doc/page_without_pic.dt.json +++ b/test/data/doc/page_without_pic.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/picture_molecule_meta/deserialized.json b/test/data/doc/picture_molecule_meta/deserialized.json index 0813f708a..e6d36bc08 100644 --- a/test/data/doc/picture_molecule_meta/deserialized.json +++ b/test/data/doc/picture_molecule_meta/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/picture_molecule_meta/input.json b/test/data/doc/picture_molecule_meta/input.json index bb292c990..8d5ec6891 100644 --- a/test/data/doc/picture_molecule_meta/input.json +++ b/test/data/doc/picture_molecule_meta/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "picture_molecule_meta", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/referenced_caption/deserialized.json b/test/data/doc/referenced_caption/deserialized.json index b0ecb2039..f2230a78d 100644 --- a/test/data/doc/referenced_caption/deserialized.json +++ b/test/data/doc/referenced_caption/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/referenced_caption/input.json b/test/data/doc/referenced_caption/input.json index e2017077c..50c9332d6 100644 --- a/test/data/doc/referenced_caption/input.json +++ b/test/data/doc/referenced_caption/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "referenced_caption", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/rich_table.out.yaml b/test/data/doc/rich_table.out.yaml index 992c08a8c..d486bbde6 100644 --- a/test/data/doc/rich_table.out.yaml +++ b/test/data/doc/rich_table.out.yaml @@ -501,4 +501,4 @@ texts: prov: [] self_ref: '#/texts/5' text: More text in the group. -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/rich_table_item_ins_norm_1.out.yaml b/test/data/doc/rich_table_item_ins_norm_1.out.yaml index 008cd1523..05597efa9 100644 --- a/test/data/doc/rich_table_item_ins_norm_1.out.yaml +++ b/test/data/doc/rich_table_item_ins_norm_1.out.yaml @@ -241,4 +241,4 @@ texts: prov: [] self_ref: '#/texts/1' text: text in italic -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/rich_table_item_ins_norm_2.out.yaml b/test/data/doc/rich_table_item_ins_norm_2.out.yaml index 4762a2ba1..0bc332819 100644 --- a/test/data/doc/rich_table_item_ins_norm_2.out.yaml +++ b/test/data/doc/rich_table_item_ins_norm_2.out.yaml @@ -251,4 +251,4 @@ texts: prov: [] self_ref: '#/texts/2' text: text before -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/rich_table_item_ins_norm_3.out.yaml b/test/data/doc/rich_table_item_ins_norm_3.out.yaml index 16c93668e..eaf069aa3 100644 --- a/test/data/doc/rich_table_item_ins_norm_3.out.yaml +++ b/test/data/doc/rich_table_item_ins_norm_3.out.yaml @@ -251,4 +251,4 @@ texts: prov: [] self_ref: '#/texts/2' text: text in italic -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/rich_table_post_text_del.out.yaml b/test/data/doc/rich_table_post_text_del.out.yaml index 06752fc27..30f0bc97c 100644 --- a/test/data/doc/rich_table_post_text_del.out.yaml +++ b/test/data/doc/rich_table_post_text_del.out.yaml @@ -491,4 +491,4 @@ texts: prov: [] self_ref: '#/texts/4' text: More text in the group. -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/roundtrip_list_item_with_inline_deserialized.yaml b/test/data/doc/roundtrip_list_item_with_inline_deserialized.yaml index b0d777bb6..18a4ae9ef 100644 --- a/test/data/doc/roundtrip_list_item_with_inline_deserialized.yaml +++ b/test/data/doc/roundtrip_list_item_with_inline_deserialized.yaml @@ -358,4 +358,4 @@ texts: prov: [] self_ref: '#/texts/23' text: final element -version: 1.10.0 +version: 1.11.0 diff --git a/test/data/doc/roundtrip_list_item_with_inline_reserialized.dclg.xml b/test/data/doc/roundtrip_list_item_with_inline_reserialized.dclg.xml index 796ed4b92..db21fb6ae 100644 --- a/test/data/doc/roundtrip_list_item_with_inline_reserialized.dclg.xml +++ b/test/data/doc/roundtrip_list_item_with_inline_reserialized.dclg.xml @@ -20,7 +20,7 @@ E=mc^2 -in line + in line Here a bold diff --git a/test/data/doc/roundtrip_list_item_with_inline_serialized.dclg.xml b/test/data/doc/roundtrip_list_item_with_inline_serialized.dclg.xml index ba9b52828..6de298fc7 100644 --- a/test/data/doc/roundtrip_list_item_with_inline_serialized.dclg.xml +++ b/test/data/doc/roundtrip_list_item_with_inline_serialized.dclg.xml @@ -18,7 +18,7 @@ E=mc^2 -in line + in line Here a bold diff --git a/test/data/doc/virtual_text_index/deserialized.json b/test/data/doc/virtual_text_index/deserialized.json index 3838f06a8..25ed806c0 100644 --- a/test/data/doc/virtual_text_index/deserialized.json +++ b/test/data/doc/virtual_text_index/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/virtual_text_index/input.json b/test/data/doc/virtual_text_index/input.json index b8c3b9a89..d45d62321 100644 --- a/test/data/doc/virtual_text_index/input.json +++ b/test/data/doc/virtual_text_index/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "virtual_text_index", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/virtual_text_list/deserialized.json b/test/data/doc/virtual_text_list/deserialized.json index c70380951..b39c0511b 100644 --- a/test/data/doc/virtual_text_list/deserialized.json +++ b/test/data/doc/virtual_text_list/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/virtual_text_list/input.json b/test/data/doc/virtual_text_list/input.json index 25c80c987..b6bf5f1ef 100644 --- a/test/data/doc/virtual_text_list/input.json +++ b/test/data/doc/virtual_text_list/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "virtual_text_list", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/virtual_text_table/deserialized.json b/test/data/doc/virtual_text_table/deserialized.json index a878507ce..98fb97e9e 100644 --- a/test/data/doc/virtual_text_table/deserialized.json +++ b/test/data/doc/virtual_text_table/deserialized.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/virtual_text_table/input.json b/test/data/doc/virtual_text_table/input.json index a5732fba5..322fcc5e7 100644 --- a/test/data/doc/virtual_text_table/input.json +++ b/test/data/doc/virtual_text_table/input.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "virtual_text_table", "furniture": { "self_ref": "#/furniture", diff --git a/test/data/doc/webvtt_example_02.json b/test/data/doc/webvtt_example_02.json index c418ac506..07e3cfbde 100644 --- a/test/data/doc/webvtt_example_02.json +++ b/test/data/doc/webvtt_example_02.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.8.0", + "version": "1.11.0", "name": "webvtt_example_02", "origin": { "mimetype": "text/vtt", diff --git a/test/data/doc/webvtt_example_04.json b/test/data/doc/webvtt_example_04.json index 6fe7d0ea9..5fd5df4f4 100644 --- a/test/data/doc/webvtt_example_04.json +++ b/test/data/doc/webvtt_example_04.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.8.0", + "version": "1.11.0", "name": "webvtt_example_04", "origin": { "mimetype": "text/vtt", diff --git a/test/data/doc/webvtt_example_05.json b/test/data/doc/webvtt_example_05.json index 76803cf6f..088766d8d 100644 --- a/test/data/doc/webvtt_example_05.json +++ b/test/data/doc/webvtt_example_05.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.8.0", + "version": "1.11.0", "name": "webvtt_example_04", "origin": { "mimetype": "text/vtt", diff --git a/test/test_deserializer_doclang.py b/test/test_deserializer_doclang.py index f13e5b9cc..7ac868048 100644 --- a/test/test_deserializer_doclang.py +++ b/test/test_deserializer_doclang.py @@ -1867,7 +1867,9 @@ def test_otsl_xml_sensitive_virtual_and_explicit_text_cells(): doc = _deserialize(doclang) assert doc.tables[0].data.grid[0][0].text == "virtual & \"quoted\" 'apostrophe'" - assert doc.tables[0].data.grid[0][1].text == "nested & bold & styled" + # The source writes ` bold & styled`; that leading space is a significant + # run boundary under the inline whitespace contract, so it survives. + assert doc.tables[0].data.grid[0][1].text == "nested & bold & styled" assert isinstance(doc.tables[0].data.grid[0][1], RichTableCell) diff --git a/test/test_inline_whitespace_contract.py b/test/test_inline_whitespace_contract.py new file mode 100644 index 000000000..3bbd41ecb --- /dev/null +++ b/test/test_inline_whitespace_contract.py @@ -0,0 +1,390 @@ +"""The InlineGroup whitespace contract. + +Runs carry their own significant whitespace; serializers concatenate them faithfully and never +insert a separator of their own. This module is the contract table: one set of inline-run +sequences checked against every text format, plus a DocLang round-trip identity check and the +load-time normalization that keeps pre-contract documents rendering as they used to. +""" + +import copy +import json +import re +import warnings +from typing import Optional + +import pytest +from pydantic import AnyUrl + +from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer +from docling_core.transforms.serializer.doclang import DocLangDocSerializer, DocLangParams +from docling_core.transforms.serializer.latex import LaTeXDocSerializer +from docling_core.types.doc.document import ( + CURRENT_VERSION, + DocItemLabel, + DoclingDocument, + Formatting, +) + +# (name, runs) where a run is (text, formatting, hyperlink) +Run = tuple[str, Optional[Formatting], Optional[str]] + +_BOLD = Formatting(bold=True) +_ITALIC = Formatting(italic=True) +_SUB = Formatting(script="sub") +_SUPER = Formatting(script="super") + +# Expected output per format. `html` is the inline-group span only, `latex`/`doclang` the body. +CONTRACT_TABLE: list[tuple[str, list[Run], dict[str, str]]] = [ + ( + "ordinal", + [("2", None, None), ("nd", _SUPER, None)], + { + "md": "2nd", + "itxt": "2nd", + "html": "2nd", + "latex": "2$^{nd}$", + "doclang": "2nd", + }, + ), + ( + "formula_unit", + [("O", None, None), ("2", _SUB, None), (".", None, None)], + { + "md": "O2.", + "itxt": "O2.", + "html": "O2.", + "latex": "O$_{2}$.", + "doclang": "O2.", + }, + ), + ( + "water", + [("H", None, None), ("2", _SUB, None), ("O", None, None)], + { + "md": "H2O", + "itxt": "H2O", + "html": "H2O", + "latex": "H$_{2}$O", + "doclang": "H2O", + }, + ), + ( + "spaced_emphasis", + [("Advanced Topics", _BOLD, None), (" in ", None, None), ("Machine Learning", _ITALIC, None)], + { + "md": "**Advanced Topics** in *Machine Learning*", + "itxt": "Advanced Topics in Machine Learning", + "html": "Advanced Topics in Machine Learning", + "latex": "\\textbf{Advanced Topics} in \\textit{Machine Learning}", + "doclang": ( + "Advanced Topics in " + "Machine Learning" + ), + }, + ), + ( + "trailing_period", + [("This is ", None, None), ("italic text", _ITALIC, None), (".", None, None)], + { + "md": "This is *italic text*.", + "itxt": "This is italic text.", + "html": "This is italic text.", + "latex": "This is \\textit{italic text}.", + "doclang": "This is italic text.", + }, + ), + ( + # Whitespace *inside* a formatted run: it must end up outside the emphasis markers, + # otherwise CommonMark renders `**bold **` as literal asterisks. + "whitespace_inside_bold", + [("bold ", _BOLD, None), ("tail", None, None)], + { + "md": "**bold** tail", + "itxt": "bold tail", + "html": "bold tail", + "latex": "\\textbf{bold} tail", + "doclang": "bold tail", + }, + ), + ( + # ... and outside the *complete* decoration stack, hyperlink included. + "whitespace_inside_bold_link", + [("bold ", _BOLD, "https://example.com/x"), ("tail", None, None)], + { + "md": "[**bold**](https://example.com/x) tail", + "itxt": "bold tail", + "html": 'bold tail', + "latex": "\\href{https://example.com/x}{\\textbf{bold}} tail", + # DocLang has no place for a hyperlink on an inline run (pre-existing gap, not + # part of this contract): the href is dropped. The whitespace still hoists. + "doclang": "bold tail", + }, + ), + ( + # A whitespace-only run survives as one space, with no empty markers. + "whitespace_only_run", + [("a", None, None), (" ", _BOLD, None), ("b", None, None)], + { + "md": "a b", + "itxt": "a b", + "html": "a b", + "latex": "a b", + "doclang": "a b", + }, + ), +] + + +def _build(runs: list[Run]) -> DoclingDocument: + doc = DoclingDocument(name="contract") + group = doc.add_inline_group() + for text, formatting, hyperlink in runs: + doc.add_text( + label=DocItemLabel.TEXT, + text=text, + parent=group, + formatting=formatting, + hyperlink=AnyUrl(hyperlink) if hyperlink else None, + ) + return doc + + +def _inline_span(doc: DoclingDocument) -> str: + html = doc.export_to_html(split_page_view=False) + match = re.search(r"(.*?)\s*$", html, re.DOTALL | re.MULTILINE) + if match is None: + match = re.search(r"(.*)", html, re.DOTALL) + assert match is not None, html + return match.group(1) + + +def _latex_body(doc: DoclingDocument) -> str: + text = LaTeXDocSerializer(doc=doc).serialize().text + body = text.split("\\begin{document}", 1)[1].split("\\end{document}", 1)[0] + return body.strip() + + +def _doclang_compact(doc: DoclingDocument) -> str: + """Serialize to DocLang without pretty-printing, so whitespace is unambiguous.""" + params = DocLangParams(include_version=False, pretty_indentation=None) + return DocLangDocSerializer(doc=doc, params=params).serialize().text.strip() + + +@pytest.mark.parametrize(("name", "runs", "expected"), CONTRACT_TABLE, ids=[c[0] for c in CONTRACT_TABLE]) +def test_contract_table(name: str, runs: list[Run], expected: dict[str, str]) -> None: + """Inline runs are concatenated faithfully in every text format.""" + doc = _build(runs) + actual = { + "md": doc.export_to_markdown(), + "itxt": doc.export_to_text(), + "html": _inline_span(doc), + "latex": _latex_body(doc), + "doclang": _doclang_compact(doc), + } + assert actual == expected + + +@pytest.mark.parametrize(("name", "runs", "expected"), CONTRACT_TABLE, ids=[c[0] for c in CONTRACT_TABLE]) +def test_doclang_roundtrip_is_lossless(name: str, runs: list[Run], expected: dict[str, str]) -> None: + """DocLang carries the run whitespace losslessly, via ``. + + Checked both pretty-printed (the default, where bare text nodes pick up indentation) and + compact, since the whitespace channel must survive the pretty printer. + """ + doc = _build(runs) + # DocLang drops hyperlinks on inline runs (pre-existing, see the table), so compare the + # whitespace rather than the full markdown for that one case. + expected_md = expected["md"] if not any(link for _, _, link in runs) else "**bold** tail" + for params in (DocLangParams(), DocLangParams(pretty_indentation=None)): + xml = DocLangDocSerializer(doc=doc, params=params).serialize().text + roundtripped = DocLangDocDeserializer().deserialize_str(xml) + assert roundtripped.export_to_markdown() == expected_md, (name, params.pretty_indentation) + assert roundtripped.export_to_text() == doc.export_to_text(), (name, params.pretty_indentation) + + +def test_no_separator_is_invented_between_runs() -> None: + """The serializers add nothing of their own: joined run text == the plain text export.""" + for name, runs, _ in CONTRACT_TABLE: + doc = _build(runs) + assert doc.export_to_text() == "".join(text for text, _, _ in runs), name + + +# -------------------------------------------------------------------------------------------- +# Load normalization for pre-contract documents +# -------------------------------------------------------------------------------------------- + +_LEGACY_VERSION = "1.8.0" + + +def _legacy_doc_dict(runs: list[str], *, version: str = _LEGACY_VERSION, label: str = "text") -> dict: + return { + "schema_name": "DoclingDocument", + "version": version, + "name": "legacy", + "body": { + "name": "_root_", + "self_ref": "#/body", + "label": "unspecified", + "children": [{"$ref": "#/groups/0"}], + }, + "furniture": {"name": "_root_", "self_ref": "#/furniture", "label": "unspecified", "children": []}, + "groups": [ + { + "self_ref": "#/groups/0", + "parent": {"$ref": "#/body"}, + "label": "inline", + "name": "group", + "content_layer": "body", + "children": [{"$ref": f"#/texts/{i}"} for i in range(len(runs))], + } + ], + "texts": [ + { + "self_ref": f"#/texts/{i}", + "parent": {"$ref": "#/groups/0"}, + "label": label, + "prov": [], + "orig": text, + "text": text, + "children": [], + "content_layer": "body", + } + for i, text in enumerate(runs) + ], + "pictures": [], + "tables": [], + "key_value_items": [], + "form_items": [], + "pages": {}, + } + + +def _load(data: dict) -> DoclingDocument: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + return DoclingDocument.model_validate(json.loads(json.dumps(data))) + + +@pytest.mark.parametrize( + ("runs", "expected_runs", "expected_md"), + [ + # Space-less DOCX-style runs stay spaced, as the old `" "` join rendered them. + (["Normal", "italic"], ["Normal ", "italic"], "Normal italic"), + # The legacy extra-space bug is retained: this used to render `H 2 O`. + (["H", "2", "O"], ["H ", "2 ", "O"], "H 2 O"), + # An existing boundary keeps its legacy double space -- the old serializer added one too. + (["left ", "right"], ["left ", "right"], "left right"), + ], +) +def test_legacy_documents_keep_their_separators(runs: list[str], expected_runs: list[str], expected_md: str) -> None: + doc = _load(_legacy_doc_dict(runs)) + assert [t.text for t in doc.texts] == expected_runs + assert doc.export_to_markdown() == expected_md + + +def test_current_version_documents_are_not_normalized() -> None: + doc = _load(_legacy_doc_dict(["H", "2", "O"], version=CURRENT_VERSION)) + assert [t.text for t in doc.texts] == ["H", "2", "O"] + assert doc.export_to_markdown() == "H2O" + + +def test_versionless_input_is_not_guessed_at() -> None: + data = _legacy_doc_dict(["H", "2", "O"]) + del data["version"] + assert [t.text for t in _load(data).texts] == ["H", "2", "O"] + + +def test_migration_is_one_way() -> None: + """Loading stamps CURRENT_VERSION, so a re-save cannot pick up a second separator.""" + once = _load(_legacy_doc_dict(["H", "2"])) + assert once.version == CURRENT_VERSION + twice = _load(json.loads(once.model_dump_json())) + assert [t.text for t in twice.texts] == [t.text for t in once.texts] == ["H ", "2"] + + +@pytest.mark.parametrize( + ("labels", "expected_md", "expected_txt"), + [ + # The separator goes on whichever neighbour is not delimiter-wrapped ... + (("text", "code"), "a `b`", "a b"), + (("code", "text"), "`a` b", "a b"), + (("text", "formula"), "a $b$", "a $b$"), + (("formula", "text"), "$a$ b", "$a$ b"), + # ... and when both are, it becomes a plain run of its own, because appending to + # either side would produce `a``b` or $a$$b$. + (("code", "code"), "`a` `b`", "a b"), + (("formula", "formula"), "$a$ $b$", "$a$ $b$"), + (("code", "formula"), "`a` $b$", "a $b$"), + (("formula", "code"), "$a$ `b`", "$a$ b"), + ], +) +def test_legacy_separator_never_lands_inside_a_delimiter( + labels: tuple[str, str], expected_md: str, expected_txt: str +) -> None: + data = _legacy_doc_dict(["a", "b"]) + for item, label in zip(data["texts"], labels): + item["label"] = label + doc = _load(data) + assert doc.export_to_markdown() == expected_md + assert doc.export_to_text() == expected_txt + + +def test_legacy_migration_does_not_touch_caller_input() -> None: + """The caller owns the dict it passes in, and repeated validation must be stable. + + The migration inserts a standalone separator run for adjacent delimited runs, so an + in-place edit would both corrupt the caller's value and compound on every re-validation. + """ + data = _legacy_doc_dict(["a", "b"]) + data["texts"][0]["label"] = "code" + data["texts"][1]["label"] = "code" + pristine = copy.deepcopy(data) + + rendered = [_load(data).export_to_markdown() for _ in range(3)] + + assert rendered == ["`a` `b`"] * 3 + assert data == pristine + + +def test_legacy_load_warns_once() -> None: + import docling_core.types.doc.document as document_module + + original = document_module._warned_legacy_inline_separators + document_module._warned_legacy_inline_separators = False + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + DoclingDocument.model_validate(_legacy_doc_dict(["a", "b"])) + DoclingDocument.model_validate(_legacy_doc_dict(["c", "d"])) + messages = [str(w.message) for w in caught if w.category is UserWarning] + assert len(messages) == 1, messages + assert _LEGACY_VERSION in messages[0] + finally: + document_module._warned_legacy_inline_separators = original + + +def test_chunk_text_follows_the_contract() -> None: + """The chunker has no inline serializer, so it inherits the Markdown join. + + That makes chunk text -- and therefore embeddings -- change with this contract. Pinned + here because the drift is otherwise invisible: nothing in a chunker golden fails, but a + persisted vector index silently stops matching. + """ + from docling_core.transforms.chunker.hierarchical_chunker import HierarchicalChunker + + doc = DoclingDocument(name="chunk") + doc.add_heading(text="Chemistry") + group = doc.add_inline_group() + for text, formatting in [ + ("Water is ", None), + ("H", None), + ("2", _SUB), + ("O", None), + (" and the ", None), + ("melting point", _BOLD), + (" is 0 C.", None), + ]: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=group, formatting=formatting) + + chunks = list(HierarchicalChunker().chunk(doc)) + assert [c.text for c in chunks] == ["Water is H2O and the **melting point** is 0 C."] From f768d757128e8aecf112eb45895122ff346b0675 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Mon, 27 Jul 2026 12:48:15 +0200 Subject: [PATCH 03/18] DCO Remediation Commit for Christoph Auer I, Christoph Auer , hereby add my Signed-off-by to this commit: 64404b3709d9e3ba7caf08957cf2ef19df29459f I, Christoph Auer , hereby add my Signed-off-by to this commit: 66040cb7b96f77699c9d74051500d645e7e758ed Signed-off-by: Christoph Auer From e4e745a3769f0ee172eb66d11ae1f5c0d4ac5892 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Tue, 18 Aug 2026 14:54:35 +0200 Subject: [PATCH 04/18] feat(compat): register 1.11->1.10 projector for the whitespace-contract bump Integrates the 1.11.0 schema bump with the downgrade-projector framework (#703) landed upstream. 1.11 and 1.10 are structurally identical; the only difference is that 1.11 inline runs own their boundary whitespace, so the projector just re-stamps the version (a 1.10 client re-applies its own join, yielding the documented harmless double space). - add @register_projector(from_minor=11, to_minor=10) + TestProjector_1_11_to_1_10 - add the auto-generated 1.11 JSON Schema snapshot - re-format the 1.11 SCHEMA_VERSION_HISTORY entry (ruff) - update the stale dclq CLI expectation (old join produced 'public class'; faithful join now yields 'public class') Signed-off-by: Christoph Auer --- docling_core/compat.py | 30 + docling_core/types/doc/common/constants.py | 6 +- docs/schemas/DoclingDocument_1_11.json | 4655 ++++++++++++++++++++ packages/dclq/tests/test_cli.py | 2 +- test/test_compat.py | 39 + 5 files changed, 4730 insertions(+), 2 deletions(-) create mode 100644 docs/schemas/DoclingDocument_1_11.json diff --git a/docling_core/compat.py b/docling_core/compat.py index c01776f00..8ea029d7a 100644 --- a/docling_core/compat.py +++ b/docling_core/compat.py @@ -198,6 +198,36 @@ def list_projectors() -> list[tuple[int, int]]: # the serialised dict without error. Full semantic fidelity is not always # possible; the goal is a document that validates rather than crashes. +# --- 1.11 → 1.10 ---------------------------------------------------------- +# Schema 1.11 (docling-core v2.86.0): InlineGroup runs carry their own +# significant whitespace and serializers concatenate them faithfully. +# +# The 1.11 and 1.10 schemas are structurally identical -- no new fields, +# labels, or lists. The only difference is *semantic*: a 1.11 run's text +# contains its boundary whitespace, whereas a 1.10 client re-inserts a +# single " " between runs at serialization time. A 1.10 client therefore +# parses a 1.11 dict without error; it just renders the boundary twice +# (e.g. "bold tail"), the documented, harmless double-space case. 1.10 +# cannot represent a tight run boundary, so there is nothing to project +# structurally -- only the version stamp is lowered so the old client's +# version-compatibility gate accepts the document. +# --------------------------------------------------------------------------- + + +@register_projector(from_minor=11, to_minor=10) +def _project_1_11_to_1_10(data: dict) -> dict: + """Downgrade a schema 1.11 document dict to schema 1.10. + + 1.11 and 1.10 share an identical structure; the difference is only in how + inline-run whitespace is represented. A 1.10 client validates the dict + as-is and re-applies its own single-space join between runs, so the sole + transform is re-stamping the version. + """ + data = dict(data) + data["version"] = "1.10.0" + return data + + # --- 1.10 → 1.9 ----------------------------------------------------------- # Schema 1.10 (docling-core v2.69.0, PR #519): # • Added FieldRegionItem, FieldHeadingItem, FieldItem, FieldValueItem diff --git a/docling_core/types/doc/common/constants.py b/docling_core/types/doc/common/constants.py index c8ba85ee2..7fc63192d 100644 --- a/docling_core/types/doc/common/constants.py +++ b/docling_core/types/doc/common/constants.py @@ -20,7 +20,11 @@ {"schema": "1.8.0", "library_version": "v2.49.0", "note": "BasePrediction, BaseMeta, meta field"}, {"schema": "1.9.0", "library_version": "v2.57.0", "note": "FineRef, DocItem.comments"}, {"schema": "1.10.0", "library_version": "v2.69.0", "note": "FieldItem types, 7 new DocItemLabel values"}, - {"schema": "1.11.0", "library_version": "v2.86.0", "note": "InlineGroup runs carry their own significant whitespace"}, + { + "schema": "1.11.0", + "library_version": "v2.86.0", + "note": "InlineGroup runs carry their own significant whitespace", + }, ] """Ordered history of DoclingDocument schema versions in docling-core v2.x. diff --git a/docs/schemas/DoclingDocument_1_11.json b/docs/schemas/DoclingDocument_1_11.json new file mode 100644 index 000000000..88833795b --- /dev/null +++ b/docs/schemas/DoclingDocument_1_11.json @@ -0,0 +1,4655 @@ +{ + "$defs": { + "BaseMeta": { + "additionalProperties": true, + "description": "Base class for metadata.", + "properties": { + "summary": { + "anyOf": [ + { + "$ref": "#/$defs/SummaryMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A condensed natural-language summary of the content rooted at this node.", + "examples": [ + { + "text": "A short company/location statement." + } + ] + }, + "language": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The detected human language of the node content, expressed as a BCP 47 code.", + "examples": [ + { + "code": "en" + } + ] + }, + "entities": { + "anyOf": [ + { + "$ref": "#/$defs/EntitiesMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Named entities extracted from the node text (persons, organisations, locations, etc.). Each mention carries the entity text, an optional type label, and an optional character span.", + "examples": [ + { + "mentions": [ + { + "charspan": [ + 0, + 3 + ], + "label": "ORG", + "text": "IBM" + } + ] + } + ] + }, + "keywords": { + "anyOf": [ + { + "$ref": "#/$defs/KeywordsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Salient terms or short keyphrases that characterise the node content. Keywords are more specific than topics and typically correspond to individual words or short multi-word expressions found in or closely related to the text. Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "transformer", + "attention mechanism", + "BERT" + ] + } + ] + }, + "topics": { + "anyOf": [ + { + "$ref": "#/$defs/TopicsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Higher-level subject categories or thematic labels inferred for the node content. Topics are broader than keywords and describe the domain or theme rather than specific terms (e.g., 'machine learning' rather than 'gradient descent'). Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "natural language processing", + "computer vision" + ] + } + ] + } + }, + "title": "BaseMeta", + "type": "object" + }, + "BoundingBox": { + "description": "BoundingBox.", + "properties": { + "l": { + "title": "L", + "type": "number" + }, + "t": { + "title": "T", + "type": "number" + }, + "r": { + "title": "R", + "type": "number" + }, + "b": { + "title": "B", + "type": "number" + }, + "coord_origin": { + "$ref": "#/$defs/CoordOrigin", + "default": "TOPLEFT" + } + }, + "required": [ + "l", + "t", + "r", + "b" + ], + "title": "BoundingBox", + "type": "object" + }, + "ChartBar": { + "description": "Represents a bar in a bar chart.\n\nAttributes:\n label (str): The label for the bar.\n values (float): The value associated with the bar.", + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "values": { + "title": "Values", + "type": "number" + } + }, + "required": [ + "label", + "values" + ], + "title": "ChartBar", + "type": "object" + }, + "ChartLine": { + "description": "Represents a line in a line chart.\n\nAttributes:\n label (str): The label for the line.\n values (list[tuple[float, float]]): A list of (x, y) coordinate pairs\n representing the line's data points.", + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "values": { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "type": "array" + }, + "title": "Values", + "type": "array" + } + }, + "required": [ + "label", + "values" + ], + "title": "ChartLine", + "type": "object" + }, + "ChartPoint": { + "description": "Represents a point in a scatter chart.\n\nAttributes:\n value (Tuple[float, float]): A (x, y) coordinate pair representing a point in a\n chart.", + "properties": { + "value": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "title": "Value", + "type": "array" + } + }, + "required": [ + "value" + ], + "title": "ChartPoint", + "type": "object" + }, + "ChartSlice": { + "description": "Represents a slice in a pie chart.\n\nAttributes:\n label (str): The label for the slice.\n value (float): The value represented by the slice.", + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "value": { + "title": "Value", + "type": "number" + } + }, + "required": [ + "label", + "value" + ], + "title": "ChartSlice", + "type": "object" + }, + "ChartStackedBar": { + "description": "Represents a stacked bar in a stacked bar chart.\n\nAttributes:\n label (list[str]): The labels for the stacked bars. Multiple values are stored\n in cases where the chart is \"double stacked,\" meaning bars are stacked both\n horizontally and vertically.\n values (list[tuple[str, int]]): A list of values representing different segments\n of the stacked bar along with their label.", + "properties": { + "label": { + "items": { + "type": "string" + }, + "title": "Label", + "type": "array" + }, + "values": { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "type": "array" + }, + "title": "Values", + "type": "array" + } + }, + "required": [ + "label", + "values" + ], + "title": "ChartStackedBar", + "type": "object" + }, + "CodeItem": { + "additionalProperties": false, + "description": "CodeItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/FloatingMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "code", + "default": "code", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "code_language": { + "$ref": "#/$defs/CodeLanguageLabel", + "default": "unknown" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "CodeItem", + "type": "object" + }, + "CodeLanguageLabel": { + "description": "CodeLanguageLabel.", + "enum": [ + "Ada", + "Awk", + "Bash", + "bc", + "C", + "C#", + "C++", + "CMake", + "COBOL", + "CSS", + "Ceylon", + "Clojure", + "Crystal", + "Cuda", + "Cython", + "D", + "Dart", + "dc", + "Dockerfile", + "DocLang", + "Elixir", + "Erlang", + "FORTRAN", + "Forth", + "Go", + "HTML", + "Haskell", + "Haxe", + "Java", + "JavaScript", + "JSON", + "Julia", + "Kotlin", + "Latex", + "Lisp", + "Lua", + "Matlab", + "MoonScript", + "Nim", + "OCaml", + "ObjectiveC", + "Octave", + "PHP", + "Pascal", + "Perl", + "Prolog", + "Python", + "Racket", + "Ruby", + "Rust", + "SML", + "SQL", + "Scala", + "Scheme", + "Swift", + "Tikz", + "TypeScript", + "unknown", + "VisualBasic", + "XML", + "YAML" + ], + "title": "CodeLanguageLabel", + "type": "string" + }, + "CodeMetaField": { + "additionalProperties": true, + "description": "Code representation for the respective item.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "title": "Text", + "type": "string" + }, + "language": { + "anyOf": [ + { + "$ref": "#/$defs/CodeLanguageLabel" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "text" + ], + "title": "CodeMetaField", + "type": "object" + }, + "ContentLayer": { + "description": "ContentLayer.", + "enum": [ + "body", + "furniture", + "background", + "invisible", + "notes" + ], + "title": "ContentLayer", + "type": "string" + }, + "CoordOrigin": { + "description": "CoordOrigin.", + "enum": [ + "TOPLEFT", + "BOTTOMLEFT" + ], + "title": "CoordOrigin", + "type": "string" + }, + "DescriptionAnnotation": { + "description": "DescriptionAnnotation.", + "properties": { + "kind": { + "const": "description", + "default": "description", + "title": "Kind", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "provenance": { + "title": "Provenance", + "type": "string" + } + }, + "required": [ + "text", + "provenance" + ], + "title": "DescriptionAnnotation", + "type": "object" + }, + "DescriptionMetaField": { + "additionalProperties": true, + "description": "Description metadata field.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "DescriptionMetaField", + "type": "object" + }, + "DocumentOrigin": { + "description": "FileSource.", + "properties": { + "mimetype": { + "title": "Mimetype", + "type": "string" + }, + "binary_hash": { + "maximum": 18446744073709551615, + "minimum": 0, + "title": "Binary Hash", + "type": "integer" + }, + "filename": { + "title": "Filename", + "type": "string" + }, + "uri": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Uri" + } + }, + "required": [ + "mimetype", + "binary_hash", + "filename" + ], + "title": "DocumentOrigin", + "type": "object" + }, + "EntitiesMetaField": { + "additionalProperties": true, + "description": "Container for extracted entity mentions.", + "properties": { + "mentions": { + "items": { + "$ref": "#/$defs/EntityMention" + }, + "minItems": 1, + "title": "Mentions", + "type": "array" + } + }, + "required": [ + "mentions" + ], + "title": "EntitiesMetaField", + "type": "object" + }, + "EntityMention": { + "additionalProperties": true, + "description": "Entity mention extracted from text.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "description": "Normalized text of the entity mention.", + "title": "Text", + "type": "string" + }, + "orig": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact source text extracted from the original charspan, analogous to TextItem.orig. This may differ from 'text' when the mention has been normalized.", + "title": "Orig" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Entity type or category.", + "title": "Label" + }, + "charspan": { + "anyOf": [ + { + "description": "Character span (0-indexed)", + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Character span (0-indexed) of the entity mention in the source text.", + "title": "Charspan" + } + }, + "required": [ + "text" + ], + "title": "EntityMention", + "type": "object" + }, + "FieldHeadingItem": { + "additionalProperties": false, + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "field_heading", + "default": "field_heading", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "level": { + "default": 1, + "maximum": 100, + "minimum": 1, + "title": "Level", + "type": "integer" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "FieldHeadingItem", + "type": "object" + }, + "FieldItem": { + "additionalProperties": false, + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "field_item", + "default": "field_item", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + } + }, + "required": [ + "self_ref" + ], + "title": "FieldItem", + "type": "object" + }, + "FieldRegionItem": { + "additionalProperties": false, + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "field_region", + "default": "field_region", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + } + }, + "required": [ + "self_ref" + ], + "title": "FieldRegionItem", + "type": "object" + }, + "FieldValueItem": { + "additionalProperties": false, + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "field_value", + "default": "field_value", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "kind": { + "default": "read_only", + "enum": [ + "read_only", + "fillable" + ], + "title": "Kind", + "type": "string" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "FieldValueItem", + "type": "object" + }, + "FineRef": { + "description": "Fine-granular reference item that can capture span range info.", + "properties": { + "$ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "$Ref", + "type": "string" + }, + "range": { + "anyOf": [ + { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Range" + } + }, + "required": [ + "$ref" + ], + "title": "FineRef", + "type": "object" + }, + "FloatingMeta": { + "additionalProperties": true, + "description": "Metadata model for floating.", + "properties": { + "summary": { + "anyOf": [ + { + "$ref": "#/$defs/SummaryMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A condensed natural-language summary of the content rooted at this node.", + "examples": [ + { + "text": "A short company/location statement." + } + ] + }, + "language": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The detected human language of the node content, expressed as a BCP 47 code.", + "examples": [ + { + "code": "en" + } + ] + }, + "entities": { + "anyOf": [ + { + "$ref": "#/$defs/EntitiesMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Named entities extracted from the node text (persons, organisations, locations, etc.). Each mention carries the entity text, an optional type label, and an optional character span.", + "examples": [ + { + "mentions": [ + { + "charspan": [ + 0, + 3 + ], + "label": "ORG", + "text": "IBM" + } + ] + } + ] + }, + "keywords": { + "anyOf": [ + { + "$ref": "#/$defs/KeywordsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Salient terms or short keyphrases that characterise the node content. Keywords are more specific than topics and typically correspond to individual words or short multi-word expressions found in or closely related to the text. Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "transformer", + "attention mechanism", + "BERT" + ] + } + ] + }, + "topics": { + "anyOf": [ + { + "$ref": "#/$defs/TopicsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Higher-level subject categories or thematic labels inferred for the node content. Topics are broader than keywords and describe the domain or theme rather than specific terms (e.g., 'machine learning' rather than 'gradient descent'). Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "natural language processing", + "computer vision" + ] + } + ] + }, + "description": { + "anyOf": [ + { + "$ref": "#/$defs/DescriptionMetaField" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "FloatingMeta", + "type": "object" + }, + "FormItem": { + "additionalProperties": false, + "description": "FormItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/FloatingMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "form", + "default": "form", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "graph": { + "$ref": "#/$defs/GraphData" + } + }, + "required": [ + "self_ref", + "graph" + ], + "title": "FormItem", + "type": "object" + }, + "Formatting": { + "description": "Formatting.", + "properties": { + "bold": { + "default": false, + "title": "Bold", + "type": "boolean" + }, + "italic": { + "default": false, + "title": "Italic", + "type": "boolean" + }, + "underline": { + "default": false, + "title": "Underline", + "type": "boolean" + }, + "strikethrough": { + "default": false, + "title": "Strikethrough", + "type": "boolean" + }, + "script": { + "$ref": "#/$defs/Script", + "default": "baseline" + } + }, + "title": "Formatting", + "type": "object" + }, + "FormulaItem": { + "additionalProperties": false, + "description": "FormulaItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "formula", + "default": "formula", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "FormulaItem", + "type": "object" + }, + "GraphCell": { + "description": "GraphCell.", + "properties": { + "label": { + "$ref": "#/$defs/GraphCellLabel" + }, + "cell_id": { + "title": "Cell Id", + "type": "integer" + }, + "text": { + "title": "Text", + "type": "string" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "prov": { + "anyOf": [ + { + "$ref": "#/$defs/ProvenanceItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "item_ref": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "label", + "cell_id", + "text", + "orig" + ], + "title": "GraphCell", + "type": "object" + }, + "GraphCellLabel": { + "description": "GraphCellLabel.", + "enum": [ + "unspecified", + "key", + "value", + "checkbox" + ], + "title": "GraphCellLabel", + "type": "string" + }, + "GraphData": { + "description": "GraphData.", + "properties": { + "cells": { + "items": { + "$ref": "#/$defs/GraphCell" + }, + "title": "Cells", + "type": "array" + }, + "links": { + "items": { + "$ref": "#/$defs/GraphLink" + }, + "title": "Links", + "type": "array" + } + }, + "title": "GraphData", + "type": "object" + }, + "GraphLink": { + "description": "GraphLink.", + "properties": { + "label": { + "$ref": "#/$defs/GraphLinkLabel" + }, + "source_cell_id": { + "title": "Source Cell Id", + "type": "integer" + }, + "target_cell_id": { + "title": "Target Cell Id", + "type": "integer" + } + }, + "required": [ + "label", + "source_cell_id", + "target_cell_id" + ], + "title": "GraphLink", + "type": "object" + }, + "GraphLinkLabel": { + "description": "GraphLinkLabel.", + "enum": [ + "unspecified", + "to_value", + "to_key", + "to_parent", + "to_child" + ], + "title": "GraphLinkLabel", + "type": "string" + }, + "GroupItem": { + "additionalProperties": false, + "description": "GroupItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "name": { + "default": "group", + "title": "Name", + "type": "string" + }, + "label": { + "$ref": "#/$defs/GroupLabel", + "default": "unspecified" + } + }, + "required": [ + "self_ref" + ], + "title": "GroupItem", + "type": "object" + }, + "GroupLabel": { + "description": "GroupLabel.", + "enum": [ + "unspecified", + "list", + "ordered_list", + "chapter", + "section", + "sheet", + "slide", + "form_area", + "key_value_area", + "comment_section", + "inline", + "picture_area" + ], + "title": "GroupLabel", + "type": "string" + }, + "HumanLanguageLabel": { + "description": "Two-letter human language primary subtags using BCP-47 values.", + "enum": [ + "aa", + "ab", + "ae", + "af", + "ak", + "am", + "an", + "ar", + "as", + "av", + "ay", + "az", + "ba", + "be", + "bg", + "bh", + "bi", + "bm", + "bn", + "bo", + "br", + "bs", + "ca", + "ce", + "ch", + "co", + "cr", + "cs", + "cu", + "cv", + "cy", + "da", + "de", + "dv", + "dz", + "ee", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "ff", + "fi", + "fj", + "fo", + "fr", + "fy", + "ga", + "gd", + "gl", + "gn", + "gu", + "gv", + "ha", + "he", + "hi", + "ho", + "hr", + "ht", + "hu", + "hy", + "hz", + "ia", + "id", + "ie", + "ig", + "ii", + "ik", + "io", + "is", + "it", + "iu", + "ja", + "jv", + "ka", + "kg", + "ki", + "kj", + "kk", + "kl", + "km", + "kn", + "ko", + "kr", + "ks", + "ku", + "kv", + "kw", + "ky", + "la", + "lb", + "lg", + "li", + "ln", + "lo", + "lt", + "lu", + "lv", + "mg", + "mh", + "mi", + "mk", + "ml", + "mn", + "mr", + "ms", + "mt", + "my", + "na", + "nb", + "nd", + "ne", + "ng", + "nl", + "nn", + "no", + "nr", + "nv", + "ny", + "oc", + "oj", + "om", + "or", + "os", + "pa", + "pi", + "pl", + "ps", + "pt", + "qu", + "rm", + "rn", + "ro", + "ru", + "rw", + "sa", + "sc", + "sd", + "se", + "sg", + "sh", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr", + "ss", + "st", + "su", + "sv", + "sw", + "ta", + "te", + "tg", + "th", + "ti", + "tk", + "tl", + "tn", + "to", + "tr", + "ts", + "tt", + "tw", + "ty", + "ug", + "uk", + "ur", + "uz", + "ve", + "vi", + "vo", + "wa", + "wo", + "xh", + "yi", + "yo", + "za", + "zh", + "zu" + ], + "title": "HumanLanguageLabel", + "type": "string" + }, + "ImageRef": { + "description": "ImageRef.", + "properties": { + "mimetype": { + "title": "Mimetype", + "type": "string" + }, + "dpi": { + "title": "Dpi", + "type": "integer" + }, + "size": { + "$ref": "#/$defs/Size" + }, + "uri": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + } + ], + "title": "Uri" + } + }, + "required": [ + "mimetype", + "dpi", + "size", + "uri" + ], + "title": "ImageRef", + "type": "object" + }, + "InlineGroup": { + "additionalProperties": false, + "description": "InlineGroup.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "name": { + "default": "group", + "title": "Name", + "type": "string" + }, + "label": { + "const": "inline", + "default": "inline", + "title": "Label", + "type": "string" + } + }, + "required": [ + "self_ref" + ], + "title": "InlineGroup", + "type": "object" + }, + "KeyValueItem": { + "additionalProperties": false, + "description": "KeyValueItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/FloatingMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "key_value_region", + "default": "key_value_region", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "graph": { + "$ref": "#/$defs/GraphData" + } + }, + "required": [ + "self_ref", + "graph" + ], + "title": "KeyValueItem", + "type": "object" + }, + "KeywordsMetaField": { + "additionalProperties": true, + "description": "Container for a list of unique keywords / keyphrases.", + "properties": { + "values": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Values", + "type": "array", + "uniqueItems": true + } + }, + "required": [ + "values" + ], + "title": "KeywordsMetaField", + "type": "object" + }, + "LanguageMetaField": { + "additionalProperties": true, + "description": "Detected human language.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "code": { + "$ref": "#/$defs/HumanLanguageLabel" + } + }, + "required": [ + "code" + ], + "title": "LanguageMetaField", + "type": "object" + }, + "ListGroup": { + "additionalProperties": false, + "description": "ListGroup.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "name": { + "default": "group", + "title": "Name", + "type": "string" + }, + "label": { + "const": "list", + "default": "list", + "title": "Label", + "type": "string" + } + }, + "required": [ + "self_ref" + ], + "title": "ListGroup", + "type": "object" + }, + "ListItem": { + "additionalProperties": false, + "description": "SectionItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "list_item", + "default": "list_item", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "enumerated": { + "default": false, + "title": "Enumerated", + "type": "boolean" + }, + "marker": { + "default": "-", + "title": "Marker", + "type": "string" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "ListItem", + "type": "object" + }, + "MiscAnnotation": { + "description": "MiscAnnotation.", + "properties": { + "kind": { + "const": "misc", + "default": "misc", + "title": "Kind", + "type": "string" + }, + "content": { + "additionalProperties": true, + "title": "Content", + "type": "object" + } + }, + "required": [ + "content" + ], + "title": "MiscAnnotation", + "type": "object" + }, + "MoleculeMetaField": { + "additionalProperties": true, + "description": "Molecule metadata field.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "smi": { + "description": "The SMILES representation of the molecule.", + "title": "Smi", + "type": "string" + } + }, + "required": [ + "smi" + ], + "title": "MoleculeMetaField", + "type": "object" + }, + "Orientation": { + "description": "Counter-clockwise rotation of a table on the page, in degrees.\n\nFollows the convention used by PIL/Pillow's ``Image.rotate``: positive\nangles rotate the table counter-clockwise. ``ROT_0`` / ``ROT_180`` keep\nrows running horizontally on the page; ``ROT_90`` / ``ROT_270`` turn\nrows into vertical stripes.", + "enum": [ + "rot_0", + "rot_90", + "rot_180", + "rot_270" + ], + "title": "Orientation", + "type": "string" + }, + "PageItem": { + "description": "PageItem.", + "properties": { + "size": { + "$ref": "#/$defs/Size" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "page_no": { + "title": "Page No", + "type": "integer" + } + }, + "required": [ + "size", + "page_no" + ], + "title": "PageItem", + "type": "object" + }, + "PictureBarChartData": { + "description": "Represents data of a bar chart.\n\nAttributes:\n kind (Literal[\"bar_chart_data\"]): The type of the chart.\n x_axis_label (str): The label for the x-axis.\n y_axis_label (str): The label for the y-axis.\n bars (list[ChartBar]): A list of bars in the chart.", + "properties": { + "kind": { + "const": "bar_chart_data", + "default": "bar_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "x_axis_label": { + "title": "X Axis Label", + "type": "string" + }, + "y_axis_label": { + "title": "Y Axis Label", + "type": "string" + }, + "bars": { + "items": { + "$ref": "#/$defs/ChartBar" + }, + "title": "Bars", + "type": "array" + } + }, + "required": [ + "title", + "x_axis_label", + "y_axis_label", + "bars" + ], + "title": "PictureBarChartData", + "type": "object" + }, + "PictureClassificationClass": { + "description": "PictureClassificationData.", + "properties": { + "class_name": { + "title": "Class Name", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + } + }, + "required": [ + "class_name", + "confidence" + ], + "title": "PictureClassificationClass", + "type": "object" + }, + "PictureClassificationData": { + "description": "PictureClassificationData.", + "properties": { + "kind": { + "const": "classification", + "default": "classification", + "title": "Kind", + "type": "string" + }, + "provenance": { + "title": "Provenance", + "type": "string" + }, + "predicted_classes": { + "items": { + "$ref": "#/$defs/PictureClassificationClass" + }, + "title": "Predicted Classes", + "type": "array" + } + }, + "required": [ + "provenance", + "predicted_classes" + ], + "title": "PictureClassificationData", + "type": "object" + }, + "PictureClassificationMetaField": { + "additionalProperties": true, + "description": "Picture classification metadata field.", + "properties": { + "predictions": { + "items": { + "$ref": "#/$defs/PictureClassificationPrediction" + }, + "minItems": 1, + "title": "Predictions", + "type": "array" + } + }, + "title": "PictureClassificationMetaField", + "type": "object" + }, + "PictureClassificationPrediction": { + "additionalProperties": true, + "description": "Picture classification instance.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "class_name": { + "title": "Class Name", + "type": "string" + } + }, + "required": [ + "class_name" + ], + "title": "PictureClassificationPrediction", + "type": "object" + }, + "PictureItem": { + "additionalProperties": false, + "description": "PictureItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/PictureMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "default": "picture", + "enum": [ + "picture", + "chart" + ], + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "annotations": { + "default": [], + "deprecated": true, + "items": { + "discriminator": { + "mapping": { + "bar_chart_data": "#/$defs/PictureBarChartData", + "classification": "#/$defs/PictureClassificationData", + "description": "#/$defs/DescriptionAnnotation", + "line_chart_data": "#/$defs/PictureLineChartData", + "misc": "#/$defs/MiscAnnotation", + "molecule_data": "#/$defs/PictureMoleculeData", + "pie_chart_data": "#/$defs/PicturePieChartData", + "scatter_chart_data": "#/$defs/PictureScatterChartData", + "stacked_bar_chart_data": "#/$defs/PictureStackedBarChartData", + "tabular_chart_data": "#/$defs/PictureTabularChartData" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/DescriptionAnnotation" + }, + { + "$ref": "#/$defs/MiscAnnotation" + }, + { + "$ref": "#/$defs/PictureClassificationData" + }, + { + "$ref": "#/$defs/PictureMoleculeData" + }, + { + "$ref": "#/$defs/PictureTabularChartData" + }, + { + "$ref": "#/$defs/PictureLineChartData" + }, + { + "$ref": "#/$defs/PictureBarChartData" + }, + { + "$ref": "#/$defs/PictureStackedBarChartData" + }, + { + "$ref": "#/$defs/PicturePieChartData" + }, + { + "$ref": "#/$defs/PictureScatterChartData" + } + ] + }, + "title": "Annotations", + "type": "array" + } + }, + "required": [ + "self_ref" + ], + "title": "PictureItem", + "type": "object" + }, + "PictureLineChartData": { + "description": "Represents data of a line chart.\n\nAttributes:\n kind (Literal[\"line_chart_data\"]): The type of the chart.\n x_axis_label (str): The label for the x-axis.\n y_axis_label (str): The label for the y-axis.\n lines (list[ChartLine]): A list of lines in the chart.", + "properties": { + "kind": { + "const": "line_chart_data", + "default": "line_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "x_axis_label": { + "title": "X Axis Label", + "type": "string" + }, + "y_axis_label": { + "title": "Y Axis Label", + "type": "string" + }, + "lines": { + "items": { + "$ref": "#/$defs/ChartLine" + }, + "title": "Lines", + "type": "array" + } + }, + "required": [ + "title", + "x_axis_label", + "y_axis_label", + "lines" + ], + "title": "PictureLineChartData", + "type": "object" + }, + "PictureMeta": { + "additionalProperties": true, + "description": "Metadata model for pictures.", + "properties": { + "summary": { + "anyOf": [ + { + "$ref": "#/$defs/SummaryMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A condensed natural-language summary of the content rooted at this node.", + "examples": [ + { + "text": "A short company/location statement." + } + ] + }, + "language": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The detected human language of the node content, expressed as a BCP 47 code.", + "examples": [ + { + "code": "en" + } + ] + }, + "entities": { + "anyOf": [ + { + "$ref": "#/$defs/EntitiesMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Named entities extracted from the node text (persons, organisations, locations, etc.). Each mention carries the entity text, an optional type label, and an optional character span.", + "examples": [ + { + "mentions": [ + { + "charspan": [ + 0, + 3 + ], + "label": "ORG", + "text": "IBM" + } + ] + } + ] + }, + "keywords": { + "anyOf": [ + { + "$ref": "#/$defs/KeywordsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Salient terms or short keyphrases that characterise the node content. Keywords are more specific than topics and typically correspond to individual words or short multi-word expressions found in or closely related to the text. Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "transformer", + "attention mechanism", + "BERT" + ] + } + ] + }, + "topics": { + "anyOf": [ + { + "$ref": "#/$defs/TopicsMetaField" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Higher-level subject categories or thematic labels inferred for the node content. Topics are broader than keywords and describe the domain or theme rather than specific terms (e.g., 'machine learning' rather than 'gradient descent'). Values are order-preserving and deduplicated.", + "examples": [ + { + "values": [ + "natural language processing", + "computer vision" + ] + } + ] + }, + "description": { + "anyOf": [ + { + "$ref": "#/$defs/DescriptionMetaField" + }, + { + "type": "null" + } + ], + "default": null + }, + "classification": { + "anyOf": [ + { + "$ref": "#/$defs/PictureClassificationMetaField" + }, + { + "type": "null" + } + ], + "default": null + }, + "molecule": { + "anyOf": [ + { + "$ref": "#/$defs/MoleculeMetaField" + }, + { + "type": "null" + } + ], + "default": null + }, + "tabular_chart": { + "anyOf": [ + { + "$ref": "#/$defs/TabularChartMetaField" + }, + { + "type": "null" + } + ], + "default": null + }, + "code": { + "anyOf": [ + { + "$ref": "#/$defs/CodeMetaField" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "PictureMeta", + "type": "object" + }, + "PictureMoleculeData": { + "description": "PictureMoleculeData.", + "properties": { + "kind": { + "const": "molecule_data", + "default": "molecule_data", + "title": "Kind", + "type": "string" + }, + "smi": { + "title": "Smi", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "class_name": { + "title": "Class Name", + "type": "string" + }, + "segmentation": { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "type": "array" + }, + "title": "Segmentation", + "type": "array" + }, + "provenance": { + "title": "Provenance", + "type": "string" + } + }, + "required": [ + "smi", + "confidence", + "class_name", + "segmentation", + "provenance" + ], + "title": "PictureMoleculeData", + "type": "object" + }, + "PicturePieChartData": { + "description": "Represents data of a pie chart.\n\nAttributes:\n kind (Literal[\"pie_chart_data\"]): The type of the chart.\n slices (list[ChartSlice]): A list of slices in the pie chart.", + "properties": { + "kind": { + "const": "pie_chart_data", + "default": "pie_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "slices": { + "items": { + "$ref": "#/$defs/ChartSlice" + }, + "title": "Slices", + "type": "array" + } + }, + "required": [ + "title", + "slices" + ], + "title": "PicturePieChartData", + "type": "object" + }, + "PictureScatterChartData": { + "description": "Represents data of a scatter chart.\n\nAttributes:\n kind (Literal[\"scatter_chart_data\"]): The type of the chart.\n x_axis_label (str): The label for the x-axis.\n y_axis_label (str): The label for the y-axis.\n points (list[ChartPoint]): A list of points in the scatter chart.", + "properties": { + "kind": { + "const": "scatter_chart_data", + "default": "scatter_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "x_axis_label": { + "title": "X Axis Label", + "type": "string" + }, + "y_axis_label": { + "title": "Y Axis Label", + "type": "string" + }, + "points": { + "items": { + "$ref": "#/$defs/ChartPoint" + }, + "title": "Points", + "type": "array" + } + }, + "required": [ + "title", + "x_axis_label", + "y_axis_label", + "points" + ], + "title": "PictureScatterChartData", + "type": "object" + }, + "PictureStackedBarChartData": { + "description": "Represents data of a stacked bar chart.\n\nAttributes:\n kind (Literal[\"stacked_bar_chart_data\"]): The type of the chart.\n x_axis_label (str): The label for the x-axis.\n y_axis_label (str): The label for the y-axis.\n stacked_bars (list[ChartStackedBar]): A list of stacked bars in the chart.", + "properties": { + "kind": { + "const": "stacked_bar_chart_data", + "default": "stacked_bar_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "x_axis_label": { + "title": "X Axis Label", + "type": "string" + }, + "y_axis_label": { + "title": "Y Axis Label", + "type": "string" + }, + "stacked_bars": { + "items": { + "$ref": "#/$defs/ChartStackedBar" + }, + "title": "Stacked Bars", + "type": "array" + } + }, + "required": [ + "title", + "x_axis_label", + "y_axis_label", + "stacked_bars" + ], + "title": "PictureStackedBarChartData", + "type": "object" + }, + "PictureTabularChartData": { + "description": "Base class for picture chart data.\n\nAttributes:\n title (str): The title of the chart.\n chart_data (TableData): Chart data in the table format.", + "properties": { + "kind": { + "const": "tabular_chart_data", + "default": "tabular_chart_data", + "title": "Kind", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "chart_data": { + "$ref": "#/$defs/TableData" + } + }, + "required": [ + "title", + "chart_data" + ], + "title": "PictureTabularChartData", + "type": "object" + }, + "ProvenanceItem": { + "description": "Provenance information for elements extracted from a textual document.\n\nA `ProvenanceItem` object acts as a lightweight pointer back into the original\ndocument for an extracted element. It applies to documents with an explicit\nor implicit layout, such as PDF, HTML, docx, or pptx.", + "properties": { + "page_no": { + "description": "Page number", + "title": "Page No", + "type": "integer" + }, + "bbox": { + "$ref": "#/$defs/BoundingBox", + "description": "Bounding box" + }, + "charspan": { + "description": "Character span (0-indexed)", + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "title": "Charspan", + "type": "array" + } + }, + "required": [ + "page_no", + "bbox", + "charspan" + ], + "title": "ProvenanceItem", + "type": "object" + }, + "RefItem": { + "description": "RefItem.", + "properties": { + "$ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "$Ref", + "type": "string" + } + }, + "required": [ + "$ref" + ], + "title": "RefItem", + "type": "object" + }, + "RichTableCell": { + "description": "RichTableCell.", + "properties": { + "bbox": { + "anyOf": [ + { + "$ref": "#/$defs/BoundingBox" + }, + { + "type": "null" + } + ], + "default": null + }, + "row_span": { + "default": 1, + "title": "Row Span", + "type": "integer" + }, + "col_span": { + "default": 1, + "title": "Col Span", + "type": "integer" + }, + "start_row_offset_idx": { + "title": "Start Row Offset Idx", + "type": "integer" + }, + "end_row_offset_idx": { + "title": "End Row Offset Idx", + "type": "integer" + }, + "start_col_offset_idx": { + "title": "Start Col Offset Idx", + "type": "integer" + }, + "end_col_offset_idx": { + "title": "End Col Offset Idx", + "type": "integer" + }, + "text": { + "title": "Text", + "type": "string" + }, + "column_header": { + "default": false, + "title": "Column Header", + "type": "boolean" + }, + "row_header": { + "default": false, + "title": "Row Header", + "type": "boolean" + }, + "row_section": { + "default": false, + "title": "Row Section", + "type": "boolean" + }, + "fillable": { + "default": false, + "title": "Fillable", + "type": "boolean" + }, + "ref": { + "$ref": "#/$defs/RefItem" + } + }, + "required": [ + "start_row_offset_idx", + "end_row_offset_idx", + "start_col_offset_idx", + "end_col_offset_idx", + "text", + "ref" + ], + "title": "RichTableCell", + "type": "object" + }, + "Script": { + "description": "Text script position.", + "enum": [ + "baseline", + "sub", + "super" + ], + "title": "Script", + "type": "string" + }, + "SectionHeaderItem": { + "additionalProperties": false, + "description": "SectionItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "section_header", + "default": "section_header", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + }, + "level": { + "default": 1, + "maximum": 100, + "minimum": 1, + "title": "Level", + "type": "integer" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "SectionHeaderItem", + "type": "object" + }, + "Size": { + "description": "Size.", + "properties": { + "width": { + "default": 0.0, + "title": "Width", + "type": "number" + }, + "height": { + "default": 0.0, + "title": "Height", + "type": "number" + } + }, + "title": "Size", + "type": "object" + }, + "SummaryMetaField": { + "additionalProperties": true, + "description": "Summary data.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "SummaryMetaField", + "type": "object" + }, + "TableCell": { + "description": "TableCell.", + "properties": { + "bbox": { + "anyOf": [ + { + "$ref": "#/$defs/BoundingBox" + }, + { + "type": "null" + } + ], + "default": null + }, + "row_span": { + "default": 1, + "title": "Row Span", + "type": "integer" + }, + "col_span": { + "default": 1, + "title": "Col Span", + "type": "integer" + }, + "start_row_offset_idx": { + "title": "Start Row Offset Idx", + "type": "integer" + }, + "end_row_offset_idx": { + "title": "End Row Offset Idx", + "type": "integer" + }, + "start_col_offset_idx": { + "title": "Start Col Offset Idx", + "type": "integer" + }, + "end_col_offset_idx": { + "title": "End Col Offset Idx", + "type": "integer" + }, + "text": { + "title": "Text", + "type": "string" + }, + "column_header": { + "default": false, + "title": "Column Header", + "type": "boolean" + }, + "row_header": { + "default": false, + "title": "Row Header", + "type": "boolean" + }, + "row_section": { + "default": false, + "title": "Row Section", + "type": "boolean" + }, + "fillable": { + "default": false, + "title": "Fillable", + "type": "boolean" + } + }, + "required": [ + "start_row_offset_idx", + "end_row_offset_idx", + "start_col_offset_idx", + "end_col_offset_idx", + "text" + ], + "title": "TableCell", + "type": "object" + }, + "TableData": { + "description": "BaseTableData.", + "properties": { + "table_cells": { + "default": [], + "items": { + "anyOf": [ + { + "$ref": "#/$defs/RichTableCell" + }, + { + "$ref": "#/$defs/TableCell" + } + ] + }, + "title": "Table Cells", + "type": "array" + }, + "num_rows": { + "default": 0, + "title": "Num Rows", + "type": "integer" + }, + "num_cols": { + "default": 0, + "title": "Num Cols", + "type": "integer" + }, + "orientation": { + "$ref": "#/$defs/Orientation", + "default": "rot_0" + } + }, + "title": "TableData", + "type": "object" + }, + "TableItem": { + "additionalProperties": false, + "description": "TableItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/FloatingMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "default": "table", + "enum": [ + "document_index", + "table" + ], + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "captions": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Captions", + "type": "array" + }, + "references": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "References", + "type": "array" + }, + "footnotes": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Footnotes", + "type": "array" + }, + "image": { + "anyOf": [ + { + "$ref": "#/$defs/ImageRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "data": { + "$ref": "#/$defs/TableData" + }, + "annotations": { + "default": [], + "deprecated": true, + "items": { + "discriminator": { + "mapping": { + "description": "#/$defs/DescriptionAnnotation", + "misc": "#/$defs/MiscAnnotation" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/DescriptionAnnotation" + }, + { + "$ref": "#/$defs/MiscAnnotation" + } + ] + }, + "title": "Annotations", + "type": "array" + } + }, + "required": [ + "self_ref", + "data" + ], + "title": "TableItem", + "type": "object" + }, + "TabularChartMetaField": { + "additionalProperties": true, + "description": "Tabular chart metadata field.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Title" + }, + "chart_data": { + "$ref": "#/$defs/TableData" + } + }, + "required": [ + "chart_data" + ], + "title": "TabularChartMetaField", + "type": "object" + }, + "TextItem": { + "additionalProperties": false, + "description": "TextItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "enum": [ + "caption", + "checkbox_selected", + "checkbox_unselected", + "footnote", + "page_footer", + "page_header", + "paragraph", + "reference", + "text", + "empty_value", + "field_key", + "field_hint", + "marker", + "handwritten_text" + ], + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + } + }, + "required": [ + "self_ref", + "label", + "orig", + "text" + ], + "title": "TextItem", + "type": "object" + }, + "TitleItem": { + "additionalProperties": false, + "description": "TitleItem.", + "properties": { + "self_ref": { + "pattern": "^#(?:/([\\w-]+)(?:/(\\d+))?)?$", + "title": "Self Ref", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "$ref": "#/$defs/RefItem" + }, + { + "type": "null" + } + ], + "default": null + }, + "children": { + "default": [], + "items": { + "$ref": "#/$defs/RefItem" + }, + "title": "Children", + "type": "array" + }, + "content_layer": { + "$ref": "#/$defs/ContentLayer", + "default": "body" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/BaseMeta" + }, + { + "type": "null" + } + ], + "default": null + }, + "label": { + "const": "title", + "default": "title", + "title": "Label", + "type": "string" + }, + "prov": { + "default": [], + "items": { + "$ref": "#/$defs/ProvenanceItem" + }, + "title": "Prov", + "type": "array" + }, + "source": { + "default": [], + "description": "The provenance of this document item. Currently, it is only used for media track provenance.", + "items": { + "discriminator": { + "mapping": { + "track": "#/$defs/TrackSource" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/TrackSource" + } + ] + }, + "title": "Source", + "type": "array" + }, + "comments": { + "default": [], + "items": { + "$ref": "#/$defs/FineRef" + }, + "title": "Comments", + "type": "array" + }, + "orig": { + "title": "Orig", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "formatting": { + "anyOf": [ + { + "$ref": "#/$defs/Formatting" + }, + { + "type": "null" + } + ], + "default": null + }, + "hyperlink": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hyperlink" + } + }, + "required": [ + "self_ref", + "orig", + "text" + ], + "title": "TitleItem", + "type": "object" + }, + "TopicsMetaField": { + "additionalProperties": true, + "description": "Container for a list of unique topics / subjects.", + "properties": { + "values": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Values", + "type": "array", + "uniqueItems": true + } + }, + "required": [ + "values" + ], + "title": "TopicsMetaField", + "type": "object" + }, + "TrackSource": { + "description": "Source metadata for a cue extracted from a media track.\n\nA `TrackSource` instance identifies a cue in a media track (audio, video, subtitles, screen-recording captions,\netc.). A *cue* here refers to any discrete segment that was pulled out of the original asset, e.g., a subtitle\nblock, an audio clip, or a timed marker in a screen-recording.", + "properties": { + "kind": { + "const": "track", + "default": "track", + "description": "Identifies this type of source.", + "title": "Kind", + "type": "string" + }, + "start_time": { + "description": "Start time offset of the track cue in seconds", + "examples": [ + 11.0, + 6.5, + 5370.0 + ], + "title": "Start Time", + "type": "number" + }, + "end_time": { + "description": "End time offset of the track cue in seconds", + "examples": [ + 12.0, + 8.2, + 5370.1 + ], + "title": "End Time", + "type": "number" + }, + "identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An identifier of the cue", + "examples": [ + "test", + "123", + "b72d946" + ], + "title": "Identifier" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The name of the voice in this track (the speaker)", + "examples": [ + "John", + "Mary", + "Speaker 1" + ], + "title": "Voice" + } + }, + "required": [ + "start_time", + "end_time" + ], + "title": "TrackSource", + "type": "object" + } + }, + "description": "DoclingDocument.", + "properties": { + "schema_name": { + "const": "DoclingDocument", + "default": "DoclingDocument", + "title": "Schema Name", + "type": "string" + }, + "version": { + "default": "1.11.0", + "pattern": "^(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)(?:-(?P(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "title": "Version", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "origin": { + "anyOf": [ + { + "$ref": "#/$defs/DocumentOrigin" + }, + { + "type": "null" + } + ], + "default": null + }, + "furniture": { + "$ref": "#/$defs/GroupItem", + "default": { + "self_ref": "#/furniture", + "parent": null, + "children": [], + "content_layer": "furniture", + "meta": null, + "name": "_root_", + "label": "unspecified" + }, + "deprecated": true + }, + "body": { + "$ref": "#/$defs/GroupItem", + "default": { + "self_ref": "#/body", + "parent": null, + "children": [], + "content_layer": "body", + "meta": null, + "name": "_root_", + "label": "unspecified" + } + }, + "groups": { + "default": [], + "items": { + "anyOf": [ + { + "$ref": "#/$defs/ListGroup" + }, + { + "$ref": "#/$defs/InlineGroup" + }, + { + "$ref": "#/$defs/GroupItem" + } + ] + }, + "title": "Groups", + "type": "array" + }, + "texts": { + "default": [], + "items": { + "anyOf": [ + { + "$ref": "#/$defs/TitleItem" + }, + { + "$ref": "#/$defs/SectionHeaderItem" + }, + { + "$ref": "#/$defs/ListItem" + }, + { + "$ref": "#/$defs/CodeItem" + }, + { + "$ref": "#/$defs/FormulaItem" + }, + { + "$ref": "#/$defs/FieldHeadingItem" + }, + { + "$ref": "#/$defs/FieldValueItem" + }, + { + "$ref": "#/$defs/TextItem" + } + ] + }, + "title": "Texts", + "type": "array" + }, + "pictures": { + "default": [], + "items": { + "$ref": "#/$defs/PictureItem" + }, + "title": "Pictures", + "type": "array" + }, + "tables": { + "default": [], + "items": { + "$ref": "#/$defs/TableItem" + }, + "title": "Tables", + "type": "array" + }, + "key_value_items": { + "default": [], + "items": { + "$ref": "#/$defs/KeyValueItem" + }, + "title": "Key Value Items", + "type": "array" + }, + "form_items": { + "default": [], + "items": { + "$ref": "#/$defs/FormItem" + }, + "title": "Form Items", + "type": "array" + }, + "field_regions": { + "default": [], + "items": { + "$ref": "#/$defs/FieldRegionItem" + }, + "title": "Field Regions", + "type": "array" + }, + "field_items": { + "default": [], + "items": { + "$ref": "#/$defs/FieldItem" + }, + "title": "Field Items", + "type": "array" + }, + "pages": { + "additionalProperties": { + "$ref": "#/$defs/PageItem" + }, + "default": {}, + "title": "Pages", + "type": "object" + } + }, + "required": [ + "name" + ], + "title": "DoclingDocument", + "type": "object" +} \ No newline at end of file diff --git a/packages/dclq/tests/test_cli.py b/packages/dclq/tests/test_cli.py index 61b7bef7b..cf5a41808 100644 --- a/packages/dclq/tests/test_cli.py +++ b/packages/dclq/tests/test_cli.py @@ -259,7 +259,7 @@ def test_text_output_collapses_formatting_and_marks_truncation(tmp_path: Path, c assert main(["show", str(path), "/formula[1]", "-B", "1", "-n"]) == 0 output = capsys.readouterr().out - assert "/code[1]-public class Main {}" in output + assert "/code[1]-public class Main {}" in output assert "/formula[1]:$$E = mc^2$$" in output assert "/d:" not in output assert "\n--\n" not in output diff --git a/test/test_compat.py b/test/test_compat.py index 88d70c6ca..a7d607624 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -243,6 +243,45 @@ def test_every_projector_has_a_test_class(self): # --------------------------------------------------------------------------- +class TestProjector_1_11_to_1_10: + """Tests for the 1.11 → 1.10 downgrade projector.""" + + def _apply(self, data: dict) -> dict: + from docling_core.compat import _project_1_11_to_1_10 + + return _project_1_11_to_1_10(data) + + def test_version_set_to_1_10(self): + result = self._apply(_minimal_doc_dict("1.11.0")) + assert result["version"] == "1.10.0" + + def test_inline_run_whitespace_is_preserved_verbatim(self): + """The projector must not touch run text: 1.11 runs own their boundary + whitespace, and a 1.10 client relies on that text being intact (it adds + its own join space on top). Stripping here would re-introduce word-merge. + """ + data = _minimal_doc_dict("1.11.0") + data["texts"] = [ + { + "self_ref": "#/texts/0", + "parent": None, + "children": [], + "content_layer": "body", + "label": "text", + "orig": "bold ", + "text": "bold ", + "prov": [], + } + ] + result = self._apply(data) + assert result["texts"][0]["text"] == "bold " + + def test_does_not_mutate_caller_input(self): + data = _minimal_doc_dict("1.11.0") + self._apply(data) + assert data["version"] == "1.11.0" + + class TestProjector_1_10_to_1_9: """Tests for the 1.10 → 1.9 downgrade projector.""" From 6a8b8374e0285a88b067535400e2af352ee8f758 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Wed, 19 Aug 2026 10:08:09 +0200 Subject: [PATCH 05/18] fix(dclq): use typer's vendored click; require typer>=0.25 typer >=0.25 vendors its own copy of click (typer._click) and raises those exceptions instead of the external click package's. The dclq CLI referenced the external click, so on typer >=0.25 bad input escaped the exit-code handlers and the context-flag callback (-C/-A/-B/--context-scope) silently no-op'd because the isinstance(param, click.Option) check no longer matched. Import the exception and Parameter types from typer._click, duck-type the option check, raise the [dclq] typer floor to >=0.25, and drop the now-unused external click dependency. Signed-off-by: Christoph Auer --- packages/dclq/dclq/cli.py | 14 +++++++++----- pyproject.toml | 3 +-- uv.lock | 4 +--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/dclq/dclq/cli.py b/packages/dclq/dclq/cli.py index 898f176ab..d045bf862 100644 --- a/packages/dclq/dclq/cli.py +++ b/packages/dclq/dclq/cli.py @@ -12,10 +12,14 @@ from types import SimpleNamespace from typing import Annotated, Any, Literal -import click import typer from lxml import etree +# typer >=0.25 vendors its own copy of click (typer._click) and raises those +# exceptions rather than the external click package's, so catch the vendored ones. +from typer._click import Parameter as _ClickParameter +from typer._click import exceptions as _click_exc + from dclq import __version__ from dclq.document import DclqError, LoadedDocument, Unit, _canonical_xpath, _is_element from docling_core.experimental.serializer.outline import ( @@ -74,9 +78,9 @@ def _main_callback( pass -def _record_context(ctx: typer.Context, param: click.Parameter, value: int | None) -> int | None: +def _record_context(ctx: typer.Context, param: _ClickParameter, value: int | None) -> int | None: """Record context flags in command-line order, matching grep semantics.""" - if value is not None and isinstance(param, click.Option): + if value is not None and getattr(param, "param_type_name", None) == "option": ctx.meta.setdefault("context_events", []).append((param.opts[0], value)) return value @@ -347,9 +351,9 @@ def main(argv: Sequence[str] | None = None) -> int: except DclqError as exc: print(f"dclq: {exc}", file=sys.stderr) return 2 - except click.exceptions.Exit as exc: + except _click_exc.Exit as exc: return exc.exit_code - except click.ClickException as exc: + except _click_exc.ClickException as exc: exc.show(file=sys.stderr) return exc.exit_code diff --git a/pyproject.toml b/pyproject.toml index 79a9bcb0c..e08d0c83c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,8 +70,7 @@ docling-serialize = "docling_core.cli.serialize:app" [project.optional-dependencies] dclq = [ "doclang>=0.7,<0.8", - "typer>=0.15.1,<0.25.0", - "click>=8.0.0", + "typer>=0.25.0,<0.27.0", "lxml>=6.0.2", ] chunking = [ diff --git a/uv.lock b/uv.lock index 784cba978..5396d6feb 100644 --- a/uv.lock +++ b/uv.lock @@ -1048,7 +1048,6 @@ chunking-openai = [ { name = "tree-sitter-typescript" }, ] dclq = [ - { name = "click" }, { name = "doclang" }, { name = "lxml" }, { name = "typer" }, @@ -1089,7 +1088,6 @@ dev = [ [package.metadata] requires-dist = [ - { name = "click", marker = "extra == 'dclq'", specifier = ">=8.0.0" }, { name = "datasets", marker = "extra == 'examples'", specifier = ">=4.0.0" }, { name = "defusedxml", specifier = ">=0.7.1,<0.8.0" }, { name = "doclang", specifier = ">=0.7,<0.8" }, @@ -1122,7 +1120,7 @@ requires-dist = [ { name = "tree-sitter-typescript", marker = "extra == 'chunking'", specifier = ">=0.23.2" }, { name = "tree-sitter-typescript", marker = "extra == 'chunking-openai'", specifier = ">=0.23.2" }, { name = "typer", specifier = ">=0.12.5,<0.27.0" }, - { name = "typer", marker = "extra == 'dclq'", specifier = ">=0.15.1,<0.25.0" }, + { name = "typer", marker = "extra == 'dclq'", specifier = ">=0.25.0,<0.27.0" }, { name = "typing-extensions", specifier = ">=4.12.2,<5.0.0" }, ] provides-extras = ["dclq", "chunking", "chunking-openai", "examples"] From 1fcf2e9f30a807996ed0e6a3f99b32db0110c8be Mon Sep 17 00:00:00 2001 From: nanhanq1 <157583847+nanhanq1@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:17:11 +0800 Subject: [PATCH 06/18] fix(doclang): keep inherited formatting on runs in inline groups is the explicit whitespace-preserving text wrapper, so inside an inline group it is a run, not a nested item: it must inherit the formatting accumulated while descending through //... . Dispatching it through _dispatch_element dropped that formatting, which surfaced after the rebase as x coming back unformatted and broke the doclang round-trip for whitespace_inside_bold / _link. Signed-off-by: nanhanq1 <157583847+nanhanq1@users.noreply.github.com> --- docling_core/transforms/deserializer/doclang.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docling_core/transforms/deserializer/doclang.py b/docling_core/transforms/deserializer/doclang.py index b129e4cfc..3ebe09900 100644 --- a/docling_core/transforms/deserializer/doclang.py +++ b/docling_core/transforms/deserializer/doclang.py @@ -1536,7 +1536,20 @@ def _parse_inline_runs( """Add text runs for ``nodes``, carrying ``formatting`` through nested formatting tags.""" for node in nodes: if isinstance(node, Element): - if node.tagName in _FORMATTING_TAG_VALUES: + if node.tagName == DocLangToken.CONTENT.value: + # is the explicit whitespace-preserving text wrapper: it is a run, + # not a nested item, so it inherits the formatting accumulated on the way in. + # Dispatching it would drop that formatting (x + # would come back unformatted). + text_content = self._collect_text(node) + if text_content: + doc.add_text( + label=DocItemLabel.TEXT, + text=text_content, + parent=parent, + formatting=formatting, + ) + elif node.tagName in _FORMATTING_TAG_VALUES: # Accumulate this tag's style and keep descending, so e.g. a # b gives "a" bold and "b" bold+italic. merged = self._merge_formatting_tag(node.tagName, formatting) From 16ba1700c39ac88dabe70f94393a5e80b7916559 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Fri, 23 Oct 2026 14:17:54 +0200 Subject: [PATCH 07/18] chore(compat): point schema 1.11.0 at the next minor release The whitespace-contract bump was first written against v2.86.0; main is now at v2.104.0, so record v2.105.0 as the placeholder release that ships schema 1.11.0. Adjust to the actual version at release time. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/compat.py | 2 +- docling_core/types/doc/common/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docling_core/compat.py b/docling_core/compat.py index 8ea029d7a..73c16d16a 100644 --- a/docling_core/compat.py +++ b/docling_core/compat.py @@ -199,7 +199,7 @@ def list_projectors() -> list[tuple[int, int]]: # possible; the goal is a document that validates rather than crashes. # --- 1.11 → 1.10 ---------------------------------------------------------- -# Schema 1.11 (docling-core v2.86.0): InlineGroup runs carry their own +# Schema 1.11 (docling-core v2.105.0): InlineGroup runs carry their own # significant whitespace and serializers concatenate them faithfully. # # The 1.11 and 1.10 schemas are structurally identical -- no new fields, diff --git a/docling_core/types/doc/common/constants.py b/docling_core/types/doc/common/constants.py index 60b3fd92f..67eb75770 100644 --- a/docling_core/types/doc/common/constants.py +++ b/docling_core/types/doc/common/constants.py @@ -22,7 +22,7 @@ {"schema": "1.10.0", "library_version": "v2.69.0", "note": "FieldItem types, 7 new DocItemLabel values"}, { "schema": "1.11.0", - "library_version": "v2.86.0", + "library_version": "v2.105.0", "note": "InlineGroup runs carry their own significant whitespace", }, ] From 0d47d03f1fe3c1815c33e5dc4252633c6c152851 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Fri, 23 Oct 2026 18:53:28 +0200 Subject: [PATCH 08/18] test: rebaseline goldens for the inline whitespace contract Regenerated with DOCLING_GEN_TEST_DATA=1. The only change is the schema version stamp 1.10.0 -> 1.11.0 in goldens added or moved on main (including the six test/ -> tests/ renames and misplaced_list_items.out.yaml). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- tests/data/doc/2408.09869v3_enriched.out.dt.json | 2 +- tests/data/doc/barchart.dt.out.json | 2 +- tests/data/doc/doc_with_kv.dt.json | 2 +- tests/data/doc/doclang_archive/load/two_pages.gt.json | 2 +- tests/data/doc/misplaced_list_items.out.yaml | 2 +- tests/data/doc/multi_captions_footnotes.deserialized.gt.json | 2 +- .../doc/multi_captions_footnotes_top_level.deserialized.gt.json | 2 +- tests/data/doc/page_with_pic.dt.json | 2 +- tests/data/doc/page_with_pic_from_files.dt.json | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/data/doc/2408.09869v3_enriched.out.dt.json b/tests/data/doc/2408.09869v3_enriched.out.dt.json index 0e03e45f1..d3ef38582 100644 --- a/tests/data/doc/2408.09869v3_enriched.out.dt.json +++ b/tests/data/doc/2408.09869v3_enriched.out.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/barchart.dt.out.json b/tests/data/doc/barchart.dt.out.json index 655c9a2ed..8859b75f9 100644 --- a/tests/data/doc/barchart.dt.out.json +++ b/tests/data/doc/barchart.dt.out.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/doc_with_kv.dt.json b/tests/data/doc/doc_with_kv.dt.json index 352900710..03e4944d2 100644 --- a/tests/data/doc/doc_with_kv.dt.json +++ b/tests/data/doc/doc_with_kv.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/doclang_archive/load/two_pages.gt.json b/tests/data/doc/doclang_archive/load/two_pages.gt.json index a19789fd6..50647c241 100644 --- a/tests/data/doc/doclang_archive/load/two_pages.gt.json +++ b/tests/data/doc/doclang_archive/load/two_pages.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "two_pages", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/misplaced_list_items.out.yaml b/tests/data/doc/misplaced_list_items.out.yaml index 3daa2763e..0771ba3bd 100644 --- a/tests/data/doc/misplaced_list_items.out.yaml +++ b/tests/data/doc/misplaced_list_items.out.yaml @@ -81,4 +81,4 @@ texts: prov: [] self_ref: '#/texts/3' text: there -version: 1.10.0 +version: 1.11.0 diff --git a/tests/data/doc/multi_captions_footnotes.deserialized.gt.json b/tests/data/doc/multi_captions_footnotes.deserialized.gt.json index 48fc4c6a6..0128145fe 100644 --- a/tests/data/doc/multi_captions_footnotes.deserialized.gt.json +++ b/tests/data/doc/multi_captions_footnotes.deserialized.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/multi_captions_footnotes_top_level.deserialized.gt.json b/tests/data/doc/multi_captions_footnotes_top_level.deserialized.gt.json index 48fc4c6a6..0128145fe 100644 --- a/tests/data/doc/multi_captions_footnotes_top_level.deserialized.gt.json +++ b/tests/data/doc/multi_captions_footnotes_top_level.deserialized.gt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/page_with_pic.dt.json b/tests/data/doc/page_with_pic.dt.json index d53c9c46a..9e548f2b4 100644 --- a/tests/data/doc/page_with_pic.dt.json +++ b/tests/data/doc/page_with_pic.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", diff --git a/tests/data/doc/page_with_pic_from_files.dt.json b/tests/data/doc/page_with_pic_from_files.dt.json index d53c9c46a..9e548f2b4 100644 --- a/tests/data/doc/page_with_pic_from_files.dt.json +++ b/tests/data/doc/page_with_pic_from_files.dt.json @@ -1,6 +1,6 @@ { "schema_name": "DoclingDocument", - "version": "1.10.0", + "version": "1.11.0", "name": "Document", "furniture": { "self_ref": "#/furniture", From d832f336c7bbb4b90b41695ab7cef5b99f66f8c7 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Fri, 23 Oct 2026 18:54:25 +0200 Subject: [PATCH 09/18] fix: join inline runs faithfully in the plain-text reader _collect_subtree_text (TextItem._get_plain_text, FloatingItem.caption_text, nested rich table cells) still joined the runs of an InlineGroup with " ", so mixed captions read "Table 1 : results". It now concatenates the runs of an inline group; every other join, including a host text item's own text and its children, keeps its space. Children of a text item are block structure (producers put a section's paragraphs under its heading), so the host text stays apart from its inline group, as on main; add_inline_group documents this. Tests that asserted the invented run spacing now expect the contract output, fixtures that built runs without boundary whitespace now carry it, and a new test pins the host-text boundary for headings and captions. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- .../transforms/serializer/markdown.py | 8 ++-- docling_core/types/doc/document.py | 4 ++ tests/test_deserializer_doclang.py | 45 ++++++++++++------- tests/test_hierarchical_chunker.py | 6 +-- tests/test_inline_whitespace_contract.py | 33 ++++++++++++++ tests/test_latex_serialization.py | 9 ++-- tests/test_serialization.py | 4 +- tests/test_serialization_doclang.py | 2 +- 8 files changed, 81 insertions(+), 30 deletions(-) diff --git a/docling_core/transforms/serializer/markdown.py b/docling_core/transforms/serializer/markdown.py index 57ff4cba3..4363c904e 100644 --- a/docling_core/transforms/serializer/markdown.py +++ b/docling_core/transforms/serializer/markdown.py @@ -173,8 +173,9 @@ def _mark_subtree_visited( def _collect_subtree_text(item: NodeItem, doc: DoclingDocument, excluded_refs: set[str]) -> str: """Collect all text from *item*'s subtree, flattening nested tables. - Returns a space-joined string of every piece of text found so that the - content of a nested table is preserved in a flat, readable form. + Returns every piece of text found, joined with spaces (the runs of an inline + group are concatenated as they are), so that the content of a nested table is + preserved in a flat, readable form. For TableItems the text is pulled from ``data.grid`` cells directly; RichTableCells are resolved through their references, without visiting the @@ -213,7 +214,8 @@ def _collect_subtree_text(item: NodeItem, doc: DoclingDocument, excluded_refs: s if child_text: parts.append(child_text) - return " ".join(parts) + # Inline runs carry their own boundary whitespace; everything else is joined as blocks. + return ("" if isinstance(item, InlineGroup) else " ").join(parts) def _longest_backtick_run(text: str) -> int: diff --git a/docling_core/types/doc/document.py b/docling_core/types/doc/document.py index 6e47fe8a3..02cd8afa1 100644 --- a/docling_core/types/doc/document.py +++ b/docling_core/types/doc/document.py @@ -1568,6 +1568,10 @@ def add_inline_group( put the boundary in the run text: ``["Advanced Topics", " in ", "Machine Learning"]``. Whitespace at the edge of a formatted run is fine -- the serializers move it outside the emphasis markers and hyperlink. + + Children of a text item are block structure: serializers keep a parent text item's own + text apart from this group (by a space or a block break). To put text flush against the + runs, make it a run of the group. """ _parent = parent or self.body cref = f"#/groups/{len(self.groups)}" diff --git a/tests/test_deserializer_doclang.py b/tests/test_deserializer_doclang.py index 74f08ff87..dda37e549 100644 --- a/tests/test_deserializer_doclang.py +++ b/tests/test_deserializer_doclang.py @@ -2610,7 +2610,7 @@ def test_mixed_text_like_archive_roundtrip( for node, _ in doc.iterate_items(root=root, with_groups=True, included_content_layers=set(ContentLayer)) ) runs = [ref.resolve(doc) for ref in inline.children] - assert [run.text for run in runs] == ["Before", "x" if "formula" in body else "bold", "after"] + assert [run.text for run in runs] == ["Before ", "x" if "formula" in body else "bold", " after"] if "formula" in body: assert runs[1].label == DocItemLabel.FORMULA else: @@ -2639,16 +2639,22 @@ def test_plain_caption_layer_roundtrip(kind: str, layer: ContentLayer, tmp_path: @pytest.mark.parametrize("kind", ["table", "picture"]) @pytest.mark.parametrize( - ("caption", "expected", "markdown"), + ("caption", "expected", "markdown", "doctags"), [ - ("Table 1: results", "Table 1: results", "Table 1: results"), - ("Table 1: results", "Table 1 : results", "Table **1** : results"), - ("Table 1: results", "Table 1 : results", "**Table** ***1*** : results"), + ("Table 1: results", "Table 1: results", "Table 1: results", "Table 1: results"), + # DocTags predates the whitespace contract: it strips each run and puts them on separate lines. + ("Table 1: results", "Table 1: results", "Table **1**: results", "Table 1 : results"), + ( + "Table 1: results", + "Table 1: results", + "**Table** ***1***: results", + "Table 1 : results", + ), ], ) @pytest.mark.parametrize("layer", [ContentLayer.BODY, ContentLayer.FURNITURE]) def test_caption_readers_after_archive_roundtrip( - kind: str, caption: str, expected: str, markdown: str, layer: ContentLayer, tmp_path: Path + kind: str, caption: str, expected: str, markdown: str, doctags: str, layer: ContentLayer, tmp_path: Path ): table_body = "AB12" if kind == "table" else "" doc = _deserialize( @@ -2665,14 +2671,15 @@ def test_caption_readers_after_archive_roundtrip( DocTagsDocSerializer(doc=doc), ]: result = serializer.serialize_captions(item=owner, layers=set(ContentLayer)).text - if isinstance(serializer, HTMLDocSerializer | DocTagsDocSerializer): + if isinstance(serializer, DocTagsDocSerializer): content = " ".join(text.strip() for text in ET.fromstring(result).itertext() if text.strip()) - assert content == expected - if isinstance(serializer, HTMLDocSerializer): - if "" in caption: - assert "1" in result - if "" in caption: - assert "" in result + assert content == doctags + elif isinstance(serializer, HTMLDocSerializer): + assert "".join(ET.fromstring(result).itertext()) == expected + if "" in caption: + assert "1" in result + if "" in caption: + assert "" in result elif isinstance(serializer, PlainTextDocSerializer): assert expected in result else: @@ -2694,10 +2701,10 @@ def test_native_caption_children_and_filters(kind: str, host_text: str): caption = doc.add_text(label=DocItemLabel.CAPTION, text=host_text) owner.captions.append(caption.get_ref()) inline = doc.add_inline_group(parent=caption) - doc.add_text(label=DocItemLabel.TEXT, text="CHILD", formatting=Formatting(bold=True), parent=inline) + doc.add_text(label=DocItemLabel.TEXT, text="CHILD ", formatting=Formatting(bold=True), parent=inline) doc.add_text(label=DocItemLabel.FORMULA, text="FORMULA", parent=inline) DoclingDocument.model_validate(doc.model_dump()) - assert owner.caption_text(doc) == " ".join(text for text in [host_text, "CHILD", "FORMULA"] if text) + assert owner.caption_text(doc) == " ".join(text for text in [host_text, "CHILD FORMULA"] if text) labels = {DocItemLabel.CAPTION, DocItemLabel.TEXT, DocItemLabel.TABLE, DocItemLabel.PICTURE} for serializer in [ MarkdownDocSerializer(doc=doc), @@ -2719,12 +2726,15 @@ def test_native_caption_children_and_filters(kind: str, host_text: str): if isinstance(serializer, HTMLDocSerializer): assert "CHILD" in result if host_text: - assert "HOST &" in result + # Host text and its inline group are joined with a space; the runs inside are not. + assert "HOST & CHILD" in result assert "&amp;" not in result elif isinstance(serializer, LaTeXDocSerializer): assert r"\textbf{CHILD}" in result elif isinstance(serializer, MarkdownDocSerializer) and not isinstance(serializer, PlainTextDocSerializer): assert "**CHILD**" in result + if host_text: + assert "HOST & **CHILD**" in result def test_rich_caption_html_direction(): @@ -2761,7 +2771,8 @@ def test_roundtrip_furniture_inline_group(): ("$q < 0$ and p <0.05 and $a<\\ln b$", ["$q < 0$ and p <0.05 and $a<\\ln b$"]), ("x]]> y", ["x]]> y"]), ("a\x18b", ["ab"]), - (" & d", ["a & b < c", "& d"]), + # The run after the CDATA section owns its leading space under the whitespace contract. + (" & d", ["a & b < c", " & d"]), ], ) def test_repair_unescaped_text(body: str, expected: list[str]): diff --git a/tests/test_hierarchical_chunker.py b/tests/test_hierarchical_chunker.py index f5d811f7d..9824dde2c 100644 --- a/tests/test_hierarchical_chunker.py +++ b/tests/test_hierarchical_chunker.py @@ -37,8 +37,8 @@ def test_chunk(): ("caption", "expected"), [ ("Table 1: results", "Table 1: results"), - ("Table 1: results", "Table **1** : results"), - ("Table x: results", "Table $x$ : results"), + ("Table 1: results", "Table **1**: results"), + ("Table x: results", "Table $x$: results"), ], ) def test_chunk_caption_stays_with_owner(kind: str, caption: str, expected: str, tmp_path: Path) -> None: @@ -74,7 +74,7 @@ def test_chunk_satellite_check_keeps_unrelated_inline_text() -> None: "After" ) chunks = [DocChunk.model_validate(chunk) for chunk in HierarchicalChunker().chunk(doc)] - assert [chunk.text for chunk in chunks] == ["Before **body**", "Table **1** : results\n\nA", "After"] + assert [chunk.text for chunk in chunks] == ["Before **body**", "Table **1**: results\n\nA", "After"] assert doc.tables[0].self_ref in {item.self_ref for item in chunks[1].meta.doc_items} diff --git a/tests/test_inline_whitespace_contract.py b/tests/test_inline_whitespace_contract.py index 57f38de36..1d7cad306 100644 --- a/tests/test_inline_whitespace_contract.py +++ b/tests/test_inline_whitespace_contract.py @@ -16,7 +16,9 @@ from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer from docling_core.transforms.serializer.doclang import DocLangDocSerializer, DocLangParams +from docling_core.transforms.serializer.html import HTMLDocSerializer from docling_core.transforms.serializer.latex import LaTeXDocSerializer +from docling_core.transforms.serializer.markdown import MarkdownDocSerializer from docling_core.types.doc.document import ( CURRENT_VERSION, DocItemLabel, @@ -208,6 +210,37 @@ def test_no_separator_is_invented_between_runs() -> None: assert doc.export_to_text() == "".join(text for text, _, _ in runs), name +def test_host_text_stays_apart_from_its_inline_group() -> None: + """Children of a text item are block structure, so the host text keeps its separator. + + Producers put a section's first paragraph under its heading; joining the two with "" would + read ``ResultsWe use bold words.``. Host boundaries match the pre-contract output; only the + runs inside the group are concatenated. + """ + doc = DoclingDocument(name="host") + heading = doc.add_heading(text="Results", level=1) + group = doc.add_inline_group(parent=heading) + for text, formatting in [("We use ", None), ("bold", _BOLD), (" words.", None)]: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=group, formatting=formatting) + caption = doc.add_text(label=DocItemLabel.CAPTION, text="Table") + picture = doc.add_picture() + picture.captions.append(caption.get_ref()) + caption_group = doc.add_inline_group(parent=caption) + doc.add_text(label=DocItemLabel.TEXT, text="1", parent=caption_group, formatting=_BOLD) + doc.add_text(label=DocItemLabel.TEXT, text=": results", parent=caption_group) + + assert heading._get_plain_text(doc) == "Results We use bold words." + assert doc.export_to_markdown().startswith("## Results\n\nWe use **bold** words.") + html = HTMLDocSerializer(doc=doc).serialize().text + assert "

Results

\nWe use bold words." in html + assert _doclang_compact(doc).startswith( + 'Results We use bold' + " words." + ) + assert picture.caption_text(doc) == "Table 1: results" + assert MarkdownDocSerializer(doc=doc).serialize_captions(item=picture).text == "Table **1**: results" + + # -------------------------------------------------------------------------------------------- # Load normalization for pre-contract documents # -------------------------------------------------------------------------------------------- diff --git a/tests/test_latex_serialization.py b/tests/test_latex_serialization.py index ec0d8a793..709dae51a 100644 --- a/tests/test_latex_serialization.py +++ b/tests/test_latex_serialization.py @@ -180,9 +180,10 @@ def test_inline_group_no_duplication(): # Formatted section heading via InlineGroup heading = doc.add_heading(text="", level=1) ig = doc.add_inline_group(parent=heading) + # Runs carry their own boundary whitespace; the serializer does not invent a separator. doc.add_text( label="text", - text="Partially formatted", + text="Partially formatted ", formatting=Formatting(italic=True), parent=ig, ) @@ -199,11 +200,11 @@ def test_inline_group_no_duplication(): # Each piece of content must appear exactly once in the body assert body.count("\\textit{Partially formatted} heading") == 1 - assert body.count("\\textbf{Term} : definition") == 1 + assert body.count("\\textbf{Term}: definition") == 1 # The content must be inside the LaTeX command, not on a standalone line assert "\\section{\\textit{Partially formatted} heading}" in body - assert "\\item \\textbf{Term} : definition" in body + assert "\\item \\textbf{Term}: definition" in body def test_latex_column_spanning_rich_cell_keeps_text_in_every_column(col_span_rich_table_doc): @@ -435,7 +436,7 @@ def test_latex_inline_group_child_meta_is_suppressed(): """Meta on items inside an InlineGroup must not produce comment blocks mid-sentence.""" doc = DoclingDocument(name="test") ig = doc.add_inline_group() - first = doc.add_text(label=DocItemLabel.TEXT, text="Alpha", parent=ig) + first = doc.add_text(label=DocItemLabel.TEXT, text="Alpha ", parent=ig) first.meta = BaseMeta(summary=SummaryMetaField(text="A summary")) doc.add_text(label=DocItemLabel.TEXT, text="Beta", parent=ig) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 17fa1ba67..37aa16ebc 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1764,7 +1764,7 @@ def test_html_formula_with_inline_group_is_not_double_escaped(inline_scope: bool parent = doc.add_inline_group(parent=doc.add_heading(text="", level=1)) formula = doc.add_formula(text="", parent=parent) group = doc.add_inline_group(parent=formula) - doc.add_text(label=DocItemLabel.TEXT, text="a ProvenanceItem: From 7a9a3343258c4af617ca3db7f79deb8390fa1562 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Fri, 23 Oct 2026 18:54:32 +0200 Subject: [PATCH 10/18] docs: note that load_from_doctags cannot restore inline run whitespace DocTags stores inline runs trimmed, so the boundary whitespace the inline whitespace contract expects is not in the source. Document the limitation instead of guessing separators back. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/types/doc/document.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docling_core/types/doc/document.py b/docling_core/types/doc/document.py index 02cd8afa1..09af94db5 100644 --- a/docling_core/types/doc/document.py +++ b/docling_core/types/doc/document.py @@ -4521,7 +4521,12 @@ def save_as_vtt( def load_from_doctags( # noqa: C901 doctag_document: DocTagsDocument, document_name: str = "Document" ) -> "DoclingDocument": - r"""Load Docling document from lists of DocTags and Images.""" + r"""Load Docling document from lists of DocTags and Images. + + Limitation: DocTags stores each ```` run trimmed, so the boundary whitespace that + the inline whitespace contract expects in the runs (see ``add_inline_group``) is not in the + source and is not restored. Serializing such an inline group joins its runs without spaces. + """ # Maps the recognized tag to a Docling label. # Code items will be given DocItemLabel.CODE tag_to_doclabel = { @@ -4900,6 +4905,8 @@ def _add_text( doc.add_table(data=table_data, caption=caption) elif tag_name == GroupLabel.INLINE: + # DocTags runs are trimmed (by the serializer and by extract_inner_text), so the + # source has lost their boundary whitespace; it is not guessed back here. inline_group = doc.add_inline_group() content = match.group("content") common_bbox = extract_bounding_box(content) From 6d543cf36fc3daca9def016b7111556225bab14a Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Fri, 23 Oct 2026 18:58:53 +0200 Subject: [PATCH 11/18] fix(md): keep the host separator before a list item's inline group The Markdown list serializer glued an inline group onto the preceding list item with no separator, so a list item "Intro" whose child group starts with "more" exported "- Intromore". Captions, headings and the plain-text reader keep a space between host text and its inline group; the list serializer now does the same when the group is the child of a text item with its own text. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/transforms/serializer/markdown.py | 6 +++++- tests/test_inline_whitespace_contract.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docling_core/transforms/serializer/markdown.py b/docling_core/transforms/serializer/markdown.py index 4363c904e..630667b45 100644 --- a/docling_core/transforms/serializer/markdown.py +++ b/docling_core/transforms/serializer/markdown.py @@ -1083,7 +1083,11 @@ def serialize( and p.spans[0].item.parent and isinstance(p.spans[0].item.parent.resolve(doc), InlineGroup) ): - my_parts[-1].text = f"{my_parts[-1].text}{p.text}" # append to last + # append to last; a group that is the child of a text item keeps the host separator + group = p.spans[0].item.parent.resolve(doc) + host = group.parent.resolve(doc) if group.parent else None + sep_host = " " if isinstance(host, TextItem) and host.text else "" + my_parts[-1].text = f"{my_parts[-1].text}{sep_host}{p.text}" my_parts[-1].spans.extend(p.spans) else: my_parts.append(p) diff --git a/tests/test_inline_whitespace_contract.py b/tests/test_inline_whitespace_contract.py index 1d7cad306..e42791691 100644 --- a/tests/test_inline_whitespace_contract.py +++ b/tests/test_inline_whitespace_contract.py @@ -241,6 +241,19 @@ def test_host_text_stays_apart_from_its_inline_group() -> None: assert MarkdownDocSerializer(doc=doc).serialize_captions(item=picture).text == "Table **1**: results" +def test_list_item_text_stays_apart_from_its_inline_group() -> None: + """A list item's own text and its inline-group child follow the same host rule in Markdown.""" + doc = DoclingDocument(name="host") + lst = doc.add_list_group() + item = doc.add_list_item(text="Intro", parent=lst) + group = doc.add_inline_group(parent=item) + for text, formatting in [("more ", None), ("x", _BOLD), (" y", None)]: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=group, formatting=formatting) + + assert doc.export_to_markdown() == "- Intro more **x** y" + assert item._get_plain_text(doc) == "Intro more x y" + + # -------------------------------------------------------------------------------------------- # Load normalization for pre-contract documents # -------------------------------------------------------------------------------------------- From 5c1e97c5fb6ffb75cdee3fb07370b799fdb1e6ff Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Fri, 23 Oct 2026 21:22:30 +0200 Subject: [PATCH 12/18] fix(compat): keep migrated separators between code or formula runs in page filters A legacy document with two adjacent code or formula runs gets a standalone " " run between them on load. It had no provenance, so filter(page_nrs=...) and export_to_markdown(page_no=...) dropped it and rendered `$a$$b$`. The separator now takes the left run's location when both runs are on the same page, so it survives a page filter exactly when both runs do. Runs on different pages keep the unlocated separator, which matches the old join. Inline group locations in DocLang and DocTags are a union of the runs, so they do not change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/types/doc/document.py | 19 +++++++++---- tests/test_inline_whitespace_contract.py | 35 ++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docling_core/types/doc/document.py b/docling_core/types/doc/document.py index 09af94db5..08ad23187 100644 --- a/docling_core/types/doc/document.py +++ b/docling_core/types/doc/document.py @@ -5692,7 +5692,7 @@ def resolve_run(child: Any) -> dict | None: runs = [(idx, run) for idx, child in enumerate(children) if (run := resolve_run(child)) and run["text"]] # Boundaries where the separator fits in neither neighbour; filled in afterwards so # the indices collected here stay valid. - standalone: list[int] = [] + standalone: list[tuple[int, list]] = [] for (_, run), (nxt_idx, nxt) in itertools.pairwise(runs): if run.get("label") not in delimited: run["text"] += " " @@ -5700,18 +5700,27 @@ def resolve_run(child: Any) -> dict | None: nxt["text"] = " " + nxt["text"] else: # Both sides are delimited: the separator would corrupt `` `a``b` `` or - # ``$a$$b$``, so it becomes a plain run of its own between them. - standalone.append(nxt_idx) + # ``$a$$b$``, so it becomes a plain run of its own between them. Page filters + # keep only located items, so it takes the left run's location when both + # runs are on one page: it then survives exactly when both of them do. + left = (run.get("prov") or [None])[-1] + right = (nxt.get("prov") or [None])[0] + same_page = ( + isinstance(left, dict) + and isinstance(right, dict) + and left.get("page_no") == right.get("page_no") + ) + standalone.append((nxt_idx, [{**left, "charspan": [0, 1]}] if same_page else [])) migrated = True - for shift, insert_at in enumerate(standalone): + for shift, (insert_at, prov) in enumerate(standalone): ref = f"#/texts/{len(texts)}" texts.append( { "self_ref": ref, "parent": {"$ref": group.get("self_ref")}, "label": DocItemLabel.TEXT.value, - "prov": [], + "prov": prov, "orig": " ", "text": " ", "children": [], diff --git a/tests/test_inline_whitespace_contract.py b/tests/test_inline_whitespace_contract.py index e42791691..a8dba43ea 100644 --- a/tests/test_inline_whitespace_contract.py +++ b/tests/test_inline_whitespace_contract.py @@ -374,6 +374,41 @@ def test_legacy_separator_never_lands_inside_a_delimiter( assert doc.export_to_text() == expected_txt +def _load_once(data: dict) -> DoclingDocument: + """Load a legacy dict, checking that the caller's value is untouched and a re-load is stable.""" + pristine = copy.deepcopy(data) + doc = _load(data) + assert data == pristine + assert _load(json.loads(doc.model_dump_json())).export_to_markdown() == doc.export_to_markdown() + return doc + + +def _locate(data: dict, pages: list[int]) -> dict: + data["pages"] = {str(p): {"page_no": p, "size": {"width": 100, "height": 100}} for p in set(pages)} + for item, page in zip(data["texts"], pages): + bbox = {"l": 0, "t": 0, "r": 10, "b": 10, "coord_origin": "BOTTOMLEFT"} + item["prov"] = [{"page_no": page, "bbox": bbox, "charspan": [0, len(item["text"])]}] + return data + + +@pytest.mark.parametrize( + ("label", "both", "first", "second"), + [("code", "`a` `b`", "`a`", "`b`"), ("formula", "$a$ $b$", "$a$", "$b$")], +) +def test_legacy_standalone_separator_survives_page_filters_with_its_runs( + label: str, both: str, first: str, second: str +) -> None: + """Page filters keep only located items; the separator must survive exactly when both runs do.""" + same_page = _load_once(_locate(_legacy_doc_dict(["a", "b"], label=label), [1, 1])) + assert same_page.filter(page_nrs={1}).export_to_markdown() == both + assert same_page.export_to_markdown(page_no=1) == both + + split = _load_once(_locate(_legacy_doc_dict(["a", "b"], label=label), [1, 2])) + assert split.filter(page_nrs={1}).export_to_markdown() == first + assert split.filter(page_nrs={2}).export_to_markdown() == second + assert split.export_to_markdown(page_no=1) == first + + def test_legacy_migration_does_not_touch_caller_input() -> None: """The caller owns the dict it passes in, and repeated validation must be stable. From c413dabac3f1c5eae15c9793ff78001fe41859aa Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Fri, 23 Oct 2026 21:27:18 +0200 Subject: [PATCH 13/18] fix(compat): count empty inline hosts and nested inline groups as legacy runs The legacy separator migration counted only direct text children with non-empty text as runs. An empty-text item whose content is a child inline group (a formula or footnote host) and an inline group nested directly in another were skipped, so `left`, host(`x`), `right` loaded as `left $x$right` instead of the old `left $x$ right`. The DocLang deserializer writes both shapes for nested mixed-content elements. A child now counts as a run when it or its subtree has text. Only a plain run with its own text takes the separator; an empty host must keep its empty text to render its group, so it is treated like a code or formula run and the separator stays outside it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/types/doc/document.py | 56 ++++++++++++------- tests/test_inline_whitespace_contract.py | 69 ++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 19 deletions(-) diff --git a/docling_core/types/doc/document.py b/docling_core/types/doc/document.py index 08ad23187..e9f997270 100644 --- a/docling_core/types/doc/document.py +++ b/docling_core/types/doc/document.py @@ -5661,24 +5661,39 @@ def _migrate_legacy_inline_separators(cls, data: Any) -> Any: if not isinstance(texts, list) or not isinstance(groups, list): return data - def resolve_run(child: Any) -> dict | None: - """Return the text item dict for an inline run reference, else None.""" - if not isinstance(child, dict): - return None - ref = child.get("$ref") or child.get("cref") - if not isinstance(ref, str) or not ref.startswith("#/texts/"): - return None - try: - item = texts[int(ref.removeprefix("#/texts/"))] - except (ValueError, IndexError): - return None - return item if isinstance(item, dict) and isinstance(item.get("text"), str) else None + def resolve(child: Any) -> dict | None: + """Return the text or group dict an inline child reference points to, else None.""" + ref = (child.get("$ref") or child.get("cref")) if isinstance(child, dict) else None + for prefix, items in (("#/texts/", texts), ("#/groups/", groups)): + if isinstance(ref, str) and ref.startswith(prefix): + try: + item = items[int(ref.removeprefix(prefix))] + except (ValueError, IndexError): + return None + return item if isinstance(item, dict) else None + return None + + def renders_text(item: dict, seen: set[int]) -> bool: + """Whether the item has text, or is an empty host or nested group whose subtree has text.""" + if id(item) in seen: # a reference cycle in malformed input + return False + seen.add(id(item)) + if isinstance(item.get("text"), str) and item["text"]: + return True + children = item.get("children") + return isinstance(children, list) and any( + (sub := resolve(child)) is not None and renders_text(sub, seen) for child in children + ) # Code and formula runs are wrapped in delimiters (`` ` ``, ``$``) by the serializers, so a # separator appended to their text would land *inside* the delimiter -- the same defect as # `**bold **`. Put it on the neighbouring plain run instead. delimited = {DocItemLabel.CODE.value, DocItemLabel.FORMULA.value} + def carries_separator(run: dict) -> bool: + """Only a plain run with its own text: an empty host's text must stay empty to render its group.""" + return isinstance(run.get("text"), str) and bool(run["text"]) and run.get("label") not in delimited + migrated = False for group in groups: if not isinstance(group, dict) or group.get("label") != GroupLabel.INLINE.value: @@ -5686,20 +5701,23 @@ def resolve_run(child: Any) -> dict | None: children = group.get("children") if not isinstance(children, list): continue - # ponytail: only `#/texts/N` children are treated as runs. A picture or table inside - # an inline group also received a separator, but whether it serialized to non-empty - # text is not knowable from the raw dict. Widen this if such a document shows up. - runs = [(idx, run) for idx, child in enumerate(children) if (run := resolve_run(child)) and run["text"]] + # shortcut: a child is a run when it or its subtree has text (an empty formula host + # holding an inline group, a nested inline group). A picture, table or empty formula also + # got the old separator, and HTML renders the empty formula, but whether such a child + # rendered text depends on the serializer. Widen this if such a document shows up. + runs = [ + (idx, run) for idx, child in enumerate(children) if (run := resolve(child)) and renders_text(run, set()) + ] # Boundaries where the separator fits in neither neighbour; filled in afterwards so # the indices collected here stay valid. standalone: list[tuple[int, list]] = [] for (_, run), (nxt_idx, nxt) in itertools.pairwise(runs): - if run.get("label") not in delimited: + if carries_separator(run): run["text"] += " " - elif nxt.get("label") not in delimited: + elif carries_separator(nxt): nxt["text"] = " " + nxt["text"] else: - # Both sides are delimited: the separator would corrupt `` `a``b` `` or + # Neither side can carry it: the separator would corrupt `` `a``b` `` or # ``$a$$b$``, so it becomes a plain run of its own between them. Page filters # keep only located items, so it takes the left run's location when both # runs are on one page: it then survives exactly when both of them do. diff --git a/tests/test_inline_whitespace_contract.py b/tests/test_inline_whitespace_contract.py index a8dba43ea..cbe5ea0e2 100644 --- a/tests/test_inline_whitespace_contract.py +++ b/tests/test_inline_whitespace_contract.py @@ -24,6 +24,7 @@ DocItemLabel, DoclingDocument, Formatting, + InlineGroup, ) # (name, runs) where a run is (text, formatting, hyperlink) @@ -409,6 +410,74 @@ def test_legacy_standalone_separator_survives_page_filters_with_its_runs( assert split.export_to_markdown(page_no=1) == first +def _host(doc: DoclingDocument, group: InlineGroup, label: DocItemLabel, runs: list[str]) -> None: + """An empty-text item whose visible content is a child inline group.""" + nested = doc.add_inline_group(parent=doc.add_text(label=label, text="", parent=group)) + for text in runs: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=nested) + + +def _nested(doc: DoclingDocument, group: InlineGroup, runs: list[str]) -> None: + nested = doc.add_inline_group(parent=group) + for text in runs: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=nested) + + +@pytest.mark.parametrize( + ("build", "expected"), + [ + pytest.param( + lambda d, g: (d.add_text(DocItemLabel.TEXT, "left", parent=g), _host(d, g, DocItemLabel.FORMULA, ["x"])), + "left $x$", + id="formula-host-last", + ), + pytest.param( + lambda d, g: ( + d.add_text(DocItemLabel.TEXT, "left", parent=g), + _host(d, g, DocItemLabel.FORMULA, ["x"]), + d.add_text(DocItemLabel.TEXT, "right", parent=g), + ), + "left $x$ right", + id="formula-host", + ), + pytest.param( + lambda d, g: ( + d.add_text(DocItemLabel.FORMULA, "l", parent=g), + _host(d, g, DocItemLabel.FORMULA, ["x"]), + d.add_text(DocItemLabel.CODE, "r", parent=g), + ), + "$l$ $x$ `r`", + id="formula-host-between-delimited", + ), + # The DocLang deserializer writes these two for nested mixed-content elements. + pytest.param( + lambda d, g: ( + d.add_text(DocItemLabel.TEXT, "left", parent=g), + _host(d, g, DocItemLabel.FOOTNOTE, ["a", "b"]), + d.add_text(DocItemLabel.TEXT, "right", parent=g), + ), + "left a b right", + id="footnote-host", + ), + pytest.param( + lambda d, g: ( + d.add_text(DocItemLabel.TEXT, "left", parent=g), + _nested(d, g, ["a", "b"]), + d.add_text(DocItemLabel.TEXT, "right", parent=g), + ), + "left a b right", + id="nested-group", + ), + ], +) +def test_legacy_separator_counts_runs_whose_text_is_in_a_child_group(build, expected: str) -> None: + """An empty host renders its child inline group, so the old join spaced it like any other run.""" + doc = DoclingDocument(name="legacy") + build(doc, doc.add_inline_group()) + data = doc.model_dump(mode="json") | {"version": _LEGACY_VERSION} + assert _load_once(data).export_to_markdown() == expected + + def test_legacy_migration_does_not_touch_caller_input() -> None: """The caller owns the dict it passes in, and repeated validation must be stable. From d0c41da54c46e23b0784bc1df9d7b3faced16ddf Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Sat, 24 Oct 2026 14:58:06 +0200 Subject: [PATCH 14/18] docs(doclang): note that pre-contract archives load without legacy separators DocLang has no schema version, so a .dclx written before schema 1.11.0 cannot be told apart from a current one and is not migrated. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/types/doc/document.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docling_core/types/doc/document.py b/docling_core/types/doc/document.py index e9f997270..ed9a13c86 100644 --- a/docling_core/types/doc/document.py +++ b/docling_core/types/doc/document.py @@ -5389,6 +5389,11 @@ def load_from_doclang_archive( ``document.xml`` is additionally capped by ``settings.max_doclang_xml_bytes`` before parsing. Archive ``assets/`` and ``pages/`` images are capped by ``settings.max_image_decoded_size`` before Pillow opens them. + + Unlike JSON, DocLang carries no schema version, so archives written by docling-core + before schema 1.11.0 (the inline whitespace contract) are loaded as current: their + inline runs were stored without boundary whitespace and now join without a separator + (``Normal**bold**tail``). Re-convert the source, or keep JSON for long-term storage. """ from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer From 92a694dd7f62890ddf1fbc94e9e10e434305119f Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Mon, 26 Oct 2026 09:01:46 +0100 Subject: [PATCH 15/18] chore(dclq): move the vendored-click fix to its own PR This reverts 6a8b8374; the change is unrelated to the whitespace contract. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- packages/dclq/dclq/cli.py | 14 +++++--------- pyproject.toml | 3 ++- uv.lock | 4 +++- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/dclq/dclq/cli.py b/packages/dclq/dclq/cli.py index d045bf862..898f176ab 100644 --- a/packages/dclq/dclq/cli.py +++ b/packages/dclq/dclq/cli.py @@ -12,14 +12,10 @@ from types import SimpleNamespace from typing import Annotated, Any, Literal +import click import typer from lxml import etree -# typer >=0.25 vendors its own copy of click (typer._click) and raises those -# exceptions rather than the external click package's, so catch the vendored ones. -from typer._click import Parameter as _ClickParameter -from typer._click import exceptions as _click_exc - from dclq import __version__ from dclq.document import DclqError, LoadedDocument, Unit, _canonical_xpath, _is_element from docling_core.experimental.serializer.outline import ( @@ -78,9 +74,9 @@ def _main_callback( pass -def _record_context(ctx: typer.Context, param: _ClickParameter, value: int | None) -> int | None: +def _record_context(ctx: typer.Context, param: click.Parameter, value: int | None) -> int | None: """Record context flags in command-line order, matching grep semantics.""" - if value is not None and getattr(param, "param_type_name", None) == "option": + if value is not None and isinstance(param, click.Option): ctx.meta.setdefault("context_events", []).append((param.opts[0], value)) return value @@ -351,9 +347,9 @@ def main(argv: Sequence[str] | None = None) -> int: except DclqError as exc: print(f"dclq: {exc}", file=sys.stderr) return 2 - except _click_exc.Exit as exc: + except click.exceptions.Exit as exc: return exc.exit_code - except _click_exc.ClickException as exc: + except click.ClickException as exc: exc.show(file=sys.stderr) return exc.exit_code diff --git a/pyproject.toml b/pyproject.toml index 03c6b49a4..930406e9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,8 @@ docling-serialize = "docling_core.cli.serialize:app" [project.optional-dependencies] dclq = [ "doclang>=0.8,<0.9", - "typer>=0.25.0,<0.27.0", + "typer>=0.15.1,<0.25.0", + "click>=8.0.0", "lxml>=6.0.2", ] chunking = [ diff --git a/uv.lock b/uv.lock index c45878306..f2d8f1813 100644 --- a/uv.lock +++ b/uv.lock @@ -1227,6 +1227,7 @@ chunking-openai = [ { name = "tree-sitter-typescript" }, ] dclq = [ + { name = "click" }, { name = "doclang" }, { name = "lxml" }, { name = "typer" }, @@ -1279,6 +1280,7 @@ typecheck = [ [package.metadata] requires-dist = [ + { name = "click", marker = "extra == 'dclq'", specifier = ">=8.0.0" }, { name = "datasets", marker = "extra == 'examples'", specifier = ">=4.0.0" }, { name = "defusedxml", specifier = ">=0.7.1,<0.8.0" }, { name = "doclang", specifier = ">=0.8,<0.9" }, @@ -1311,7 +1313,7 @@ requires-dist = [ { name = "tree-sitter-typescript", marker = "extra == 'chunking'", specifier = ">=0.23.2" }, { name = "tree-sitter-typescript", marker = "extra == 'chunking-openai'", specifier = ">=0.23.2" }, { name = "typer", specifier = ">=0.12.5,<0.27.0" }, - { name = "typer", marker = "extra == 'dclq'", specifier = ">=0.25.0,<0.27.0" }, + { name = "typer", marker = "extra == 'dclq'", specifier = ">=0.15.1,<0.25.0" }, { name = "typing-extensions", specifier = ">=4.12.2,<5.0.0" }, ] provides-extras = ["dclq", "chunking", "chunking-openai", "examples"] From 31e7aa168f6296cbcf5d1f31c79d31ea3023d47c Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Mon, 26 Oct 2026 10:53:56 +0100 Subject: [PATCH 16/18] fix(compat): don't run the legacy inline migration in project_to project_to validated the projected dict under its target stamp. For 1.10.0 that ran the load-time separator migration on runs that already carry their whitespace, so 'Normal ', 'bold', ' tail' came back as 'Normal ', 'bold ', ' tail'. Validate under the source stamp and restore the target stamp afterwards. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/compat.py | 4 +++- tests/test_compat.py | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docling_core/compat.py b/docling_core/compat.py index 73c16d16a..6b57ca592 100644 --- a/docling_core/compat.py +++ b/docling_core/compat.py @@ -179,7 +179,9 @@ def project_to(doc: DoclingDocument, target_version: str) -> DoclingDocument: # The current SDK's version validator normalises the version field to # CURRENT_VERSION on success. We restore the target version afterwards so # callers and downstream serialisation correctly reflect the projected schema. - result = DoclingDocument.model_validate(data) + # Validate under the source stamp: the projected stamp would trigger load-time upgrades + # (the inline separator migration) on data that is already current. + result = DoclingDocument.model_validate({**data, "version": doc.version}) object.__setattr__(result, "version", data["version"]) return result diff --git a/tests/test_compat.py b/tests/test_compat.py index 24d5f3d36..2438a93f6 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -44,7 +44,7 @@ FIRST_SUPPORTED_MINOR, SCHEMA_VERSION_HISTORY, ) -from docling_core.types.doc.document import DoclingDocument +from docling_core.types.doc.document import DocItemLabel, DoclingDocument, Formatting # Root of the repository, used to locate docs/schemas/. _REPO_ROOT = Path(__file__).parent.parent @@ -322,6 +322,20 @@ def test_does_not_mutate_caller_input(self): self._apply(data) assert data["version"] == "1.11.0" + def test_project_to_does_not_run_the_legacy_migration(self): + """The 1.10 stamp must not make project_to re-inject legacy separators.""" + doc = DoclingDocument(name="projected") + group = doc.add_inline_group() + doc.add_text(label=DocItemLabel.TEXT, text="Normal ", parent=group) + doc.add_text(label=DocItemLabel.TEXT, text="bold", parent=group, formatting=Formatting(bold=True)) + doc.add_text(label=DocItemLabel.TEXT, text=" tail", parent=group) + + result = project_to(doc, target_version="1.10.0") + + assert result.version == "1.10.0" + assert [t.text for t in result.texts] == ["Normal ", "bold", " tail"] + assert result.export_to_markdown() == "Normal **bold** tail" + class TestProjector_1_10_to_1_9: """Tests for the 1.10 → 1.9 downgrade projector.""" From 0f0042f13322744edb4d77f23736f5a0ca52401a Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Mon, 26 Oct 2026 11:09:37 +0100 Subject: [PATCH 17/18] fix(compat): strip inline runs when projecting 1.11 to 1.10 A 1.10 client joins inline runs with " ", so runs that keep their boundary whitespace rendered with double spaces and with the space inside the markup (**bold **, which CommonMark does not render as bold). The projector now strips each inline-group child's text to the shape old producers emitted. Whitespace-only runs become "", which old serializers skip, so no refs move. orig and text outside inline groups are left alone. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/compat.py | 31 +++++++++++++++++++-------- tests/test_compat.py | 48 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/docling_core/compat.py b/docling_core/compat.py index 6b57ca592..925580702 100644 --- a/docling_core/compat.py +++ b/docling_core/compat.py @@ -207,12 +207,12 @@ def list_projectors() -> list[tuple[int, int]]: # The 1.11 and 1.10 schemas are structurally identical -- no new fields, # labels, or lists. The only difference is *semantic*: a 1.11 run's text # contains its boundary whitespace, whereas a 1.10 client re-inserts a -# single " " between runs at serialization time. A 1.10 client therefore -# parses a 1.11 dict without error; it just renders the boundary twice -# (e.g. "bold tail"), the documented, harmless double-space case. 1.10 -# cannot represent a tight run boundary, so there is nothing to project -# structurally -- only the version stamp is lowered so the old client's -# version-compatibility gate accepts the document. +# single " " between runs at serialization time. Left as-is, a 1.10 client +# renders every boundary twice and keeps the space inside the markup +# (``**bold **``, which CommonMark does not render as bold). The projector +# therefore strips inline runs to the shape old producers emitted; the old +# join puts one space back. 1.10 cannot represent a tight run boundary, so +# ``H2O`` comes back as the old ``H 2 O``. # --------------------------------------------------------------------------- @@ -221,12 +221,25 @@ def _project_1_11_to_1_10(data: dict) -> dict: """Downgrade a schema 1.11 document dict to schema 1.10. 1.11 and 1.10 share an identical structure; the difference is only in how - inline-run whitespace is represented. A 1.10 client validates the dict - as-is and re-applies its own single-space join between runs, so the sole - transform is re-stamping the version. + inline-run whitespace is represented. A 1.10 client joins runs with ``" "``, + so the text of each inline-group child is stripped. A whitespace-only run + becomes ``""``, which old serializers skip, so no refs need renumbering. + ``orig`` is left alone. """ data = dict(data) data["version"] = "1.10.0" + inline_runs = { + child.get("$ref", child.get("cref")) + for group in data.get("groups", []) + if group.get("label") == "inline" + for child in group.get("children", []) + } + data["texts"] = [ + {**item, "text": item["text"].strip()} + if item.get("self_ref") in inline_runs and isinstance(item.get("text"), str) + else item + for item in data.get("texts", []) + ] return data diff --git a/tests/test_compat.py b/tests/test_compat.py index 2438a93f6..2c14a987f 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -296,11 +296,8 @@ def test_version_set_to_1_10(self): result = self._apply(_minimal_doc_dict("1.11.0")) assert result["version"] == "1.10.0" - def test_inline_run_whitespace_is_preserved_verbatim(self): - """The projector must not touch run text: 1.11 runs own their boundary - whitespace, and a 1.10 client relies on that text being intact (it adds - its own join space on top). Stripping here would re-introduce word-merge. - """ + def test_plain_text_keeps_its_whitespace(self): + """Only inline-group runs are stripped; other text items stay as they are.""" data = _minimal_doc_dict("1.11.0") data["texts"] = [ { @@ -317,13 +314,44 @@ def test_inline_run_whitespace_is_preserved_verbatim(self): result = self._apply(data) assert result["texts"][0]["text"] == "bold " + def test_inline_runs_are_stripped_to_the_old_shape(self): + """A 1.10 client joins runs with " ", so runs must not carry boundary whitespace.""" + doc = DoclingDocument(name="runs") + caption = doc.add_text(label=DocItemLabel.CAPTION, text=" Caption ") + heading = doc.add_heading(text="Heading ") + group = doc.add_inline_group() + doc.add_text(label=DocItemLabel.TEXT, text="Some ", parent=group) + doc.add_text(label=DocItemLabel.TEXT, text="bold ", parent=group, formatting=Formatting(bold=True)) + doc.add_text(label=DocItemLabel.TEXT, text="text with a ", parent=group) + doc.add_text(label=DocItemLabel.TEXT, text="link ", parent=group, hyperlink="https://example.com") + doc.add_text(label=DocItemLabel.TEXT, text=" ", parent=group) + doc.add_text(label=DocItemLabel.TEXT, text="core", parent=group, formatting=Formatting(bold=True)) + data = json.loads(doc.model_dump_json()) + + result = self._apply(data) + + assert len(result["texts"]) == len(data["texts"]) + assert [t["text"] for t in result["texts"]] == [ + " Caption ", + "Heading ", + "Some", + "bold", + "text with a", + "link", + "", + "core", + ] + assert [t["orig"] for t in result["texts"]] == [t["orig"] for t in data["texts"]] + assert data["texts"][2]["text"] == "Some " + assert caption.text == " Caption " and heading.text == "Heading " + def test_does_not_mutate_caller_input(self): data = _minimal_doc_dict("1.11.0") self._apply(data) assert data["version"] == "1.11.0" def test_project_to_does_not_run_the_legacy_migration(self): - """The 1.10 stamp must not make project_to re-inject legacy separators.""" + """project_to keeps the old-shape runs; reloading them on this core restores the spacing.""" doc = DoclingDocument(name="projected") group = doc.add_inline_group() doc.add_text(label=DocItemLabel.TEXT, text="Normal ", parent=group) @@ -331,10 +359,12 @@ def test_project_to_does_not_run_the_legacy_migration(self): doc.add_text(label=DocItemLabel.TEXT, text=" tail", parent=group) result = project_to(doc, target_version="1.10.0") + dumped = json.loads(result.model_dump_json()) - assert result.version == "1.10.0" - assert [t.text for t in result.texts] == ["Normal ", "bold", " tail"] - assert result.export_to_markdown() == "Normal **bold** tail" + assert dumped["version"] == "1.10.0" + assert [t["text"] for t in dumped["texts"]] == ["Normal", "bold", "tail"] + reloaded = DoclingDocument.model_validate(dumped) + assert reloaded.export_to_markdown() == "Normal **bold** tail" class TestProjector_1_10_to_1_9: From 01481d48af8e515aa74fe51bd1ad6e84797b7b61 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Mon, 26 Oct 2026 11:36:55 +0100 Subject: [PATCH 18/18] fix(compat): reuse the stripped separator run when migrating Between two code or formula runs the migration inserts a plain " " run. The 1.11 -> 1.10 projector strips it to "", and the next migration skipped the empty run and inserted another, so every projection round trip added a run. Refill an empty plain run between the two neighbours instead. Old serializers skipped empty runs, so legacy output is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christoph Auer --- docling_core/types/doc/document.py | 17 +++++++++++++++- tests/test_compat.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docling_core/types/doc/document.py b/docling_core/types/doc/document.py index ed9a13c86..af5d3fa36 100644 --- a/docling_core/types/doc/document.py +++ b/docling_core/types/doc/document.py @@ -5716,11 +5716,26 @@ def carries_separator(run: dict) -> bool: # Boundaries where the separator fits in neither neighbour; filled in afterwards so # the indices collected here stay valid. standalone: list[tuple[int, list]] = [] - for (_, run), (nxt_idx, nxt) in itertools.pairwise(runs): + for (idx, run), (nxt_idx, nxt) in itertools.pairwise(runs): + # An empty plain run between them is the separator the 1.11 -> 1.10 projector + # stripped; refilling it keeps repeated projection and migration from adding runs. + empty = next( + ( + between + for child in children[idx + 1 : nxt_idx] + if (between := resolve(child)) is not None + and between.get("text") == "" + and between.get("label") == DocItemLabel.TEXT.value + and not between.get("children") + ), + None, + ) if carries_separator(run): run["text"] += " " elif carries_separator(nxt): nxt["text"] = " " + nxt["text"] + elif empty is not None: + empty["text"] = " " else: # Neither side can carry it: the separator would corrupt `` `a``b` `` or # ``$a$$b$``, so it becomes a plain run of its own between them. Page filters diff --git a/tests/test_compat.py b/tests/test_compat.py index 2c14a987f..662aa2054 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -350,6 +350,37 @@ def test_does_not_mutate_caller_input(self): self._apply(data) assert data["version"] == "1.11.0" + @pytest.mark.parametrize( + "runs", + [ + [("code", "a"), ("code", "b")], + [("formula", "x"), ("formula", "y")], + [("code", "a"), ("formula", "x"), ("code", "b")], + [("text", "Install with "), ("code", "pip"), ("text", ", then")], + [("text", "a"), ("text", " "), ("text", "b")], + ], + ) + def test_repeated_projection_and_migration_do_not_drift(self, runs): + """Down-projecting and reloading again must not add runs or change the output.""" + doc = DoclingDocument(name="drift") + group = doc.add_inline_group() + for kind, text in runs: + if kind == "code": + doc.add_code(text=text, parent=group) + elif kind == "formula": + doc.add_formula(text=text, parent=group) + else: + doc.add_text(label=DocItemLabel.TEXT, text=text, parent=group) + + def round_trip(d: DoclingDocument) -> DoclingDocument: + return DoclingDocument.model_validate(json.loads(project_to(d, "1.10.0").model_dump_json())) + + first = round_trip(doc) + again = first + for _ in range(3): + again = round_trip(again) + assert again.export_to_dict() == first.export_to_dict() + def test_project_to_does_not_run_the_legacy_migration(self): """project_to keeps the old-shape runs; reloading them on this core restores the spacing.""" doc = DoclingDocument(name="projected")