diff --git a/docling/backend/html_backend.py b/docling/backend/html_backend.py index b756f0bae4..203e5abb70 100644 --- a/docling/backend/html_backend.py +++ b/docling/backend/html_backend.py @@ -340,30 +340,23 @@ class AnnotatedText(BaseModel): class AnnotatedTextList(list): def to_single_text_element(self) -> AnnotatedText: - current_h = None current_text = "" - current_f = None current_code = False current_source_tag_id = None + # The single element can only keep a formatting or a hyperlink that + # applies to all of its text, e.g. not the link of one word in a heading. + text_runs = [at for at in self if at.text.strip()] + current_f = text_runs[0].formatting if text_runs else None + if any(at.formatting != current_f for at in text_runs): + current_f = None + current_h = text_runs[0].hyperlink if text_runs else None + if any(at.hyperlink != current_h for at in text_runs): + current_h = None for at in self: t = at.text - h = at.hyperlink - f = at.formatting c = at.code s = at.source_tag_id current_text += t.strip() + " " - if f is not None and current_f is None: - current_f = f - elif f is not None and current_f is not None and f != current_f: - _log.warning( - f"Clashing formatting: '{f}' and '{current_f}'! Chose '{current_f}'" - ) - if h is not None and current_h is None: - current_h = h - elif h is not None and current_h is not None and h != current_h: - _log.warning( - f"Clashing hyperlinks: '{h}' and '{current_h}'! Chose '{current_h}'" - ) if s is not None and current_source_tag_id is None: current_source_tag_id = s elif ( diff --git a/tests/data/epub/groundtruth/epub_purvis_poetry.epub.json b/tests/data/epub/groundtruth/epub_purvis_poetry.epub.json index 00c3c6cec0..b72b1347c9 100644 --- a/tests/data/epub/groundtruth/epub_purvis_poetry.epub.json +++ b/tests/data/epub/groundtruth/epub_purvis_poetry.epub.json @@ -1553,13 +1553,6 @@ "prov": [], "orig": "To the Hibernia", "text": "To the Hibernia", - "formatting": { - "bold": false, - "italic": true, - "underline": false, - "strikethrough": false, - "script": "baseline" - }, "level": 1 }, { diff --git a/tests/data/epub/groundtruth/epub_purvis_poetry.epub.md b/tests/data/epub/groundtruth/epub_purvis_poetry.epub.md index 40f6c60103..103bb53cc5 100644 --- a/tests/data/epub/groundtruth/epub_purvis_poetry.epub.md +++ b/tests/data/epub/groundtruth/epub_purvis_poetry.epub.md @@ -218,7 +218,7 @@ The idol of my soul- Then blame me not-this burst of grief I cannot now control. -## *To the Hibernia* +## To the Hibernia O, speed thee! speed thee! gallant bark, Across the bounding wave; diff --git a/tests/data/html/groundtruth/hyperlink_05.html.json b/tests/data/html/groundtruth/hyperlink_05.html.json index 8b8f4e70c5..bd56f4e876 100644 --- a/tests/data/html/groundtruth/hyperlink_05.html.json +++ b/tests/data/html/groundtruth/hyperlink_05.html.json @@ -92,8 +92,7 @@ "label": "caption", "prov": [], "orig": "This is an example caption for the image.", - "text": "This is an example caption for the image.", - "hyperlink": "#caption" + "text": "This is an example caption for the image." } ], "pictures": [ diff --git a/tests/data/html/groundtruth/wiki_duck.html.json b/tests/data/html/groundtruth/wiki_duck.html.json index f8543618a1..4ab710880d 100644 --- a/tests/data/html/groundtruth/wiki_duck.html.json +++ b/tests/data/html/groundtruth/wiki_duck.html.json @@ -10830,8 +10830,7 @@ "label": "caption", "prov": [], "orig": "Pacific black duck displaying the characteristic upending \"duck\"", - "text": "Pacific black duck displaying the characteristic upending \"duck\"", - "hyperlink": "/wiki/Pacific_black_duck" + "text": "Pacific black duck displaying the characteristic upending \"duck\"" }, { "self_ref": "#/texts/308", @@ -11582,8 +11581,7 @@ "label": "caption", "prov": [], "orig": "Male mallard .", - "text": "Male mallard .", - "hyperlink": "/wiki/Mallard" + "text": "Male mallard ." }, { "self_ref": "#/texts/359", @@ -11595,8 +11593,7 @@ "label": "caption", "prov": [], "orig": "Wood ducks .", - "text": "Wood ducks .", - "hyperlink": "/wiki/Wood_duck" + "text": "Wood ducks ." }, { "self_ref": "#/texts/360", @@ -12042,8 +12039,7 @@ "label": "caption", "prov": [], "orig": "Mallard landing in approach", - "text": "Mallard landing in approach", - "hyperlink": "/wiki/Mallard" + "text": "Mallard landing in approach" }, { "self_ref": "#/texts/390", @@ -12881,8 +12877,7 @@ "label": "caption", "prov": [], "orig": "Male Mandarin duck", - "text": "Male Mandarin duck", - "hyperlink": "/wiki/Mandarin_duck" + "text": "Male Mandarin duck" }, { "self_ref": "#/texts/445", @@ -13272,8 +13267,7 @@ "label": "caption", "prov": [], "orig": "Flying steamer ducks in Ushuaia , Argentina", - "text": "Flying steamer ducks in Ushuaia , Argentina", - "hyperlink": "/wiki/Flying_steamer_ducks" + "text": "Flying steamer ducks in Ushuaia , Argentina" }, { "self_ref": "#/texts/475", @@ -13658,8 +13652,7 @@ "label": "caption", "prov": [], "orig": "Female mallard in Cornwall , England", - "text": "Female mallard in Cornwall , England", - "hyperlink": "/wiki/Cornwall" + "text": "Female mallard in Cornwall , England" }, { "self_ref": "#/texts/503", @@ -13767,8 +13760,7 @@ "label": "caption", "prov": [], "orig": "Pecten along the bill", - "text": "Pecten along the bill", - "hyperlink": "/wiki/Pecten_(biology)" + "text": "Pecten along the bill" }, { "self_ref": "#/texts/508", @@ -13780,8 +13772,7 @@ "label": "caption", "prov": [], "orig": "Mallard duckling preening", - "text": "Mallard duckling preening", - "hyperlink": "/wiki/Mallard" + "text": "Mallard duckling preening" }, { "self_ref": "#/texts/509", @@ -14183,8 +14174,7 @@ "label": "caption", "prov": [], "orig": "A Muscovy duckling", - "text": "A Muscovy duckling", - "hyperlink": "/wiki/Muscovy_duck" + "text": "A Muscovy duckling" }, { "self_ref": "#/texts/539", @@ -16109,8 +16099,7 @@ "label": "caption", "prov": [], "orig": "Indian Runner ducks , a common breed of domestic ducks", - "text": "Indian Runner ducks , a common breed of domestic ducks", - "hyperlink": "/wiki/Indian_Runner_duck" + "text": "Indian Runner ducks , a common breed of domestic ducks" }, { "self_ref": "#/texts/675", @@ -16420,15 +16409,7 @@ "label": "caption", "prov": [], "orig": "Three black-colored ducks in the coat of arms of Maaninka [ 49 ]", - "text": "Three black-colored ducks in the coat of arms of Maaninka [ 49 ]", - "formatting": { - "bold": false, - "italic": false, - "underline": false, - "strikethrough": false, - "script": "super" - }, - "hyperlink": "/wiki/Maaninka" + "text": "Three black-colored ducks in the coat of arms of Maaninka [ 49 ]" }, { "self_ref": "#/texts/696", diff --git a/tests/data/html/groundtruth/wiki_duck.html.md b/tests/data/html/groundtruth/wiki_duck.html.md index 9b2f3e7d68..f1b8de9d77 100644 --- a/tests/data/html/groundtruth/wiki_duck.html.md +++ b/tests/data/html/groundtruth/wiki_duck.html.md @@ -321,7 +321,7 @@ Pecten along the bill -[Mallard duckling preening](/wiki/Mallard) +Mallard duckling preening Ducks eat food sources such as [grasses](/wiki/Poaceae) , aquatic plants, fish, insects, small amphibians, worms, and small [molluscs](/wiki/Mollusc) . diff --git a/tests/test_backend_html.py b/tests/test_backend_html.py index c694297263..a95813f010 100644 --- a/tests/test_backend_html.py +++ b/tests/test_backend_html.py @@ -168,6 +168,58 @@ def test_heading_levels(): assert found_lvl_1 and found_lvl_2 +@pytest.mark.parametrize( + ("heading", "bold", "hyperlink"), + [ + # Formatting or a link on one word does not apply to the whole heading + ("

Section bold heading

", False, None), + ("

Docs and more

", False, None), + # Formatting or a link on all of the text is kept + ("

All bold

", True, None), + ( + "

Docs

", + False, + "https://example.com/", + ), + ], +) +def test_heading_keeps_only_formatting_of_all_its_text( + heading: str, bold: bool, hyperlink: str | None +): + html = f"{heading}

text

".encode() + in_doc = InputDocument( + path_or_stream=BytesIO(html), + format=InputFormat.HTML, + backend=HTMLDocumentBackend, + filename="test", + ) + doc = in_doc._backend.convert() + headers = [item for item in doc.texts if isinstance(item, SectionHeaderItem)] + assert len(headers) == 1 + header = headers[0] + assert bool(header.formatting and header.formatting.bold) is bold + assert (str(header.hyperlink) if header.hyperlink else None) == hyperlink + + +def test_caption_with_partial_link_has_no_hyperlink(): + html = ( + b"
" + b"
Female mallard in Cornwall," + b" England
" + ) + in_doc = InputDocument( + path_or_stream=BytesIO(html), + format=InputFormat.HTML, + backend=HTMLDocumentBackend, + filename="test", + ) + doc = in_doc._backend.convert() + captions = [item for item in doc.texts if item.label == DocItemLabel.CAPTION] + assert len(captions) == 1 + assert captions[0].text.startswith("Female mallard in Cornwall") + assert captions[0].hyperlink is None + + def test_table_header_rowspan_without_body_does_not_crash(): # A table whose only row is a `th` with rowspan (no body rows to span into) # used to raise IndexError: get_html_table_row_col counts no rows for an