diff --git a/docling/backend/html_backend.py b/docling/backend/html_backend.py
index 82c44f2253..fa5779e326 100644
--- a/docling/backend/html_backend.py
+++ b/docling/backend/html_backend.py
@@ -124,6 +124,16 @@ def _warn_headers_without_origin() -> None:
# Using Unicode Private Use Area to avoid conflicts with actual content
_BR_SENTINEL = "\ue000"
+
+def _resolve_line_breaks(text: str) -> str:
+ """Replace the sentinels of
tags with a line break.
+
+ For text that is one paragraph (heading, caption, list item): consecutive
+ tags give one line break.
+ """
+ return re.sub(rf" *{_BR_SENTINEL}[ {_BR_SENTINEL}]*", "\n", text).strip()
+
+
DEFAULT_IMAGE_WIDTH = 128
DEFAULT_IMAGE_HEIGHT = 128
_MHTML_SYNTHETIC_BASE = "thismessage:/"
@@ -384,7 +394,7 @@ def to_single_text_element(self) -> AnnotatedText:
current_code = c if c else current_code
return AnnotatedText(
- text=current_text.strip(),
+ text=_resolve_line_breaks(current_text),
hyperlink=current_h,
formatting=current_f,
code=current_code,
@@ -2871,9 +2881,9 @@ def _add_list_item_with_content(
tag, ignore_list=True, find_parent_annotation=True
)
min_parts = parts.simplify_text_elements()
- item_text = re.sub(
- r"\s+|\n+", " ", "".join([el.text for el in min_parts])
- ).strip()
+ item_text = _resolve_line_breaks(
+ re.sub(r"\s+|\n+", " ", "".join([el.text for el in min_parts]))
+ )
if not item_text:
return None
@@ -2895,7 +2905,12 @@ def _add_list_item_with_content(
with self._use_inline_group(min_parts, doc):
compacted_parts = self._compact_adjacent_single_char_parts(min_parts)
for annotated_text, source_tag_ids in compacted_parts:
- text_part = re.sub(r"\s+|\n+", " ", annotated_text.text).strip()
+ text_part = _resolve_line_breaks(
+ re.sub(r"\s+|\n+", " ", annotated_text.text)
+ )
+ if not text_part:
+ # The part only held
tags
+ continue
clean_text = HTMLDocumentBackend._clean_unicode(text_part)
# Apply extra formatting if provided
@@ -2940,7 +2955,7 @@ def _add_list_item_with_content(
else:
# Simple content - single text element
annotated_text = min_parts[0]
- text = re.sub(r"\s+|\n+", " ", annotated_text.text).strip()
+ text = _resolve_line_breaks(re.sub(r"\s+|\n+", " ", annotated_text.text))
clean_text = HTMLDocumentBackend._clean_unicode(text)
prov = self._make_text_prov(
text=clean_text,
@@ -3460,7 +3475,12 @@ def _handle_block(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: # noq
language_hint = self._code_language_hint(tag)
with self._use_inline_group(annotated_texts, doc) as inline_ref:
for annotated_text in annotated_texts:
- text_orig = annotated_text.text.strip()
+ # In preformatted text, each
is a newline
+ text_orig = re.sub(
+ rf" ?{_BR_SENTINEL} ?", "\n", annotated_text.text
+ ).strip()
+ if not text_orig:
+ continue
text_clean = HTMLDocumentBackend._clean_unicode(text_orig)
prov = self._make_prov(
text=text_clean,
@@ -3883,7 +3903,7 @@ def _extract_text_excluding_tag_obj_ids(
) -> str:
def _extract(node: PageElement) -> list[str]:
if isinstance(node, NavigableString):
- return [str(node)]
+ return [str(node).replace(_BR_SENTINEL, "\n")]
if isinstance(node, Tag):
if id(node) in excluded_obj_ids:
return []
diff --git a/tests/test_backend_html.py b/tests/test_backend_html.py
index 40502e5a91..4ba99db06c 100644
--- a/tests/test_backend_html.py
+++ b/tests/test_backend_html.py
@@ -2274,6 +2274,52 @@ def test_html_newline_handling():
)
+def test_html_line_break_outside_paragraphs():
+ # A
outside a paragraph or a table cell, e.g. in a heading or a list
+ # item, left the internal sentinel (U+E000) in the text. It is now a line
+ # break, as in a paragraph. The text of a checkbox stays on one line.
+ src = (
+ b"
| A | B |

Code" + b"
line
bold" + b"
code