From 57075d90ddc1381adad1542d22d5e796a91e8282 Mon Sep 17 00:00:00 2001 From: Krina-Paghadar Date: Fri, 23 Oct 2026 15:52:49 +0530 Subject: [PATCH 1/2] fix(md): keep backslash escapes inside the surrounding text Marko splits an escaped character (snake\_case, \$5, 1\.) into its own Literal node. Each node became a separate text item of an inline group and the exporters join those items with a space, so the text came out as "snake _ case" and "1 . not a list". Merge text runs that touch each other before the items are created. Signed-off-by: Krina-Paghadar Co-Authored-By: Claude Sonnet 5.5 --- docling/backend/md_backend.py | 46 ++++++++++++++++++++++++++++++++++ tests/test_backend_markdown.py | 34 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/docling/backend/md_backend.py b/docling/backend/md_backend.py index 7e77cb8054..3a6629515f 100644 --- a/docling/backend/md_backend.py +++ b/docling/backend/md_backend.py @@ -164,6 +164,51 @@ def _only_plain_line_breaks(children: list) -> bool: ) +def _merge_adjacent_text_runs(root: marko.element.Element) -> None: + """Merge neighbouring RawText/Literal runs of the AST into single RawText runs. + + Marko splits a backslash escape (``snake\\_case``, ``\\$5``, ``1\\.``) out of the + surrounding text into its own ``Literal`` node. Each run becomes a separate + text item of one inline group, which the exporters join with a space + (``snake _ case``). Runs that touch each other in the source are one piece of + text, so they are merged here, before the items are created. + + Args: + root: The root of the marko AST. Its descendants are changed in place. + """ + if not _MARKO_AVAILABLE: + return + stack: list = [root] + while stack: + element = stack.pop() + children = getattr(element, "children", None) + if not isinstance(children, list): + continue + merged: list = [] + for child in children: + if ( + merged + and isinstance(child, marko.inline.RawText | marko.inline.Literal) + and isinstance(merged[-1], marko.inline.RawText) + ): + # RawText is HTML-unescaped when the item is created, so an + # escaped ampersand must stay an entity to survive that step. + piece = ( + "&" + if isinstance(child, marko.inline.Literal) + and (child.children == "&") + else child.children + ) + merged[-1].children += piece + continue + if isinstance(child, marko.inline.Literal): + piece = "&" if child.children == "&" else child.children + child = marko.inline.RawText(piece) + merged.append(child) + stack.append(child) + element.children = merged + + class MarkdownDocumentBackend(DeclarativeDocumentBackend): @staticmethod def _apply_formatting( @@ -1162,6 +1207,7 @@ def convert(self) -> DoclingDocument: # children per cell, enabling RichTableCell for formatted content. marko_parser = Markdown(extensions=[_GFM_EXTENSION]) parsed_ast = marko_parser.parse(self.markdown) + _merge_adjacent_text_runs(parsed_ast) self._iterate_elements( element=parsed_ast, depth=0, diff --git a/tests/test_backend_markdown.py b/tests/test_backend_markdown.py index 95bbaecc55..7ed621bdb5 100644 --- a/tests/test_backend_markdown.py +++ b/tests/test_backend_markdown.py @@ -1306,3 +1306,37 @@ def test_rich_table_cell_no_stray_body_items(): # The only direct body child must be the table itself. body_crefs = {ref.cref for ref in doc.body.children} assert body_crefs == {"#/tables/0"} + + +def test_backslash_escapes_stay_in_their_text_run(): + r"""A backslash escape is part of the text around it, not a separate item. + + Marko splits ``\_`` into its own node. Each node became one item of an + inline group, and the exporters join those items with a space, so + ``snake\_case`` was exported as ``snake _ case``. + """ + opt = MarkdownBackendOptions() + + def items(markdown: str): + doc = _convert_markdown(markdown, opt) + return [(str(t.label), t.text) for t in doc.texts] + + assert items(r"snake\_case here") == [("text", "snake_case here")] + assert items(r"Price is \$5 and 50\% off") == [("text", "Price is $5 and 50% off")] + assert items(r"\*not emphasis\*") == [("text", "*not emphasis*")] + assert items(r"1\. not a list") == [("text", "1. not a list")] + assert items(r"- item with \* star") == [("list_item", "item with * star")] + assert items(r"# Head \# x") == [("title", "Head # x")] + assert items(r"> quote \* star") == [("text", "quote * star")] + + # An escaped ampersand stays literal text, not an HTML entity + assert items(r"a \& b") == [("text", "a & b")] + + # Runs that are really separate (inline code, emphasis) stay separate items + assert items(r"Use `x` and snake\_case") == [ + ("text", "Use"), + ("code", "x"), + ("text", "and snake_case"), + ] + doc = _convert_markdown(r"**bold \* star** plain", opt) + assert [t.text for t in doc.texts] == ["bold * star", "plain"] From 48f1cf3de33e1d8d34f9d9b6abc9606bc25c4ee7 Mon Sep 17 00:00:00 2001 From: Krina-Paghadar Date: Sun, 25 Oct 2026 14:30:07 +0530 Subject: [PATCH 2/2] test(md): update the Markdown groundtruth for merged escape runs The groundtruth held the old output of the backend: "2 . strange" for an escaped marker, and inline groups numbered in the old order. Regenerated with docling-core 2.103.1 (the locked version) and DOCLING_GEN_TEST_DATA=1. The only content change is "2. strange" and the marker of an ordered list item that has an escaped character. The other lines are renumbered group references. Signed-off-by: Krina-Paghadar Co-Authored-By: Claude Sonnet 5.5 --- .../md/groundtruth/escaped_characters.md.json | 2 +- .../groundtruth/inline_and_formatting.md.yaml | 156 +++++++++--------- .../md/groundtruth/mixed_without_h1.md.md | 2 +- .../md/groundtruth/mixed_without_h1.md.yaml | 4 +- 4 files changed, 82 insertions(+), 82 deletions(-) diff --git a/tests/data/md/groundtruth/escaped_characters.md.json b/tests/data/md/groundtruth/escaped_characters.md.json index 287450224f..907df2fd38 100644 --- a/tests/data/md/groundtruth/escaped_characters.md.json +++ b/tests/data/md/groundtruth/escaped_characters.md.json @@ -167,7 +167,7 @@ "orig": "& < > \" '", "text": "& < > \" '", "enumerated": true, - "marker": "" + "marker": "1." }, { "self_ref": "#/texts/5", diff --git a/tests/data/md/groundtruth/inline_and_formatting.md.yaml b/tests/data/md/groundtruth/inline_and_formatting.md.yaml index c6181a41cb..c746bee498 100644 --- a/tests/data/md/groundtruth/inline_and_formatting.md.yaml +++ b/tests/data/md/groundtruth/inline_and_formatting.md.yaml @@ -170,38 +170,46 @@ groups: parent: $ref: '#/body' self_ref: '#/groups/13' +- children: + - $ref: '#/groups/15' + content_layer: body + label: unspecified + name: rich_cell_group_0_0_0 + parent: + $ref: '#/tables/0' + self_ref: '#/groups/14' - children: - $ref: '#/texts/53' content_layer: body label: inline name: group parent: - $ref: '#/groups/15' - self_ref: '#/groups/14' + $ref: '#/groups/14' + self_ref: '#/groups/15' - children: - - $ref: '#/groups/14' + - $ref: '#/groups/17' content_layer: body label: unspecified - name: rich_cell_group_0_0_0 + name: rich_cell_group_0_1_0 parent: $ref: '#/tables/0' - self_ref: '#/groups/15' + self_ref: '#/groups/16' - children: - $ref: '#/texts/54' content_layer: body label: inline name: group parent: - $ref: '#/groups/17' - self_ref: '#/groups/16' + $ref: '#/groups/16' + self_ref: '#/groups/17' - children: - - $ref: '#/groups/16' + - $ref: '#/groups/19' content_layer: body label: unspecified - name: rich_cell_group_0_1_0 + name: rich_cell_group_0_0_2 parent: $ref: '#/tables/0' - self_ref: '#/groups/17' + self_ref: '#/groups/18' - children: - $ref: '#/texts/55' - $ref: '#/texts/56' @@ -209,16 +217,16 @@ groups: label: inline name: group parent: - $ref: '#/groups/19' - self_ref: '#/groups/18' + $ref: '#/groups/18' + self_ref: '#/groups/19' - children: - - $ref: '#/groups/18' + - $ref: '#/groups/21' content_layer: body label: unspecified - name: rich_cell_group_0_0_2 + name: rich_cell_group_0_0_3 parent: $ref: '#/tables/0' - self_ref: '#/groups/19' + self_ref: '#/groups/20' - children: - $ref: '#/texts/57' - $ref: '#/texts/58' @@ -226,16 +234,16 @@ groups: label: inline name: group parent: - $ref: '#/groups/21' - self_ref: '#/groups/20' + $ref: '#/groups/20' + self_ref: '#/groups/21' - children: - - $ref: '#/groups/20' + - $ref: '#/groups/23' content_layer: body label: unspecified - name: rich_cell_group_0_0_3 + name: rich_cell_group_0_0_4 parent: $ref: '#/tables/0' - self_ref: '#/groups/21' + self_ref: '#/groups/22' - children: - $ref: '#/texts/59' - $ref: '#/texts/60' @@ -243,16 +251,16 @@ groups: label: inline name: group parent: - $ref: '#/groups/23' - self_ref: '#/groups/22' + $ref: '#/groups/22' + self_ref: '#/groups/23' - children: - - $ref: '#/groups/22' + - $ref: '#/groups/25' content_layer: body label: unspecified - name: rich_cell_group_0_0_4 + name: rich_cell_group_0_0_5 parent: $ref: '#/tables/0' - self_ref: '#/groups/23' + self_ref: '#/groups/24' - children: - $ref: '#/texts/61' - $ref: '#/texts/62' @@ -263,16 +271,16 @@ groups: label: inline name: group parent: - $ref: '#/groups/25' - self_ref: '#/groups/24' + $ref: '#/groups/24' + self_ref: '#/groups/25' - children: - - $ref: '#/groups/24' + - $ref: '#/groups/27' content_layer: body label: unspecified - name: rich_cell_group_0_0_5 + name: rich_cell_group_0_0_6 parent: $ref: '#/tables/0' - self_ref: '#/groups/25' + self_ref: '#/groups/26' - children: - $ref: '#/texts/66' - $ref: '#/texts/67' @@ -281,16 +289,16 @@ groups: label: inline name: group parent: - $ref: '#/groups/27' - self_ref: '#/groups/26' + $ref: '#/groups/26' + self_ref: '#/groups/27' - children: - - $ref: '#/groups/26' + - $ref: '#/groups/29' content_layer: body label: unspecified - name: rich_cell_group_0_0_6 + name: rich_cell_group_0_0_7 parent: $ref: '#/tables/0' - self_ref: '#/groups/27' + self_ref: '#/groups/28' - children: - $ref: '#/texts/69' - $ref: '#/texts/70' @@ -298,15 +306,7 @@ groups: label: inline name: group parent: - $ref: '#/groups/29' - self_ref: '#/groups/28' -- children: - - $ref: '#/groups/28' - content_layer: body - label: unspecified - name: rich_cell_group_0_0_7 - parent: - $ref: '#/tables/0' + $ref: '#/groups/28' self_ref: '#/groups/29' key_value_items: [] name: inline_and_formatting @@ -321,14 +321,14 @@ tables: - annotations: [] captions: [] children: - - $ref: '#/groups/15' - - $ref: '#/groups/17' - - $ref: '#/groups/19' - - $ref: '#/groups/21' - - $ref: '#/groups/23' - - $ref: '#/groups/25' - - $ref: '#/groups/27' - - $ref: '#/groups/29' + - $ref: '#/groups/14' + - $ref: '#/groups/16' + - $ref: '#/groups/18' + - $ref: '#/groups/20' + - $ref: '#/groups/22' + - $ref: '#/groups/24' + - $ref: '#/groups/26' + - $ref: '#/groups/28' content_layer: body data: grid: @@ -518,7 +518,7 @@ tables: end_row_offset_idx: 1 fillable: false ref: - $ref: '#/groups/15' + $ref: '#/groups/14' row_header: false row_section: false row_span: 1 @@ -531,7 +531,7 @@ tables: end_row_offset_idx: 1 fillable: false ref: - $ref: '#/groups/17' + $ref: '#/groups/16' row_header: false row_section: false row_span: 1 @@ -566,7 +566,7 @@ tables: end_row_offset_idx: 3 fillable: false ref: - $ref: '#/groups/19' + $ref: '#/groups/18' row_header: false row_section: false row_span: 1 @@ -590,7 +590,7 @@ tables: end_row_offset_idx: 4 fillable: false ref: - $ref: '#/groups/21' + $ref: '#/groups/20' row_header: false row_section: false row_span: 1 @@ -614,7 +614,7 @@ tables: end_row_offset_idx: 5 fillable: false ref: - $ref: '#/groups/23' + $ref: '#/groups/22' row_header: false row_section: false row_span: 1 @@ -638,7 +638,7 @@ tables: end_row_offset_idx: 6 fillable: false ref: - $ref: '#/groups/25' + $ref: '#/groups/24' row_header: false row_section: false row_span: 1 @@ -662,7 +662,7 @@ tables: end_row_offset_idx: 7 fillable: false ref: - $ref: '#/groups/27' + $ref: '#/groups/26' row_header: false row_section: false row_span: 1 @@ -686,7 +686,7 @@ tables: end_row_offset_idx: 8 fillable: false ref: - $ref: '#/groups/29' + $ref: '#/groups/28' row_header: false row_section: false row_span: 1 @@ -1330,7 +1330,7 @@ texts: label: text orig: Bold Heading parent: - $ref: '#/groups/14' + $ref: '#/groups/15' prov: [] self_ref: '#/texts/53' text: Bold Heading @@ -1345,7 +1345,7 @@ texts: label: text orig: Italic Heading parent: - $ref: '#/groups/16' + $ref: '#/groups/17' prov: [] self_ref: '#/texts/54' text: Italic Heading @@ -1360,7 +1360,7 @@ texts: label: text orig: C parent: - $ref: '#/groups/18' + $ref: '#/groups/19' prov: [] self_ref: '#/texts/55' text: C @@ -1369,7 +1369,7 @@ texts: label: text orig: Cadre parent: - $ref: '#/groups/18' + $ref: '#/groups/19' prov: [] self_ref: '#/texts/56' text: Cadre @@ -1378,7 +1378,7 @@ texts: label: text orig: foo parent: - $ref: '#/groups/20' + $ref: '#/groups/21' prov: [] self_ref: '#/texts/57' text: foo @@ -1393,7 +1393,7 @@ texts: label: text orig: bar parent: - $ref: '#/groups/20' + $ref: '#/groups/21' prov: [] self_ref: '#/texts/58' text: bar @@ -1408,7 +1408,7 @@ texts: label: text orig: A parent: - $ref: '#/groups/22' + $ref: '#/groups/23' prov: [] self_ref: '#/texts/59' text: A @@ -1423,7 +1423,7 @@ texts: label: text orig: B parent: - $ref: '#/groups/22' + $ref: '#/groups/23' prov: [] self_ref: '#/texts/60' text: B @@ -1438,7 +1438,7 @@ texts: label: text orig: italic parent: - $ref: '#/groups/24' + $ref: '#/groups/25' prov: [] self_ref: '#/texts/61' text: italic @@ -1447,7 +1447,7 @@ texts: label: text orig: and parent: - $ref: '#/groups/24' + $ref: '#/groups/25' prov: [] self_ref: '#/texts/62' text: and @@ -1462,7 +1462,7 @@ texts: label: text orig: bold parent: - $ref: '#/groups/24' + $ref: '#/groups/25' prov: [] self_ref: '#/texts/63' text: bold @@ -1471,7 +1471,7 @@ texts: label: text orig: and parent: - $ref: '#/groups/24' + $ref: '#/groups/25' prov: [] self_ref: '#/texts/64' text: and @@ -1486,7 +1486,7 @@ texts: label: text orig: strikethrough parent: - $ref: '#/groups/24' + $ref: '#/groups/25' prov: [] self_ref: '#/texts/65' text: strikethrough @@ -1495,7 +1495,7 @@ texts: label: text orig: run parent: - $ref: '#/groups/26' + $ref: '#/groups/27' prov: [] self_ref: '#/texts/66' text: run @@ -1507,7 +1507,7 @@ texts: label: code orig: build parent: - $ref: '#/groups/26' + $ref: '#/groups/27' prov: [] references: [] self_ref: '#/texts/67' @@ -1517,7 +1517,7 @@ texts: label: text orig: now parent: - $ref: '#/groups/26' + $ref: '#/groups/27' prov: [] self_ref: '#/texts/68' text: now @@ -1526,7 +1526,7 @@ texts: label: text orig: visit parent: - $ref: '#/groups/28' + $ref: '#/groups/29' prov: [] self_ref: '#/texts/69' text: visit @@ -1536,7 +1536,7 @@ texts: label: text orig: Docling parent: - $ref: '#/groups/28' + $ref: '#/groups/29' prov: [] self_ref: '#/texts/70' text: Docling diff --git a/tests/data/md/groundtruth/mixed_without_h1.md.md b/tests/data/md/groundtruth/mixed_without_h1.md.md index 47232f4441..884981ef3b 100644 --- a/tests/data/md/groundtruth/mixed_without_h1.md.md +++ b/tests/data/md/groundtruth/mixed_without_h1.md.md @@ -5,6 +5,6 @@ Content before first heading - A. first - subitem - B. second -- 2 . strange +- 2. strange The end! diff --git a/tests/data/md/groundtruth/mixed_without_h1.md.yaml b/tests/data/md/groundtruth/mixed_without_h1.md.yaml index b3037f9d61..6f0944c3b6 100644 --- a/tests/data/md/groundtruth/mixed_without_h1.md.yaml +++ b/tests/data/md/groundtruth/mixed_without_h1.md.yaml @@ -111,12 +111,12 @@ texts: enumerated: false label: list_item marker: '' - orig: 2 . strange + orig: 2. strange parent: $ref: '#/groups/1' prov: [] self_ref: '#/texts/5' - text: 2 . strange + text: 2. strange - children: [] content_layer: body label: text