From 2a69dc3dab26fee8c702dc1c7853851471337c9a Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:34:59 +0900 Subject: [PATCH 1/2] fix(md): keep inline code spans inside GFM table cells A CodeSpan in a table cell closed the table mid-row, so one GFM table became two and the text before the span was dropped. Signed-off-by: kevin9327 <5299031+kevin9327@users.noreply.github.com> --- docling/backend/md_backend.py | 57 ++++++++++++++++++++-------------- tests/test_backend_markdown.py | 32 +++++++++++++++++++ 2 files changed, 66 insertions(+), 23 deletions(-) diff --git a/docling/backend/md_backend.py b/docling/backend/md_backend.py index 90399e0a29..b195c9aa29 100644 --- a/docling/backend/md_backend.py +++ b/docling/backend/md_backend.py @@ -712,32 +712,43 @@ def _iterate_elements( # noqa: C901 self._pending_soft_line_break = False elif isinstance(element, marko.inline.CodeSpan): - self._close_table(doc) - _log.debug(" - Code Span: %s", element.children) - snippet_text = str(element.children).strip() - # If this CodeSpan is the only content of a list item / heading, Marko won't - # emit RawText. Flush pending creations here to avoid leaking payloads. - if creation_stack and snippet_text: - parent_item = self._flush_creation_stack( - doc=doc, - creation_stack=creation_stack, - snippet_text=snippet_text, - parent_item=parent_item, - list_ordered_flag_by_ref=list_ordered_flag_by_ref, - list_start_by_ref=list_start_by_ref, - list_item_counter_by_ref=list_item_counter_by_ref, - list_last_item_by_ref=list_last_item_by_ref, + snippet_text = str(element.children) + if self.in_table: + # Keep the span in the current row. Closing the table here used + # to split one GFM table into two and drop the text before the + # span. + cell_text = unescape(snippet_text) + if self.md_table_buffer: + self.md_table_buffer[-1] += cell_text + else: + self.md_table_buffer.append(cell_text) + else: + self._close_table(doc) + _log.debug(" - Code Span: %s", element.children) + snippet_text = snippet_text.strip() + # If this CodeSpan is the only content of a list item / heading, Marko won't + # emit RawText. Flush pending creations here to avoid leaking payloads. + if creation_stack and snippet_text: + parent_item = self._flush_creation_stack( + doc=doc, + creation_stack=creation_stack, + snippet_text=snippet_text, + parent_item=parent_item, + list_ordered_flag_by_ref=list_ordered_flag_by_ref, + list_start_by_ref=list_start_by_ref, + list_item_counter_by_ref=list_item_counter_by_ref, + list_last_item_by_ref=list_last_item_by_ref, + formatting=formatting, + hyperlink=hyperlink, + ) + # Represent CodeSpan as the container's text; avoid adding a duplicate CodeItem. + return + doc.add_code( + parent=parent_item, + text=snippet_text, formatting=formatting, hyperlink=hyperlink, ) - # Represent CodeSpan as the container's text; avoid adding a duplicate CodeItem. - return - doc.add_code( - parent=parent_item, - text=snippet_text, - formatting=formatting, - hyperlink=hyperlink, - ) elif ( isinstance(element, marko.block.CodeBlock | marko.block.FencedCode) diff --git a/tests/test_backend_markdown.py b/tests/test_backend_markdown.py index 2c35fbc32a..331feab50f 100644 --- a/tests/test_backend_markdown.py +++ b/tests/test_backend_markdown.py @@ -238,6 +238,38 @@ def test_code_block_language_detection(): ] +def test_convert_table_keeps_inline_code_spans(): + """A code span inside a GFM table cell must not close the table. + + CodeSpan used to flush the half-built table, so a cell like + ``run `build` now`` split the table in two and dropped the word + before the span. + """ + markdown = """| Command | Description | +| --- | --- | +| run `build` now | builds it | +| clean | removes it | +""" + conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) + assert conv_result.status == ConversionStatus.SUCCESS + + assert len(conv_result.document.tables) == 1 + table_data = conv_result.document.tables[0].data + assert table_data.num_rows == 3 + assert table_data.num_cols == 2 + cells = [cell.text for cell in table_data.table_cells] + assert cells[0] == "Command" + assert cells[1] == "Description" + # Adjacent RawText is still stripped before concatenation, so spaces + # around the span may collapse. The span itself must stay in this cell. + assert "run" in cells[2] + assert "build" in cells[2] + assert "now" in cells[2] + assert cells[3] == "builds it" + assert cells[4] == "clean" + assert cells[5] == "removes it" + + def test_convert_table_has_no_duplicate_cells(): """ Regression test: From d18232f9b86b954187091b438f2030dcdd64ce95 Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sat, 26 Sep 2026 09:02:23 +0900 Subject: [PATCH 2/2] fix(md): keep a code span's cell whole, its pipes and entities literal Review: a RawText segment was stripped before it joined the row, so "run `build` now" became "runbuildnow"; a pipe inside a code span split the cell; and the span's text was decoded once on the way in and again in _close_table. The table buffer now has one contract: _close_table decodes it exactly once, a RawText keeps its whitespace and its pipe entities, and literal text such as a code span is entity-encoded with its pipes as |. The debug line fires in both paths, the comment states the rule, the fixture gains a code-span row, and the test asserts every cell, including a pipe and an entity inside a span. Co-Authored-By: Claude Fable 5.1 Signed-off-by: kevin9327 <5299031+kevin9327@users.noreply.github.com> --- docling/backend/md_backend.py | 48 ++++++++++++------- .../groundtruth/inline_and_formatting.md.md | 1 + .../groundtruth/inline_and_formatting.md.yaml | 48 ++++++++++++++++++- .../data/md/sources/inline_and_formatting.md | 1 + tests/test_backend_markdown.py | 34 ++++++------- 5 files changed, 98 insertions(+), 34 deletions(-) diff --git a/docling/backend/md_backend.py b/docling/backend/md_backend.py index b195c9aa29..b64b879bca 100644 --- a/docling/backend/md_backend.py +++ b/docling/backend/md_backend.py @@ -3,6 +3,7 @@ from __future__ import annotations +import html import logging import re import warnings @@ -216,6 +217,12 @@ def _starts_pipeless_table(element: marko.block.Paragraph) -> bool: MarkdownDocumentBackend._split_table_row(lines[1]) ) + # md_table_buffer holds the rows of the table being read, one string per + # row, in a form _split_table_row can split on "|" and _close_table decodes + # exactly once with unescape(): a RawText is decoded on the way in except + # for the pipe entities, so a pipe that is cell content survives the split; + # literal text, such as a code span, is entity-encoded on the way in and + # its pipes become |. @staticmethod def _unescape_except_pipe(text: str) -> str: def replace(match): @@ -320,6 +327,20 @@ def __init__( ) from e return + @staticmethod + def _encode_table_literal(text: str) -> str: + """Encode literal text for md_table_buffer: entities are escaped so the + decode in _close_table returns the text as written, and a pipe is not + a column separator.""" + return MarkdownDocumentBackend._escape_pipes(html.escape(text, quote=False)) + + def _append_table_text(self, text: str) -> None: + """Add text to the current row of md_table_buffer.""" + if self.md_table_buffer: + self.md_table_buffer[-1] += text + else: + self.md_table_buffer.append(text) + def _close_table(self, doc: DoclingDocument): self.in_pipeless_table = False if self.in_table: @@ -655,14 +676,12 @@ def _iterate_elements( # noqa: C901 if is_table_row: self.in_table = True if self.in_table and original_text: - table_text = self._unescape_except_pipe(original_text) + # Whitespace is kept: a cell can be several nodes ("run ", a + # code span, " now"), and _split_table_row strips the cell once. + cell_text = self._unescape_except_pipe(original_text) if is_escape: - table_text = self._escape_pipes(table_text) - # If we're in a table, keep adding text (for formatted content in cells) - if self.md_table_buffer: - self.md_table_buffer[len(self.md_table_buffer) - 1] += table_text - else: - self.md_table_buffer.append(table_text) + cell_text = self._escape_pipes(cell_text) + self._append_table_text(cell_text) elif snippet_text: # Not in table - close any pending table and process as regular text self._close_table(doc) @@ -712,19 +731,16 @@ def _iterate_elements( # noqa: C901 self._pending_soft_line_break = False elif isinstance(element, marko.inline.CodeSpan): + _log.debug(" - Code Span: %s", element.children) snippet_text = str(element.children) if self.in_table: - # Keep the span in the current row. Closing the table here used - # to split one GFM table into two and drop the text before the - # span. - cell_text = unescape(snippet_text) - if self.md_table_buffer: - self.md_table_buffer[-1] += cell_text - else: - self.md_table_buffer.append(cell_text) + # A CodeSpan does not delimit cells; keep its content in the + # current buffer slot. Its text is literal, so it is encoded for + # the one decode in _close_table, and a pipe in it is not a + # column separator. + self._append_table_text(self._encode_table_literal(snippet_text)) else: self._close_table(doc) - _log.debug(" - Code Span: %s", element.children) snippet_text = snippet_text.strip() # If this CodeSpan is the only content of a list item / heading, Marko won't # emit RawText. Flush pending creations here to avoid leaking payloads. diff --git a/tests/data/md/groundtruth/inline_and_formatting.md.md b/tests/data/md/groundtruth/inline_and_formatting.md.md index 260a4eeabf..e09260802d 100644 --- a/tests/data/md/groundtruth/inline_and_formatting.md.md +++ b/tests/data/md/groundtruth/inline_and_formatting.md.md @@ -34,3 +34,4 @@ Some *`formatted_code`* | foo bar | y | | A B | z | | italic and bold | w | +| run build now | code in a cell | diff --git a/tests/data/md/groundtruth/inline_and_formatting.md.yaml b/tests/data/md/groundtruth/inline_and_formatting.md.yaml index 1e1bd8509f..a63db7606c 100644 --- a/tests/data/md/groundtruth/inline_and_formatting.md.yaml +++ b/tests/data/md/groundtruth/inline_and_formatting.md.yaml @@ -169,7 +169,7 @@ groups: key_value_items: [] name: inline_and_formatting origin: - binary_hash: 1050321783461772863 + binary_hash: 4724758855956816018 filename: inline_and_formatting.md mimetype: text/markdown pages: {} @@ -314,8 +314,30 @@ tables: start_col_offset_idx: 1 start_row_offset_idx: 5 text: w + - - col_span: 1 + column_header: false + end_col_offset_idx: 1 + end_row_offset_idx: 7 + fillable: false + row_header: false + row_section: false + row_span: 1 + start_col_offset_idx: 0 + start_row_offset_idx: 6 + text: run build now + - col_span: 1 + column_header: false + end_col_offset_idx: 2 + end_row_offset_idx: 7 + fillable: false + row_header: false + row_section: false + row_span: 1 + start_col_offset_idx: 1 + start_row_offset_idx: 6 + text: code in a cell num_cols: 2 - num_rows: 6 + num_rows: 7 orientation: rot_0 table_cells: - col_span: 1 @@ -450,6 +472,28 @@ tables: start_col_offset_idx: 1 start_row_offset_idx: 5 text: w + - col_span: 1 + column_header: false + end_col_offset_idx: 1 + end_row_offset_idx: 7 + fillable: false + row_header: false + row_section: false + row_span: 1 + start_col_offset_idx: 0 + start_row_offset_idx: 6 + text: run build now + - col_span: 1 + column_header: false + end_col_offset_idx: 2 + end_row_offset_idx: 7 + fillable: false + row_header: false + row_section: false + row_span: 1 + start_col_offset_idx: 1 + start_row_offset_idx: 6 + text: code in a cell footnotes: [] label: table parent: diff --git a/tests/data/md/sources/inline_and_formatting.md b/tests/data/md/sources/inline_and_formatting.md index dfc617dede..c0889f1c3e 100644 --- a/tests/data/md/sources/inline_and_formatting.md +++ b/tests/data/md/sources/inline_and_formatting.md @@ -34,3 +34,4 @@ Some *`formatted_code`* | foo **bar** | y | | **A** **B** | z | | *italic* and **bold** | w | +| run `build` now | code in a cell | diff --git a/tests/test_backend_markdown.py b/tests/test_backend_markdown.py index 331feab50f..842eb494b5 100644 --- a/tests/test_backend_markdown.py +++ b/tests/test_backend_markdown.py @@ -239,15 +239,16 @@ def test_code_block_language_detection(): def test_convert_table_keeps_inline_code_spans(): - """A code span inside a GFM table cell must not close the table. + """A code span inside a GFM table cell is part of that cell. - CodeSpan used to flush the half-built table, so a cell like - ``run `build` now`` split the table in two and dropped the word - before the span. + The span keeps the words around it, a pipe inside it is not a column + separator, and its text is literal, so an entity in it is not decoded. """ markdown = """| Command | Description | | --- | --- | | run `build` now | builds it | +| `a | b` | keeps the pipe | +| `&` | stays literal | | clean | removes it | """ conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) @@ -255,19 +256,20 @@ def test_convert_table_keeps_inline_code_spans(): assert len(conv_result.document.tables) == 1 table_data = conv_result.document.tables[0].data - assert table_data.num_rows == 3 + assert table_data.num_rows == 5 assert table_data.num_cols == 2 - cells = [cell.text for cell in table_data.table_cells] - assert cells[0] == "Command" - assert cells[1] == "Description" - # Adjacent RawText is still stripped before concatenation, so spaces - # around the span may collapse. The span itself must stay in this cell. - assert "run" in cells[2] - assert "build" in cells[2] - assert "now" in cells[2] - assert cells[3] == "builds it" - assert cells[4] == "clean" - assert cells[5] == "removes it" + assert [cell.text for cell in table_data.table_cells] == [ + "Command", + "Description", + "run build now", + "builds it", + "a | b", + "keeps the pipe", + "&", + "stays literal", + "clean", + "removes it", + ] def test_convert_table_has_no_duplicate_cells():