diff --git a/docling/backend/md_backend.py b/docling/backend/md_backend.py index 90399e0a29..b64b879bca 100644 --- a/docling/backend/md_backend.py +++ b/docling/backend/md_backend.py @@ -3,6 +3,7 @@ from __future__ import annotations +import html import logging import re import warnings @@ -216,6 +217,12 @@ def _starts_pipeless_table(element: marko.block.Paragraph) -> bool: MarkdownDocumentBackend._split_table_row(lines[1]) ) + # md_table_buffer holds the rows of the table being read, one string per + # row, in a form _split_table_row can split on "|" and _close_table decodes + # exactly once with unescape(): a RawText is decoded on the way in except + # for the pipe entities, so a pipe that is cell content survives the split; + # literal text, such as a code span, is entity-encoded on the way in and + # its pipes become |. @staticmethod def _unescape_except_pipe(text: str) -> str: def replace(match): @@ -320,6 +327,20 @@ def __init__( ) from e return + @staticmethod + def _encode_table_literal(text: str) -> str: + """Encode literal text for md_table_buffer: entities are escaped so the + decode in _close_table returns the text as written, and a pipe is not + a column separator.""" + return MarkdownDocumentBackend._escape_pipes(html.escape(text, quote=False)) + + def _append_table_text(self, text: str) -> None: + """Add text to the current row of md_table_buffer.""" + if self.md_table_buffer: + self.md_table_buffer[-1] += text + else: + self.md_table_buffer.append(text) + def _close_table(self, doc: DoclingDocument): self.in_pipeless_table = False if self.in_table: @@ -655,14 +676,12 @@ def _iterate_elements( # noqa: C901 if is_table_row: self.in_table = True if self.in_table and original_text: - table_text = self._unescape_except_pipe(original_text) + # Whitespace is kept: a cell can be several nodes ("run ", a + # code span, " now"), and _split_table_row strips the cell once. + cell_text = self._unescape_except_pipe(original_text) if is_escape: - table_text = self._escape_pipes(table_text) - # If we're in a table, keep adding text (for formatted content in cells) - if self.md_table_buffer: - self.md_table_buffer[len(self.md_table_buffer) - 1] += table_text - else: - self.md_table_buffer.append(table_text) + cell_text = self._escape_pipes(cell_text) + self._append_table_text(cell_text) elif snippet_text: # Not in table - close any pending table and process as regular text self._close_table(doc) @@ -712,32 +731,40 @@ def _iterate_elements( # noqa: C901 self._pending_soft_line_break = False elif isinstance(element, marko.inline.CodeSpan): - self._close_table(doc) _log.debug(" - Code Span: %s", element.children) - snippet_text = str(element.children).strip() - # If this CodeSpan is the only content of a list item / heading, Marko won't - # emit RawText. Flush pending creations here to avoid leaking payloads. - if creation_stack and snippet_text: - parent_item = self._flush_creation_stack( - doc=doc, - creation_stack=creation_stack, - snippet_text=snippet_text, - parent_item=parent_item, - list_ordered_flag_by_ref=list_ordered_flag_by_ref, - list_start_by_ref=list_start_by_ref, - list_item_counter_by_ref=list_item_counter_by_ref, - list_last_item_by_ref=list_last_item_by_ref, + snippet_text = str(element.children) + if self.in_table: + # A CodeSpan does not delimit cells; keep its content in the + # current buffer slot. Its text is literal, so it is encoded for + # the one decode in _close_table, and a pipe in it is not a + # column separator. + self._append_table_text(self._encode_table_literal(snippet_text)) + else: + self._close_table(doc) + snippet_text = snippet_text.strip() + # If this CodeSpan is the only content of a list item / heading, Marko won't + # emit RawText. Flush pending creations here to avoid leaking payloads. + if creation_stack and snippet_text: + parent_item = self._flush_creation_stack( + doc=doc, + creation_stack=creation_stack, + snippet_text=snippet_text, + parent_item=parent_item, + list_ordered_flag_by_ref=list_ordered_flag_by_ref, + list_start_by_ref=list_start_by_ref, + list_item_counter_by_ref=list_item_counter_by_ref, + list_last_item_by_ref=list_last_item_by_ref, + formatting=formatting, + hyperlink=hyperlink, + ) + # Represent CodeSpan as the container's text; avoid adding a duplicate CodeItem. + return + doc.add_code( + parent=parent_item, + text=snippet_text, formatting=formatting, hyperlink=hyperlink, ) - # Represent CodeSpan as the container's text; avoid adding a duplicate CodeItem. - return - doc.add_code( - parent=parent_item, - text=snippet_text, - formatting=formatting, - hyperlink=hyperlink, - ) elif ( isinstance(element, marko.block.CodeBlock | marko.block.FencedCode) diff --git a/tests/data/md/groundtruth/inline_and_formatting.md.md b/tests/data/md/groundtruth/inline_and_formatting.md.md index 260a4eeabf..e09260802d 100644 --- a/tests/data/md/groundtruth/inline_and_formatting.md.md +++ b/tests/data/md/groundtruth/inline_and_formatting.md.md @@ -34,3 +34,4 @@ Some *`formatted_code`* | foo bar | y | | A B | z | | italic and bold | w | +| run build now | code in a cell | diff --git a/tests/data/md/groundtruth/inline_and_formatting.md.yaml b/tests/data/md/groundtruth/inline_and_formatting.md.yaml index 1e1bd8509f..a63db7606c 100644 --- a/tests/data/md/groundtruth/inline_and_formatting.md.yaml +++ b/tests/data/md/groundtruth/inline_and_formatting.md.yaml @@ -169,7 +169,7 @@ groups: key_value_items: [] name: inline_and_formatting origin: - binary_hash: 1050321783461772863 + binary_hash: 4724758855956816018 filename: inline_and_formatting.md mimetype: text/markdown pages: {} @@ -314,8 +314,30 @@ tables: start_col_offset_idx: 1 start_row_offset_idx: 5 text: w + - - col_span: 1 + column_header: false + end_col_offset_idx: 1 + end_row_offset_idx: 7 + fillable: false + row_header: false + row_section: false + row_span: 1 + start_col_offset_idx: 0 + start_row_offset_idx: 6 + text: run build now + - col_span: 1 + column_header: false + end_col_offset_idx: 2 + end_row_offset_idx: 7 + fillable: false + row_header: false + row_section: false + row_span: 1 + start_col_offset_idx: 1 + start_row_offset_idx: 6 + text: code in a cell num_cols: 2 - num_rows: 6 + num_rows: 7 orientation: rot_0 table_cells: - col_span: 1 @@ -450,6 +472,28 @@ tables: start_col_offset_idx: 1 start_row_offset_idx: 5 text: w + - col_span: 1 + column_header: false + end_col_offset_idx: 1 + end_row_offset_idx: 7 + fillable: false + row_header: false + row_section: false + row_span: 1 + start_col_offset_idx: 0 + start_row_offset_idx: 6 + text: run build now + - col_span: 1 + column_header: false + end_col_offset_idx: 2 + end_row_offset_idx: 7 + fillable: false + row_header: false + row_section: false + row_span: 1 + start_col_offset_idx: 1 + start_row_offset_idx: 6 + text: code in a cell footnotes: [] label: table parent: diff --git a/tests/data/md/sources/inline_and_formatting.md b/tests/data/md/sources/inline_and_formatting.md index dfc617dede..c0889f1c3e 100644 --- a/tests/data/md/sources/inline_and_formatting.md +++ b/tests/data/md/sources/inline_and_formatting.md @@ -34,3 +34,4 @@ Some *`formatted_code`* | foo **bar** | y | | **A** **B** | z | | *italic* and **bold** | w | +| run `build` now | code in a cell | diff --git a/tests/test_backend_markdown.py b/tests/test_backend_markdown.py index 2c35fbc32a..842eb494b5 100644 --- a/tests/test_backend_markdown.py +++ b/tests/test_backend_markdown.py @@ -238,6 +238,40 @@ def test_code_block_language_detection(): ] +def test_convert_table_keeps_inline_code_spans(): + """A code span inside a GFM table cell is part of that cell. + + The span keeps the words around it, a pipe inside it is not a column + separator, and its text is literal, so an entity in it is not decoded. + """ + markdown = """| Command | Description | +| --- | --- | +| run `build` now | builds it | +| `a | b` | keeps the pipe | +| `&` | stays literal | +| clean | removes it | +""" + conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) + assert conv_result.status == ConversionStatus.SUCCESS + + assert len(conv_result.document.tables) == 1 + table_data = conv_result.document.tables[0].data + assert table_data.num_rows == 5 + assert table_data.num_cols == 2 + assert [cell.text for cell in table_data.table_cells] == [ + "Command", + "Description", + "run build now", + "builds it", + "a | b", + "keeps the pipe", + "&", + "stays literal", + "clean", + "removes it", + ] + + def test_convert_table_has_no_duplicate_cells(): """ Regression test: