From 9134a0886efd6abd33c7123e0e4e2614c3fa3a0e Mon Sep 17 00:00:00 2001 From: Siddharth-DWT Date: Wed, 21 Oct 2026 17:11:56 +0530 Subject: [PATCH 1/2] fix(markdown): keep GFM task-list checkboxes and paragraph [x] markers marko's GFM paragraph strips a leading "[x] " / "[ ] " from every paragraph and records it in `checked`, which the Markdown backend never read. Task-list items lost their state, and any paragraph starting with the marker lost that text. Only a list item's first paragraph now treats the marker as a task marker; the backend turns those items into checkbox_selected / checkbox_unselected items in the list, as the HTML backend does. Elsewhere the marker stays text. Fixes #4930 Signed-off-by: Siddharth-DWT --- docling/backend/md_backend.py | 50 ++++++++++++++++++++++++++++- tests/test_backend_markdown.py | 58 ++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/docling/backend/md_backend.py b/docling/backend/md_backend.py index 99cbcaf23a..f54e54bc0a 100644 --- a/docling/backend/md_backend.py +++ b/docling/backend/md_backend.py @@ -64,6 +64,30 @@ from marko.helpers import MarkoExtension class _GfmParagraph(_gfm_el.Paragraph): + # GFM task-list markers ("[x] " / "[ ] ") belong only at the start of a list + # item's first paragraph. marko's GFM paragraph strips them from every + # paragraph, which would delete the text of e.g. "[x] I accept the terms". + @classmethod + def parse(cls, source): + state = source.state + opens_list_item = isinstance(state, marko.block.ListItem) and not getattr( + state, "_docling_has_paragraph", False + ) + if isinstance(state, marko.block.ListItem): + setattr(state, "_docling_has_paragraph", True) + result = super().parse(source) + if hasattr(result, "__dict__") and not isinstance( + result, marko.block.BlockElement + ): + setattr(result, "docling_task_list_candidate", opens_list_item) + return result + + def __init__(self, lines: list[str]) -> None: + if getattr(lines, "docling_task_list_candidate", False): + super().__init__(lines) + else: + marko.block.Paragraph.__init__(self, lines) + @classmethod def break_paragraph( cls, source: marko.source.Source, lazy: bool = False @@ -129,6 +153,7 @@ class _ListItemCreationPayload(BaseModel): kind: Literal["list_item"] = "list_item" enumerated: bool marker: str = "" + checkbox: DocItemLabel | None = None _CreationPayload = Annotated[ @@ -607,7 +632,18 @@ def _create_list_item( marker: str = "", formatting: Formatting | None = None, hyperlink: AnyUrl | Path | None = None, + checkbox: DocItemLabel | None = None, ): + if checkbox is not None: + # A GFM task-list item: a checkbox in the list, as the HTML backend + # produces for
  • . + return doc.add_text( + label=checkbox, + text=text, + parent=parent_item, + formatting=formatting, + hyperlink=hyperlink, + ) item = doc.add_list_item( text=text, enumerated=enumerated, @@ -676,6 +712,7 @@ def _flush_creation_stack( marker=to_create.marker, formatting=formatting, hyperlink=hyperlink, + checkbox=to_create.checkbox, ) if parent_ref: list_last_item_by_ref[parent_ref] = cast(ListItem, parent_item) @@ -788,6 +825,14 @@ def _iterate_elements( # noqa: C901 start = list_start_by_ref.get(parent_ref, 1) count = list_item_counter_by_ref.get(parent_ref, 0) marker = f"{start + count}." + checked: bool | None = getattr(child, "checked", None) + checkbox: DocItemLabel | None = None + if checked is not None: + checkbox = ( + DocItemLabel.CHECKBOX_SELECTED + if checked + else DocItemLabel.CHECKBOX_UNSELECTED + ) non_list_children: list[marko.element.Element] = [ item for item in child.children @@ -808,6 +853,7 @@ def _iterate_elements( # noqa: C901 marker=marker, formatting=formatting, hyperlink=hyperlink, + checkbox=checkbox, ) if parent_ref: list_last_item_by_ref[parent_ref] = cast(ListItem, parent_item) @@ -816,7 +862,9 @@ def _iterate_elements( # noqa: C901 ) else: creation_stack.append( - _ListItemCreationPayload(enumerated=enumerated, marker=marker) + _ListItemCreationPayload( + enumerated=enumerated, marker=marker, checkbox=checkbox + ) ) elif isinstance(element, marko.inline.Image): diff --git a/tests/test_backend_markdown.py b/tests/test_backend_markdown.py index 95bbaecc55..1e2442619a 100644 --- a/tests/test_backend_markdown.py +++ b/tests/test_backend_markdown.py @@ -1306,3 +1306,61 @@ def test_rich_table_cell_no_stray_body_items(): # The only direct body child must be the table itself. body_crefs = {ref.cref for ref in doc.body.children} assert body_crefs == {"#/tables/0"} + + +def test_task_list_items_become_checkboxes(): + """GFM task-list items keep their state, as the HTML backend does (#4930).""" + markdown = "- [x] done item\n- [ ] open item\n- [X] upper case\n- plain item\n" + conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) + assert conv_result.status == ConversionStatus.SUCCESS + + doc = conv_result.document + assert [(t.label, t.text) for t in doc.texts] == [ + (DocItemLabel.CHECKBOX_SELECTED, "done item"), + (DocItemLabel.CHECKBOX_UNSELECTED, "open item"), + (DocItemLabel.CHECKBOX_SELECTED, "upper case"), + (DocItemLabel.LIST_ITEM, "plain item"), + ] + # the checkboxes stay inside the list + assert {t.parent.cref for t in doc.texts} == {"#/groups/0"} + assert doc.export_to_markdown() == ( + "- [x] done item\n- [ ] open item\n- [x] upper case\n- plain item" + ) + + +def test_nested_task_list_items_become_checkboxes(): + markdown = "- [x] parent\n - [ ] child\n- plain\n" + conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) + assert conv_result.status == ConversionStatus.SUCCESS + + doc = conv_result.document + assert [(t.label, t.text) for t in doc.texts] == [ + (DocItemLabel.CHECKBOX_SELECTED, "parent"), + (DocItemLabel.CHECKBOX_UNSELECTED, "child"), + (DocItemLabel.LIST_ITEM, "plain"), + ] + assert doc.export_to_markdown() == "- [x] parent\n - [ ] child\n- plain" + + +@pytest.mark.parametrize( + "markdown", + [ + "[x] I accept the terms\n", + "[ ] Not yet decided\n", + "> [x] quoted\n", + "- item\n\n [x] second paragraph of the item\n", + ], +) +def test_task_markers_outside_a_list_items_first_paragraph_are_text(markdown): + """GFM task markers only apply to a list item's first paragraph; elsewhere the + brackets are text and must not be dropped (#4930).""" + conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) + assert conv_result.status == ConversionStatus.SUCCESS + + texts = [t.text for t in conv_result.document.texts] + marker = "[x]" if "[x]" in markdown else "[ ]" + assert any(text.startswith(f"{marker} ") for text in texts), texts + assert not any( + t.label in (DocItemLabel.CHECKBOX_SELECTED, DocItemLabel.CHECKBOX_UNSELECTED) + for t in conv_result.document.texts + ) From 36e2adb928a1bb046c57d7bccae010fc07b665d7 Mon Sep 17 00:00:00 2001 From: Siddharth-DWT Date: Sat, 24 Oct 2026 16:16:25 +0530 Subject: [PATCH 2/2] fix(markdown): keep task-list markers as text instead of checkbox items Review feedback: turning task items into checkbox text items dropped ordered list numbers on export, broke the HTML list structure, still lost the marker when the item did not start with a paragraph, and probed attributes. Base _GfmParagraph on marko's plain Paragraph instead of GFM's, so the "[x] " / "[ ] " marker is never stripped. Task items stay list items with the marker in their text, which keeps numbering and both exports intact, and no paragraph loses its leading marker. Signed-off-by: Siddharth-DWT --- docling/backend/md_backend.py | 56 ++++--------------------- tests/test_backend_markdown.py | 75 +++++++++++++++++----------------- 2 files changed, 44 insertions(+), 87 deletions(-) diff --git a/docling/backend/md_backend.py b/docling/backend/md_backend.py index f54e54bc0a..eec4aac3e4 100644 --- a/docling/backend/md_backend.py +++ b/docling/backend/md_backend.py @@ -63,30 +63,12 @@ from marko.ext.gfm import elements as _gfm_el from marko.helpers import MarkoExtension - class _GfmParagraph(_gfm_el.Paragraph): - # GFM task-list markers ("[x] " / "[ ] ") belong only at the start of a list - # item's first paragraph. marko's GFM paragraph strips them from every - # paragraph, which would delete the text of e.g. "[x] I accept the terms". - @classmethod - def parse(cls, source): - state = source.state - opens_list_item = isinstance(state, marko.block.ListItem) and not getattr( - state, "_docling_has_paragraph", False - ) - if isinstance(state, marko.block.ListItem): - setattr(state, "_docling_has_paragraph", True) - result = super().parse(source) - if hasattr(result, "__dict__") and not isinstance( - result, marko.block.BlockElement - ): - setattr(result, "docling_task_list_candidate", opens_list_item) - return result - - def __init__(self, lines: list[str]) -> None: - if getattr(lines, "docling_task_list_candidate", False): - super().__init__(lines) - else: - marko.block.Paragraph.__init__(self, lines) + class _GfmParagraph(marko.block.Paragraph): + # Based on marko's plain Paragraph rather than the GFM one: GFM's strips a + # leading task-list marker ("[x] " / "[ ] ") and keeps it only in + # `checked`, which a DoclingDocument list item cannot carry, so the marker + # was lost. Here it stays part of the text, in list items and elsewhere. + override = True @classmethod def break_paragraph( @@ -153,7 +135,6 @@ class _ListItemCreationPayload(BaseModel): kind: Literal["list_item"] = "list_item" enumerated: bool marker: str = "" - checkbox: DocItemLabel | None = None _CreationPayload = Annotated[ @@ -632,18 +613,7 @@ def _create_list_item( marker: str = "", formatting: Formatting | None = None, hyperlink: AnyUrl | Path | None = None, - checkbox: DocItemLabel | None = None, ): - if checkbox is not None: - # A GFM task-list item: a checkbox in the list, as the HTML backend - # produces for
  • . - return doc.add_text( - label=checkbox, - text=text, - parent=parent_item, - formatting=formatting, - hyperlink=hyperlink, - ) item = doc.add_list_item( text=text, enumerated=enumerated, @@ -712,7 +682,6 @@ def _flush_creation_stack( marker=to_create.marker, formatting=formatting, hyperlink=hyperlink, - checkbox=to_create.checkbox, ) if parent_ref: list_last_item_by_ref[parent_ref] = cast(ListItem, parent_item) @@ -825,14 +794,6 @@ def _iterate_elements( # noqa: C901 start = list_start_by_ref.get(parent_ref, 1) count = list_item_counter_by_ref.get(parent_ref, 0) marker = f"{start + count}." - checked: bool | None = getattr(child, "checked", None) - checkbox: DocItemLabel | None = None - if checked is not None: - checkbox = ( - DocItemLabel.CHECKBOX_SELECTED - if checked - else DocItemLabel.CHECKBOX_UNSELECTED - ) non_list_children: list[marko.element.Element] = [ item for item in child.children @@ -853,7 +814,6 @@ def _iterate_elements( # noqa: C901 marker=marker, formatting=formatting, hyperlink=hyperlink, - checkbox=checkbox, ) if parent_ref: list_last_item_by_ref[parent_ref] = cast(ListItem, parent_item) @@ -862,9 +822,7 @@ def _iterate_elements( # noqa: C901 ) else: creation_stack.append( - _ListItemCreationPayload( - enumerated=enumerated, marker=marker, checkbox=checkbox - ) + _ListItemCreationPayload(enumerated=enumerated, marker=marker) ) elif isinstance(element, marko.inline.Image): diff --git a/tests/test_backend_markdown.py b/tests/test_backend_markdown.py index 1e2442619a..29d5558fe0 100644 --- a/tests/test_backend_markdown.py +++ b/tests/test_backend_markdown.py @@ -1308,59 +1308,58 @@ def test_rich_table_cell_no_stray_body_items(): assert body_crefs == {"#/tables/0"} -def test_task_list_items_become_checkboxes(): - """GFM task-list items keep their state, as the HTML backend does (#4930).""" - markdown = "- [x] done item\n- [ ] open item\n- [X] upper case\n- plain item\n" +def _md_convert(markdown: str): conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) assert conv_result.status == ConversionStatus.SUCCESS + return conv_result.document + + +def test_task_list_items_keep_their_markers(): + """GFM task-list markers stay in the list item text instead of being dropped (#4930).""" + doc = _md_convert( + "- [x] done item\n- [ ] open item\n- [X] upper case\n- plain item\n" + ) - doc = conv_result.document assert [(t.label, t.text) for t in doc.texts] == [ - (DocItemLabel.CHECKBOX_SELECTED, "done item"), - (DocItemLabel.CHECKBOX_UNSELECTED, "open item"), - (DocItemLabel.CHECKBOX_SELECTED, "upper case"), + (DocItemLabel.LIST_ITEM, "[x] done item"), + (DocItemLabel.LIST_ITEM, "[ ] open item"), + (DocItemLabel.LIST_ITEM, "[X] upper case"), (DocItemLabel.LIST_ITEM, "plain item"), ] - # the checkboxes stay inside the list - assert {t.parent.cref for t in doc.texts} == {"#/groups/0"} assert doc.export_to_markdown() == ( - "- [x] done item\n- [ ] open item\n- [x] upper case\n- plain item" + "- [x] done item\n- [ ] open item\n- [X] upper case\n- plain item" ) + html = doc.export_to_html() + assert "
  • [x] done item
    • " in html + assert "
  • plain item
    • " in html -def test_nested_task_list_items_become_checkboxes(): - markdown = "- [x] parent\n - [ ] child\n- plain\n" - conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) - assert conv_result.status == ConversionStatus.SUCCESS +def test_ordered_task_list_items_keep_their_numbers(): + doc = _md_convert("7. [x] seven\n8. [ ] eight\n9. nine\n") + + assert all(t.label == DocItemLabel.LIST_ITEM for t in doc.texts) + assert doc.export_to_markdown() == "7. [x] seven\n8. [ ] eight\n9. nine" + + +def test_nested_task_list_items_keep_their_markers(): + doc = _md_convert("- [x] parent\n - [ ] child\n- plain\n") - doc = conv_result.document - assert [(t.label, t.text) for t in doc.texts] == [ - (DocItemLabel.CHECKBOX_SELECTED, "parent"), - (DocItemLabel.CHECKBOX_UNSELECTED, "child"), - (DocItemLabel.LIST_ITEM, "plain"), - ] assert doc.export_to_markdown() == "- [x] parent\n - [ ] child\n- plain" @pytest.mark.parametrize( - "markdown", + ("markdown", "expected"), [ - "[x] I accept the terms\n", - "[ ] Not yet decided\n", - "> [x] quoted\n", - "- item\n\n [x] second paragraph of the item\n", + ("[x] I accept the terms\n", "[x] I accept the terms"), + ("[ ] Not yet decided\n", "[ ] Not yet decided"), + ("> [x] quoted\n", "[x] quoted"), + ( + "- item\n\n [x] second paragraph of the item\n", + "[x] second paragraph of the item", + ), + ("- # Title\n\n [x] ordinary second block\n", "[x] ordinary second block"), ], ) -def test_task_markers_outside_a_list_items_first_paragraph_are_text(markdown): - """GFM task markers only apply to a list item's first paragraph; elsewhere the - brackets are text and must not be dropped (#4930).""" - conv_result = get_converter().convert_string(markdown, format=InputFormat.MD) - assert conv_result.status == ConversionStatus.SUCCESS - - texts = [t.text for t in conv_result.document.texts] - marker = "[x]" if "[x]" in markdown else "[ ]" - assert any(text.startswith(f"{marker} ") for text in texts), texts - assert not any( - t.label in (DocItemLabel.CHECKBOX_SELECTED, DocItemLabel.CHECKBOX_UNSELECTED) - for t in conv_result.document.texts - ) +def test_task_markers_outside_list_items_are_kept(markdown, expected): + """A leading "[x] " / "[ ] " is not dropped from other paragraphs (#4930).""" + assert expected in [t.text for t in _md_convert(markdown).texts]