fix(pdf): preserve nested lists and Word circle markers

docling/models/postprocessing/list_marker_processor.py

@@ -81,6 +81,11 @@ def __init__(self, infer_enumerated: bool = True):
             re.compile(f"^({pattern})" + r"\s(.+)", re.DOTALL)
             for pattern in self._bullet_patterns
         ]
+        # Word's ASCII circle is ambiguous in prose; only strip it from list items.
+        self._compiled_list_bullet_item_patterns = [
+            *self._compiled_bullet_item_patterns,
+            re.compile(r"^(o)\s(.+)", re.DOTALL),
+        ]
         self._compiled_numbered_item_patterns = [
             re.compile(f"^({pattern})" + r"\s(.+)", re.DOTALL)
             for pattern in self._numbered_patterns
@@ -106,6 +111,17 @@ def _is_numbered_item(self, text: str) -> bool:
             pattern.match(text) for pattern in self._compiled_numbered_item_patterns
         )
 
+    def has_same_numbered_marker_style(self, first: str, second: str) -> bool:
+        """Check whether two different numbered markers use the same format."""
+        first_match = self._find_match(first, self._compiled_numbered_item_patterns)
+        second_match = self._find_match(second, self._compiled_numbered_item_patterns)
+        return (
+            first_match is not None
+            and second_match is not None
+            and first_match.re is second_match.re
+            and first_match[1] != second_match[1]
+        )
+
     @classmethod
     def _create_list_item(
         cls,
@@ -192,7 +208,7 @@ def process_list_item(self, item: ListItem) -> ListItem:
 
         is_enumerated: bool = False
         if mtch := self._find_match(
-            text=item.orig, patterns=self._compiled_bullet_item_patterns
+            text=item.orig, patterns=self._compiled_list_bullet_item_patterns
         ):
             is_enumerated = False
         elif mtch := self._find_match(

docling/models/stages/reading_order/readingorder_model.py

@@ -733,6 +733,9 @@ def materialize_siblings(
                 isinstance(parent, GroupItem) and parent.label == GroupLabel.FORM_AREA
             ) or isinstance(parent, FieldRegionItem)
             current_list: GroupItem | None = None
+            list_stack: list[tuple[float, GroupItem, float]] = []
+            previous_list_element: ReadingOrderPageElement | None = None
+            indent_tolerance = 5.0
             siblings = [
                 rel
                 for rel in ordered_siblings[parent_ref]
@@ -746,11 +749,64 @@ def materialize_siblings(
                 rel = entry
                 element = id_to_elem[rel.ref.cref]
                 page_height = page_no_to_pages[element.page_no].size.height  # type: ignore
-                if split_lists_on_non_list and not (
+                is_list_item = (
                     isinstance(element, TextElement)
                     and element.label == DocItemLabel.LIST_ITEM
-                ):
+                )
+                line_height = element.cluster.bbox.height
+                if is_list_item:
+                    line_height = min(
+                        (cell.rect.height for cell in element.cluster.cells),
+                        default=line_height,
+                    )
+                    # A move right and back up is a new column, not a sublist.
+                    if current_list is None or (
+                        previous_list_element is not None
+                        and rel.page_no == previous_list_element.page_no
+                        and rel.l > previous_list_element.l + indent_tolerance
+                        and rel.t >= previous_list_element.t - indent_tolerance
+                    ):
+                        current_list = None
+                        list_stack.clear()
+                    left = element.cluster.bbox.l
+                    # Small layout-box differences do not make a new list level.
+                    while list_stack and left < list_stack[-1][0] - indent_tolerance:
+                        list_stack.pop()
+                    if list_stack:
+                        current_list = list_stack[-1][1]
+                        first_item = current_list.children[0].resolve(out_doc)
+                        # A leading quotation mark can hang left of the list marker.
+                        if (
+                            isinstance(first_item, TextItem)
+                            and first_item.orig.startswith(
+                                ("«", "\u201c", "\u2018", '"')
+                            )
+                            and first_item.prov
+                            and list_stack[-1][0] == first_item.prov[0].bbox.l
+                            and 0 < left - list_stack[-1][0] <= list_stack[-1][2]
+                            and self.list_item_processor.has_same_numbered_marker_style(
+                                first_item.orig[1:], element.text or ""
+                            )
+                        ):
+                            list_stack[-1] = (left, current_list, list_stack[-1][2])
+                        if left > list_stack[-1][0] + indent_tolerance:
+                            current_list = out_doc.add_group(
+                                label=GroupLabel.LIST,
+                                name="list",
+                                parent=current_list.children[-1].resolve(out_doc),
+                            )
+                            list_stack.append((left, current_list, line_height))
+                    else:
+                        current_list = None
+                elif (
+                    split_lists_on_non_list or isinstance(element, FigureElement)
+                ) and element.label not in {
+                    DocItemLabel.PAGE_HEADER,
+                    DocItemLabel.PAGE_FOOTER,
+                    DocItemLabel.FOOTNOTE,
+                }:
                     current_list = None
+                    list_stack.clear()
 
                 if isinstance(element, TextElement):
                     if element.label == DocItemLabel.CODE:
@@ -773,6 +829,12 @@ def materialize_siblings(
                         parent=parent,
                         page_height=page_height,
                     )
+                    if is_list_item and not list_stack and current_list is not None:
+                        list_stack.append(
+                            (element.cluster.bbox.l, current_list, line_height)
+                        )
+                    if is_list_item:
+                        previous_list_element = rel
 
                     if element.field_item is None:
                         for merged_cid in el_merges_mapping.get(rel.cid, []):
@@ -1034,7 +1096,12 @@ def _handle_text_element(
                 parent=parent,
             )
         else:
-            current_list = None
+            if label not in [
+                DocItemLabel.PAGE_HEADER,
+                DocItemLabel.PAGE_FOOTER,
+                DocItemLabel.FOOTNOTE,
+            ]:
+                current_list = None
 
             content_layer = ContentLayer.BODY
             if element.label in [DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER]:

docs/concepts/docling_document.md

@@ -53,6 +53,15 @@ Below example shows how all items under the heading "Let's swim" (`#/texts/5`) a
 "Let's swim" are both text items and groups, which contain the list elements. The group items are stored in the
 top-level `groups` field.
 
+For PDFs, list nesting is inferred from horizontal indentation. A nested list group
+is a child of the preceding item at the outer level, and an outdent returns to an outer list.
+Page headers, footers, and footnotes do not interrupt a list that continues on the next page.
+An opening quotation mark that hangs left of a numbered list does not add a list level.
+
+Recognised list markers are stored in `ListItem.marker` and removed from `ListItem.text`;
+`ListItem.orig` retains the source text. Word's `o` circle marker is recognised on
+items labelled as list items, but does not turn ordinary prose into a list.
+
 ![doc_hierarchy_2](../assets/docling_doc_hierarchy_2.png)
 
 <!--