Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 114 additions & 7 deletions docling/backend/html_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,66 @@ def _parse_ol_start(value: str) -> Optional[int]:
return start if start <= _OL_START_MAX else None


# The marker kinds of the `type` attribute of an `<ol>` (case-sensitive).
_OL_TYPES: Final = frozenset({"1", "a", "A", "i", "I"})
# The CSS `list-style-type` keywords with the `type` of the same marker kind.
_LIST_STYLE_TYPES: Final = {
"decimal": "1",
"lower-alpha": "a",
"lower-latin": "a",
"upper-alpha": "A",
"upper-latin": "A",
"lower-roman": "i",
"upper-roman": "I",
}
_LIST_STYLE_KEYWORD_RE: Final = re.compile(
r"list-style(?:-type)?\s*:\s*(?P<value>[^;]*)", re.IGNORECASE
)
_ROMAN_NUMERALS: Final = (
(1000, "m"),
(900, "cm"),
(500, "d"),
(400, "cd"),
(100, "c"),
(90, "xc"),
(50, "l"),
(40, "xl"),
(10, "x"),
(9, "ix"),
(5, "v"),
(4, "iv"),
(1, "i"),
)


def _format_ol_marker(ordinal: int, list_type: str) -> str:
"""Return the marker of an ordered list item, as a browser renders it.

Letters count a, ..., z, aa, ab, ... and Roman numerals cover 1 to 3999, as the
CSS `lower-alpha` and `lower-roman` counter styles. Outside that range, the
marker is a decimal number.

Args:
ordinal: The ordinal value of the list item.
list_type: The `type` attribute of the list, one of `_OL_TYPES`.

Returns:
The marker, e.g. `"c."` for the ordinal 3 and the type `"a"`.
"""
label = str(ordinal)
if list_type in ("a", "A") and ordinal >= 1:
label = ""
while ordinal:
ordinal, rest = divmod(ordinal - 1, 26)
label = chr(ord("a") + rest) + label
elif list_type in ("i", "I") and 1 <= ordinal <= 3999:
label = ""
for value, numeral in _ROMAN_NUMERALS:
count, ordinal = divmod(ordinal, value)
label += numeral * count
return f"{label.upper() if list_type.isupper() else label}."


_CUSTOM_CHECKBOX_CLASSES: Final = {"checkbox", "checkbox-box", "checkbox-input"}
_CHECKBOX_MARK_TEXTS: Final = {"x", "✓", "✔", "☑"}
_CHECKBOX_CONTAINER_CLASSES: Final = {
Expand Down Expand Up @@ -3069,6 +3129,8 @@ def _handle_list( # noqa: C901
added_refs: list[RefItem] = []
tag_name = tag.name.lower()
start: Optional[int] = None
list_type = "1"
step = 1
name: str = ""
is_ordered = tag_name == "ol"
is_description = tag_name == "dl"
Expand All @@ -3079,6 +3141,17 @@ def _handle_list( # noqa: C901
start_attr = tag.get("start")
if isinstance(start_attr, str):
start = _parse_ol_start(start_attr)
list_type = self._get_ol_type(tag)
if tag.has_attr("reversed"):
step = -1
items = tag.find_all("li", recursive=False)
if start is None and (
list_type != "1"
or step == -1
or any(li.has_attr("value") for li in items)
):
# A reversed list counts down from its number of items
start = len(items) if step == -1 else 1
name = "ordered list" + (f" start {start}" if start is not None else "")
else:
name = "list"
Expand Down Expand Up @@ -3120,6 +3193,8 @@ def open_list_group(group_start: Optional[int]) -> ListGroup:

# Track the number of list items added (not all children)
list_item_counter: int = 0
# The ordinal value of the next list item, if the list has numbers
ordinal = start

# Handle description lists (<dl> with <dt> and <dd>)
if is_description:
Expand Down Expand Up @@ -3222,17 +3297,25 @@ def open_list_group(group_start: Optional[int]) -> ListGroup:

else:
if list_group is None:
if is_ordered and start is None:
start = 1
list_group = open_list_group(
start + list_item_counter if start is not None else None
)
if is_ordered and ordinal is None:
ordinal = list_item_counter + 1
list_group = open_list_group(ordinal)

# The `value` attribute of an item sets its ordinal value, and
# the items after it continue from there
value_attr = li.get("value")
if (
ordinal is not None
and isinstance(value_attr, str)
and (value := _parse_ol_start(value_attr)) is not None
):
ordinal = value

# 1) determine the marker: an explicit marker set on the item
# wins, otherwise use the counter
marker: str = self._get_list_style_marker(li) or (
f"{start + list_item_counter}."
if is_ordered and start is not None
_format_ol_marker(ordinal, list_type)
if is_ordered and ordinal is not None
else ""
)

Expand Down Expand Up @@ -3288,6 +3371,8 @@ def open_list_group(group_start: Optional[int]) -> ListGroup:
# Increment counter only when a list item is actually added
if list_item:
list_item_counter += 1
if ordinal is not None:
ordinal += step
prev_item = list_item

if list_item or inputs_in_li or custom_checkboxes_in_li:
Expand Down Expand Up @@ -3325,6 +3410,28 @@ def open_list_group(group_start: Optional[int]) -> ListGroup:
self.level -= 1
return added_refs

@staticmethod
def _get_ol_type(tag: Tag) -> str:
"""Return the kind of marker of the items of an `<ol>`, one of `_OL_TYPES`.

A `list-style-type` keyword in the inline style wins over the `type`
attribute, as in browsers. Other keywords and stylesheets are ignored.
"""
style = tag.get("style")
matches = (
list(_LIST_STYLE_KEYWORD_RE.finditer(style))
if isinstance(style, str)
else []
)
if matches:
for keyword in matches[-1].group("value").lower().split():
if keyword in _LIST_STYLE_TYPES:
return _LIST_STYLE_TYPES[keyword]
type_attr = tag.get("type")
return (
type_attr if isinstance(type_attr, str) and type_attr in _OL_TYPES else "1"
)

@staticmethod
def _get_list_style_marker(li: Tag) -> str:
"""Return the marker that an inline string `list-style-type` sets on `li`.
Expand Down
47 changes: 47 additions & 0 deletions tests/test_backend_html.py
Original file line number Diff line number Diff line change
Expand Up @@ -797,6 +797,53 @@ def test_ordered_list_item_marker_from_list_style_string(items: str, expected: s
assert doc.export_to_markdown() == expected


@pytest.mark.parametrize(
("items", "expected"),
[
# The `type` attribute sets the kind of marker
('<ol type="a"><li>x</li><li>y</li></ol>', ["a.", "b."]),
('<ol type="A" start="26"><li>x</li><li>y</li></ol>', ["Z.", "AA."]),
('<ol type="i" start="3"><li>x</li><li>y</li></ol>', ["iii.", "iv."]),
(
'<ol type="I" start="3999"><li>x</li><li>y</li></ol>',
["MMMCMXCIX.", "4000."],
),
# A `list-style-type` keyword in the inline style wins over the `type`
('<ol style="list-style-type: lower-alpha;"><li>x</li></ol>', ["a."]),
(
'<ol type="a" style="list-style-type: upper-roman"><li>x</li></ol>',
["I."],
),
# A reversed list counts down, by default from its number of items
("<ol reversed><li>x</li><li>y</li><li>z</li></ol>", ["3.", "2.", "1."]),
('<ol reversed start="10" type="a"><li>x</li><li>y</li></ol>', ["j.", "i."]),
# The `value` attribute sets the number of an item and of the items after it
('<ol><li>x</li><li value="5">y</li><li>z</li></ol>', ["1.", "5.", "6."]),
(
'<ol reversed><li>x</li><li value="10">y</li><li>z</li></ol>',
["3.", "10.", "9."],
),
# The numbering continues after content that interrupts the list
('<ol type="a"><li>x</li><p>between</p><li>y</li></ol>', ["a.", "b."]),
# An unknown type is ignored, as in browsers
('<ol type="b"><li>x</li><li>y</li></ol>', ["", ""]),
],
)
def test_ordered_list_item_marker_from_attributes(items: str, expected: list[str]):
html = f"<html><body>{items}</body></html>".encode()
in_doc = InputDocument(
path_or_stream=BytesIO(html),
format=InputFormat.HTML,
backend=HTMLDocumentBackend,
filename="test",
)
doc = in_doc._backend.convert()
markers = [
item.marker for item in doc.texts if item.label == DocItemLabel.LIST_ITEM
]
assert markers == expected


def test_list_item_markers_survive_html_export_round_trip():
# Docling's HTML export writes the original list item markers as a CSS
# string list-style-type. Converting that HTML again must keep them.
Expand Down
Loading