Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docling/utils/dots_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,11 @@
re.IGNORECASE,
)
# Literal spans and code spans pass through; `\\` keeps `\\*` from unescaping.
# A code span opens only at a full backtick run, read atomically with `(?=(`+))\1`:
# a backtracking `(`+)` takes cubic time on a long unclosed run.
_MARKDOWN_ESCAPE_PATTERN = re.compile(
rf"{_LITERAL_INLINE_PATTERN.pattern}|(`+)[\s\S]*?(?<!`)\1(?!`)|\\([*_])|\\\\",
rf"{_LITERAL_INLINE_PATTERN.pattern}"
r"|(?<!`)(?=(`+))\1[\s\S]*?(?<!`)\1(?!`)|\\([*_])|\\\\",
re.IGNORECASE,
)

Expand Down
42 changes: 34 additions & 8 deletions docling/utils/nemotron_parse_v3_5_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

from __future__ import annotations

import html
import logging
import re
from dataclasses import dataclass
Expand Down Expand Up @@ -42,7 +43,7 @@
)
from PIL import Image as PILImage

from docling.utils.chandra_utils import _add_html_table
from docling.utils.chandra_utils import _MAX_TABLE_GRID_CELLS, _add_html_table
from docling.utils.code_language import detect_code_language
from docling.utils.dots_utils import (
_add_inline_text,
Expand Down Expand Up @@ -86,6 +87,11 @@
"mbox": "",
"&": "&amp;",
}
# Inline tags that a table cell keeps as markup, and LaTeX text commands that it maps;
# all other text is escaped.
_CELL_MARKUP = re.compile(
rf"{_LATEX_TEXT.pattern}|(?i:</?(?:b|br|code|em|i|strong|sub|sup|u)\s*/?>)"
)
_CHART_OPEN = re.compile(r'<chart\s+type="([^"]*)"\s*>')
_CHART_TABLE_KEY = re.compile(r"table\[(\w+)\]")

Expand Down Expand Up @@ -244,7 +250,30 @@ def _unwrap_command(text: str, command: str, arg_count: int) -> list[str] | None


def _span(value: str) -> int:
return int(value) if value.strip().isdigit() and int(value) > 0 else 1
digits = value.strip().lstrip("0")
if not digits.isdecimal():
return 1
# Above the limit _add_html_table rejects the table; do not build a huge grid first.
if len(digits) > len(str(_MAX_TABLE_GRID_CELLS)):
return _MAX_TABLE_GRID_CELLS + 1
return min(int(digits), _MAX_TABLE_GRID_CELLS + 1)


def _escape_cell_markup(text: str) -> str:
"""Escape cell text so that it cannot change the table; keep inline tags."""
parts: list[str] = []
pos = 0
for match in _CELL_MARKUP.finditer(text):
parts.append(html.escape(text[pos : match.start()], quote=False))
command = match.group(1) or match.group(2)
parts.append(
match.group()
if command is None
else _LATEX_TEXT_REPLACEMENTS.get(command, command)
)
pos = match.end()
parts.append(html.escape(text[pos:], quote=False))
return "".join(parts)


def _parse_latex_cell(text: str) -> _LatexCell:
Expand All @@ -259,12 +288,7 @@ def _parse_latex_cell(text: str) -> _LatexCell:
header, text = True, args[0]
else:
break
text = _LATEX_TEXT.sub(
lambda match: _LATEX_TEXT_REPLACEMENTS.get(
match.group(1) or match.group(2), match.group(2) or ""
),
text,
)
text = _escape_cell_markup(text)
return _LatexCell(
text=text.strip(), row_span=row_span, col_span=col_span, header=header
)
Expand Down Expand Up @@ -304,6 +328,8 @@ def latex_tabular_to_html(latex: str) -> str:
continue
# Text under a multirow: the predicted row span was too long.
owner[0].row_span = row_index - owner[1]
if col + cell.col_span > _MAX_TABLE_GRID_CELLS:
raise ValueError(f"LaTeX table exceeds {_MAX_TABLE_GRID_CELLS} columns")
for owned_col in range(col, col + cell.col_span):
owners[owned_col] = (cell, row_index)
cells.append(cell)
Expand Down
3 changes: 3 additions & 0 deletions tests/test_dots_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,9 @@ def test_markdown_emphasis_preserves_math_code_and_html(page_size: Size):
("$x_1 * y_2 * z_3$", "$x_1 * y_2 * z_3$"),
("______", "______"),
("a *\n=\nb", "a *\n=\nb"),
(r"``a\_b` c`` and d\_e", r"``a\_b` c`` and d_e"),
# An unclosed run took cubic time before the code-span match was atomic.
("`" * 5000 + r" a\_b", "`" * 5000 + " a_b"),
],
)
def test_literal_markers_stay_literal(text: str, expected: str, page_size: Size):
Expand Down
35 changes: 35 additions & 0 deletions tests/test_nemotron_parse_v3_5.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,41 @@ def test_table_maps_multirow_multicolumn_and_thead() -> None:
assert [cells[2, col].text for col in range(4)] == ["Kin $", "145", "", "5 % < 6"]


def test_table_with_oversized_span_is_rejected_without_building_it() -> None:
# Building the span grid used memory linear in the predicted span value.
table = r"\begin{tabular}{c}\multicolumn{1000000000}{c}{x}\\\end{tabular}"

data = _parse(_block("Table", table)).tables[0].data

assert (data.num_rows, data.num_cols) == (0, 0)


def test_table_cell_markup_stays_cell_text() -> None:
table = (
r"\begin{tabular}{cc}"
r'a</td></tr><tr><td colspan="9">b & x<sup>2</sup> \& y\\'
r"\end{tabular}"
)

data = _parse(_block("Table", table)).tables[0].data

assert (data.num_rows, data.num_cols) == (1, 2)
assert data.grid[0][0].text == 'a</td></tr><tr><td colspan="9">b'
assert data.grid[0][1].text == "x2 & y"


def test_table_cell_latex_angle_brackets_stay_cell_text() -> None:
table = (
r"\begin{tabular}{cc}"
r"\textless{}b\textgreater{}x\textless{}/b\textgreater{} & <code>a<b</code>\\"
r"\end{tabular}"
)

data = _parse(_block("Table", table)).tables[0].data

assert [cell.text for cell in data.grid[0]] == ["<b>x</b>", "a<b"]


def test_table_cell_under_multirow_with_text_ends_the_span() -> None:
table = r"\begin{tabular}{cc}\multirow{3}{*}{A} & 1\\ \mbox{} & 2\\ B & 3\\\end{tabular}"

Expand Down
Loading