Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docling/utils/nemotron_parse_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from docling.backend.md_backend import MarkdownDocumentBackend
from docling.datamodel.base_models import InputFormat
from docling.datamodel.document import InputDocument
from docling.utils.vlm_utils import strip_math_delimiters

_log = logging.getLogger(__name__)

Expand Down Expand Up @@ -321,7 +322,7 @@ def parse_nemotron_parse_v2(
else:
label = _TEXT_LABELS.get(region.label, DocItemLabel.TEXT)
text = (
raw_text
strip_math_delimiters(raw_text)
if label == DocItemLabel.FORMULA
else _markdown_to_text(raw_text)
)
Expand Down
13 changes: 13 additions & 0 deletions docling/utils/vlm_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,19 @@ def parse_markdown_heading(text: str) -> tuple[str, int | None]:
return match.group(2), len(match.group(1))


def strip_math_delimiters(text: str) -> str:
"""Remove one matching outer math pair without interpreting its LaTeX."""
text = text.strip()
for opening, closing in (("$$", "$$"), (r"\[", r"\]"), (r"\(", r"\)"), ("$", "$")):
if (
len(text) >= len(opening) + len(closing)
and text.startswith(opening)
and text.endswith(closing)
):
return text[len(opening) : -len(closing)].strip()
return text


def strip_stop_strings(texts: list[str], stop_strings: list[str]) -> list[str]:
"""Strip stop strings from decoded texts.

Expand Down
2 changes: 1 addition & 1 deletion tach.toml
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ depends_on = []
[[modules]]
path = "docling.utils.nemotron_parse_utils"
layer = "core"
depends_on = ["docling.backend", "docling.datamodel"]
depends_on = ["docling.backend", "docling.datamodel", "docling.utils"]

[[modules]]
path = "docling.utils"
Expand Down
34 changes: 34 additions & 0 deletions tests/test_nemotron_formula_exports.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# SPDX-FileCopyrightText: The Docling Contributors
# SPDX-License-Identifier: MIT

import pytest
from docling_core.types.doc import FormulaItem, Size

from docling.utils.nemotron_parse_utils import parse_nemotron_parse_v2


@pytest.mark.parametrize(
"opening, closing",
[("$", "$"), ("$$", "$$"), (r"\(", r"\)"), (r"\[", r"\]"), ("", "")],
)
def test_formula_export_has_one_delimiter_pair(opening: str, closing: str) -> None:
latex = r"\left\{x_1 * y_2 : x > 0\right\}"
raw = f"{opening}{latex}{closing}"
source = f"<x_0.2><y_0.2>{raw}<x_0.8><y_0.4><class_Formula>"
size = Size(width=600, height=800)
doc = parse_nemotron_parse_v2(source, size, size, page_no=1)
formula = doc.texts[0]
assert isinstance(formula, FormulaItem)
assert formula.text == latex
assert formula.orig == raw
assert formula.prov
assert doc.export_to_markdown() == f"$${latex}$$"


def test_multiline_formula_keeps_latex_and_line_breaks() -> None:
latex = "\\begin{aligned}\nx &= y \\\\\nz &= w\n\\end{aligned}"
source = f"<x_0.2><y_0.2>$$\n{latex}\n$$<x_0.8><y_0.4><class_Formula>"
size = Size(width=600, height=800)
doc = parse_nemotron_parse_v2(source, size, size, page_no=1)
assert doc.texts[0].text == latex
assert doc.export_to_markdown() == f"$${latex}$$"
2 changes: 1 addition & 1 deletion tests/test_nemotron_parse_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ def test_parse_nemotron_markdown_and_semantic_classes() -> None:
item.text for item in document.texts if item.label == DocItemLabel.LIST_ITEM
] == ["First item", "Second item"]
assert any(
item.label == DocItemLabel.FORMULA and item.text == r"\(x^2\)"
item.label == DocItemLabel.FORMULA and item.text == r"x^2"
for item in document.texts
)
assert len(document.pictures) == 2
Expand Down
Loading