Skip to content

fix(jats): preserve table footnotes #4818

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AryanRG wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(jats): preserve table footnotes #4818

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
45474bb
fix(jats): preserve table footnotes
AryanRG Oct 14, 2026
ad75068
fix(jats): support table footnotes with released core
AryanRG Oct 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 100 additions & 7 deletions docling/backend/xml/jats_backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
NodeItem,
TableCell,
TableData,
TableItem,
TextItem,
)
from docling_core.types.doc.document import Formatting, Script
Expand Down Expand Up @@ -787,8 +788,20 @@ def _walk_inline_formula(
node: etree._Element,
formatting: Formatting | None = None,
hyperlink: AnyUrl | Path | None = None,
*,
resolve_ext_links: bool = False,
paragraph_breaks: bool = False,
) -> list[InlineSegment]:
"""Walk inline content of ``node`` into text and formula segments.

With ``paragraph_breaks``, a space separates nonempty direct ``p`` children
of ``node`` only; recursive calls do not inherit it.
"""
current = JatsDocumentBackend._merge_formatting(formatting, node.tag)
if resolve_ext_links and node.tag == "ext-link":
ext_link = JatsDocumentBackend._parse_ext_link_href(node.get(_XLINK_HREF))
if ext_link is not None:
hyperlink = ext_link
segments: list[InlineSegment] = []
if node.text:
text = JatsDocumentBackend._normalize_xml_line_breaks(node.text)
Expand All @@ -801,6 +814,7 @@ def _walk_inline_formula(
hyperlink=hyperlink,
)
)
seen_paragraph = False
for child in node:
tag = child.tag
if not isinstance(tag, str) or tag.endswith("}math"):
Expand All @@ -817,9 +831,42 @@ def _walk_inline_formula(
)
)
else:
segments.extend(
JatsDocumentBackend._walk_inline_formula(child, current, hyperlink)
child_segments = JatsDocumentBackend._walk_inline_formula(
child,
current,
hyperlink,
resolve_ext_links=resolve_ext_links,
)
if paragraph_breaks and tag == "p":
nonempty = any(
s.label != DocItemLabel.TEXT or s.text.strip()
for s in child_segments
)
if nonempty:
# An earlier paragraph implies segments is nonempty.
last = segments[-1] if seen_paragraph else None
first = child_segments[0]
if (
last is not None
and not (
last.label == DocItemLabel.TEXT
and last.text[-1:].isspace()
)
and not (
first.label == DocItemLabel.TEXT
and first.text[:1].isspace()
)
):
segments.append(
InlineSegment(
label=DocItemLabel.TEXT,
text=" ",
formatting=current,
hyperlink=hyperlink,
)
)
seen_paragraph = True
segments.extend(child_segments)
if child.tail:
tail = JatsDocumentBackend._normalize_xml_line_breaks(child.tail)
if tail:
Expand Down Expand Up @@ -1141,11 +1188,11 @@ def parse_table_data(element: Tag) -> TableData | None:

def _add_table(
self, doc: DoclingDocument, parent: NodeItem, table_xml_component: Table
) -> None:
) -> TableItem | None:
soup = BeautifulSoup(table_xml_component["content"], "html.parser")
table_tag = soup.find("table")
if not isinstance(table_tag, Tag):
return
return None

data = JatsDocumentBackend.parse_table_data(table_tag)
# TODO: format label vs caption once styling is supported
Expand All @@ -1158,9 +1205,51 @@ def _add_table(
else None
)
if data is not None:
doc.add_table(data=data, parent=parent, caption=table_caption)
return doc.add_table(data=data, parent=parent, caption=table_caption)

return
return None

@staticmethod
def _add_table_footnotes(
doc: DoclingDocument, table_item: TableItem, node: etree._Element
) -> None:
"""Attach each direct ``table-wrap-foot/fn`` as one table FOOTNOTE item."""
for fn in node.xpath("table-wrap-foot/fn"):
# Coalesce equivalent text runs (e.g. a literal <xref>) first.
merged: list[InlineSegment] = []
JatsDocumentBackend._extend_segments(
merged,
JatsDocumentBackend._walk_inline_formula(
fn, resolve_ext_links=True, paragraph_breaks=True
),
)
segments = JatsDocumentBackend._strip_segments(merged)
if not segments:
continue
# Keep table-note text distinguishable from body text for export
# filtering and chunk metadata; formulas retain their own label.
segments = [
(
replace(segment, label=DocItemLabel.FOOTNOTE)
if segment.label == DocItemLabel.TEXT
else segment
)
for segment in segments
]
# Released core table serializers skip referenced footnote wrappers.
# Keep all content in child runs so it is also exported by traversal.
footnote = doc.add_text(

@github-actions github-actions Bot Oct 15, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

question: Empty-text FOOTNOTE wrapper depends on one docling-core serializer behavior

Each note is a FOOTNOTE item with text="". The real text sits in child runs, and table_item.footnotes points to the empty wrapper. The code comment says this only works because released core skips referenced wrappers but still walks their children. pyproject.toml allows docling-core>=2.98.0,<3.0.0. The PR evidence covers only 2.100.0.

Please show two things:

  1. The oldest allowed core (2.98.0) still exports these notes in Markdown and HTML.
  2. A core release that prints table.footnotes text by reference does not add an empty line or duplicate the note. In that case the child runs would print a second time.

If one of these fails, either raise the core lower bound or add a test that fails on the problem. If you cannot show both, say that this layout is a deliberate workaround.

@AryanRG AryanRG Oct 16, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, I checked both points against ad75068.

  1. Oldest allowed core (2.98.0). With docling-core[chunking]==2.98.0, tests/test_backend_jats_table_footnotes.py passes: 22 passed. The chunking extra is needed for the HierarchicalChunker import and matches the project's test dependency. I also exported 12 synthetic JATS cases containing 15 table notes—plain text, literal xrefs, labels, styled/linked/formula runs, multiple paragraphs, and multiple tables—to Markdown and HTML, before and after a JSON round trip. Every expected note fragment appears exactly once, with no empty footnote div or whitespace-only line.

  2. A core that prints table.footnotes by reference. The released table serializers I checked (2.98.0, 2.100.0, and 2.101.1) and current docling-core main do not call serialize_footnotes. The closest candidate is docling-core#675 (open at 11bb2a7, based on core 2.86.0, below Docling's current 2.98.0 floor, so the current converter cannot import against it). Running the JATS backend directly against that head, note text is not duplicated: its hook reads only the wrapper's empty text, while traversal emits the child runs once. It does add empty output, though: one <div class="footnote"></div> per note in HTML, and a whitespace-only line in Markdown when a table has two or more notes.

So the oldest supported release works, but the future-serializer case is only partly clean. The empty-text wrapper is a deliberate workaround for current released serializers, which skip referenced footnote items while still walking their children. Raising the lower bound would not help because no checked release has the by-reference behavior. If that behavior lands in core, this layout should be revisited. I can add a guard test here that fails on an empty footnote div or whitespace-only line if that would be useful.

label=DocItemLabel.FOOTNOTE, text="", parent=table_item
)
# _emit_inline groups multiple runs; a single run needs the same
# inline scope, especially when it is a formula.
container = (
doc.add_inline_group(parent=footnote)
if len(segments) == 1
else footnote
)
JatsDocumentBackend._emit_inline(doc, container, segments)
table_item.footnotes.append(footnote.get_ref())

def _add_tables(
self, doc: DoclingDocument, parent: NodeItem, node: etree._Element
Expand Down Expand Up @@ -1197,9 +1286,13 @@ def _add_tables(
table["label"] = node.xpath("label")[0].text

try:
self._add_table(doc, parent, table)
table_item = self._add_table(doc, parent, table)
except Exception:
_log.warning(f"Skipping unsupported table in {self.file!s}")
return

if table_item is not None:
self._add_table_footnotes(doc, table_item, node)

return

Expand Down
Loading
Loading