diff --git a/docling/backend/asciidoc_backend.py b/docling/backend/asciidoc_backend.py index 63c984b44b..5ff05f88ae 100644 --- a/docling/backend/asciidoc_backend.py +++ b/docling/backend/asciidoc_backend.py @@ -1,1109 +1,1119 @@ -# SPDX-FileCopyrightText: The Docling Contributors -# SPDX-License-Identifier: MIT - -import logging -import re -from collections.abc import Iterator -from dataclasses import dataclass -from io import BytesIO -from pathlib import Path -from typing import Final, Optional, Union - -from docling_core.types.doc import ( - ContentLayer, - DocItemLabel, - DoclingDocument, - DocumentOrigin, - Formatting, - GroupItem, - GroupLabel, - ImageRef, - ListItem, - NodeItem, - TableCell, - TableData, - TextItem, -) - -from docling.backend.abstract_backend import DeclarativeDocumentBackend -from docling.backend.utils.image_resource_loader import ImageResourceLoader -from docling.datamodel.backend_options import AsciiDocBackendOptions -from docling.datamodel.base_models import InputFormat -from docling.datamodel.document import InputDocument -from docling.exceptions import DocumentLoadError -from docling.utils.code_language import CodeLanguageLabel, detect_code_language -from docling.utils.text_decoding import decode_text - -_log = logging.getLogger(__name__) - -# A line starting with exactly two slashes is a comment and never body content. -# A line of four or more slashes opens a delimited comment block: every line up -# to an identical line is a comment. -_LINE_COMMENT_RE = re.compile(r"^//(?!/)") -_COMMENT_BLOCK_DELIMITER_RE = re.compile(r"/{4,}") - -# Cell format specifier that may precede a "|" delimiter, e.g. "^.^h" in -# "^.^h|Header": span (3*, 2+, .2+, 2.3+), alignment (<, ^, >, .^), style -# (a/d/e/h/l/m/s). AsciiDoc writes the span as [colspan][.rowspan] followed by -# "+" or "*", and either number may be omitted, so ".2+" is a rowspan on its -# own. Requiring at least one of the two keeps a bare "+" from matching. -_CELL_SPEC: Final = r"(?:(?:\d+(?:\.\d+)?|\.\d+)[*+])*[<^>]?(?:\.[<^>])?[adehlms]?" -# An ordered marker with a letter has one letter only ("a.", "B."), so a line -# that starts with an abbreviation such as "Dr." or "Fig." stays text. -_LIST_ITEM_PATTERN: Final = r"^(\s*)(\*+|-|\.+|\d+\.|[a-zA-Z]\.)\s+(.*)" - -# Inline formatting marks, unconstrained (doubled, valid anywhere) before -# constrained (single, valid only at word boundaries, so "snake_case" and -# "2*3*4" stay plain text), as in Asciidoctor. A constrained span does not -# contain its own mark, so each scan stops at the next mark on the line. -_INLINE_FORMAT_RE: Final = re.compile( - r"``(?P.+?)``" - r"|\*\*(?P.+?)\*\*" - r"|__(?P.+?)__" - r"|(?[^`\s](?:[^`]*?[^`\s])?)`(?!\w)" - r"|(?[^*\s](?:[^*]*?[^*\s])?)\*(?!\w)" - r"|(?[^_\s](?:[^_]*?[^_\s])?)_(?!\w)" -) - - -@dataclass(frozen=True) -class _InlineRun: - text: str - bold: bool = False - italic: bool = False - code: bool = False - - -@dataclass(frozen=True) -class _LiteralBlock: - text: str - language: str | None = None # set for [source,lang] listing blocks - - -@dataclass(frozen=True) -class _CommentBlock: - """An AsciiDoc comment, with its // or //// delimiters removed. - - One block per comment: a run of consecutive // lines is one comment, and - so is everything between a pair of //// delimiters. - """ - - text: str - - -class AsciiDocBackend(DeclarativeDocumentBackend): - def __init__( - self, - in_doc: "InputDocument", - path_or_stream: Union[BytesIO, Path], - options: Optional[AsciiDocBackendOptions] = None, - ): - if options is None: - options = AsciiDocBackendOptions() - super().__init__(in_doc, path_or_stream, options) - - self.path_or_stream = path_or_stream - self.options: AsciiDocBackendOptions - self._image_loader = ImageResourceLoader( - enable_local_fetch=options.enable_local_fetch, - enable_remote_fetch=options.enable_remote_fetch, - ) - - # A leading BOM is dropped. Kept, it prefixes the first line, so a - # document title ("= Title") is no longer recognized as one and the BOM - # reaches the output. - try: - self.lines = decode_text(self.path_or_stream, options.encoding).split("\n") - self.valid = True - - except DocumentLoadError: - # Already carries a message naming what could not be decoded. - raise - except Exception as e: - raise DocumentLoadError( - f"Could not initialize AsciiDoc backend for file with hash {self.document_hash}." - ) from e - return - - def is_valid(self) -> bool: - return self.valid - - @classmethod - def supports_pagination(cls) -> bool: - return False - - def unload(self): - return - - @classmethod - def supported_formats(cls) -> set[InputFormat]: - return {InputFormat.ASCIIDOC} - - def convert(self) -> DoclingDocument: - """ - Parses the ASCII into a structured document model. - """ - - origin = DocumentOrigin( - filename=self.file.name or "file", - mimetype="text/asciidoc", - binary_hash=self.document_hash, - ) - - doc = DoclingDocument(name=self.file.stem or "file", origin=origin) - - doc = self._parse(doc) - - return doc - - def _flush_blank_line( - self, - doc: DoclingDocument, - caption_data: list[str], - text_data: list[str], - parent: Union[GroupItem, None], - ) -> tuple[list[str], list[str]]: - """Flush both accumulators on a blank line. - - Plain text accumulated before a block title is emitted first to - preserve reading order, then any pending caption is emitted as a - bold paragraph. - """ - text_data = self._flush_text_data(doc=doc, text_data=text_data, parent=parent) - if caption_data: - caption_data = self._flush_caption_as_paragraph(doc, caption_data, parent) - return caption_data, text_data - - def _flush_caption_as_paragraph( - self, - doc: DoclingDocument, - caption_data: list[str], - parent: Union[GroupItem, None], - ) -> list[str]: - """Emit a pending block title as a bold paragraph. - - Used for block titles whose target is not a floating item (picture, - table, or code block), which have no caption slot. - """ - doc.add_text( - text=" ".join(caption_data), - label=DocItemLabel.PARAGRAPH, - parent=parent, - formatting=Formatting(bold=True), - ) - return [] - - def _parse(self, doc: DoclingDocument): - """Orchestrate parsing and populate `doc` from the source lines. - - Handles titles, section headers, text paragraphs, lists, tables, - pictures, literal (code) blocks, and block titles. - - Block titles (AsciiDoc lines that start with `.` immediately - followed by text, e.g. `.Procedure`) are treated as follows: - - * FloatingItem targets (picture, table, code block): the block - title is created as a `CAPTION`-labelled `TextItem` and - attached to the floating item via its `caption` parameter, so it - participates in the standard caption relationship. - * All other targets (lists, paragraphs, ...): the block title is - emitted as a bold `PARAGRAPH` `TextItem` inserted - immediately before the element it precedes. This preserves - reading order while acknowledging that non-floating items (e.g. - `GroupItem`) have no caption slot. - """ - - in_list = False - in_table = False - in_delimited_table = False - - text_data: list[str] = [] - table_data: list[list[str]] = [] - caption_data: list[str] = [] - last_list_item: ListItem | None = None - list_continuation = False - # The marker kind of the list group at each level ("." for ordered, - # "*" or "-" for bullets), for the list being parsed. Reset when a new - # list starts, so a level that holds a section header is never mistaken - # for a list. - list_kinds: dict[int, str] = {} - - parents: dict[int, Union[GroupItem, None]] = {} - indents: dict[int, Union[GroupItem, None]] = {} - - for i in range(10): - parents[i] = None - indents[i] = None - - for block in self._iter_blocks(self.lines): - if not isinstance(block, str): - text_data, caption_data, in_list, last_list_item, list_continuation = ( - self._add_block( - doc=doc, - block=block, - text_data=text_data, - caption_data=caption_data, - in_list=in_list, - last_list_item=last_list_item, - list_continuation=list_continuation, - parents=parents, - ) - ) - continue - - line = block - stripped_line = line.strip() - if in_delimited_table and self._continue_table(table_data, line): - continue - - in_list, last_list_item, list_continuation = self._close_list_if_needed( - line=line, - in_list=in_list, - parents=parents, - last_list_item=last_list_item, - list_continuation=list_continuation, - is_continuation_block=False, - ) - - # Block attributes and anchors (e.g. "[cols=\"2,1\"]", "[NOTE]", - # "[[some-id]]") are metadata that applies to the block that - # follows; they are never body content, so skip them. - if ( - re.match(r"^\[\[.+\]\]$", stripped_line) - or re.match(r"^\[[^\[\]]+\]$", stripped_line) - or re.match(r"^:[\w.-]+:(\s.*)?$", stripped_line) - ): - continue - - # Title - if self._is_title(line): - item = self._parse_title(line) - level = item["level"] - - parents[level] = doc.add_text( - text=item["text"], label=DocItemLabel.TITLE - ) - - # Section headers - elif self._is_section_header(line): - # A heading ends the paragraph being accumulated: without - # this flush the text before the heading is appended to the - # text after it and lands in the wrong section. - text_data = self._flush_text_data( - doc=doc, - text_data=text_data, - parent=self._get_current_parent(parents), - ) - item = self._parse_section_header(line) - level = item["level"] - - ancestor = next( - (parents[k] for k in range(level - 1, -1, -1) if parents[k]), - None, - ) - parents[level] = doc.add_heading( - text=item["text"], level=item["level"], parent=ancestor - ) - for k, v in parents.items(): - if k > level: - parents[k] = None - - # Lists - elif self._is_list_item(line): - _log.debug(f"line: {line}") - item = self._parse_list_item(line) - _log.debug(f"parsed list-item: {item}") - - level = self._get_current_level(parents) - - if not in_list: - in_list = True - # GroupItem has no caption slot; emit the pending block - # title as a bold paragraph immediately before the list. - if caption_data: - doc.add_text( - text=" ".join(caption_data), - label=DocItemLabel.PARAGRAPH, - parent=parents[level], - formatting=Formatting(bold=True), - ) - caption_data = [] - - parents[level + 1] = doc.add_group( - parent=parents[level], name="list", label=GroupLabel.LIST - ) - indents[level + 1] = item["indent"] - list_kinds = {level + 1: self._list_kind(item)} - - elif in_list and item["indent"] > indents[level]: - # A nested list belongs to the item above it. As a sibling - # of the items, it is counted as an item of the outer list, - # and the next ordered item is numbered one too high. - parents[level + 1] = doc.add_group( - parent=last_list_item, name="list", label=GroupLabel.LIST - ) - indents[level + 1] = item["indent"] - list_kinds[level + 1] = self._list_kind(item) - - elif in_list and item["indent"] < indents[level]: - while level > 0 and item["indent"] < indents[level]: - # Only pop the current level if there is an outer group - # to fall back to; otherwise keep it as the list root. - if indents[level - 1] is None: - break - parents[level] = None - indents[level] = None - level -= 1 - - self._match_list_kind( - doc=doc, - item=item, - parents=parents, - indents=indents, - list_kinds=list_kinds, - last_list_item=last_list_item, - ) - - last_list_item = doc.add_list_item( - item["text"], - enumerated=item["numbered"], - marker=(item["marker"] if item["marker"][:-1].isdigit() else None), - parent=self._get_current_parent(parents), - ) - list_continuation = False - - elif in_list and stripped_line == "+": - list_continuation = True - continue - - # Tables - elif line.strip() == "|===": # start or end of table - if in_table: - self._add_table_if_nonempty( - doc, table_data, caption_data, self._get_current_parent(parents) - ) - caption_data = [] - in_table = False - in_delimited_table = False - table_data = [] - else: - in_table = True - in_delimited_table = True - - elif self._is_table_line(line): # within a table - in_table = True - table_data.append(self._parse_table_line(line)) - - elif in_table and not self._is_table_line(line): # end of table - self._add_table_if_nonempty( - doc, table_data, caption_data, self._get_current_parent(parents) - ) - caption_data = [] - in_table = False - table_data = [] - - # Picture - elif self._is_picture(line): - caption_data, in_list, last_list_item = self._add_picture_item( - doc=doc, - line=line, - caption_data=caption_data, - in_list=in_list, - last_list_item=last_list_item, - parents=parents, - ) - list_continuation = False - - # Caption - elif self._is_caption(line) and len(caption_data) == 0: - item = self._parse_caption(line) - caption_data.append(item["text"]) - - elif len(line.strip()) > 0 and len(caption_data) > 0: - # A block title is a single line, so this line starts the - # paragraph it titles. A paragraph has no caption slot. - parent = self._get_current_parent(parents) - text_data = self._flush_text_data( - doc=doc, text_data=text_data, parent=parent - ) - caption_data = self._flush_caption_as_paragraph( - doc, caption_data, parent - ) - item = self._parse_text(line) - text_data.append(item["text"]) - - elif len(line.strip()) == 0: - caption_data, text_data = self._flush_blank_line( - doc, - caption_data, - text_data, - self._get_current_parent(parents), - ) - - elif len(line.strip()) > 0: # allow multiline texts - item = self._parse_text(line) - text_data.append(item["text"]) - - text_data = self._flush_text_data( - doc=doc, text_data=text_data, parent=self._get_current_parent(parents) - ) - - if in_table: - # An unclosed table still consumes its pending caption below. - self._add_table_if_nonempty( - doc, table_data, caption_data, self._get_current_parent(parents) - ) - elif caption_data: - # A block title never followed by a floating element must still - # reach the document. - self._flush_caption_as_paragraph( - doc, caption_data, self._get_current_parent(parents) - ) - - return doc - - @staticmethod - def _get_current_level(parents): - for k, v in parents.items(): - if v is None and k > 0: - return k - 1 - - return 0 - - @staticmethod - def _get_current_parent(parents): - for k, v in parents.items(): - if v is None and k > 0: - return parents[k - 1] - - return None - - _SOURCE_ATTR_RE = re.compile(r"^\[source(?:,\s*([\w+#.-]+))?[^\]]*\]$") - _CONTENT_BLOCK_DELIMITERS = ("====", "****", "____", "--", "++++") - - @staticmethod - def _has_matching_closer(lines: list[str], open_idx: int, delimiter: str) -> bool: - """Return True if `delimiter` appears on a line of its own after `open_idx`.""" - for j in range(open_idx + 1, len(lines)): - if lines[j].strip() == delimiter: - return True - return False - - @staticmethod - def _is_line_comment(line: str) -> bool: - """Is `line` a // comment line rather than text or a //// block delimiter?""" - return _LINE_COMMENT_RE.match(line) is not None - - @staticmethod - def _make_comment(comment_lines: list[str]) -> _CommentBlock | None: - """Build a comment block, or None when the comment carries no text.""" - text = "\n".join(comment_lines).strip() - return _CommentBlock(text=text) if text else None - - @classmethod - def _iter_blocks( - cls, lines: list[str] - ) -> Iterator[str | _LiteralBlock | _CommentBlock]: - block_data: list[str] | None = None - block_delimiter: str | None = None - block_language: str | None = None - comment_data: list[str] | None = None - comment_delimiter: str | None = None - # Content block delimiters still waiting for their closer, innermost - # last. The lines between a pair run through this loop like any other, - # so a comment inside such a block is a comment and not body text. - open_content_blocks: list[str] = [] - - i = 0 - n = len(lines) - while i < n: - line = lines[i] - stripped = line.strip() - - if block_data is not None: - # Inside a verbatim block: only the matching closer ends it, - # and a "//" line here is content rather than a comment. - if stripped == block_delimiter: - yield _LiteralBlock( - text="\n".join(block_data), language=block_language - ) - block_data = None - block_delimiter = None - block_language = None - else: - block_data.append(line.rstrip("\r\n")) - i += 1 - continue - - # A comment block hides every construct, so it is checked ahead of - # the block openers: a "----" between //// delimiters is comment - # text, not the start of a listing. - if comment_data is not None: - if stripped == comment_delimiter: - comment = cls._make_comment(comment_data) - if comment is not None: - yield comment - comment_data = None - comment_delimiter = None - else: - comment_data.append(line.rstrip("\r\n")) - i += 1 - continue - - if _COMMENT_BLOCK_DELIMITER_RE.fullmatch(stripped): - comment_data = [] - comment_delimiter = stripped - i += 1 - continue - - if cls._is_line_comment(line): - # Consecutive // lines read as one comment, so take the run. - comment_lines = [] - while i < n and cls._is_line_comment(lines[i]): - comment_lines.append(lines[i].rstrip("\r\n")[2:].strip()) - i += 1 - comment = cls._make_comment(comment_lines) - if comment is not None: - yield comment - continue - - if stripped in {"....", "----"}: - # "...." is a literal block; "----" a listing block. A - # "[source,lang]" attribute consumed just before a "----" - # labels the listing's language. - block_data = [] - block_delimiter = stripped - block_language = None - i += 1 - continue - - if open_content_blocks and stripped == open_content_blocks[-1]: - # Closer of the innermost content block. Yield a trailing blank - # so the following text starts a fresh paragraph. - open_content_blocks.pop() - yield "" - i += 1 - continue - - if stripped in cls._CONTENT_BLOCK_DELIMITERS: - if cls._has_matching_closer(lines, i, stripped): - open_content_blocks.append(stripped) - else: - # Stray delimiter (no closer): break the paragraph, do not - # render as text. The line must be consumed either way; - # leaving `i` on it loops over the same line forever. - yield "" - i += 1 - continue - - source_attr = cls._SOURCE_ATTR_RE.match(stripped) - if source_attr is not None and i + 1 < n and lines[i + 1].strip() == "----": - block_data = [] - block_delimiter = "----" - block_language = source_attr.group(1) - i += 2 - continue - - yield line - i += 1 - - if block_data is not None: - yield _LiteralBlock(text="\n".join(block_data), language=block_language) - if comment_data is not None: - # An unclosed //// comments out the rest of the document, which is - # how Asciidoctor reads it. - comment = cls._make_comment(comment_data) - if comment is not None: - yield comment - - def _add_picture_item( - self, - *, - doc: DoclingDocument, - line: str, - caption_data: list[str], - in_list: bool, - last_list_item: ListItem | None, - parents: dict[int, GroupItem | None], - ) -> tuple[list[str], bool, ListItem | None]: - """Add a picture with its pending caption, if any.""" - caption: Optional[TextItem] = None - if len(caption_data) > 0: - caption = doc.add_text( - text=" ".join(caption_data), label=DocItemLabel.CAPTION - ) - - caption_data = [] - - item = self._parse_picture(line) - - image: Optional[ImageRef] = None - if "uri" in item and self.options.fetch_images: - base_path = ( - str(self.options.source_uri) - if self.options.source_uri is not None - else None - ) - image = self._image_loader.load_image_ref(item["uri"], base_path) - doc.add_picture( - image=image, - caption=caption, - parent=last_list_item if in_list else self._get_current_parent(parents), - ) - return caption_data, in_list, last_list_item - - def _add_block( - self, - *, - doc: DoclingDocument, - block: _LiteralBlock | _CommentBlock, - text_data: list[str], - caption_data: list[str], - in_list: bool, - last_list_item: ListItem | None, - list_continuation: bool, - parents: dict[int, GroupItem | None], - ) -> tuple[list[str], list[str], bool, ListItem | None, bool]: - """Add a delimited block and return the parser state after it. - - A comment is markup, not content: it neither ends the list nor the - paragraph it sits in, so the state passes through unchanged. A - literal block flushes the pending text and captions first. - """ - if isinstance(block, _CommentBlock): - self._add_comment_block( - doc=doc, - block=block, - in_list=in_list, - last_list_item=last_list_item, - parents=parents, - ) - return text_data, caption_data, in_list, last_list_item, list_continuation - return self._add_literal_block( - doc=doc, - block=block, - text_data=text_data, - caption_data=caption_data, - in_list=in_list, - last_list_item=last_list_item, - list_continuation=list_continuation, - parents=parents, - ) - - def _add_comment_block( - self, - *, - doc: DoclingDocument, - block: _CommentBlock, - in_list: bool, - last_list_item: ListItem | None, - parents: dict[int, GroupItem | None], - ) -> None: - """Add a comment to the notes content layer. - - A comment between two lines of one paragraph lands just before that - paragraph, since flushing the text to place it would split the body - text in two. - """ - if not self.options.capture_comments: - return - doc.add_text( - text=block.text, - label=DocItemLabel.TEXT, - content_layer=ContentLayer.NOTES, - parent=(last_list_item if in_list else self._get_current_parent(parents)), - ) - - def _add_literal_block( - self, - *, - doc: DoclingDocument, - block: _LiteralBlock, - text_data: list[str], - caption_data: list[str], - in_list: bool, - last_list_item: ListItem | None, - list_continuation: bool, - parents: dict[int, GroupItem | None], - ) -> tuple[list[str], list[str], bool, ListItem | None, bool]: - """Add a literal/listing block, flushing pending text and captions.""" - in_list, last_list_item, list_continuation = self._close_list_if_needed( - line="", - in_list=in_list, - parents=parents, - last_list_item=last_list_item, - list_continuation=list_continuation, - is_continuation_block=True, - ) - text_data = self._flush_text_data( - doc=doc, - text_data=text_data, - parent=self._get_current_parent(parents), - ) - caption: Optional[TextItem] = None - if caption_data: - caption = doc.add_text( - text=" ".join(caption_data), - label=DocItemLabel.CAPTION, - ) - caption_data = [] - code_language = ( - detect_code_language(block.text, hint=block.language) - if block.language is not None - else None - ) - doc.add_code( - text=block.text, - code_language=code_language, - caption=caption, - parent=(last_list_item if in_list else self._get_current_parent(parents)), - ) - return text_data, caption_data, in_list, last_list_item, False - - @classmethod - def _close_list_if_needed( - cls, - *, - line: str, - in_list: bool, - parents: dict[int, GroupItem | None], - last_list_item: ListItem | None, - list_continuation: bool, - is_continuation_block: bool, - ) -> tuple[bool, ListItem | None, bool]: - if ( - not in_list - or cls._is_list_item(line) - or line.strip() in {"", "+"} - or (list_continuation and (is_continuation_block or cls._is_picture(line))) - ): - return in_list, last_list_item, list_continuation - - # Close every level of the list, not only the innermost one. Otherwise - # the text after a nested list is added to the outer list group. - level = cls._get_current_level(parents) - while ( - isinstance(group := parents[level], GroupItem) - and group.label == GroupLabel.LIST - ): - parents[level] = None - level -= 1 - return False, None, False - - @classmethod - def _match_list_kind( - cls, - *, - doc: DoclingDocument, - item: dict, - parents: dict[int, GroupItem | None], - indents: dict[int, GroupItem | None], - list_kinds: dict[int, str], - last_list_item: ListItem | None, - ) -> None: - """Keep items with different list markers out of the same list group. - - AsciiDoc nests lists by marker, not by indentation. The item returns to - the closest enclosing list with its marker at the same indent, or else - opens a nested list under the item above it. - """ - level = cls._get_current_level(parents) - kind = cls._list_kind(item) - if list_kinds.get(level) == kind: - return - outer = level - 1 - while outer in list_kinds and indents[outer] == item["indent"]: - if list_kinds[outer] == kind: - for closed in range(outer + 1, level + 1): - parents[closed] = None - indents[closed] = None - return - outer -= 1 - parents[level + 1] = doc.add_group( - parent=last_list_item, name="list", label=GroupLabel.LIST - ) - indents[level + 1] = item["indent"] - list_kinds[level + 1] = kind - - @staticmethod - def _list_kind(item: dict) -> str: - # All ordered markers ("." and "1.") count as one kind; a bullet's kind - # is its marker character, so "*" and "-" lists nest like Asciidoctor. - return "." if item["numbered"] else item["marker"][0] - - @staticmethod - def _flush_text_data( - *, - doc: DoclingDocument, - text_data: list[str], - parent: GroupItem | None, - ) -> list[str]: - if len(text_data) == 0: - return [] - text = " ".join(text_data) - runs = AsciiDocBackend._parse_inline(text) - if not any(run.bold or run.italic or run.code for run in runs): - doc.add_text(text=text, label=DocItemLabel.PARAGRAPH, parent=parent) - return [] - # Mixed formatting needs one item per run, kept together in an inline - # group like the Markdown backend does. - group = doc.add_inline_group(parent=parent) - for run in runs: - run_text = run.text.strip() - if not run_text: - continue - if run.code: - doc.add_code(text=run_text, parent=group) - else: - doc.add_text( - text=run_text, - label=DocItemLabel.TEXT, - parent=group, - formatting=( - Formatting(bold=run.bold, italic=run.italic) - if run.bold or run.italic - else None - ), - ) - return [] - - @staticmethod - def _parse_inline( - text: str, bold: bool = False, italic: bool = False - ) -> list[_InlineRun]: - """Split text into runs of plain, bold, italic and monospace text.""" - runs: list[_InlineRun] = [] - position = 0 - for match in _INLINE_FORMAT_RE.finditer(text): - if match.start() > position: - runs.append( - _InlineRun(text[position : match.start()], bold=bold, italic=italic) - ) - if (code := match.group("umono") or match.group("mono")) is not None: - runs.append(_InlineRun(code, code=True)) - elif (inner := match.group("ubold") or match.group("bold")) is not None: - runs.extend(AsciiDocBackend._parse_inline(inner, True, italic)) - else: - inner = match.group("uitalic") or match.group("italic") - runs.extend(AsciiDocBackend._parse_inline(inner, bold, True)) - position = match.end() - if position < len(text): - runs.append(_InlineRun(text[position:], bold=bold, italic=italic)) - return runs - - # ========= Title - @staticmethod - def _is_title(line): - return re.match(r"^= ", line) - - @staticmethod - def _strip_closing_marker(text: str, marker: str) -> str: - """Drop the closing marks of a symmetric title such as "== Title ==". - - Asciidoctor accepts a title closed by the same marks that open it, - separated by whitespace, and leaves them out of the title text. - """ - return re.sub(rf"[ \t]+{re.escape(marker)}$", "", text) - - @staticmethod - def _parse_title(line): - text = AsciiDocBackend._strip_closing_marker(line[2:].strip(), "=") - return {"type": "title", "text": text, "level": 0} - - # ========= Section headers - @staticmethod - def _is_section_header(line): - return re.match(r"^==+\s+", line) - - @staticmethod - def _parse_section_header(line): - match = re.match(r"^(=+)\s+(.*)", line) - - marker = match.group(1) # The list marker (e.g., "*", "-", "1.") - text = match.group(2) # The actual text of the list item - - header_level = marker.count("=") # number of '=' represents level - return { - "type": "header", - "level": header_level - 1, - "text": AsciiDocBackend._strip_closing_marker(text.strip(), marker), - } - - # ========= Lists - @staticmethod - def _is_list_item(line): - return re.match(_LIST_ITEM_PATTERN, line) - - @staticmethod - def _parse_list_item(line): - """Extract the item marker (number or bullet symbol) and the text of the item.""" - - match = re.match(_LIST_ITEM_PATTERN, line) - if match: - indent = match.group(1) - marker = match.group(2) # The list marker (e.g., "*", "-", "1.") - text = match.group(3) # The actual text of the list item - indent_width = len(indent) - if marker.startswith((".", "*")): - indent_width += len(marker) - 1 - - if marker.startswith("*") or marker == "-": - return { - "type": "list_item", - "marker": marker, - "text": text.strip(), - "numbered": False, - "indent": indent_width, - } - else: - return { - "type": "list_item", - "marker": marker, - "text": text.strip(), - "numbered": True, - "indent": indent_width, - } - else: - # Fallback if no match - return { - "type": "list_item", - "marker": "-", - "text": line, - "numbered": False, - "indent": 0, - } - - # ========= Tables - @staticmethod - def _is_table_line(line): - # A cell may be the only one on its line: vertical AsciiDoc tables - # commonly write one cell per line, so a second "|" is not required. - return re.match(rf"^{_CELL_SPEC}\|", line) is not None - - @classmethod - def _continue_table(cls, table_data: list[list[str]], line: str) -> bool: - """Consume a line between |=== delimiters that is not a row. - - A blank line only separates rows, and a line without a leading "|" - continues the cell above it. Returns False for any other line. - """ - stripped = line.strip() - if stripped == "|===" or cls._is_table_line(line): - return False - if stripped and not table_data: - return False - if stripped: - table_data[-1][-1] = f"{table_data[-1][-1]} {stripped}".lstrip() - return True - - @staticmethod - def _parse_table_line(line): - # Drop cell specifiers glued to a "|" (e.g. "^.^h"); anchored to - # whitespace so content ending in a style letter (e.g. "Eth") survives. - line = re.sub(rf"(^|\s){_CELL_SPEC}(?=\|)", r"\1", line) - # Split on unescaped "|" and remove the leading empty string from the - # first "|"; a "\|" is a pipe in the cell text and loses its backslash. - cells = re.split(r"(? None: - if not table_data: - return - - caption = None - if caption_data: - caption = doc.add_text( - text=" ".join(caption_data), label=DocItemLabel.CAPTION - ) - - data = AsciiDocBackend._populate_table_as_grid(table_data) - doc.add_table(data=data, parent=parent, caption=caption) - - @staticmethod - def _populate_table_as_grid(table_data): - num_rows = len(table_data) - - # Adjust the table data into a grid format - num_cols = max((len(row) for row in table_data), default=0) - - data = TableData(num_rows=num_rows, num_cols=num_cols, table_cells=[]) - for row_idx, row in enumerate(table_data): - # Pad rows with empty strings to match column count - # grid.append(row + [''] * (max_cols - len(row))) - - for col_idx, text in enumerate(row): - row_span = 1 - col_span = 1 - - cell = TableCell( - text=text, - row_span=row_span, - col_span=col_span, - start_row_offset_idx=row_idx, - end_row_offset_idx=row_idx + row_span, - start_col_offset_idx=col_idx, - end_col_offset_idx=col_idx + col_span, - column_header=row_idx == 0, - row_header=False, - ) - data.table_cells.append(cell) - - return data - - # ========= Pictures - @staticmethod - def _is_picture(line): - return re.match(r"^image::", line) - - @staticmethod - def _parse_picture(line): - """ - Parse an image macro, extracting its path and attributes. - Syntax: image::path/to/image.png[Alt Text, width=200, height=150, align=center] - """ - mtch = re.match(r"^image::(.+)\[(.*)\]$", line) - if mtch: - picture_path = mtch.group(1).strip() - attributes = mtch.group(2).split(",") - picture_info = {"type": "picture", "uri": picture_path} - - # Extract optional attributes (alt text, width, height, alignment) - if attributes: - alt_parts = [attributes[0].strip()] if attributes[0] else [""] - for attr in attributes[1:]: - if "=" in attr: - key, value = attr.split("=", 1) - picture_info[key.strip()] = value.strip() - else: - alt_parts.append(attr.strip()) - picture_info["alt"] = ", ".join(alt_parts) - - return picture_info - - return {"type": "picture", "uri": line} - - # ========= Captions - @staticmethod - def _is_caption(line): - return re.match(r"^\.(\S.*)", line) - - @staticmethod - def _parse_caption(line): - mtch = re.match(r"^\.(.+)", line) - if mtch: - text = mtch.group(1) - return {"type": "caption", "text": text} - - return {"type": "caption", "text": ""} - - # ========= Plain text - @staticmethod - def _parse_text(line): - return {"type": "text", "text": line.strip()} +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +import logging +import re +from collections.abc import Iterator +from dataclasses import dataclass +from io import BytesIO +from pathlib import Path +from typing import Final, Optional, Union + +from docling_core.types.doc import ( + ContentLayer, + DocItemLabel, + DoclingDocument, + DocumentOrigin, + Formatting, + GroupItem, + GroupLabel, + ImageRef, + ListItem, + NodeItem, + TableCell, + TableData, + TextItem, +) + +from docling.backend.abstract_backend import DeclarativeDocumentBackend +from docling.backend.utils.image_resource_loader import ImageResourceLoader +from docling.datamodel.backend_options import AsciiDocBackendOptions +from docling.datamodel.base_models import InputFormat +from docling.datamodel.document import InputDocument +from docling.exceptions import DocumentLoadError +from docling.utils.code_language import CodeLanguageLabel, detect_code_language +from docling.utils.text_decoding import decode_text + +_log = logging.getLogger(__name__) + +# A line starting with exactly two slashes is a comment and never body content. +# A line of four or more slashes opens a delimited comment block: every line up +# to an identical line is a comment. +_LINE_COMMENT_RE = re.compile(r"^//(?!/)") +_COMMENT_BLOCK_DELIMITER_RE = re.compile(r"/{4,}") +# A line of four or more hyphens opens a listing block and a line of four or +# more dots opens a literal block: the count is a minimum, so "-----" opens a +# listing just as "----" does. +_VERBATIM_DELIMITER_RE = re.compile(r"-{4,}|\.{4,}") +# A listing block takes a "[source,lang]" attribute; a literal block does not. +_LISTING_DELIMITER_RE = re.compile(r"-{4,}") + +# Cell format specifier that may precede a "|" delimiter, e.g. "^.^h" in +# "^.^h|Header": span (3*, 2+, .2+, 2.3+), alignment (<, ^, >, .^), style +# (a/d/e/h/l/m/s). AsciiDoc writes the span as [colspan][.rowspan] followed by +# "+" or "*", and either number may be omitted, so ".2+" is a rowspan on its +# own. Requiring at least one of the two keeps a bare "+" from matching. +_CELL_SPEC: Final = r"(?:(?:\d+(?:\.\d+)?|\.\d+)[*+])*[<^>]?(?:\.[<^>])?[adehlms]?" +# An ordered marker with a letter has one letter only ("a.", "B."), so a line +# that starts with an abbreviation such as "Dr." or "Fig." stays text. +_LIST_ITEM_PATTERN: Final = r"^(\s*)(\*+|-|\.+|\d+\.|[a-zA-Z]\.)\s+(.*)" + +# Inline formatting marks, unconstrained (doubled, valid anywhere) before +# constrained (single, valid only at word boundaries, so "snake_case" and +# "2*3*4" stay plain text), as in Asciidoctor. A constrained span does not +# contain its own mark, so each scan stops at the next mark on the line. +_INLINE_FORMAT_RE: Final = re.compile( + r"``(?P.+?)``" + r"|\*\*(?P.+?)\*\*" + r"|__(?P.+?)__" + r"|(?[^`\s](?:[^`]*?[^`\s])?)`(?!\w)" + r"|(?[^*\s](?:[^*]*?[^*\s])?)\*(?!\w)" + r"|(?[^_\s](?:[^_]*?[^_\s])?)_(?!\w)" +) + + +@dataclass(frozen=True) +class _InlineRun: + text: str + bold: bool = False + italic: bool = False + code: bool = False + + +@dataclass(frozen=True) +class _LiteralBlock: + text: str + language: str | None = None # set for [source,lang] listing blocks + + +@dataclass(frozen=True) +class _CommentBlock: + """An AsciiDoc comment, with its // or //// delimiters removed. + + One block per comment: a run of consecutive // lines is one comment, and + so is everything between a pair of //// delimiters. + """ + + text: str + + +class AsciiDocBackend(DeclarativeDocumentBackend): + def __init__( + self, + in_doc: "InputDocument", + path_or_stream: Union[BytesIO, Path], + options: Optional[AsciiDocBackendOptions] = None, + ): + if options is None: + options = AsciiDocBackendOptions() + super().__init__(in_doc, path_or_stream, options) + + self.path_or_stream = path_or_stream + self.options: AsciiDocBackendOptions + self._image_loader = ImageResourceLoader( + enable_local_fetch=options.enable_local_fetch, + enable_remote_fetch=options.enable_remote_fetch, + ) + + # A leading BOM is dropped. Kept, it prefixes the first line, so a + # document title ("= Title") is no longer recognized as one and the BOM + # reaches the output. + try: + self.lines = decode_text(self.path_or_stream, options.encoding).split("\n") + self.valid = True + + except DocumentLoadError: + # Already carries a message naming what could not be decoded. + raise + except Exception as e: + raise DocumentLoadError( + f"Could not initialize AsciiDoc backend for file with hash {self.document_hash}." + ) from e + return + + def is_valid(self) -> bool: + return self.valid + + @classmethod + def supports_pagination(cls) -> bool: + return False + + def unload(self): + return + + @classmethod + def supported_formats(cls) -> set[InputFormat]: + return {InputFormat.ASCIIDOC} + + def convert(self) -> DoclingDocument: + """ + Parses the ASCII into a structured document model. + """ + + origin = DocumentOrigin( + filename=self.file.name or "file", + mimetype="text/asciidoc", + binary_hash=self.document_hash, + ) + + doc = DoclingDocument(name=self.file.stem or "file", origin=origin) + + doc = self._parse(doc) + + return doc + + def _flush_blank_line( + self, + doc: DoclingDocument, + caption_data: list[str], + text_data: list[str], + parent: Union[GroupItem, None], + ) -> tuple[list[str], list[str]]: + """Flush both accumulators on a blank line. + + Plain text accumulated before a block title is emitted first to + preserve reading order, then any pending caption is emitted as a + bold paragraph. + """ + text_data = self._flush_text_data(doc=doc, text_data=text_data, parent=parent) + if caption_data: + caption_data = self._flush_caption_as_paragraph(doc, caption_data, parent) + return caption_data, text_data + + def _flush_caption_as_paragraph( + self, + doc: DoclingDocument, + caption_data: list[str], + parent: Union[GroupItem, None], + ) -> list[str]: + """Emit a pending block title as a bold paragraph. + + Used for block titles whose target is not a floating item (picture, + table, or code block), which have no caption slot. + """ + doc.add_text( + text=" ".join(caption_data), + label=DocItemLabel.PARAGRAPH, + parent=parent, + formatting=Formatting(bold=True), + ) + return [] + + def _parse(self, doc: DoclingDocument): + """Orchestrate parsing and populate `doc` from the source lines. + + Handles titles, section headers, text paragraphs, lists, tables, + pictures, literal (code) blocks, and block titles. + + Block titles (AsciiDoc lines that start with `.` immediately + followed by text, e.g. `.Procedure`) are treated as follows: + + * FloatingItem targets (picture, table, code block): the block + title is created as a `CAPTION`-labelled `TextItem` and + attached to the floating item via its `caption` parameter, so it + participates in the standard caption relationship. + * All other targets (lists, paragraphs, ...): the block title is + emitted as a bold `PARAGRAPH` `TextItem` inserted + immediately before the element it precedes. This preserves + reading order while acknowledging that non-floating items (e.g. + `GroupItem`) have no caption slot. + """ + + in_list = False + in_table = False + in_delimited_table = False + + text_data: list[str] = [] + table_data: list[list[str]] = [] + caption_data: list[str] = [] + last_list_item: ListItem | None = None + list_continuation = False + # The marker kind of the list group at each level ("." for ordered, + # "*" or "-" for bullets), for the list being parsed. Reset when a new + # list starts, so a level that holds a section header is never mistaken + # for a list. + list_kinds: dict[int, str] = {} + + parents: dict[int, Union[GroupItem, None]] = {} + indents: dict[int, Union[GroupItem, None]] = {} + + for i in range(10): + parents[i] = None + indents[i] = None + + for block in self._iter_blocks(self.lines): + if not isinstance(block, str): + text_data, caption_data, in_list, last_list_item, list_continuation = ( + self._add_block( + doc=doc, + block=block, + text_data=text_data, + caption_data=caption_data, + in_list=in_list, + last_list_item=last_list_item, + list_continuation=list_continuation, + parents=parents, + ) + ) + continue + + line = block + stripped_line = line.strip() + if in_delimited_table and self._continue_table(table_data, line): + continue + + in_list, last_list_item, list_continuation = self._close_list_if_needed( + line=line, + in_list=in_list, + parents=parents, + last_list_item=last_list_item, + list_continuation=list_continuation, + is_continuation_block=False, + ) + + # Block attributes and anchors (e.g. "[cols=\"2,1\"]", "[NOTE]", + # "[[some-id]]") are metadata that applies to the block that + # follows; they are never body content, so skip them. + if ( + re.match(r"^\[\[.+\]\]$", stripped_line) + or re.match(r"^\[[^\[\]]+\]$", stripped_line) + or re.match(r"^:[\w.-]+:(\s.*)?$", stripped_line) + ): + continue + + # Title + if self._is_title(line): + item = self._parse_title(line) + level = item["level"] + + parents[level] = doc.add_text( + text=item["text"], label=DocItemLabel.TITLE + ) + + # Section headers + elif self._is_section_header(line): + # A heading ends the paragraph being accumulated: without + # this flush the text before the heading is appended to the + # text after it and lands in the wrong section. + text_data = self._flush_text_data( + doc=doc, + text_data=text_data, + parent=self._get_current_parent(parents), + ) + item = self._parse_section_header(line) + level = item["level"] + + ancestor = next( + (parents[k] for k in range(level - 1, -1, -1) if parents[k]), + None, + ) + parents[level] = doc.add_heading( + text=item["text"], level=item["level"], parent=ancestor + ) + for k, v in parents.items(): + if k > level: + parents[k] = None + + # Lists + elif self._is_list_item(line): + _log.debug(f"line: {line}") + item = self._parse_list_item(line) + _log.debug(f"parsed list-item: {item}") + + level = self._get_current_level(parents) + + if not in_list: + in_list = True + # GroupItem has no caption slot; emit the pending block + # title as a bold paragraph immediately before the list. + if caption_data: + doc.add_text( + text=" ".join(caption_data), + label=DocItemLabel.PARAGRAPH, + parent=parents[level], + formatting=Formatting(bold=True), + ) + caption_data = [] + + parents[level + 1] = doc.add_group( + parent=parents[level], name="list", label=GroupLabel.LIST + ) + indents[level + 1] = item["indent"] + list_kinds = {level + 1: self._list_kind(item)} + + elif in_list and item["indent"] > indents[level]: + # A nested list belongs to the item above it. As a sibling + # of the items, it is counted as an item of the outer list, + # and the next ordered item is numbered one too high. + parents[level + 1] = doc.add_group( + parent=last_list_item, name="list", label=GroupLabel.LIST + ) + indents[level + 1] = item["indent"] + list_kinds[level + 1] = self._list_kind(item) + + elif in_list and item["indent"] < indents[level]: + while level > 0 and item["indent"] < indents[level]: + # Only pop the current level if there is an outer group + # to fall back to; otherwise keep it as the list root. + if indents[level - 1] is None: + break + parents[level] = None + indents[level] = None + level -= 1 + + self._match_list_kind( + doc=doc, + item=item, + parents=parents, + indents=indents, + list_kinds=list_kinds, + last_list_item=last_list_item, + ) + + last_list_item = doc.add_list_item( + item["text"], + enumerated=item["numbered"], + marker=(item["marker"] if item["marker"][:-1].isdigit() else None), + parent=self._get_current_parent(parents), + ) + list_continuation = False + + elif in_list and stripped_line == "+": + list_continuation = True + continue + + # Tables + elif line.strip() == "|===": # start or end of table + if in_table: + self._add_table_if_nonempty( + doc, table_data, caption_data, self._get_current_parent(parents) + ) + caption_data = [] + in_table = False + in_delimited_table = False + table_data = [] + else: + in_table = True + in_delimited_table = True + + elif self._is_table_line(line): # within a table + in_table = True + table_data.append(self._parse_table_line(line)) + + elif in_table and not self._is_table_line(line): # end of table + self._add_table_if_nonempty( + doc, table_data, caption_data, self._get_current_parent(parents) + ) + caption_data = [] + in_table = False + table_data = [] + + # Picture + elif self._is_picture(line): + caption_data, in_list, last_list_item = self._add_picture_item( + doc=doc, + line=line, + caption_data=caption_data, + in_list=in_list, + last_list_item=last_list_item, + parents=parents, + ) + list_continuation = False + + # Caption + elif self._is_caption(line) and len(caption_data) == 0: + item = self._parse_caption(line) + caption_data.append(item["text"]) + + elif len(line.strip()) > 0 and len(caption_data) > 0: + # A block title is a single line, so this line starts the + # paragraph it titles. A paragraph has no caption slot. + parent = self._get_current_parent(parents) + text_data = self._flush_text_data( + doc=doc, text_data=text_data, parent=parent + ) + caption_data = self._flush_caption_as_paragraph( + doc, caption_data, parent + ) + item = self._parse_text(line) + text_data.append(item["text"]) + + elif len(line.strip()) == 0: + caption_data, text_data = self._flush_blank_line( + doc, + caption_data, + text_data, + self._get_current_parent(parents), + ) + + elif len(line.strip()) > 0: # allow multiline texts + item = self._parse_text(line) + text_data.append(item["text"]) + + text_data = self._flush_text_data( + doc=doc, text_data=text_data, parent=self._get_current_parent(parents) + ) + + if in_table: + # An unclosed table still consumes its pending caption below. + self._add_table_if_nonempty( + doc, table_data, caption_data, self._get_current_parent(parents) + ) + elif caption_data: + # A block title never followed by a floating element must still + # reach the document. + self._flush_caption_as_paragraph( + doc, caption_data, self._get_current_parent(parents) + ) + + return doc + + @staticmethod + def _get_current_level(parents): + for k, v in parents.items(): + if v is None and k > 0: + return k - 1 + + return 0 + + @staticmethod + def _get_current_parent(parents): + for k, v in parents.items(): + if v is None and k > 0: + return parents[k - 1] + + return None + + _SOURCE_ATTR_RE = re.compile(r"^\[source(?:,\s*([\w+#.-]+))?[^\]]*\]$") + _CONTENT_BLOCK_DELIMITERS = ("====", "****", "____", "--", "++++") + + @staticmethod + def _has_matching_closer(lines: list[str], open_idx: int, delimiter: str) -> bool: + """Return True if `delimiter` appears on a line of its own after `open_idx`.""" + for j in range(open_idx + 1, len(lines)): + if lines[j].strip() == delimiter: + return True + return False + + @staticmethod + def _is_line_comment(line: str) -> bool: + """Is `line` a // comment line rather than text or a //// block delimiter?""" + return _LINE_COMMENT_RE.match(line) is not None + + @staticmethod + def _make_comment(comment_lines: list[str]) -> _CommentBlock | None: + """Build a comment block, or None when the comment carries no text.""" + text = "\n".join(comment_lines).strip() + return _CommentBlock(text=text) if text else None + + @classmethod + def _iter_blocks( + cls, lines: list[str] + ) -> Iterator[str | _LiteralBlock | _CommentBlock]: + block_data: list[str] | None = None + block_delimiter: str | None = None + block_language: str | None = None + comment_data: list[str] | None = None + comment_delimiter: str | None = None + # Content block delimiters still waiting for their closer, innermost + # last. The lines between a pair run through this loop like any other, + # so a comment inside such a block is a comment and not body text. + open_content_blocks: list[str] = [] + + i = 0 + n = len(lines) + while i < n: + line = lines[i] + stripped = line.strip() + + if block_data is not None: + # Inside a verbatim block: only the matching closer ends it, + # and a "//" line here is content rather than a comment. + if stripped == block_delimiter: + yield _LiteralBlock( + text="\n".join(block_data), language=block_language + ) + block_data = None + block_delimiter = None + block_language = None + else: + block_data.append(line.rstrip("\r\n")) + i += 1 + continue + + # A comment block hides every construct, so it is checked ahead of + # the block openers: a "----" between //// delimiters is comment + # text, not the start of a listing. + if comment_data is not None: + if stripped == comment_delimiter: + comment = cls._make_comment(comment_data) + if comment is not None: + yield comment + comment_data = None + comment_delimiter = None + else: + comment_data.append(line.rstrip("\r\n")) + i += 1 + continue + + if _COMMENT_BLOCK_DELIMITER_RE.fullmatch(stripped): + comment_data = [] + comment_delimiter = stripped + i += 1 + continue + + if cls._is_line_comment(line): + # Consecutive // lines read as one comment, so take the run. + comment_lines = [] + while i < n and cls._is_line_comment(lines[i]): + comment_lines.append(lines[i].rstrip("\r\n")[2:].strip()) + i += 1 + comment = cls._make_comment(comment_lines) + if comment is not None: + yield comment + continue + + if _VERBATIM_DELIMITER_RE.fullmatch(stripped): + # "...." is a literal block; "----" a listing block. A + # "[source,lang]" attribute consumed just before a listing + # labels the listing's language. + block_data = [] + block_delimiter = stripped + block_language = None + i += 1 + continue + + if open_content_blocks and stripped == open_content_blocks[-1]: + # Closer of the innermost content block. Yield a trailing blank + # so the following text starts a fresh paragraph. + open_content_blocks.pop() + yield "" + i += 1 + continue + + if stripped in cls._CONTENT_BLOCK_DELIMITERS: + if cls._has_matching_closer(lines, i, stripped): + open_content_blocks.append(stripped) + else: + # Stray delimiter (no closer): break the paragraph, do not + # render as text. The line must be consumed either way; + # leaving `i` on it loops over the same line forever. + yield "" + i += 1 + continue + + source_attr = cls._SOURCE_ATTR_RE.match(stripped) + if ( + source_attr is not None + and i + 1 < n + and _LISTING_DELIMITER_RE.fullmatch(lines[i + 1].strip()) + ): + block_data = [] + block_delimiter = lines[i + 1].strip() + block_language = source_attr.group(1) + i += 2 + continue + + yield line + i += 1 + + if block_data is not None: + yield _LiteralBlock(text="\n".join(block_data), language=block_language) + if comment_data is not None: + # An unclosed //// comments out the rest of the document, which is + # how Asciidoctor reads it. + comment = cls._make_comment(comment_data) + if comment is not None: + yield comment + + def _add_picture_item( + self, + *, + doc: DoclingDocument, + line: str, + caption_data: list[str], + in_list: bool, + last_list_item: ListItem | None, + parents: dict[int, GroupItem | None], + ) -> tuple[list[str], bool, ListItem | None]: + """Add a picture with its pending caption, if any.""" + caption: Optional[TextItem] = None + if len(caption_data) > 0: + caption = doc.add_text( + text=" ".join(caption_data), label=DocItemLabel.CAPTION + ) + + caption_data = [] + + item = self._parse_picture(line) + + image: Optional[ImageRef] = None + if "uri" in item and self.options.fetch_images: + base_path = ( + str(self.options.source_uri) + if self.options.source_uri is not None + else None + ) + image = self._image_loader.load_image_ref(item["uri"], base_path) + doc.add_picture( + image=image, + caption=caption, + parent=last_list_item if in_list else self._get_current_parent(parents), + ) + return caption_data, in_list, last_list_item + + def _add_block( + self, + *, + doc: DoclingDocument, + block: _LiteralBlock | _CommentBlock, + text_data: list[str], + caption_data: list[str], + in_list: bool, + last_list_item: ListItem | None, + list_continuation: bool, + parents: dict[int, GroupItem | None], + ) -> tuple[list[str], list[str], bool, ListItem | None, bool]: + """Add a delimited block and return the parser state after it. + + A comment is markup, not content: it neither ends the list nor the + paragraph it sits in, so the state passes through unchanged. A + literal block flushes the pending text and captions first. + """ + if isinstance(block, _CommentBlock): + self._add_comment_block( + doc=doc, + block=block, + in_list=in_list, + last_list_item=last_list_item, + parents=parents, + ) + return text_data, caption_data, in_list, last_list_item, list_continuation + return self._add_literal_block( + doc=doc, + block=block, + text_data=text_data, + caption_data=caption_data, + in_list=in_list, + last_list_item=last_list_item, + list_continuation=list_continuation, + parents=parents, + ) + + def _add_comment_block( + self, + *, + doc: DoclingDocument, + block: _CommentBlock, + in_list: bool, + last_list_item: ListItem | None, + parents: dict[int, GroupItem | None], + ) -> None: + """Add a comment to the notes content layer. + + A comment between two lines of one paragraph lands just before that + paragraph, since flushing the text to place it would split the body + text in two. + """ + if not self.options.capture_comments: + return + doc.add_text( + text=block.text, + label=DocItemLabel.TEXT, + content_layer=ContentLayer.NOTES, + parent=(last_list_item if in_list else self._get_current_parent(parents)), + ) + + def _add_literal_block( + self, + *, + doc: DoclingDocument, + block: _LiteralBlock, + text_data: list[str], + caption_data: list[str], + in_list: bool, + last_list_item: ListItem | None, + list_continuation: bool, + parents: dict[int, GroupItem | None], + ) -> tuple[list[str], list[str], bool, ListItem | None, bool]: + """Add a literal/listing block, flushing pending text and captions.""" + in_list, last_list_item, list_continuation = self._close_list_if_needed( + line="", + in_list=in_list, + parents=parents, + last_list_item=last_list_item, + list_continuation=list_continuation, + is_continuation_block=True, + ) + text_data = self._flush_text_data( + doc=doc, + text_data=text_data, + parent=self._get_current_parent(parents), + ) + caption: Optional[TextItem] = None + if caption_data: + caption = doc.add_text( + text=" ".join(caption_data), + label=DocItemLabel.CAPTION, + ) + caption_data = [] + code_language = ( + detect_code_language(block.text, hint=block.language) + if block.language is not None + else None + ) + doc.add_code( + text=block.text, + code_language=code_language, + caption=caption, + parent=(last_list_item if in_list else self._get_current_parent(parents)), + ) + return text_data, caption_data, in_list, last_list_item, False + + @classmethod + def _close_list_if_needed( + cls, + *, + line: str, + in_list: bool, + parents: dict[int, GroupItem | None], + last_list_item: ListItem | None, + list_continuation: bool, + is_continuation_block: bool, + ) -> tuple[bool, ListItem | None, bool]: + if ( + not in_list + or cls._is_list_item(line) + or line.strip() in {"", "+"} + or (list_continuation and (is_continuation_block or cls._is_picture(line))) + ): + return in_list, last_list_item, list_continuation + + # Close every level of the list, not only the innermost one. Otherwise + # the text after a nested list is added to the outer list group. + level = cls._get_current_level(parents) + while ( + isinstance(group := parents[level], GroupItem) + and group.label == GroupLabel.LIST + ): + parents[level] = None + level -= 1 + return False, None, False + + @classmethod + def _match_list_kind( + cls, + *, + doc: DoclingDocument, + item: dict, + parents: dict[int, GroupItem | None], + indents: dict[int, GroupItem | None], + list_kinds: dict[int, str], + last_list_item: ListItem | None, + ) -> None: + """Keep items with different list markers out of the same list group. + + AsciiDoc nests lists by marker, not by indentation. The item returns to + the closest enclosing list with its marker at the same indent, or else + opens a nested list under the item above it. + """ + level = cls._get_current_level(parents) + kind = cls._list_kind(item) + if list_kinds.get(level) == kind: + return + outer = level - 1 + while outer in list_kinds and indents[outer] == item["indent"]: + if list_kinds[outer] == kind: + for closed in range(outer + 1, level + 1): + parents[closed] = None + indents[closed] = None + return + outer -= 1 + parents[level + 1] = doc.add_group( + parent=last_list_item, name="list", label=GroupLabel.LIST + ) + indents[level + 1] = item["indent"] + list_kinds[level + 1] = kind + + @staticmethod + def _list_kind(item: dict) -> str: + # All ordered markers ("." and "1.") count as one kind; a bullet's kind + # is its marker character, so "*" and "-" lists nest like Asciidoctor. + return "." if item["numbered"] else item["marker"][0] + + @staticmethod + def _flush_text_data( + *, + doc: DoclingDocument, + text_data: list[str], + parent: GroupItem | None, + ) -> list[str]: + if len(text_data) == 0: + return [] + text = " ".join(text_data) + runs = AsciiDocBackend._parse_inline(text) + if not any(run.bold or run.italic or run.code for run in runs): + doc.add_text(text=text, label=DocItemLabel.PARAGRAPH, parent=parent) + return [] + # Mixed formatting needs one item per run, kept together in an inline + # group like the Markdown backend does. + group = doc.add_inline_group(parent=parent) + for run in runs: + run_text = run.text.strip() + if not run_text: + continue + if run.code: + doc.add_code(text=run_text, parent=group) + else: + doc.add_text( + text=run_text, + label=DocItemLabel.TEXT, + parent=group, + formatting=( + Formatting(bold=run.bold, italic=run.italic) + if run.bold or run.italic + else None + ), + ) + return [] + + @staticmethod + def _parse_inline( + text: str, bold: bool = False, italic: bool = False + ) -> list[_InlineRun]: + """Split text into runs of plain, bold, italic and monospace text.""" + runs: list[_InlineRun] = [] + position = 0 + for match in _INLINE_FORMAT_RE.finditer(text): + if match.start() > position: + runs.append( + _InlineRun(text[position : match.start()], bold=bold, italic=italic) + ) + if (code := match.group("umono") or match.group("mono")) is not None: + runs.append(_InlineRun(code, code=True)) + elif (inner := match.group("ubold") or match.group("bold")) is not None: + runs.extend(AsciiDocBackend._parse_inline(inner, True, italic)) + else: + inner = match.group("uitalic") or match.group("italic") + runs.extend(AsciiDocBackend._parse_inline(inner, bold, True)) + position = match.end() + if position < len(text): + runs.append(_InlineRun(text[position:], bold=bold, italic=italic)) + return runs + + # ========= Title + @staticmethod + def _is_title(line): + return re.match(r"^= ", line) + + @staticmethod + def _strip_closing_marker(text: str, marker: str) -> str: + """Drop the closing marks of a symmetric title such as "== Title ==". + + Asciidoctor accepts a title closed by the same marks that open it, + separated by whitespace, and leaves them out of the title text. + """ + return re.sub(rf"[ \t]+{re.escape(marker)}$", "", text) + + @staticmethod + def _parse_title(line): + text = AsciiDocBackend._strip_closing_marker(line[2:].strip(), "=") + return {"type": "title", "text": text, "level": 0} + + # ========= Section headers + @staticmethod + def _is_section_header(line): + return re.match(r"^==+\s+", line) + + @staticmethod + def _parse_section_header(line): + match = re.match(r"^(=+)\s+(.*)", line) + + marker = match.group(1) # The list marker (e.g., "*", "-", "1.") + text = match.group(2) # The actual text of the list item + + header_level = marker.count("=") # number of '=' represents level + return { + "type": "header", + "level": header_level - 1, + "text": AsciiDocBackend._strip_closing_marker(text.strip(), marker), + } + + # ========= Lists + @staticmethod + def _is_list_item(line): + return re.match(_LIST_ITEM_PATTERN, line) + + @staticmethod + def _parse_list_item(line): + """Extract the item marker (number or bullet symbol) and the text of the item.""" + + match = re.match(_LIST_ITEM_PATTERN, line) + if match: + indent = match.group(1) + marker = match.group(2) # The list marker (e.g., "*", "-", "1.") + text = match.group(3) # The actual text of the list item + indent_width = len(indent) + if marker.startswith((".", "*")): + indent_width += len(marker) - 1 + + if marker.startswith("*") or marker == "-": + return { + "type": "list_item", + "marker": marker, + "text": text.strip(), + "numbered": False, + "indent": indent_width, + } + else: + return { + "type": "list_item", + "marker": marker, + "text": text.strip(), + "numbered": True, + "indent": indent_width, + } + else: + # Fallback if no match + return { + "type": "list_item", + "marker": "-", + "text": line, + "numbered": False, + "indent": 0, + } + + # ========= Tables + @staticmethod + def _is_table_line(line): + # A cell may be the only one on its line: vertical AsciiDoc tables + # commonly write one cell per line, so a second "|" is not required. + return re.match(rf"^{_CELL_SPEC}\|", line) is not None + + @classmethod + def _continue_table(cls, table_data: list[list[str]], line: str) -> bool: + """Consume a line between |=== delimiters that is not a row. + + A blank line only separates rows, and a line without a leading "|" + continues the cell above it. Returns False for any other line. + """ + stripped = line.strip() + if stripped == "|===" or cls._is_table_line(line): + return False + if stripped and not table_data: + return False + if stripped: + table_data[-1][-1] = f"{table_data[-1][-1]} {stripped}".lstrip() + return True + + @staticmethod + def _parse_table_line(line): + # Drop cell specifiers glued to a "|" (e.g. "^.^h"); anchored to + # whitespace so content ending in a style letter (e.g. "Eth") survives. + line = re.sub(rf"(^|\s){_CELL_SPEC}(?=\|)", r"\1", line) + # Split on unescaped "|" and remove the leading empty string from the + # first "|"; a "\|" is a pipe in the cell text and loses its backslash. + cells = re.split(r"(? None: + if not table_data: + return + + caption = None + if caption_data: + caption = doc.add_text( + text=" ".join(caption_data), label=DocItemLabel.CAPTION + ) + + data = AsciiDocBackend._populate_table_as_grid(table_data) + doc.add_table(data=data, parent=parent, caption=caption) + + @staticmethod + def _populate_table_as_grid(table_data): + num_rows = len(table_data) + + # Adjust the table data into a grid format + num_cols = max((len(row) for row in table_data), default=0) + + data = TableData(num_rows=num_rows, num_cols=num_cols, table_cells=[]) + for row_idx, row in enumerate(table_data): + # Pad rows with empty strings to match column count + # grid.append(row + [''] * (max_cols - len(row))) + + for col_idx, text in enumerate(row): + row_span = 1 + col_span = 1 + + cell = TableCell( + text=text, + row_span=row_span, + col_span=col_span, + start_row_offset_idx=row_idx, + end_row_offset_idx=row_idx + row_span, + start_col_offset_idx=col_idx, + end_col_offset_idx=col_idx + col_span, + column_header=row_idx == 0, + row_header=False, + ) + data.table_cells.append(cell) + + return data + + # ========= Pictures + @staticmethod + def _is_picture(line): + return re.match(r"^image::", line) + + @staticmethod + def _parse_picture(line): + """ + Parse an image macro, extracting its path and attributes. + Syntax: image::path/to/image.png[Alt Text, width=200, height=150, align=center] + """ + mtch = re.match(r"^image::(.+)\[(.*)\]$", line) + if mtch: + picture_path = mtch.group(1).strip() + attributes = mtch.group(2).split(",") + picture_info = {"type": "picture", "uri": picture_path} + + # Extract optional attributes (alt text, width, height, alignment) + if attributes: + alt_parts = [attributes[0].strip()] if attributes[0] else [""] + for attr in attributes[1:]: + if "=" in attr: + key, value = attr.split("=", 1) + picture_info[key.strip()] = value.strip() + else: + alt_parts.append(attr.strip()) + picture_info["alt"] = ", ".join(alt_parts) + + return picture_info + + return {"type": "picture", "uri": line} + + # ========= Captions + @staticmethod + def _is_caption(line): + return re.match(r"^\.(\S.*)", line) + + @staticmethod + def _parse_caption(line): + mtch = re.match(r"^\.(.+)", line) + if mtch: + text = mtch.group(1) + return {"type": "caption", "text": text} + + return {"type": "caption", "text": ""} + + # ========= Plain text + @staticmethod + def _parse_text(line): + return {"type": "text", "text": line.strip()} diff --git a/tests/test_backend_asciidoc.py b/tests/test_backend_asciidoc.py index 3b227cf375..51474d5f33 100644 --- a/tests/test_backend_asciidoc.py +++ b/tests/test_backend_asciidoc.py @@ -1,1279 +1,1300 @@ -# SPDX-FileCopyrightText: The Docling Contributors -# SPDX-License-Identifier: MIT - -import glob -import threading -import time -from io import BytesIO -from pathlib import Path - -import pytest -from docling_core.types.doc import ( - CodeItem, - CodeLanguageLabel, - ContentLayer, - DocItemLabel, - DoclingDocument, - Formatting, - ImageRefMode, - ListItem, -) - -from docling.backend.abstract_backend import DeclarativeDocumentBackend -from docling.backend.asciidoc_backend import AsciiDocBackend -from docling.datamodel.backend_options import AsciiDocBackendOptions -from docling.datamodel.base_models import InputFormat -from docling.datamodel.document import InputDocument - -from .test_data_gen_flag import GEN_TEST_DATA -from .verify_utils import verify_document, verify_export - - -def _get_backend(fname: Path) -> DeclarativeDocumentBackend: - in_doc = InputDocument( - path_or_stream=fname, - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - ) - - doc_backend = in_doc._backend - assert isinstance(doc_backend, DeclarativeDocumentBackend) - return doc_backend - - -def test_list_dedent_to_base_does_not_crash() -> None: - # A list that starts indented and then dedents back to the base level used - # to raise "TypeError: '<' not supported between instances of 'int' and - # 'NoneType'": the dedent loop walked past level 0, where the base indent is - # never set. It should keep both items instead. - src = b" * a\n* b\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="dedent.asciidoc", - ) - doc = in_doc._backend.convert() - - assert [item.text for item in doc.texts] == ["a", "b"] - - -def test_rowspan_only_cell_specifier_keeps_the_row() -> None: - # AsciiDoc writes a span as [colspan][.rowspan] followed by "+" or "*", and - # either number may be omitted, so ".2+" is a rowspan on its own. The cell - # specifier pattern required a leading digit, so _is_table_line rejected the - # line, the block loop read that as the end of the table, and the row after - # it leaked into the document as literal text. - src = b"|===\n|A |B\n.2+|tall |x\n|y\n|===\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="rowspan.asciidoc", - ) - doc = in_doc._backend.convert() - - assert doc.tables, "the table was dropped entirely" - table = doc.tables[0] - # The trailing single-cell row "|y" is kept: cells flow into the grid, so it - # lands in row 3, column 1, with the second column left empty. - assert (table.data.num_rows, table.data.num_cols) == (3, 2) - assert [cell.text for cell in table.data.table_cells] == [ - "A", - "B", - "tall", - "x", - "y", - ] - assert [item.text for item in doc.texts] == [] - - -def test_incomplete_table_does_not_emit_an_empty_table() -> None: - # A trailing row with fewer cells than the table's column count is kept and - # padded with an empty cell (matching how Asciidoctor fills the grid), so no - # empty phantom table is emitted and no cell content is lost. - for row, first in ((b"|3", "3"), (b"2+|wide", "wide")): - src = b"|===\n|A |B\n" + row + b"\n|===\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="single-cell-row.adoc", - ) - doc = in_doc._backend.convert() - - assert len(doc.tables) == 1 - assert (doc.tables[0].data.num_rows, doc.tables[0].data.num_cols) == (2, 2) - assert [cell.text for cell in doc.tables[0].data.table_cells] == [ - "A", - "B", - first, - ] - - -def test_single_cell_per_line_table_is_not_truncated() -> None: - # Vertical AsciiDoc tables write one cell per line. A single-| line used to - # be read as the end of the table: the first such line was silently - # dropped, the remaining ones leaked into the document as literal text, and - # the closing |=== reopened an empty table that vanished at EOF. - src = b"|===\n|Name\n|Age\n|Alice\n|30\n|===\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="vertical-table.adoc", - ) - doc = in_doc._backend.convert() - - assert len(doc.tables) == 1 - assert (doc.tables[0].data.num_rows, doc.tables[0].data.num_cols) == (4, 1) - assert [cell.text for cell in doc.tables[0].data.table_cells] == [ - "Name", - "Age", - "Alice", - "30", - ] - assert [item.text for item in doc.texts] == [] - - -def test_blank_line_inside_table_does_not_split_it() -> None: - # Inside |=== ... |=== a blank line only separates rows: Asciidoctor's own - # examples put one after the header row. It used to end the table, so the - # header and the body came out as two tables and the second lost the - # caption. - for src in ( - b".Formats\n|===\n|Format |Extension\n\n|Markdown |.md\n|JSON |.json\n|===\n", - b".Formats\n|===\n\n|Format |Extension\n\n|Markdown |.md\n\n|JSON |.json\n|===\n", - ): - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="blank-line-table.adoc", - ) - doc = in_doc._backend.convert() - - assert len(doc.tables) == 1 - table = doc.tables[0] - assert (table.data.num_rows, table.data.num_cols) == (3, 2) - assert [cell.text for cell in table.data.table_cells] == [ - "Format", - "Extension", - "Markdown", - ".md", - "JSON", - ".json", - ] - assert table.caption_text(doc) == "Formats" - - -def test_wrapped_cell_text_stays_in_its_cell() -> None: - # A line without a leading "|" inside a table continues the cell above it. - # It used to end the table: the line was dropped and the rows after it - # became a second table. - src = ( - b"|===\n|Format |Extension\n|Markdown |.md or\n(also .markdown)\n" - b"|JSON |.json\n|===\nAfter.\n" - ) - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="wrapped-cell-table.adoc", - ) - doc = in_doc._backend.convert() - - assert len(doc.tables) == 1 - assert [cell.text for cell in doc.tables[0].data.table_cells] == [ - "Format", - "Extension", - "Markdown", - ".md or (also .markdown)", - "JSON", - ".json", - ] - assert [item.text for item in doc.texts] == ["After."] - - -def test_escaped_pipe_stays_inside_its_cell() -> None: - # A backslash-escaped pipe is cell content: Asciidoctor partitions cells on - # unescaped pipes only and drops the backslash from the output. Splitting on - # every "|" cut the cell at the escaped pipe, left the backslash in the text - # and gave the row more columns than the header. - src = ( - b"|===\n|Option |Values\n" - b"|--format |json\\|yaml\n" - b"|--level |debug\\|info\\|warn\n|===\n" - ) - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="escaped-pipe.adoc", - ) - doc = in_doc._backend.convert() - - assert len(doc.tables) == 1 - table = doc.tables[0] - assert (table.data.num_rows, table.data.num_cols) == (3, 2) - assert [cell.text for cell in table.data.table_cells] == [ - "Option", - "Values", - "--format", - "json|yaml", - "--level", - "debug|info|warn", - ] - - -def test_source_listing_block_becomes_code_item() -> None: - # "[source,python] / ---- / ... / ----" is a listing block: it must become a - # code item carrying the declared language, not a paragraph with the block - # markers leaked into the text. - src = b"[source,python]\n----\nx = 1\ny = 2\n----\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="listing.adoc", - ) - doc = in_doc._backend.convert() - - code_items = [item for item in doc.texts if item.label == DocItemLabel.CODE] - assert len(code_items) == 1 - assert code_items[0].text == "x = 1\ny = 2" - assert code_items[0].code_language == CodeLanguageLabel.PYTHON - # no marker or attribute text may leak into the body - assert [item.text for item in doc.texts if item.label != DocItemLabel.CODE] == [] - - -def test_listing_block_without_language_stays_code() -> None: - src = b"----\nplain listing\n----\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="listing_no_lang.adoc", - ) - doc = in_doc._backend.convert() - - code_items = [item for item in doc.texts if item.label == DocItemLabel.CODE] - assert len(code_items) == 1 - assert code_items[0].text == "plain listing" - assert [item.text for item in doc.texts if item.label != DocItemLabel.CODE] == [] - - -def test_content_block_delimiters_do_not_leak() -> None: - # Example (====), sidebar (****), open (--) blocks carry regular content: - # the delimiter lines must be consumed, not echoed into the text. - src = b"====\nexample content\n====\n\n****\nsidebar text\n****\n\n--\nopen content\n--\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="content_blocks.adoc", - ) - doc = in_doc._backend.convert() - - texts = [item.text for item in doc.texts] - assert "example content" in texts - assert "sidebar text" in texts - assert "open content" in texts - for leaked in ("====", "****", "--"): - assert not any(leaked in text for text in texts) - - -def test_stray_dashes_are_not_swallowed() -> None: - # A lone "--" line without a matching closer (e.g. a changelog separator) - # must not be treated as an open-block delimiter consuming the document. - src = b"before\n--\nafter\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="stray_dashes.adoc", - ) - doc = in_doc._backend.convert() - - # the content after the stray "--" must survive; the unmatched open-block - # marker breaks the paragraph instead of rendering as text (matching - # asciidoctor, where an unclosed open block starts a separate block) - texts = [item.text for item in doc.texts] - assert texts == ["before", "after"] - - -def test_block_attributes_and_anchors_are_not_emitted_as_text() -> None: - # "[cols=...]", "[[id]]" and admonition names like "[NOTE]" are block - # metadata for whatever follows; they used to leak into the document body - # as literal text items. - src = ( - b":_mod-docs-content-type: PROCEDURE\n" - b":experimental:\n" - b"[[table-id]]\n" - b'[cols="2"]\n' - b"|===\n| a | b\n| c | d\n|===\n" - b"\n" - b"[NOTE]\nThis is a caveat.\n" - ) - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="attributes.adoc", - ) - doc = in_doc._backend.convert() - - assert len(doc.tables) == 1 - assert [item.text for item in doc.texts] == ["This is a caveat."] - - -def test_unclosed_table_at_end_keeps_caption() -> None: - src = b".End table\n|===\n|A |B" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="unclosed-table.adoc", - ) - doc = in_doc._backend.convert() - - assert len(doc.tables) == 1 - assert [caption.resolve(doc).text for caption in doc.tables[0].captions] == [ - "End table" - ] - - -def test_auto_numbered_list_keeps_items_and_following_text() -> None: - source = b"""= Installation Guide - -== Steps - -. Download the archive -. Unpack it -. Run the installer - -== Troubleshooting - -If the installer fails, check the log file. -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="ordered-list.adoc", - ) - doc = in_doc._backend.convert() - - list_items = [item for item in doc.texts if isinstance(item, ListItem)] - assert [item.text for item in list_items] == [ - "Download the archive", - "Unpack it", - "Run the installer", - ] - assert all(item.enumerated for item in list_items) - assert "If the installer fails, check the log file." in doc.export_to_markdown() - - -def test_line_starting_with_an_abbreviation_is_not_a_list_item() -> None: - source = b"""= Review - -The report was reviewed. -Dr. Smith approved it on Monday. - -Fig. 2 shows the architecture. - -a. First option -b. Second option -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="abbreviation.adoc", - ) - doc = in_doc._backend.convert() - - paragraphs = [item.text for item in doc.texts if item.label == "paragraph"] - assert paragraphs == [ - "The report was reviewed. Dr. Smith approved it on Monday.", - "Fig. 2 shows the architecture.", - ] - list_items = [item for item in doc.texts if isinstance(item, ListItem)] - assert [item.text for item in list_items] == ["First option", "Second option"] - assert all(item.enumerated for item in list_items) - - -def test_nested_list_belongs_to_the_item_above_it() -> None: - # A nested list used to be added next to the items of the outer list - # instead of under the item above it. The outer list then counted it as an - # item, so "Run the installer" was exported as "3." instead of "2.". - source = b""". Download the archive -.. Pick the file for your platform -. Run the installer -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="nested-ordered-list.adoc", - ) - doc = in_doc._backend.convert() - - download, pick, run = (item for item in doc.texts if isinstance(item, ListItem)) - nested_list = pick.parent.resolve(doc) - assert nested_list.parent == download.get_ref() - assert run.parent == download.parent - assert doc.export_to_markdown() == ( - "1. Download the archive\n" - " 1. Pick the file for your platform\n" - "2. Run the installer" - ) - - -def test_nested_bullet_list_keeps_items_nested_and_in_order() -> None: - # "**" and "***" mark nested bullet items, the same way ".." does for - # ordered lists. They used to fall through to paragraph text, which lost the - # nesting and moved that text after the rest of the list. - source = b"""* apple -* banana -** banana split -*** with cherries -** banana bread -* cherry -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="nested-list.adoc", - ) - doc = in_doc._backend.convert() - - list_items = [item for item in doc.texts if isinstance(item, ListItem)] - assert [item.text for item in list_items] == [ - "apple", - "banana", - "banana split", - "with cherries", - "banana bread", - "cherry", - ] - assert not any(item.enumerated for item in list_items) - assert doc.export_to_markdown() == ( - "- apple\n" - "- banana\n" - " - banana split\n" - " - with cherries\n" - " - banana bread\n" - "- cherry" - ) - - -def test_text_after_a_nested_list_is_not_part_of_the_list() -> None: - # Ending a list used to close only its innermost level, so the paragraph - # after a nested list was added to the outer list group, and so was the - # next list. - source = b"""* apple -** apple pie - -After the list. - -* mango -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="nested-list-end.adoc", - ) - doc = in_doc._backend.convert() - - paragraph = next(item for item in doc.texts if item.text == "After the list.") - assert paragraph.parent == doc.body.get_ref() - mango = next(item for item in doc.texts if item.text == "mango") - assert mango.parent.resolve(doc).parent == doc.body.get_ref() - assert doc.export_to_markdown() == ( - "- apple\n - apple pie\n\nAfter the list.\n\n- mango" - ) - - -def test_bullet_items_under_an_ordered_list_are_nested() -> None: - # AsciiDoc nests lists by marker, not by indentation: a "*" item after a - # "." item opens a nested bullet list, and the next "." item returns to the - # ordered list, also after a deeper "**" item. Both kinds used to share one - # list group, so "a" was exported as item "3." and "z" joined the bullets. - source = b""". one -. two - -* a -** b - -Separate paragraph. - -. x -* y -** y2 -. z -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="mixed-list.adoc", - ) - doc = in_doc._backend.convert() - - list_items = {item.text: item for item in doc.texts if isinstance(item, ListItem)} - assert [text for text, item in list_items.items() if item.enumerated] == [ - "one", - "two", - "x", - "z", - ] - assert list_items["x"].parent == list_items["z"].parent - assert list_items["y"].parent != list_items["x"].parent - assert doc.export_to_markdown() == ( - "1. one\n" - "2. two\n" - " - a\n" - " - b\n" - "\n" - "Separate paragraph.\n" - "\n" - "1. x\n" - " - y\n" - " - y2\n" - "2. z" - ) - - -def test_different_bullet_markers_are_nested() -> None: - # "*" and "-" are different list markers, so a "-" item after a "*" item - # opens a nested list, as Asciidoctor renders it. Both used to share one - # bullet list. An item returns to the closest enclosing list with its own - # marker, also across more than one level ("d" below). - source = b"""- a -* b -** c -- d - -Separate paragraph. - -. one -* two -- three -. four -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="bullet-markers.adoc", - ) - doc = in_doc._backend.convert() - - list_items = {item.text: item for item in doc.texts if isinstance(item, ListItem)} - assert list_items["a"].parent == list_items["d"].parent - assert list_items["b"].parent.resolve(doc).parent == list_items["a"].get_ref() - assert list_items["one"].parent == list_items["four"].parent - assert doc.export_to_markdown() == ( - "- a\n" - " - b\n" - " - c\n" - "- d\n" - "\n" - "Separate paragraph.\n" - "\n" - "1. one\n" - " - two\n" - " - three\n" - "2. four" - ) - - -def test_literal_block_keeps_its_content_and_following_text() -> None: - source = b"""= Guide - -== One - -Before the block. - -.... -raw literal -second line -.... - -== Two - -After the block. -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="literal-block.adoc", - ) - doc = in_doc._backend.convert() - - code_items = [item for item in doc.texts if isinstance(item, CodeItem)] - assert [item.text for item in code_items] == ["raw literal\nsecond line"] - assert "After the block." in doc.export_to_markdown() - - -def test_literal_block_attaches_caption_to_code_item() -> None: - # A block title before a literal block must be attached as a proper caption - # on the CodeItem (FloatingItem), not emitted as a standalone orphan. - source = b""".Literal example -.... -raw literal -.... - -image::next.png[] -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="captioned-literal-block.adoc", - ) - doc = in_doc._backend.convert() - - code_items = [item for item in doc.texts if isinstance(item, CodeItem)] - assert len(code_items) == 1 - assert code_items[0].text == "raw literal" - # The block title must be structurally linked as the code item's caption. - assert len(code_items[0].captions) == 1 - assert code_items[0].captions[0].resolve(doc).text == "Literal example" - - -def test_block_title_before_list_renders_as_bold_paragraph() -> None: - # A block title preceding a list (GroupItem) has no caption slot; it must - # be emitted as a bold PARAGRAPH immediately before the list. - source = b""".Steps - -. First step -. Second step -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="block-title-list.adoc", - ) - doc = in_doc._backend.convert() - - from docling_core.types.doc import Formatting - - non_list_texts = [item for item in doc.texts if not isinstance(item, ListItem)] - assert len(non_list_texts) == 1 - title_item = non_list_texts[0] - assert title_item.label == DocItemLabel.PARAGRAPH - assert title_item.text == "Steps" - assert title_item.formatting == Formatting(bold=True) - - # The block title must appear before the list items in export order. - md = doc.export_to_markdown() - assert md.index("**Steps**") < md.index("First step") - - -def test_parse_picture() -> None: - line = ( - "image::images/example1.png[Example Image, width=200, height=150, align=center]" - ) - res = AsciiDocBackend._parse_picture(line) - assert res - assert res.get("width", 0) == "200" - assert res.get("height", 0) == "150" - assert res.get("uri", "") == "images/example1.png" - - line = "image::renamed-bookmark.png[Renamed bookmark]" - res = AsciiDocBackend._parse_picture(line) - assert res - assert "width" not in res - assert "height" not in res - assert res.get("uri", "") == "renamed-bookmark.png" - - line = "image::images/screenshot.png[A screenshot showing a dialog box, containing text fields, buttons, and validation errors, width=604, height=422]" - res = AsciiDocBackend._parse_picture(line) - assert res - assert res.get("width", 0) == "604" - assert res.get("height", 0) == "422" - assert res.get("uri", "") == "images/screenshot.png" - assert ( - res.get("alt", "") - == "A screenshot showing a dialog box, containing text fields, buttons, and validation errors" - ) - - -def test_table_cell_format_specifiers() -> None: - # A header row whose cells carry alignment + style specifiers ("^.^h|") - # must still be detected as a table line and parsed into clean cells. - line = "^.^h|Field ^.^h| Description" - assert AsciiDocBackend._is_table_line(line) - assert AsciiDocBackend._parse_table_line(line) == ["Field", "Description"] - - # A column-spanning specifier ("2+^|") is dropped from the cell text. - assert AsciiDocBackend._parse_table_line("2+^|Spanned ^|Next") == [ - "Spanned", - "Next", - ] - - -def test_table_cell_content_preserved() -> None: - # Single-letter cells that coincide with style operators (s, h, m, ...) and - # words ending in one (Eth) must not be mistaken for cell specifiers. - assert AsciiDocBackend._parse_table_line("| s | Strong") == ["s", "Strong"] - assert AsciiDocBackend._parse_table_line("| eth | Eth | Ethernet") == [ - "eth", - "Eth", - "Ethernet", - ] - - -def test_empty_table_does_not_crash() -> None: - # An empty table must yield an empty grid rather than raising. - data = AsciiDocBackend._populate_table_as_grid([]) - assert data.num_rows == 0 - assert data.num_cols == 0 - - -def test_non_numeric_image_dimensions_do_not_crash() -> None: - # image width/height can be non-numeric in real AsciiDoc (e.g. "50%", "auto"). - # convert() used int(item["width"]) directly, so such an image raised - # ValueError and failed the whole document; it must fall back to the default - # size and keep converting the rest of the content. - from io import BytesIO - - adoc = ( - b"= Title\n\n" - b"Intro text.\n\n" - b"image::diagram.png[Architecture, width=50%, height=auto]\n\n" - b"Text after the image.\n" - ) - in_doc = InputDocument( - path_or_stream=BytesIO(adoc), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="dims.adoc", - ) - doc = in_doc._backend.convert() - - md = doc.export_to_markdown() - assert "Intro text." in md - assert "Text after the image." in md - - assert doc.pictures[0].image is None - - -def test_local_images_are_embedded_and_missing_images_do_not_break_export( - tmp_path: Path, -) -> None: - import pytest - - in_path = Path("tests/data/asciidoc/sources/asciidoc_03.asciidoc") - options = AsciiDocBackendOptions( - fetch_images=True, - enable_local_fetch=True, - source_uri=in_path, - ) - in_doc = InputDocument( - path_or_stream=in_path, - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - backend_options=options, - ) - with pytest.warns(UserWarning, match="Could not process an image"): - doc = in_doc._backend.convert() - - assert doc.pictures[0].image is not None - assert doc.pictures[0].image.uri.scheme == "data" - assert doc.pictures[1].image is None - assert doc.pictures[2].image is None - - output_path = tmp_path / "asciidoc_03.md" - doc.save_as_markdown(output_path, image_mode=ImageRefMode.EMBEDDED) - assert "data:image/png;base64," in output_path.read_text(encoding="utf-8") - - -def test_images_not_fetched_when_fetch_images_is_false() -> None: - """Images are not loaded when fetch_images is False (the default). - - No image data should be loaded even when the source file is on disk and - enable_local_fetch would otherwise allow it. - """ - in_path = Path("tests/data/asciidoc/sources/asciidoc_03.asciidoc") - doc = _get_backend(in_path).convert() - - assert all(pic.image is None for pic in doc.pictures) - - -def test_asciidocs_examples() -> None: - fnames = sorted(glob.glob("./tests/data/asciidoc/sources/*.asciidoc")) - - for fname in fnames: - in_path = Path(fname) - gt_path = Path("./tests/data/asciidoc/groundtruth/") / f"{in_path.name}" - - doc_backend = _get_backend(in_path) - doc = doc_backend.convert() - - pred_md = doc.export_to_markdown(compact_tables=True) - - # Verify markdown export - assert verify_export(pred_md, str(gt_path) + ".md", generate=GEN_TEST_DATA) - - -def test_utf8_bom_does_not_hide_the_document_title(tmp_path: Path) -> None: - """A leading UTF-8 BOM must not survive into the first line. - - Decoding with plain utf-8 kept it, so "= Title" started with U+FEFF, was no - longer recognized as the document title, and the BOM reached the exported - text. Both the stream and the file path are covered, since each decodes - separately. - """ - adoc_bytes = "\ufeff= Document Title\n\nSome body text.\n".encode() - - in_doc = InputDocument( - path_or_stream=BytesIO(adoc_bytes), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="bom.adoc", - ) - stream_doc = in_doc._backend.convert() - - adoc_file = tmp_path / "bom.adoc" - adoc_file.write_bytes(adoc_bytes) - file_doc = _get_backend(adoc_file).convert() - - for doc in (stream_doc, file_doc): - assert doc.texts[0].label == "title" - assert doc.texts[0].text == "Document Title" - - -def test_inline_formatting_in_paragraphs() -> None: - # Bold, italic and monospace marks used to stay in the text as literal - # "*", "_" and "`". Single marks only count at word boundaries, so - # "snake_case" and "2*3*4" stay plain, as in Asciidoctor. - source = b"""Some *bold*, _italic_ and `mono` text, with **st**rong. - -*_both_* and snake_case_name and 2*3*4. - -No marks here. -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="inline-formatting.adoc", - ) - doc = in_doc._backend.convert() - - styled = { - item.text: (item.formatting.bold, item.formatting.italic) - for item in doc.texts - if item.formatting is not None - } - assert styled == { - "bold": (True, False), - "italic": (False, True), - "st": (True, False), - "both": (True, True), - } - assert [item.text for item in doc.texts if isinstance(item, CodeItem)] == ["mono"] - plain = [item.text for item in doc.texts if item.formatting is None] - assert "and snake_case_name and 2*3*4." in plain - assert "No marks here." in plain - - -def test_inline_formatting_marks_and_spans() -> None: - source = b"""Text with __ui__, ``um`` and *a b c* marks. - -Nested **bold _and italic_** text. -""" - in_doc = InputDocument( - path_or_stream=BytesIO(source), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="inline-spans.adoc", - ) - doc = in_doc._backend.convert() - - styled = [ - (item.text, item.formatting.bold, item.formatting.italic) - for item in doc.texts - if item.formatting is not None - ] - assert styled == [ - ("ui", False, True), - ("a b c", True, False), - ("bold", True, False), - ("and italic", True, True), - ] - assert [item.text for item in doc.texts if isinstance(item, CodeItem)] == ["um"] - - -@pytest.mark.parametrize("mark", ["*", "_", "`"]) -def test_many_unclosed_inline_marks_stay_plain_text(mark: str) -> None: - # A long line of opening marks that never close must convert in linear - # time and keep the marks as plain text. - line = f" {mark}a" * 16_000 - in_doc = InputDocument( - path_or_stream=BytesIO(line.encode()), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="unclosed-marks.adoc", - ) - - start = time.monotonic() - doc = in_doc._backend.convert() - elapsed = time.monotonic() - start - - assert [item.text for item in doc.texts] == [line.strip()] - assert doc.texts[0].formatting is None - assert elapsed < 2.0 - - -def test_heading_flushes_pending_paragraph() -> None: - # text accumulated before a section header used to be appended to the - # text after the header and attributed to the wrong section - src = b"= Doc\n== S1\n=== S1.1\nbody\n== S2\nbody2\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="section-flush.adoc", - ) - doc = in_doc._backend.convert() - - paras = { - item.text: item.parent.resolve(doc).text - for item, _ in doc.iterate_items() - if item.label.value == "paragraph" - } - assert paras == {"body": "S1.1", "body2": "S2"} - - -def test_symmetric_titles_drop_their_closing_marks() -> None: - # Asciidoctor accepts titles closed by the marks that open them - # ("== Title =="); the closing marks used to end up in the heading text. - # Marks of a different length are part of the title, as in Asciidoctor. - src = b"= Doc =\n\n== Intro ==\n\ntext\n\n=== Details ==\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="symmetric.adoc", - ) - doc = in_doc._backend.convert() - - headings = [ - (item.label, item.text) - for item in doc.texts - if item.label in (DocItemLabel.TITLE, DocItemLabel.SECTION_HEADER) - ] - assert headings == [ - (DocItemLabel.TITLE, "Doc"), - (DocItemLabel.SECTION_HEADER, "Intro"), - (DocItemLabel.SECTION_HEADER, "Details =="), - ] - - -def test_block_title_without_floating_target_keeps_content() -> None: - # A block title followed by a paragraph used to absorb every following - # line as "multiline caption" and drop all of it at EOF. - src = b"para one\n\n.Procedure\nDo the thing.\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="block-title-paragraph.adoc", - ) - doc = in_doc._backend.convert() - - texts = [item.text for item, _ in doc.iterate_items()] - assert "para one" in texts - assert any("Do the thing." in t for t in texts) - - -def test_block_title_blank_line_separates_paragraphs() -> None: - src = b".Overview\n\nBody text.\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="block-title-blank.adoc", - ) - doc = in_doc._backend.convert() - - texts = [item.text for item, _ in doc.iterate_items()] - assert "Overview" in texts - assert "Body text." in texts - - -def test_text_before_block_title_is_not_merged_with_text_after() -> None: - # Plain text followed by a block title and more text on the next line: - # the pre-title paragraph must be emitted before the title, not merged - # with the paragraph that follows the blank line after the title. - src = b"text1\n.Caption\ntext2\n\ntext3\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="text-caption-text.adoc", - ) - doc = in_doc._backend.convert() - - texts = [item.text for item, _ in doc.iterate_items()] - assert texts == ["text1", "Caption", "text2", "text3"] - - -def test_block_title_is_not_merged_into_the_paragraph_it_titles() -> None: - # A block title is a single line; the paragraph after it is ordinary text. - doc = _convert_adoc( - b"Intro.\n\n.Block title\nTitled paragraph,\nsecond line.\n\nAfter.\n" - ) - - assert [(item.text, item.formatting) for item in doc.texts] == [ - ("Intro.", None), - ("Block title", Formatting(bold=True)), - ("Titled paragraph, second line.", None), - ("After.", None), - ] - - -def test_content_block_delimiters_not_leaked() -> None: - # paired example/sidebar/open/passthrough/quote/listing delimiters used - # to render as literal text around their content - src = b"====\nexample content\n====\nafter\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="example-block.adoc", - ) - doc = in_doc._backend.convert() - - texts = [item.text for item, _ in doc.iterate_items()] - assert any("example content" in t for t in texts) - assert not any("====" in t for t in texts) - assert "after" in texts - - -def test_content_block_delimiter_variants() -> None: - # "****", "____", and "++++" are content-block delimiters handled by - # _CONTENT_BLOCK_DELIMITERS; their inner lines must be preserved and the - # delimiter lines must not appear in the output. - for delim in (b"****", b"____", b"++++"): - src = delim + b"\ninner\n" + delim + b"\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="delim.adoc", - ) - doc = in_doc._backend.convert() - texts = [item.text for item, _ in doc.iterate_items()] - assert any("inner" in t for t in texts), f"{delim!r}: {texts}" - assert not any(delim.decode() in t for t in texts), f"{delim!r}: {texts}" - - -def test_line_comments_not_body_text() -> None: - # a line starting with // is a comment; it used to render as a paragraph - src = b"// a comment line\nreal para\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="comments.adoc", - ) - doc = in_doc._backend.convert() - - texts = [item.text for item, _ in doc.iterate_items()] - assert not any("comment line" in t for t in texts) - assert "real para" in texts - - -def test_comment_block_hides_content() -> None: - # //// delimited comment blocks hide everything up to the closer - src = b"////\nhidden text\n////\nvisible\n" - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="comment-block.adoc", - ) - doc = in_doc._backend.convert() - - texts = [item.text for item, _ in doc.iterate_items()] - assert not any("hidden" in t for t in texts) - assert "visible" in texts - - -def _convert_adoc( - src: bytes, options: AsciiDocBackendOptions | None = None -) -> DoclingDocument: - in_doc = InputDocument( - path_or_stream=BytesIO(src), - format=InputFormat.ASCIIDOC, - backend=AsciiDocBackend, - filename="comments.adoc", - backend_options=options, - ) - return in_doc._backend.convert() - - -def _notes(doc: DoclingDocument) -> list[str]: - return [ - item.text - for item, _ in doc.iterate_items( - included_content_layers={ContentLayer.NOTES}, - ) - ] - - -def test_comments_are_kept_in_the_notes_layer() -> None: - doc = _convert_adoc(b"// a review note\nreal para\n") - - assert _notes(doc) == ["a review note"] - assert "real para" in [item.text for item, _ in doc.iterate_items()] - - -def test_comment_run_is_one_note_with_its_lines_kept() -> None: - # consecutive // lines are one comment; the line breaks inside it survive - doc = _convert_adoc(b"// ask legal about this\n// and check the date\n\nbody\n") - - assert _notes(doc) == ["ask legal about this\nand check the date"] - - -def test_comment_block_is_kept_verbatim_in_the_notes_layer() -> None: - # the content of a //// block may be disabled markup, so it is not reflowed - doc = _convert_adoc(b"////\n* disabled item\n* another\n////\nvisible\n") - - assert _notes(doc) == ["* disabled item\n* another"] - assert "visible" in [item.text for item, _ in doc.iterate_items()] - - -def test_empty_comments_add_no_note() -> None: - doc = _convert_adoc(b"//\n//\n\nbody\n") - - assert _notes(doc) == [] - - -def test_comments_stay_out_of_default_exports() -> None: - # markdown and HTML leave the notes layer out, so #4394's output stands - doc = _convert_adoc(b"// a review note\nreal para\n") - - assert doc.export_to_markdown() == "real para" - assert "a review note" not in doc.export_to_html() - assert "a review note" in doc.export_to_markdown( - included_content_layers={ContentLayer.BODY, ContentLayer.NOTES} - ) - - -def test_capture_comments_disabled_drops_comments() -> None: - doc = _convert_adoc( - b"// a review note\nreal para\n", - AsciiDocBackendOptions(capture_comments=False), - ) - - assert _notes(doc) == [] - assert "real para" in [item.text for item, _ in doc.iterate_items()] - - -def test_comment_inside_a_content_block_is_a_note_not_body_text() -> None: - # the lines of an example/sidebar/quote/passthrough/open block used to be - # passed through unfiltered, so a comment in one leaked as body text - for delim in (b"====", b"****", b"____", b"++++", b"--"): - src = delim + b"\n// a note inside\ninner text\n" + delim + b"\n" - doc = _convert_adoc(src) - - texts = [item.text for item, _ in doc.iterate_items()] - assert _notes(doc) == ["a note inside"], f"{delim!r}" - assert "inner text" in texts, f"{delim!r}: {texts}" - assert not any("a note inside" in t for t in texts), f"{delim!r}: {texts}" - - -def test_comment_block_hides_a_verbatim_delimiter() -> None: - # a "----" between //// delimiters is comment text; it used to open a - # listing block and leak the commented-out lines as a code item - doc = _convert_adoc(b"before\n\n////\n----\nhidden code\n----\n////\n\nafter\n") - - texts = [item.text for item, _ in doc.iterate_items()] - assert texts == ["before", "after"], texts - assert _notes(doc) == ["----\nhidden code\n----"] - - -def test_comment_does_not_break_the_list_it_sits_in() -> None: - doc = _convert_adoc(b"* item one\n// a note on item one\n* item two\n") - - items = [item.text for item, _ in doc.iterate_items() if isinstance(item, ListItem)] - assert items == ["item one", "item two"] - assert _notes(doc) == ["a note on item one"] - - -def test_comment_markers_inside_a_listing_stay_code() -> None: - doc = _convert_adoc(b"----\n// not a comment, this is code\ncode()\n----\n") - - assert _notes(doc) == [] - codes = [item.text for item, _ in doc.iterate_items() if isinstance(item, CodeItem)] - assert codes == ["// not a comment, this is code\ncode()"] - - -def test_unclosed_comment_block_comments_out_the_rest() -> None: - doc = _convert_adoc(b"visible before\n\n////\nhidden rest\nmore hidden\n") - - assert [item.text for item, _ in doc.iterate_items()] == ["visible before"] - assert _notes(doc) == ["hidden rest\nmore hidden"] - - -def test_three_slashes_are_body_text_not_a_comment() -> None: - # only exactly two slashes start a line comment, as in Asciidoctor - doc = _convert_adoc(b"/// not a comment\n//// nor this\n\n// a note\n") - - assert [item.text for item, _ in doc.iterate_items()] == [ - "/// not a comment //// nor this" - ] - assert _notes(doc) == ["a note"] - - -@pytest.mark.parametrize("delimiter", ["/////", "///////"]) -def test_longer_comment_delimiter_needs_an_identical_closer(delimiter: str) -> None: - src = f"before\n\n{delimiter}\nhidden\n////\nstill hidden\n{delimiter}\n\nafter\n" - doc = _convert_adoc(src.encode()) - - assert [item.text for item, _ in doc.iterate_items()] == ["before", "after"] - assert _notes(doc) == ["hidden\n////\nstill hidden"] - - -def _convert_within(src: bytes, seconds: float = 10.0) -> DoclingDocument: - """Convert `src`, failing rather than hanging if the parser never returns. - - The work runs on a daemon thread so a parser stuck in a loop cannot hold up - the test session: the deadline turns it into a plain assertion failure and - the thread dies with the interpreter. - """ - result: list[DoclingDocument] = [] - worker = threading.Thread( - target=lambda: result.append(_convert_adoc(src)), daemon=True - ) - worker.start() - worker.join(seconds) - assert not worker.is_alive(), ( - f"convert() did not return within {seconds}s for {src!r}; " - "the parser is looping over one line" - ) - return result[0] - - -def test_unclosed_content_block_delimiter_terminates() -> None: - # A content block delimiter with no closer used to leave the line cursor in - # place, so _iter_blocks looped over that one line forever and convert() - # never returned. The delimiter is markup, so it breaks the paragraph and - # the text around it survives. - for delim in (b"====", b"****", b"____", b"++++", b"--"): - doc = _convert_within(b"before\n" + delim + b"\nafter\n") - - texts = [item.text for item, _ in doc.iterate_items()] - assert "before" in texts, f"{delim!r}: {texts}" - assert "after" in texts, f"{delim!r}: {texts}" - assert not any(delim.decode() in t for t in texts), f"{delim!r}: {texts}" +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +import glob +import threading +import time +from io import BytesIO +from pathlib import Path + +import pytest +from docling_core.types.doc import ( + CodeItem, + CodeLanguageLabel, + ContentLayer, + DocItemLabel, + DoclingDocument, + Formatting, + ImageRefMode, + ListItem, +) + +from docling.backend.abstract_backend import DeclarativeDocumentBackend +from docling.backend.asciidoc_backend import AsciiDocBackend +from docling.datamodel.backend_options import AsciiDocBackendOptions +from docling.datamodel.base_models import InputFormat +from docling.datamodel.document import InputDocument + +from .test_data_gen_flag import GEN_TEST_DATA +from .verify_utils import verify_document, verify_export + + +def _get_backend(fname: Path) -> DeclarativeDocumentBackend: + in_doc = InputDocument( + path_or_stream=fname, + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + ) + + doc_backend = in_doc._backend + assert isinstance(doc_backend, DeclarativeDocumentBackend) + return doc_backend + + +def test_list_dedent_to_base_does_not_crash() -> None: + # A list that starts indented and then dedents back to the base level used + # to raise "TypeError: '<' not supported between instances of 'int' and + # 'NoneType'": the dedent loop walked past level 0, where the base indent is + # never set. It should keep both items instead. + src = b" * a\n* b\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="dedent.asciidoc", + ) + doc = in_doc._backend.convert() + + assert [item.text for item in doc.texts] == ["a", "b"] + + +def test_rowspan_only_cell_specifier_keeps_the_row() -> None: + # AsciiDoc writes a span as [colspan][.rowspan] followed by "+" or "*", and + # either number may be omitted, so ".2+" is a rowspan on its own. The cell + # specifier pattern required a leading digit, so _is_table_line rejected the + # line, the block loop read that as the end of the table, and the row after + # it leaked into the document as literal text. + src = b"|===\n|A |B\n.2+|tall |x\n|y\n|===\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="rowspan.asciidoc", + ) + doc = in_doc._backend.convert() + + assert doc.tables, "the table was dropped entirely" + table = doc.tables[0] + # The trailing single-cell row "|y" is kept: cells flow into the grid, so it + # lands in row 3, column 1, with the second column left empty. + assert (table.data.num_rows, table.data.num_cols) == (3, 2) + assert [cell.text for cell in table.data.table_cells] == [ + "A", + "B", + "tall", + "x", + "y", + ] + assert [item.text for item in doc.texts] == [] + + +def test_incomplete_table_does_not_emit_an_empty_table() -> None: + # A trailing row with fewer cells than the table's column count is kept and + # padded with an empty cell (matching how Asciidoctor fills the grid), so no + # empty phantom table is emitted and no cell content is lost. + for row, first in ((b"|3", "3"), (b"2+|wide", "wide")): + src = b"|===\n|A |B\n" + row + b"\n|===\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="single-cell-row.adoc", + ) + doc = in_doc._backend.convert() + + assert len(doc.tables) == 1 + assert (doc.tables[0].data.num_rows, doc.tables[0].data.num_cols) == (2, 2) + assert [cell.text for cell in doc.tables[0].data.table_cells] == [ + "A", + "B", + first, + ] + + +def test_single_cell_per_line_table_is_not_truncated() -> None: + # Vertical AsciiDoc tables write one cell per line. A single-| line used to + # be read as the end of the table: the first such line was silently + # dropped, the remaining ones leaked into the document as literal text, and + # the closing |=== reopened an empty table that vanished at EOF. + src = b"|===\n|Name\n|Age\n|Alice\n|30\n|===\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="vertical-table.adoc", + ) + doc = in_doc._backend.convert() + + assert len(doc.tables) == 1 + assert (doc.tables[0].data.num_rows, doc.tables[0].data.num_cols) == (4, 1) + assert [cell.text for cell in doc.tables[0].data.table_cells] == [ + "Name", + "Age", + "Alice", + "30", + ] + assert [item.text for item in doc.texts] == [] + + +def test_blank_line_inside_table_does_not_split_it() -> None: + # Inside |=== ... |=== a blank line only separates rows: Asciidoctor's own + # examples put one after the header row. It used to end the table, so the + # header and the body came out as two tables and the second lost the + # caption. + for src in ( + b".Formats\n|===\n|Format |Extension\n\n|Markdown |.md\n|JSON |.json\n|===\n", + b".Formats\n|===\n\n|Format |Extension\n\n|Markdown |.md\n\n|JSON |.json\n|===\n", + ): + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="blank-line-table.adoc", + ) + doc = in_doc._backend.convert() + + assert len(doc.tables) == 1 + table = doc.tables[0] + assert (table.data.num_rows, table.data.num_cols) == (3, 2) + assert [cell.text for cell in table.data.table_cells] == [ + "Format", + "Extension", + "Markdown", + ".md", + "JSON", + ".json", + ] + assert table.caption_text(doc) == "Formats" + + +def test_wrapped_cell_text_stays_in_its_cell() -> None: + # A line without a leading "|" inside a table continues the cell above it. + # It used to end the table: the line was dropped and the rows after it + # became a second table. + src = ( + b"|===\n|Format |Extension\n|Markdown |.md or\n(also .markdown)\n" + b"|JSON |.json\n|===\nAfter.\n" + ) + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="wrapped-cell-table.adoc", + ) + doc = in_doc._backend.convert() + + assert len(doc.tables) == 1 + assert [cell.text for cell in doc.tables[0].data.table_cells] == [ + "Format", + "Extension", + "Markdown", + ".md or (also .markdown)", + "JSON", + ".json", + ] + assert [item.text for item in doc.texts] == ["After."] + + +def test_escaped_pipe_stays_inside_its_cell() -> None: + # A backslash-escaped pipe is cell content: Asciidoctor partitions cells on + # unescaped pipes only and drops the backslash from the output. Splitting on + # every "|" cut the cell at the escaped pipe, left the backslash in the text + # and gave the row more columns than the header. + src = ( + b"|===\n|Option |Values\n" + b"|--format |json\\|yaml\n" + b"|--level |debug\\|info\\|warn\n|===\n" + ) + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="escaped-pipe.adoc", + ) + doc = in_doc._backend.convert() + + assert len(doc.tables) == 1 + table = doc.tables[0] + assert (table.data.num_rows, table.data.num_cols) == (3, 2) + assert [cell.text for cell in table.data.table_cells] == [ + "Option", + "Values", + "--format", + "json|yaml", + "--level", + "debug|info|warn", + ] + + +def test_source_listing_block_becomes_code_item() -> None: + # "[source,python] / ---- / ... / ----" is a listing block: it must become a + # code item carrying the declared language, not a paragraph with the block + # markers leaked into the text. + src = b"[source,python]\n----\nx = 1\ny = 2\n----\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="listing.adoc", + ) + doc = in_doc._backend.convert() + + code_items = [item for item in doc.texts if item.label == DocItemLabel.CODE] + assert len(code_items) == 1 + assert code_items[0].text == "x = 1\ny = 2" + assert code_items[0].code_language == CodeLanguageLabel.PYTHON + # no marker or attribute text may leak into the body + assert [item.text for item in doc.texts if item.label != DocItemLabel.CODE] == [] + + +def test_delimiter_longer_than_four_characters_opens_the_block() -> None: + # A listing block opens on four or more hyphens and a literal block on four + # or more dots, so the marker of a valid block is not always four characters + # long. Only "----" and "...." opened a block, so the markers of a longer + # block stayed in the document as paragraphs and the code became prose. + src = b"[source,python]\n-----\nx = 1\n-----\n\n.....\nraw\n.....\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="long-delimiter.adoc", + ) + doc = in_doc._backend.convert() + + code_items = [item for item in doc.texts if item.label == DocItemLabel.CODE] + assert [item.text for item in code_items] == ["x = 1", "raw"] + assert code_items[0].code_language == CodeLanguageLabel.PYTHON + # no marker or attribute text may leak into the body + assert [item.text for item in doc.texts if item.label != DocItemLabel.CODE] == [] + + +def test_listing_block_without_language_stays_code() -> None: + src = b"----\nplain listing\n----\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="listing_no_lang.adoc", + ) + doc = in_doc._backend.convert() + + code_items = [item for item in doc.texts if item.label == DocItemLabel.CODE] + assert len(code_items) == 1 + assert code_items[0].text == "plain listing" + assert [item.text for item in doc.texts if item.label != DocItemLabel.CODE] == [] + + +def test_content_block_delimiters_do_not_leak() -> None: + # Example (====), sidebar (****), open (--) blocks carry regular content: + # the delimiter lines must be consumed, not echoed into the text. + src = b"====\nexample content\n====\n\n****\nsidebar text\n****\n\n--\nopen content\n--\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="content_blocks.adoc", + ) + doc = in_doc._backend.convert() + + texts = [item.text for item in doc.texts] + assert "example content" in texts + assert "sidebar text" in texts + assert "open content" in texts + for leaked in ("====", "****", "--"): + assert not any(leaked in text for text in texts) + + +def test_stray_dashes_are_not_swallowed() -> None: + # A lone "--" line without a matching closer (e.g. a changelog separator) + # must not be treated as an open-block delimiter consuming the document. + src = b"before\n--\nafter\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="stray_dashes.adoc", + ) + doc = in_doc._backend.convert() + + # the content after the stray "--" must survive; the unmatched open-block + # marker breaks the paragraph instead of rendering as text (matching + # asciidoctor, where an unclosed open block starts a separate block) + texts = [item.text for item in doc.texts] + assert texts == ["before", "after"] + + +def test_block_attributes_and_anchors_are_not_emitted_as_text() -> None: + # "[cols=...]", "[[id]]" and admonition names like "[NOTE]" are block + # metadata for whatever follows; they used to leak into the document body + # as literal text items. + src = ( + b":_mod-docs-content-type: PROCEDURE\n" + b":experimental:\n" + b"[[table-id]]\n" + b'[cols="2"]\n' + b"|===\n| a | b\n| c | d\n|===\n" + b"\n" + b"[NOTE]\nThis is a caveat.\n" + ) + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="attributes.adoc", + ) + doc = in_doc._backend.convert() + + assert len(doc.tables) == 1 + assert [item.text for item in doc.texts] == ["This is a caveat."] + + +def test_unclosed_table_at_end_keeps_caption() -> None: + src = b".End table\n|===\n|A |B" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="unclosed-table.adoc", + ) + doc = in_doc._backend.convert() + + assert len(doc.tables) == 1 + assert [caption.resolve(doc).text for caption in doc.tables[0].captions] == [ + "End table" + ] + + +def test_auto_numbered_list_keeps_items_and_following_text() -> None: + source = b"""= Installation Guide + +== Steps + +. Download the archive +. Unpack it +. Run the installer + +== Troubleshooting + +If the installer fails, check the log file. +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="ordered-list.adoc", + ) + doc = in_doc._backend.convert() + + list_items = [item for item in doc.texts if isinstance(item, ListItem)] + assert [item.text for item in list_items] == [ + "Download the archive", + "Unpack it", + "Run the installer", + ] + assert all(item.enumerated for item in list_items) + assert "If the installer fails, check the log file." in doc.export_to_markdown() + + +def test_line_starting_with_an_abbreviation_is_not_a_list_item() -> None: + source = b"""= Review + +The report was reviewed. +Dr. Smith approved it on Monday. + +Fig. 2 shows the architecture. + +a. First option +b. Second option +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="abbreviation.adoc", + ) + doc = in_doc._backend.convert() + + paragraphs = [item.text for item in doc.texts if item.label == "paragraph"] + assert paragraphs == [ + "The report was reviewed. Dr. Smith approved it on Monday.", + "Fig. 2 shows the architecture.", + ] + list_items = [item for item in doc.texts if isinstance(item, ListItem)] + assert [item.text for item in list_items] == ["First option", "Second option"] + assert all(item.enumerated for item in list_items) + + +def test_nested_list_belongs_to_the_item_above_it() -> None: + # A nested list used to be added next to the items of the outer list + # instead of under the item above it. The outer list then counted it as an + # item, so "Run the installer" was exported as "3." instead of "2.". + source = b""". Download the archive +.. Pick the file for your platform +. Run the installer +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="nested-ordered-list.adoc", + ) + doc = in_doc._backend.convert() + + download, pick, run = (item for item in doc.texts if isinstance(item, ListItem)) + nested_list = pick.parent.resolve(doc) + assert nested_list.parent == download.get_ref() + assert run.parent == download.parent + assert doc.export_to_markdown() == ( + "1. Download the archive\n" + " 1. Pick the file for your platform\n" + "2. Run the installer" + ) + + +def test_nested_bullet_list_keeps_items_nested_and_in_order() -> None: + # "**" and "***" mark nested bullet items, the same way ".." does for + # ordered lists. They used to fall through to paragraph text, which lost the + # nesting and moved that text after the rest of the list. + source = b"""* apple +* banana +** banana split +*** with cherries +** banana bread +* cherry +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="nested-list.adoc", + ) + doc = in_doc._backend.convert() + + list_items = [item for item in doc.texts if isinstance(item, ListItem)] + assert [item.text for item in list_items] == [ + "apple", + "banana", + "banana split", + "with cherries", + "banana bread", + "cherry", + ] + assert not any(item.enumerated for item in list_items) + assert doc.export_to_markdown() == ( + "- apple\n" + "- banana\n" + " - banana split\n" + " - with cherries\n" + " - banana bread\n" + "- cherry" + ) + + +def test_text_after_a_nested_list_is_not_part_of_the_list() -> None: + # Ending a list used to close only its innermost level, so the paragraph + # after a nested list was added to the outer list group, and so was the + # next list. + source = b"""* apple +** apple pie + +After the list. + +* mango +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="nested-list-end.adoc", + ) + doc = in_doc._backend.convert() + + paragraph = next(item for item in doc.texts if item.text == "After the list.") + assert paragraph.parent == doc.body.get_ref() + mango = next(item for item in doc.texts if item.text == "mango") + assert mango.parent.resolve(doc).parent == doc.body.get_ref() + assert doc.export_to_markdown() == ( + "- apple\n - apple pie\n\nAfter the list.\n\n- mango" + ) + + +def test_bullet_items_under_an_ordered_list_are_nested() -> None: + # AsciiDoc nests lists by marker, not by indentation: a "*" item after a + # "." item opens a nested bullet list, and the next "." item returns to the + # ordered list, also after a deeper "**" item. Both kinds used to share one + # list group, so "a" was exported as item "3." and "z" joined the bullets. + source = b""". one +. two + +* a +** b + +Separate paragraph. + +. x +* y +** y2 +. z +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="mixed-list.adoc", + ) + doc = in_doc._backend.convert() + + list_items = {item.text: item for item in doc.texts if isinstance(item, ListItem)} + assert [text for text, item in list_items.items() if item.enumerated] == [ + "one", + "two", + "x", + "z", + ] + assert list_items["x"].parent == list_items["z"].parent + assert list_items["y"].parent != list_items["x"].parent + assert doc.export_to_markdown() == ( + "1. one\n" + "2. two\n" + " - a\n" + " - b\n" + "\n" + "Separate paragraph.\n" + "\n" + "1. x\n" + " - y\n" + " - y2\n" + "2. z" + ) + + +def test_different_bullet_markers_are_nested() -> None: + # "*" and "-" are different list markers, so a "-" item after a "*" item + # opens a nested list, as Asciidoctor renders it. Both used to share one + # bullet list. An item returns to the closest enclosing list with its own + # marker, also across more than one level ("d" below). + source = b"""- a +* b +** c +- d + +Separate paragraph. + +. one +* two +- three +. four +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="bullet-markers.adoc", + ) + doc = in_doc._backend.convert() + + list_items = {item.text: item for item in doc.texts if isinstance(item, ListItem)} + assert list_items["a"].parent == list_items["d"].parent + assert list_items["b"].parent.resolve(doc).parent == list_items["a"].get_ref() + assert list_items["one"].parent == list_items["four"].parent + assert doc.export_to_markdown() == ( + "- a\n" + " - b\n" + " - c\n" + "- d\n" + "\n" + "Separate paragraph.\n" + "\n" + "1. one\n" + " - two\n" + " - three\n" + "2. four" + ) + + +def test_literal_block_keeps_its_content_and_following_text() -> None: + source = b"""= Guide + +== One + +Before the block. + +.... +raw literal +second line +.... + +== Two + +After the block. +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="literal-block.adoc", + ) + doc = in_doc._backend.convert() + + code_items = [item for item in doc.texts if isinstance(item, CodeItem)] + assert [item.text for item in code_items] == ["raw literal\nsecond line"] + assert "After the block." in doc.export_to_markdown() + + +def test_literal_block_attaches_caption_to_code_item() -> None: + # A block title before a literal block must be attached as a proper caption + # on the CodeItem (FloatingItem), not emitted as a standalone orphan. + source = b""".Literal example +.... +raw literal +.... + +image::next.png[] +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="captioned-literal-block.adoc", + ) + doc = in_doc._backend.convert() + + code_items = [item for item in doc.texts if isinstance(item, CodeItem)] + assert len(code_items) == 1 + assert code_items[0].text == "raw literal" + # The block title must be structurally linked as the code item's caption. + assert len(code_items[0].captions) == 1 + assert code_items[0].captions[0].resolve(doc).text == "Literal example" + + +def test_block_title_before_list_renders_as_bold_paragraph() -> None: + # A block title preceding a list (GroupItem) has no caption slot; it must + # be emitted as a bold PARAGRAPH immediately before the list. + source = b""".Steps + +. First step +. Second step +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="block-title-list.adoc", + ) + doc = in_doc._backend.convert() + + from docling_core.types.doc import Formatting + + non_list_texts = [item for item in doc.texts if not isinstance(item, ListItem)] + assert len(non_list_texts) == 1 + title_item = non_list_texts[0] + assert title_item.label == DocItemLabel.PARAGRAPH + assert title_item.text == "Steps" + assert title_item.formatting == Formatting(bold=True) + + # The block title must appear before the list items in export order. + md = doc.export_to_markdown() + assert md.index("**Steps**") < md.index("First step") + + +def test_parse_picture() -> None: + line = ( + "image::images/example1.png[Example Image, width=200, height=150, align=center]" + ) + res = AsciiDocBackend._parse_picture(line) + assert res + assert res.get("width", 0) == "200" + assert res.get("height", 0) == "150" + assert res.get("uri", "") == "images/example1.png" + + line = "image::renamed-bookmark.png[Renamed bookmark]" + res = AsciiDocBackend._parse_picture(line) + assert res + assert "width" not in res + assert "height" not in res + assert res.get("uri", "") == "renamed-bookmark.png" + + line = "image::images/screenshot.png[A screenshot showing a dialog box, containing text fields, buttons, and validation errors, width=604, height=422]" + res = AsciiDocBackend._parse_picture(line) + assert res + assert res.get("width", 0) == "604" + assert res.get("height", 0) == "422" + assert res.get("uri", "") == "images/screenshot.png" + assert ( + res.get("alt", "") + == "A screenshot showing a dialog box, containing text fields, buttons, and validation errors" + ) + + +def test_table_cell_format_specifiers() -> None: + # A header row whose cells carry alignment + style specifiers ("^.^h|") + # must still be detected as a table line and parsed into clean cells. + line = "^.^h|Field ^.^h| Description" + assert AsciiDocBackend._is_table_line(line) + assert AsciiDocBackend._parse_table_line(line) == ["Field", "Description"] + + # A column-spanning specifier ("2+^|") is dropped from the cell text. + assert AsciiDocBackend._parse_table_line("2+^|Spanned ^|Next") == [ + "Spanned", + "Next", + ] + + +def test_table_cell_content_preserved() -> None: + # Single-letter cells that coincide with style operators (s, h, m, ...) and + # words ending in one (Eth) must not be mistaken for cell specifiers. + assert AsciiDocBackend._parse_table_line("| s | Strong") == ["s", "Strong"] + assert AsciiDocBackend._parse_table_line("| eth | Eth | Ethernet") == [ + "eth", + "Eth", + "Ethernet", + ] + + +def test_empty_table_does_not_crash() -> None: + # An empty table must yield an empty grid rather than raising. + data = AsciiDocBackend._populate_table_as_grid([]) + assert data.num_rows == 0 + assert data.num_cols == 0 + + +def test_non_numeric_image_dimensions_do_not_crash() -> None: + # image width/height can be non-numeric in real AsciiDoc (e.g. "50%", "auto"). + # convert() used int(item["width"]) directly, so such an image raised + # ValueError and failed the whole document; it must fall back to the default + # size and keep converting the rest of the content. + from io import BytesIO + + adoc = ( + b"= Title\n\n" + b"Intro text.\n\n" + b"image::diagram.png[Architecture, width=50%, height=auto]\n\n" + b"Text after the image.\n" + ) + in_doc = InputDocument( + path_or_stream=BytesIO(adoc), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="dims.adoc", + ) + doc = in_doc._backend.convert() + + md = doc.export_to_markdown() + assert "Intro text." in md + assert "Text after the image." in md + + assert doc.pictures[0].image is None + + +def test_local_images_are_embedded_and_missing_images_do_not_break_export( + tmp_path: Path, +) -> None: + import pytest + + in_path = Path("tests/data/asciidoc/sources/asciidoc_03.asciidoc") + options = AsciiDocBackendOptions( + fetch_images=True, + enable_local_fetch=True, + source_uri=in_path, + ) + in_doc = InputDocument( + path_or_stream=in_path, + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + backend_options=options, + ) + with pytest.warns(UserWarning, match="Could not process an image"): + doc = in_doc._backend.convert() + + assert doc.pictures[0].image is not None + assert doc.pictures[0].image.uri.scheme == "data" + assert doc.pictures[1].image is None + assert doc.pictures[2].image is None + + output_path = tmp_path / "asciidoc_03.md" + doc.save_as_markdown(output_path, image_mode=ImageRefMode.EMBEDDED) + assert "data:image/png;base64," in output_path.read_text(encoding="utf-8") + + +def test_images_not_fetched_when_fetch_images_is_false() -> None: + """Images are not loaded when fetch_images is False (the default). + + No image data should be loaded even when the source file is on disk and + enable_local_fetch would otherwise allow it. + """ + in_path = Path("tests/data/asciidoc/sources/asciidoc_03.asciidoc") + doc = _get_backend(in_path).convert() + + assert all(pic.image is None for pic in doc.pictures) + + +def test_asciidocs_examples() -> None: + fnames = sorted(glob.glob("./tests/data/asciidoc/sources/*.asciidoc")) + + for fname in fnames: + in_path = Path(fname) + gt_path = Path("./tests/data/asciidoc/groundtruth/") / f"{in_path.name}" + + doc_backend = _get_backend(in_path) + doc = doc_backend.convert() + + pred_md = doc.export_to_markdown(compact_tables=True) + + # Verify markdown export + assert verify_export(pred_md, str(gt_path) + ".md", generate=GEN_TEST_DATA) + + +def test_utf8_bom_does_not_hide_the_document_title(tmp_path: Path) -> None: + """A leading UTF-8 BOM must not survive into the first line. + + Decoding with plain utf-8 kept it, so "= Title" started with U+FEFF, was no + longer recognized as the document title, and the BOM reached the exported + text. Both the stream and the file path are covered, since each decodes + separately. + """ + adoc_bytes = "\ufeff= Document Title\n\nSome body text.\n".encode() + + in_doc = InputDocument( + path_or_stream=BytesIO(adoc_bytes), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="bom.adoc", + ) + stream_doc = in_doc._backend.convert() + + adoc_file = tmp_path / "bom.adoc" + adoc_file.write_bytes(adoc_bytes) + file_doc = _get_backend(adoc_file).convert() + + for doc in (stream_doc, file_doc): + assert doc.texts[0].label == "title" + assert doc.texts[0].text == "Document Title" + + +def test_inline_formatting_in_paragraphs() -> None: + # Bold, italic and monospace marks used to stay in the text as literal + # "*", "_" and "`". Single marks only count at word boundaries, so + # "snake_case" and "2*3*4" stay plain, as in Asciidoctor. + source = b"""Some *bold*, _italic_ and `mono` text, with **st**rong. + +*_both_* and snake_case_name and 2*3*4. + +No marks here. +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="inline-formatting.adoc", + ) + doc = in_doc._backend.convert() + + styled = { + item.text: (item.formatting.bold, item.formatting.italic) + for item in doc.texts + if item.formatting is not None + } + assert styled == { + "bold": (True, False), + "italic": (False, True), + "st": (True, False), + "both": (True, True), + } + assert [item.text for item in doc.texts if isinstance(item, CodeItem)] == ["mono"] + plain = [item.text for item in doc.texts if item.formatting is None] + assert "and snake_case_name and 2*3*4." in plain + assert "No marks here." in plain + + +def test_inline_formatting_marks_and_spans() -> None: + source = b"""Text with __ui__, ``um`` and *a b c* marks. + +Nested **bold _and italic_** text. +""" + in_doc = InputDocument( + path_or_stream=BytesIO(source), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="inline-spans.adoc", + ) + doc = in_doc._backend.convert() + + styled = [ + (item.text, item.formatting.bold, item.formatting.italic) + for item in doc.texts + if item.formatting is not None + ] + assert styled == [ + ("ui", False, True), + ("a b c", True, False), + ("bold", True, False), + ("and italic", True, True), + ] + assert [item.text for item in doc.texts if isinstance(item, CodeItem)] == ["um"] + + +@pytest.mark.parametrize("mark", ["*", "_", "`"]) +def test_many_unclosed_inline_marks_stay_plain_text(mark: str) -> None: + # A long line of opening marks that never close must convert in linear + # time and keep the marks as plain text. + line = f" {mark}a" * 16_000 + in_doc = InputDocument( + path_or_stream=BytesIO(line.encode()), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="unclosed-marks.adoc", + ) + + start = time.monotonic() + doc = in_doc._backend.convert() + elapsed = time.monotonic() - start + + assert [item.text for item in doc.texts] == [line.strip()] + assert doc.texts[0].formatting is None + assert elapsed < 2.0 + + +def test_heading_flushes_pending_paragraph() -> None: + # text accumulated before a section header used to be appended to the + # text after the header and attributed to the wrong section + src = b"= Doc\n== S1\n=== S1.1\nbody\n== S2\nbody2\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="section-flush.adoc", + ) + doc = in_doc._backend.convert() + + paras = { + item.text: item.parent.resolve(doc).text + for item, _ in doc.iterate_items() + if item.label.value == "paragraph" + } + assert paras == {"body": "S1.1", "body2": "S2"} + + +def test_symmetric_titles_drop_their_closing_marks() -> None: + # Asciidoctor accepts titles closed by the marks that open them + # ("== Title =="); the closing marks used to end up in the heading text. + # Marks of a different length are part of the title, as in Asciidoctor. + src = b"= Doc =\n\n== Intro ==\n\ntext\n\n=== Details ==\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="symmetric.adoc", + ) + doc = in_doc._backend.convert() + + headings = [ + (item.label, item.text) + for item in doc.texts + if item.label in (DocItemLabel.TITLE, DocItemLabel.SECTION_HEADER) + ] + assert headings == [ + (DocItemLabel.TITLE, "Doc"), + (DocItemLabel.SECTION_HEADER, "Intro"), + (DocItemLabel.SECTION_HEADER, "Details =="), + ] + + +def test_block_title_without_floating_target_keeps_content() -> None: + # A block title followed by a paragraph used to absorb every following + # line as "multiline caption" and drop all of it at EOF. + src = b"para one\n\n.Procedure\nDo the thing.\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="block-title-paragraph.adoc", + ) + doc = in_doc._backend.convert() + + texts = [item.text for item, _ in doc.iterate_items()] + assert "para one" in texts + assert any("Do the thing." in t for t in texts) + + +def test_block_title_blank_line_separates_paragraphs() -> None: + src = b".Overview\n\nBody text.\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="block-title-blank.adoc", + ) + doc = in_doc._backend.convert() + + texts = [item.text for item, _ in doc.iterate_items()] + assert "Overview" in texts + assert "Body text." in texts + + +def test_text_before_block_title_is_not_merged_with_text_after() -> None: + # Plain text followed by a block title and more text on the next line: + # the pre-title paragraph must be emitted before the title, not merged + # with the paragraph that follows the blank line after the title. + src = b"text1\n.Caption\ntext2\n\ntext3\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="text-caption-text.adoc", + ) + doc = in_doc._backend.convert() + + texts = [item.text for item, _ in doc.iterate_items()] + assert texts == ["text1", "Caption", "text2", "text3"] + + +def test_block_title_is_not_merged_into_the_paragraph_it_titles() -> None: + # A block title is a single line; the paragraph after it is ordinary text. + doc = _convert_adoc( + b"Intro.\n\n.Block title\nTitled paragraph,\nsecond line.\n\nAfter.\n" + ) + + assert [(item.text, item.formatting) for item in doc.texts] == [ + ("Intro.", None), + ("Block title", Formatting(bold=True)), + ("Titled paragraph, second line.", None), + ("After.", None), + ] + + +def test_content_block_delimiters_not_leaked() -> None: + # paired example/sidebar/open/passthrough/quote/listing delimiters used + # to render as literal text around their content + src = b"====\nexample content\n====\nafter\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="example-block.adoc", + ) + doc = in_doc._backend.convert() + + texts = [item.text for item, _ in doc.iterate_items()] + assert any("example content" in t for t in texts) + assert not any("====" in t for t in texts) + assert "after" in texts + + +def test_content_block_delimiter_variants() -> None: + # "****", "____", and "++++" are content-block delimiters handled by + # _CONTENT_BLOCK_DELIMITERS; their inner lines must be preserved and the + # delimiter lines must not appear in the output. + for delim in (b"****", b"____", b"++++"): + src = delim + b"\ninner\n" + delim + b"\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="delim.adoc", + ) + doc = in_doc._backend.convert() + texts = [item.text for item, _ in doc.iterate_items()] + assert any("inner" in t for t in texts), f"{delim!r}: {texts}" + assert not any(delim.decode() in t for t in texts), f"{delim!r}: {texts}" + + +def test_line_comments_not_body_text() -> None: + # a line starting with // is a comment; it used to render as a paragraph + src = b"// a comment line\nreal para\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="comments.adoc", + ) + doc = in_doc._backend.convert() + + texts = [item.text for item, _ in doc.iterate_items()] + assert not any("comment line" in t for t in texts) + assert "real para" in texts + + +def test_comment_block_hides_content() -> None: + # //// delimited comment blocks hide everything up to the closer + src = b"////\nhidden text\n////\nvisible\n" + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="comment-block.adoc", + ) + doc = in_doc._backend.convert() + + texts = [item.text for item, _ in doc.iterate_items()] + assert not any("hidden" in t for t in texts) + assert "visible" in texts + + +def _convert_adoc( + src: bytes, options: AsciiDocBackendOptions | None = None +) -> DoclingDocument: + in_doc = InputDocument( + path_or_stream=BytesIO(src), + format=InputFormat.ASCIIDOC, + backend=AsciiDocBackend, + filename="comments.adoc", + backend_options=options, + ) + return in_doc._backend.convert() + + +def _notes(doc: DoclingDocument) -> list[str]: + return [ + item.text + for item, _ in doc.iterate_items( + included_content_layers={ContentLayer.NOTES}, + ) + ] + + +def test_comments_are_kept_in_the_notes_layer() -> None: + doc = _convert_adoc(b"// a review note\nreal para\n") + + assert _notes(doc) == ["a review note"] + assert "real para" in [item.text for item, _ in doc.iterate_items()] + + +def test_comment_run_is_one_note_with_its_lines_kept() -> None: + # consecutive // lines are one comment; the line breaks inside it survive + doc = _convert_adoc(b"// ask legal about this\n// and check the date\n\nbody\n") + + assert _notes(doc) == ["ask legal about this\nand check the date"] + + +def test_comment_block_is_kept_verbatim_in_the_notes_layer() -> None: + # the content of a //// block may be disabled markup, so it is not reflowed + doc = _convert_adoc(b"////\n* disabled item\n* another\n////\nvisible\n") + + assert _notes(doc) == ["* disabled item\n* another"] + assert "visible" in [item.text for item, _ in doc.iterate_items()] + + +def test_empty_comments_add_no_note() -> None: + doc = _convert_adoc(b"//\n//\n\nbody\n") + + assert _notes(doc) == [] + + +def test_comments_stay_out_of_default_exports() -> None: + # markdown and HTML leave the notes layer out, so #4394's output stands + doc = _convert_adoc(b"// a review note\nreal para\n") + + assert doc.export_to_markdown() == "real para" + assert "a review note" not in doc.export_to_html() + assert "a review note" in doc.export_to_markdown( + included_content_layers={ContentLayer.BODY, ContentLayer.NOTES} + ) + + +def test_capture_comments_disabled_drops_comments() -> None: + doc = _convert_adoc( + b"// a review note\nreal para\n", + AsciiDocBackendOptions(capture_comments=False), + ) + + assert _notes(doc) == [] + assert "real para" in [item.text for item, _ in doc.iterate_items()] + + +def test_comment_inside_a_content_block_is_a_note_not_body_text() -> None: + # the lines of an example/sidebar/quote/passthrough/open block used to be + # passed through unfiltered, so a comment in one leaked as body text + for delim in (b"====", b"****", b"____", b"++++", b"--"): + src = delim + b"\n// a note inside\ninner text\n" + delim + b"\n" + doc = _convert_adoc(src) + + texts = [item.text for item, _ in doc.iterate_items()] + assert _notes(doc) == ["a note inside"], f"{delim!r}" + assert "inner text" in texts, f"{delim!r}: {texts}" + assert not any("a note inside" in t for t in texts), f"{delim!r}: {texts}" + + +def test_comment_block_hides_a_verbatim_delimiter() -> None: + # a "----" between //// delimiters is comment text; it used to open a + # listing block and leak the commented-out lines as a code item + doc = _convert_adoc(b"before\n\n////\n----\nhidden code\n----\n////\n\nafter\n") + + texts = [item.text for item, _ in doc.iterate_items()] + assert texts == ["before", "after"], texts + assert _notes(doc) == ["----\nhidden code\n----"] + + +def test_comment_does_not_break_the_list_it_sits_in() -> None: + doc = _convert_adoc(b"* item one\n// a note on item one\n* item two\n") + + items = [item.text for item, _ in doc.iterate_items() if isinstance(item, ListItem)] + assert items == ["item one", "item two"] + assert _notes(doc) == ["a note on item one"] + + +def test_comment_markers_inside_a_listing_stay_code() -> None: + doc = _convert_adoc(b"----\n// not a comment, this is code\ncode()\n----\n") + + assert _notes(doc) == [] + codes = [item.text for item, _ in doc.iterate_items() if isinstance(item, CodeItem)] + assert codes == ["// not a comment, this is code\ncode()"] + + +def test_unclosed_comment_block_comments_out_the_rest() -> None: + doc = _convert_adoc(b"visible before\n\n////\nhidden rest\nmore hidden\n") + + assert [item.text for item, _ in doc.iterate_items()] == ["visible before"] + assert _notes(doc) == ["hidden rest\nmore hidden"] + + +def test_three_slashes_are_body_text_not_a_comment() -> None: + # only exactly two slashes start a line comment, as in Asciidoctor + doc = _convert_adoc(b"/// not a comment\n//// nor this\n\n// a note\n") + + assert [item.text for item, _ in doc.iterate_items()] == [ + "/// not a comment //// nor this" + ] + assert _notes(doc) == ["a note"] + + +@pytest.mark.parametrize("delimiter", ["/////", "///////"]) +def test_longer_comment_delimiter_needs_an_identical_closer(delimiter: str) -> None: + src = f"before\n\n{delimiter}\nhidden\n////\nstill hidden\n{delimiter}\n\nafter\n" + doc = _convert_adoc(src.encode()) + + assert [item.text for item, _ in doc.iterate_items()] == ["before", "after"] + assert _notes(doc) == ["hidden\n////\nstill hidden"] + + +def _convert_within(src: bytes, seconds: float = 10.0) -> DoclingDocument: + """Convert `src`, failing rather than hanging if the parser never returns. + + The work runs on a daemon thread so a parser stuck in a loop cannot hold up + the test session: the deadline turns it into a plain assertion failure and + the thread dies with the interpreter. + """ + result: list[DoclingDocument] = [] + worker = threading.Thread( + target=lambda: result.append(_convert_adoc(src)), daemon=True + ) + worker.start() + worker.join(seconds) + assert not worker.is_alive(), ( + f"convert() did not return within {seconds}s for {src!r}; " + "the parser is looping over one line" + ) + return result[0] + + +def test_unclosed_content_block_delimiter_terminates() -> None: + # A content block delimiter with no closer used to leave the line cursor in + # place, so _iter_blocks looped over that one line forever and convert() + # never returned. The delimiter is markup, so it breaks the paragraph and + # the text around it survives. + for delim in (b"====", b"****", b"____", b"++++", b"--"): + doc = _convert_within(b"before\n" + delim + b"\nafter\n") + + texts = [item.text for item, _ in doc.iterate_items()] + assert "before" in texts, f"{delim!r}: {texts}" + assert "after" in texts, f"{delim!r}: {texts}" + assert not any(delim.decode() in t for t in texts), f"{delim!r}: {texts}"