diff --git a/docling/models/postprocessing/list_marker_processor.py b/docling/models/postprocessing/list_marker_processor.py index 14302005c3..23b169f755 100644 --- a/docling/models/postprocessing/list_marker_processor.py +++ b/docling/models/postprocessing/list_marker_processor.py @@ -81,6 +81,11 @@ def __init__(self, infer_enumerated: bool = True): re.compile(f"^({pattern})" + r"\s(.+)", re.DOTALL) for pattern in self._bullet_patterns ] + # Word's ASCII circle is ambiguous in prose; only strip it from list items. + self._compiled_list_bullet_item_patterns = [ + *self._compiled_bullet_item_patterns, + re.compile(r"^(o)\s(.+)", re.DOTALL), + ] self._compiled_numbered_item_patterns = [ re.compile(f"^({pattern})" + r"\s(.+)", re.DOTALL) for pattern in self._numbered_patterns @@ -106,6 +111,17 @@ def _is_numbered_item(self, text: str) -> bool: pattern.match(text) for pattern in self._compiled_numbered_item_patterns ) + def has_same_numbered_marker_style(self, first: str, second: str) -> bool: + """Check whether two different numbered markers use the same format.""" + first_match = self._find_match(first, self._compiled_numbered_item_patterns) + second_match = self._find_match(second, self._compiled_numbered_item_patterns) + return ( + first_match is not None + and second_match is not None + and first_match.re is second_match.re + and first_match[1] != second_match[1] + ) + @classmethod def _create_list_item( cls, @@ -192,7 +208,7 @@ def process_list_item(self, item: ListItem) -> ListItem: is_enumerated: bool = False if mtch := self._find_match( - text=item.orig, patterns=self._compiled_bullet_item_patterns + text=item.orig, patterns=self._compiled_list_bullet_item_patterns ): is_enumerated = False elif mtch := self._find_match( diff --git a/docling/models/stages/reading_order/readingorder_model.py b/docling/models/stages/reading_order/readingorder_model.py index 51479b1631..5ade243802 100644 --- a/docling/models/stages/reading_order/readingorder_model.py +++ b/docling/models/stages/reading_order/readingorder_model.py @@ -733,6 +733,9 @@ def materialize_siblings( isinstance(parent, GroupItem) and parent.label == GroupLabel.FORM_AREA ) or isinstance(parent, FieldRegionItem) current_list: GroupItem | None = None + list_stack: list[tuple[float, GroupItem, float]] = [] + previous_list_element: ReadingOrderPageElement | None = None + indent_tolerance = 5.0 siblings = [ rel for rel in ordered_siblings[parent_ref] @@ -746,11 +749,64 @@ def materialize_siblings( rel = entry element = id_to_elem[rel.ref.cref] page_height = page_no_to_pages[element.page_no].size.height # type: ignore - if split_lists_on_non_list and not ( + is_list_item = ( isinstance(element, TextElement) and element.label == DocItemLabel.LIST_ITEM - ): + ) + line_height = element.cluster.bbox.height + if is_list_item: + line_height = min( + (cell.rect.height for cell in element.cluster.cells), + default=line_height, + ) + # A move right and back up is a new column, not a sublist. + if current_list is None or ( + previous_list_element is not None + and rel.page_no == previous_list_element.page_no + and rel.l > previous_list_element.l + indent_tolerance + and rel.t >= previous_list_element.t - indent_tolerance + ): + current_list = None + list_stack.clear() + left = element.cluster.bbox.l + # Small layout-box differences do not make a new list level. + while list_stack and left < list_stack[-1][0] - indent_tolerance: + list_stack.pop() + if list_stack: + current_list = list_stack[-1][1] + first_item = current_list.children[0].resolve(out_doc) + # A leading quotation mark can hang left of the list marker. + if ( + isinstance(first_item, TextItem) + and first_item.orig.startswith( + ("«", "\u201c", "\u2018", '"') + ) + and first_item.prov + and list_stack[-1][0] == first_item.prov[0].bbox.l + and 0 < left - list_stack[-1][0] <= list_stack[-1][2] + and self.list_item_processor.has_same_numbered_marker_style( + first_item.orig[1:], element.text or "" + ) + ): + list_stack[-1] = (left, current_list, list_stack[-1][2]) + if left > list_stack[-1][0] + indent_tolerance: + current_list = out_doc.add_group( + label=GroupLabel.LIST, + name="list", + parent=current_list.children[-1].resolve(out_doc), + ) + list_stack.append((left, current_list, line_height)) + else: + current_list = None + elif ( + split_lists_on_non_list or isinstance(element, FigureElement) + ) and element.label not in { + DocItemLabel.PAGE_HEADER, + DocItemLabel.PAGE_FOOTER, + DocItemLabel.FOOTNOTE, + }: current_list = None + list_stack.clear() if isinstance(element, TextElement): if element.label == DocItemLabel.CODE: @@ -773,6 +829,12 @@ def materialize_siblings( parent=parent, page_height=page_height, ) + if is_list_item and not list_stack and current_list is not None: + list_stack.append( + (element.cluster.bbox.l, current_list, line_height) + ) + if is_list_item: + previous_list_element = rel if element.field_item is None: for merged_cid in el_merges_mapping.get(rel.cid, []): @@ -1034,7 +1096,12 @@ def _handle_text_element( parent=parent, ) else: - current_list = None + if label not in [ + DocItemLabel.PAGE_HEADER, + DocItemLabel.PAGE_FOOTER, + DocItemLabel.FOOTNOTE, + ]: + current_list = None content_layer = ContentLayer.BODY if element.label in [DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER]: diff --git a/docs/concepts/docling_document.md b/docs/concepts/docling_document.md index 4b20b7f7c4..bcbdacccd9 100644 --- a/docs/concepts/docling_document.md +++ b/docs/concepts/docling_document.md @@ -53,6 +53,15 @@ Below example shows how all items under the heading "Let's swim" (`#/texts/5`) a "Let's swim" are both text items and groups, which contain the list elements. The group items are stored in the top-level `groups` field. +For PDFs, list nesting is inferred from horizontal indentation. A nested list group +is a child of the preceding item at the outer level, and an outdent returns to an outer list. +Page headers, footers, and footnotes do not interrupt a list that continues on the next page. +An opening quotation mark that hangs left of a numbered list does not add a list level. + +Recognised list markers are stored in `ListItem.marker` and removed from `ListItem.text`; +`ListItem.orig` retains the source text. Word's `o` circle marker is recognised on +items labelled as list items, but does not turn ordinary prose into a list. + ![doc_hierarchy_2](../assets/docling_doc_hierarchy_2.png) +- c. Structure predicted by TableFormer: + Figure 1: Picture of a table with subtle, complex features such as (1) multi-column headers, (2) cell with multi-row text and (3) cells with no content. Image from PubTabNet evaluation set, filename: 'PMC2944238 004 02'. | 0 | 1 2 | 1 2 | 1 | 1 | @@ -249,6 +250,7 @@ end object detection with transformers. In Andrea Vedaldi, Horst Bischof, Thomas - [13] Thotreingam Kasar, Philippine Barlas, Sebastien Adam, Cl´ement Chatelain, and Thierry Paquet. Learning to detect tables in scanned document images using line information. In 2013 12th International Conference on Document Analysis and Recognition, pages 1185-1189. IEEE, 2013. 2 - [14] Pratik Kayal, Mrinal Anand, Harsh Desai, and Mayank Singh. Icdar 2021 competition on scientific table image recognition to latex, 2021. 2 - [15] Harold WKuhn. The hungarian method for the assignment problem. Naval research logistics quarterly, 2(1-2):83-97, 1955. 6 + - [16] Girish Kulkarni, Visruth Premraj, Vicente Ordonez, Sagnik Dhar, Siming Li, Yejin Choi, Alexander C. Berg, and Tamara L. Berg. Babytalk: Understanding and generating simple image descriptions. IEEE Transactions on Pattern Analysis and Machine Intelligence, 35(12):2891-2903, 2013. 4 - [17] Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou, and Zhoujun Li. Tablebank: A benchmark dataset for table detection and recognition, 2019. 2, 3 - [18] Yiren Li, Zheng Huang, Junchi Yan, Yi Zhou, Fan Ye, and Xianhui Liu. Gfte: Graph-based financial table extraction. In Alberto Del Bimbo, Rita Cucchiara, Stan Sclaroff, Giovanni Maria Farinella, Tao Mei, Marco Bertini, Hugo Jair Escalante, and Roberto Vezzani, editors, Pattern Recognition. ICPR International Workshops and Challenges, pages 644-658, Cham, 2021. Springer International Publishing. 2, 3 diff --git a/tests/data/pdf/groundtruth/2206.01062.json b/tests/data/pdf/groundtruth/2206.01062.json index 729d9c0695..02ca6db6ad 100644 --- a/tests/data/pdf/groundtruth/2206.01062.json +++ b/tests/data/pdf/groundtruth/2206.01062.json @@ -98,6 +98,9 @@ { "$ref": "#/groups/1" }, + { + "$ref": "#/groups/2" + }, { "$ref": "#/texts/309" }, @@ -186,7 +189,7 @@ "$ref": "#/texts/349" }, { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, { "$ref": "#/texts/356" @@ -300,7 +303,7 @@ "$ref": "#/texts/420" }, { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, { "$ref": "#/texts/434" @@ -312,7 +315,10 @@ "$ref": "#/texts/452" }, { - "$ref": "#/groups/4" + "$ref": "#/groups/5" + }, + { + "$ref": "#/groups/6" } ], "content_layer": "body", @@ -351,7 +357,18 @@ "children": [ { "$ref": "#/texts/307" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/2", + "parent": { + "$ref": "#/body" + }, + "children": [ { "$ref": "#/texts/308" } @@ -361,7 +378,7 @@ "label": "list" }, { - "self_ref": "#/groups/2", + "self_ref": "#/groups/3", "parent": { "$ref": "#/body" }, @@ -390,7 +407,7 @@ "label": "list" }, { - "self_ref": "#/groups/3", + "self_ref": "#/groups/4", "parent": { "$ref": "#/body" }, @@ -440,7 +457,7 @@ "label": "list" }, { - "self_ref": "#/groups/4", + "self_ref": "#/groups/5", "parent": { "$ref": "#/body" }, @@ -462,7 +479,18 @@ }, { "$ref": "#/texts/458" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/6", + "parent": { + "$ref": "#/body" + }, + "children": [ { "$ref": "#/texts/459" }, @@ -8861,7 +8889,7 @@ { "self_ref": "#/texts/308", "parent": { - "$ref": "#/groups/1" + "$ref": "#/groups/2" }, "children": [], "content_layer": "body", @@ -10057,7 +10085,7 @@ { "self_ref": "#/texts/350", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10086,7 +10114,7 @@ { "self_ref": "#/texts/351", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10115,7 +10143,7 @@ { "self_ref": "#/texts/352", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10144,7 +10172,7 @@ { "self_ref": "#/texts/353", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10173,7 +10201,7 @@ { "self_ref": "#/texts/354", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10202,7 +10230,7 @@ { "self_ref": "#/texts/355", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -12051,7 +12079,7 @@ { "self_ref": "#/texts/421", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12080,7 +12108,7 @@ { "self_ref": "#/texts/422", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12109,7 +12137,7 @@ { "self_ref": "#/texts/423", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12138,7 +12166,7 @@ { "self_ref": "#/texts/424", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12167,7 +12195,7 @@ { "self_ref": "#/texts/425", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12196,7 +12224,7 @@ { "self_ref": "#/texts/426", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12225,7 +12253,7 @@ { "self_ref": "#/texts/427", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12254,7 +12282,7 @@ { "self_ref": "#/texts/428", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12283,7 +12311,7 @@ { "self_ref": "#/texts/429", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12312,7 +12340,7 @@ { "self_ref": "#/texts/430", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12341,7 +12369,7 @@ { "self_ref": "#/texts/431", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12370,7 +12398,7 @@ { "self_ref": "#/texts/432", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12399,7 +12427,7 @@ { "self_ref": "#/texts/433", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -12941,7 +12969,7 @@ { "self_ref": "#/texts/453", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -12970,7 +12998,7 @@ { "self_ref": "#/texts/454", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -12999,7 +13027,7 @@ { "self_ref": "#/texts/455", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -13028,7 +13056,7 @@ { "self_ref": "#/texts/456", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -13057,7 +13085,7 @@ { "self_ref": "#/texts/457", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -13086,7 +13114,7 @@ { "self_ref": "#/texts/458", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -13115,7 +13143,7 @@ { "self_ref": "#/texts/459", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -13144,7 +13172,7 @@ { "self_ref": "#/texts/460", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -13173,7 +13201,7 @@ { "self_ref": "#/texts/461", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -13202,7 +13230,7 @@ { "self_ref": "#/texts/462", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", diff --git a/tests/data/pdf/groundtruth/2206.01062.md b/tests/data/pdf/groundtruth/2206.01062.md index 84ceda6dd2..79d7b0a9e2 100644 --- a/tests/data/pdf/groundtruth/2206.01062.md +++ b/tests/data/pdf/groundtruth/2206.01062.md @@ -50,6 +50,7 @@ In this paper, we present the DocLayNet dataset. It provides pageby-page layout [1 https://developer.ibm.com/exchanges/data/all/doclaynet](https://developer.ibm.com/exchanges/data/all/doclaynet) - This enables experimentation with annotation uncertainty and quality control analysis. + - (5) Pre-defined Train-, Test- & Validation-set: Like DocBank, we provide fixed train-, test- & validation-sets to ensure proportional representation of the class-labels. Further, we prevent leakage of unique layouts across sets, which has a large effect on model accuracy scores. All aspects outlined above are detailed in Section 3. In Section 4, we will elaborate on how we designed and executed this large-scale human annotation campaign. We will also share key insights and lessons learned that might prove helpful for other parties planning to set up annotation campaigns. @@ -285,6 +286,7 @@ Diaconu, Mai Thanh Minh, Marc, albinxavi, fatih, oleg, and wanghao yang. ultraly - [17] Yuxin Wu, Alexander Kirillov, Francisco Massa, Wan-Yen Lo, and Ross Girshick. Detectron2, 2019. - [18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 1513715145, feb 2021. - [19] Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, and Ming Zhou. Layoutlm: Pre-training of text and layout for document image understanding. In Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD, pages 1192-1200, New York, USA, 2020. Association for Computing Machinery. + - [20] Shoubin Li, Xuyan Ma, Shuaiqun Pan, Jun Hu, Lin Shi, and Qing Wang. Vtlayout: Fusion of visual and text features for document layout analysis, 2021. - [21] Peng Zhang, Can Li, Liang Qiao, Zhanzhan Cheng, Shiliang Pu, Yi Niu, and Fei Wu. Vsr: A unified framework for document layout analysis combining vision, semantics and relations, 2021. - [22] Peter W J Staar, Michele Dolfi, Christoph Auer, and Costas Bekas. Corpus conversion service: A machine learning platform to ingest documents at scale. In Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD, pages 774-782. ACM, 2018. diff --git a/tests/data/pdf/groundtruth/2305.03393v1.json b/tests/data/pdf/groundtruth/2305.03393v1.json index 32bf80c493..7983b94f4a 100644 --- a/tests/data/pdf/groundtruth/2305.03393v1.json +++ b/tests/data/pdf/groundtruth/2305.03393v1.json @@ -278,14 +278,8 @@ { "$ref": "#/texts/342" }, - { - "$ref": "#/groups/4" - }, { "$ref": "#/texts/356" - }, - { - "$ref": "#/groups/5" } ], "content_layer": "body", @@ -376,18 +370,7 @@ }, { "$ref": "#/texts/340" - } - ], - "content_layer": "body", - "name": "list", - "label": "list" - }, - { - "self_ref": "#/groups/4", - "parent": { - "$ref": "#/body" - }, - "children": [ + }, { "$ref": "#/texts/343" }, @@ -426,18 +409,7 @@ }, { "$ref": "#/texts/355" - } - ], - "content_layer": "body", - "name": "list", - "label": "list" - }, - { - "self_ref": "#/groups/5", - "parent": { - "$ref": "#/body" - }, - "children": [ + }, { "$ref": "#/texts/357" }, @@ -9841,7 +9813,7 @@ { "self_ref": "#/texts/343", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -9870,7 +9842,7 @@ { "self_ref": "#/texts/344", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -9899,7 +9871,7 @@ { "self_ref": "#/texts/345", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -9928,7 +9900,7 @@ { "self_ref": "#/texts/346", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -9957,7 +9929,7 @@ { "self_ref": "#/texts/347", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -9986,7 +9958,7 @@ { "self_ref": "#/texts/348", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10015,7 +9987,7 @@ { "self_ref": "#/texts/349", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10044,7 +10016,7 @@ { "self_ref": "#/texts/350", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10073,7 +10045,7 @@ { "self_ref": "#/texts/351", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10102,7 +10074,7 @@ { "self_ref": "#/texts/352", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10131,7 +10103,7 @@ { "self_ref": "#/texts/353", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10160,7 +10132,7 @@ { "self_ref": "#/texts/354", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10189,7 +10161,7 @@ { "self_ref": "#/texts/355", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10245,7 +10217,7 @@ { "self_ref": "#/texts/357", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10274,7 +10246,7 @@ { "self_ref": "#/texts/358", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10303,7 +10275,7 @@ { "self_ref": "#/texts/359", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10332,7 +10304,7 @@ { "self_ref": "#/texts/360", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10361,7 +10333,7 @@ { "self_ref": "#/texts/361", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -10390,7 +10362,7 @@ { "self_ref": "#/texts/362", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", diff --git a/tests/data/pdf/groundtruth/2305.03393v1.md b/tests/data/pdf/groundtruth/2305.03393v1.md index 38906b1dc4..016635bb9b 100644 --- a/tests/data/pdf/groundtruth/2305.03393v1.md +++ b/tests/data/pdf/groundtruth/2305.03393v1.md @@ -169,7 +169,6 @@ Secondly, OTSL has more inherent structure and a significantly restricted vocabu 2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Fornés, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545561. Springer International Publishing, Cham (2022) 3. Chi, Z., Huang, H., Xu, H.D., Yu, H., Yin, W., Mao, X.L.: Complicated table structure recognition. arXiv preprint arXiv:1908.04729 (2019) 4. Deng, Y., Rosenberg, D., Mann, G.: Challenges in end-to-end neural scientific table recognition. In: 2019 International Conference on Document Analysis and Recognition (ICDAR). pp. 894-901. IEEE (2019) - 5. Kayal, P., Anand, M., Desai, H., Singh, M.: Tables to latex: structure and content extraction from scientific tables. International Journal on Document Analysis and Recognition (IJDAR) pp. 1-10 (2022) 6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 18681873. IEEE (2022) 7. Li, M., Cui, L., Huang, S., Wei, F., Zhou, M., Li, Z.: Tablebank: A benchmark dataset for table detection and recognition (2019) @@ -183,7 +182,6 @@ Secondly, OTSL has more inherent structure and a significantly restricted vocabu 15. Staar, P.W.J., Dolfi, M., Auer, C., Bekas, C.: Corpus conversion service: A machine learning platform to ingest documents at scale. In: Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. pp. 774-782. KDD '18, Association for Computing Machinery, New York, NY, USA (2018). https://doi.org/10.1145/3219819.3219834, https://doi.org/10. 1145/3219819.3219834 16. Wang, X.: Tabular Abstraction, Editing, and Formatting. Ph.D. thesis, CAN (1996), aAINN09397 17. Xue, W., Li, Q., Tao, D.: Res2tim: Reconstruct syntactic structures from table images. In: 2019 International Conference on Document Analysis and Recognition (ICDAR). pp. 749-755. IEEE (2019) - 18. Xue, W., Yu, B., Wang, W., Tao, D., Li, Q.: Tgrnet: A table graph reconstruction network for table structure recognition. In: Proceedings of the IEEE/CVF International Conference on Computer Vision. pp. 1295-1304 (2021) 19. Ye, J., Qi, X., He, Y., Chen, Y., Gu, D., Gao, P., Xiao, R.: Pingan-vcgroup's solution for icdar 2021 competition on scientific literature parsing task b: Table recognition to html (2021). https://doi.org/10.48550/ARXIV.2105.01848, https://arxiv.org/abs/2105.01848 20. Zhang, Z., Zhang, J., Du, J., Wang, F.: Split, embed and merge: An accurate table structure recognizer. Pattern Recognition 126, 108565 (2022) diff --git a/tests/data/pdf/groundtruth/elsevier-00.json b/tests/data/pdf/groundtruth/elsevier-00.json index 1b853fc191..f554718ded 100644 --- a/tests/data/pdf/groundtruth/elsevier-00.json +++ b/tests/data/pdf/groundtruth/elsevier-00.json @@ -623,6 +623,9 @@ { "$ref": "#/groups/2" }, + { + "$ref": "#/groups/3" + }, { "$ref": "#/texts/243" }, @@ -633,19 +636,22 @@ "$ref": "#/texts/245" }, { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, { "$ref": "#/texts/247" }, { - "$ref": "#/groups/4" + "$ref": "#/groups/5" + }, + { + "$ref": "#/groups/6" }, { "$ref": "#/texts/275" }, { - "$ref": "#/groups/5" + "$ref": "#/groups/7" }, { "$ref": "#/texts/282" @@ -734,7 +740,18 @@ }, { "$ref": "#/texts/212" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/3", + "parent": { + "$ref": "#/body" + }, + "children": [ { "$ref": "#/texts/213" }, @@ -831,7 +848,7 @@ "label": "list" }, { - "self_ref": "#/groups/3", + "self_ref": "#/groups/4", "parent": { "$ref": "#/body" }, @@ -845,7 +862,7 @@ "label": "list" }, { - "self_ref": "#/groups/4", + "self_ref": "#/groups/5", "parent": { "$ref": "#/body" }, @@ -897,7 +914,18 @@ }, { "$ref": "#/texts/263" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/6", + "parent": { + "$ref": "#/body" + }, + "children": [ { "$ref": "#/texts/264" }, @@ -937,7 +965,7 @@ "label": "list" }, { - "self_ref": "#/groups/5", + "self_ref": "#/groups/7", "parent": { "$ref": "#/body" }, @@ -7086,7 +7114,7 @@ { "self_ref": "#/texts/213", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7116,7 +7144,7 @@ { "self_ref": "#/texts/214", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7146,7 +7174,7 @@ { "self_ref": "#/texts/215", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7176,7 +7204,7 @@ { "self_ref": "#/texts/216", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7206,7 +7234,7 @@ { "self_ref": "#/texts/217", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7236,7 +7264,7 @@ { "self_ref": "#/texts/218", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7266,7 +7294,7 @@ { "self_ref": "#/texts/219", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7296,7 +7324,7 @@ { "self_ref": "#/texts/220", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7326,7 +7354,7 @@ { "self_ref": "#/texts/221", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7356,7 +7384,7 @@ { "self_ref": "#/texts/222", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7385,7 +7413,7 @@ { "self_ref": "#/texts/223", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7414,7 +7442,7 @@ { "self_ref": "#/texts/224", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7443,7 +7471,7 @@ { "self_ref": "#/texts/225", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7472,7 +7500,7 @@ { "self_ref": "#/texts/226", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7501,7 +7529,7 @@ { "self_ref": "#/texts/227", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7531,7 +7559,7 @@ { "self_ref": "#/texts/228", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7561,7 +7589,7 @@ { "self_ref": "#/texts/229", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7590,7 +7618,7 @@ { "self_ref": "#/texts/230", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7620,7 +7648,7 @@ { "self_ref": "#/texts/231", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7650,7 +7678,7 @@ { "self_ref": "#/texts/232", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7680,7 +7708,7 @@ { "self_ref": "#/texts/233", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7710,7 +7738,7 @@ { "self_ref": "#/texts/234", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7739,7 +7767,7 @@ { "self_ref": "#/texts/235", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7769,7 +7797,7 @@ { "self_ref": "#/texts/236", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7799,7 +7827,7 @@ { "self_ref": "#/texts/237", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7829,7 +7857,7 @@ { "self_ref": "#/texts/238", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7859,7 +7887,7 @@ { "self_ref": "#/texts/239", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7889,7 +7917,7 @@ { "self_ref": "#/texts/240", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7919,7 +7947,7 @@ { "self_ref": "#/texts/241", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -7949,7 +7977,7 @@ { "self_ref": "#/texts/242", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -8060,7 +8088,7 @@ { "self_ref": "#/texts/246", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/4" }, "children": [], "content_layer": "body", @@ -8117,7 +8145,7 @@ { "self_ref": "#/texts/248", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8147,7 +8175,7 @@ { "self_ref": "#/texts/249", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8177,7 +8205,7 @@ { "self_ref": "#/texts/250", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8207,7 +8235,7 @@ { "self_ref": "#/texts/251", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8237,7 +8265,7 @@ { "self_ref": "#/texts/252", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8266,7 +8294,7 @@ { "self_ref": "#/texts/253", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8296,7 +8324,7 @@ { "self_ref": "#/texts/254", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8326,7 +8354,7 @@ { "self_ref": "#/texts/255", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8355,7 +8383,7 @@ { "self_ref": "#/texts/256", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8385,7 +8413,7 @@ { "self_ref": "#/texts/257", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8414,7 +8442,7 @@ { "self_ref": "#/texts/258", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8444,7 +8472,7 @@ { "self_ref": "#/texts/259", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8473,7 +8501,7 @@ { "self_ref": "#/texts/260", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8503,7 +8531,7 @@ { "self_ref": "#/texts/261", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8533,7 +8561,7 @@ { "self_ref": "#/texts/262", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8563,7 +8591,7 @@ { "self_ref": "#/texts/263", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -8593,7 +8621,7 @@ { "self_ref": "#/texts/264", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8623,7 +8651,7 @@ { "self_ref": "#/texts/265", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8653,7 +8681,7 @@ { "self_ref": "#/texts/266", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8682,7 +8710,7 @@ { "self_ref": "#/texts/267", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8712,7 +8740,7 @@ { "self_ref": "#/texts/268", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8742,7 +8770,7 @@ { "self_ref": "#/texts/269", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8771,7 +8799,7 @@ { "self_ref": "#/texts/270", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8801,7 +8829,7 @@ { "self_ref": "#/texts/271", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8831,7 +8859,7 @@ { "self_ref": "#/texts/272", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8861,7 +8889,7 @@ { "self_ref": "#/texts/273", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8891,7 +8919,7 @@ { "self_ref": "#/texts/274", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -8948,7 +8976,7 @@ { "self_ref": "#/texts/276", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/7" }, "children": [], "content_layer": "body", @@ -8978,7 +9006,7 @@ { "self_ref": "#/texts/277", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/7" }, "children": [], "content_layer": "body", @@ -9008,7 +9036,7 @@ { "self_ref": "#/texts/278", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/7" }, "children": [], "content_layer": "body", @@ -9038,7 +9066,7 @@ { "self_ref": "#/texts/279", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/7" }, "children": [], "content_layer": "body", @@ -9067,7 +9095,7 @@ { "self_ref": "#/texts/280", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/7" }, "children": [], "content_layer": "body", @@ -9096,7 +9124,7 @@ { "self_ref": "#/texts/281", "parent": { - "$ref": "#/groups/5" + "$ref": "#/groups/7" }, "children": [], "content_layer": "body", diff --git a/tests/data/pdf/groundtruth/elsevier-00.md b/tests/data/pdf/groundtruth/elsevier-00.md index 26b5747666..651434766d 100644 --- a/tests/data/pdf/groundtruth/elsevier-00.md +++ b/tests/data/pdf/groundtruth/elsevier-00.md @@ -380,6 +380,7 @@ No data was used for the research described in the article. - [Araújo, A.G., Perevalov, O.V., Jukov, R.A., 1988. Folha 4 da Carta Geol´ogica de Angola `a escala 1:1 000 000. República Popular de Angola. Minist´erio da Industria. Instituto Nacional de Geologia.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref4) - [Brilha, J., 2016. Inventory and quantitative assessment of geosites and geodiversity sites: a review. Geoheritage 8, 119-134.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref5) - [Brilha, J., 2018. Geoheritage: inventories and evaluation. In: Reynard, Brilha (Eds.), Geoheritage: Assessment, Protection. and Management. Elsevier, pp. 69-85.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref6) + - [Brognon, G.P., Verrier, G.R., 1966. Oil and geology in Cuanza Basin of Angola. AAPG (Am. Assoc. Pet. Geol.) Bull. 50 (1), 108-158.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref7) - [Brownfield, M.E., Charpentier, R.R., 2006. Geology and Total Petroleum Systems of the West-Central Coastal Province (7203), West Africa, 2207-B. U.S. Geological Survey Bulletin, pp. 1-52.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref8) - [Buta-Neto, A., Tavares, T.S., Quesne, D., Guiraud, M., Meister, C., David, B., Morais, M.- L., 2006. Synth`ese pr´eliminaire des travaux men´es sur le bassin de Benguela (Sud Angola): implications s´edimentologiques et structurales. Afr. Geosci. Rev. 13 (3), 239-250.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref9) @@ -431,6 +432,7 @@ No data was used for the research described in the article. - [Neto, K., Henriques, M.H., 2022. Geoconservation in Africa: state of the art and future challenges. Gondwana Res. 111, 1-15.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref52) - [Neto, M.G.M., 1961. As bacias sedimentares de Benguela e Moçˆamedes. Bol. Serv. Geol. Minas Angola 3, 63-93.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref53) - [Neto, M.G.M., 1964. O sedimentar costeiro de Angola. Algumas notas sobre o estado actual do seu conhecimento. Junta de Investigaç˜ao do Ultramar, pp. 1-232.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref54) + - [Peres, A.M., Moreira, A.F.D., 1970. Notícia Explicativa da Folha 185 - Vila Nova Do Seles da Carta Geol´ogica de Angola, `a escala 1: 100 000. Direcç˜ao Provincial dos Serviços de Geologia e Minas, 42 p.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref55) - [Peres, A.M., Moreira, A.F.D., Lap˜ao, L.G.P., Galv˜ao, C.F., Sousa, J.P., Pereira, E.P., 1970. Carta Geol´ogica 185 - Vila Nova do Seles `a escala 1:100 000. Direcç˜ao Provincial dos Serviços de Geologia e Minas.](http://refhub.elsevier.com/S1464-343X%2826%2900366-3/sref56) - P´erez-Díaz, L., Eagles, G., 2017. South Atlantic paleobathymetry since early Cretaceous. Sci. Rep. 7, 11819. https://doi.org/10.1038/s41598-017-11959-7. diff --git a/tests/data/pdf/groundtruth/redp5110_sampled.json b/tests/data/pdf/groundtruth/redp5110_sampled.json index 7c49f6ada6..57affabd5d 100644 --- a/tests/data/pdf/groundtruth/redp5110_sampled.json +++ b/tests/data/pdf/groundtruth/redp5110_sampled.json @@ -369,7 +369,7 @@ "$ref": "#/texts/173" }, { - "$ref": "#/groups/9" + "$ref": "#/groups/10" }, { "$ref": "#/pictures/10" @@ -381,7 +381,7 @@ "$ref": "#/texts/177" }, { - "$ref": "#/groups/10" + "$ref": "#/groups/11" }, { "$ref": "#/texts/179" @@ -390,7 +390,7 @@ "$ref": "#/pictures/11" }, { - "$ref": "#/groups/11" + "$ref": "#/groups/12" }, { "$ref": "#/pictures/12" @@ -399,11 +399,14 @@ "$ref": "#/texts/188" }, { - "$ref": "#/groups/12" + "$ref": "#/groups/13" }, { "$ref": "#/pictures/13" }, + { + "$ref": "#/groups/14" + }, { "$ref": "#/pictures/14" }, @@ -644,7 +647,18 @@ "children": [ { "$ref": "#/texts/165" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/9", + "parent": { + "$ref": "#/texts/165" + }, + "children": [ { "$ref": "#/texts/166" }, @@ -663,7 +677,7 @@ "label": "list" }, { - "self_ref": "#/groups/9", + "self_ref": "#/groups/10", "parent": { "$ref": "#/body" }, @@ -677,7 +691,7 @@ "label": "list" }, { - "self_ref": "#/groups/10", + "self_ref": "#/groups/11", "parent": { "$ref": "#/body" }, @@ -691,7 +705,7 @@ "label": "list" }, { - "self_ref": "#/groups/11", + "self_ref": "#/groups/12", "parent": { "$ref": "#/body" }, @@ -705,14 +719,25 @@ "label": "list" }, { - "self_ref": "#/groups/12", + "self_ref": "#/groups/13", "parent": { "$ref": "#/body" }, "children": [ { "$ref": "#/texts/189" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/14", + "parent": { + "$ref": "#/body" + }, + "children": [ { "$ref": "#/texts/191" } @@ -5264,7 +5289,11 @@ "parent": { "$ref": "#/groups/8" }, - "children": [], + "children": [ + { + "$ref": "#/groups/9" + } + ], "content_layer": "body", "label": "list_item", "prov": [ @@ -5291,7 +5320,7 @@ { "self_ref": "#/texts/166", "parent": { - "$ref": "#/groups/8" + "$ref": "#/groups/9" }, "children": [], "content_layer": "body", @@ -5320,7 +5349,7 @@ { "self_ref": "#/texts/167", "parent": { - "$ref": "#/groups/8" + "$ref": "#/groups/9" }, "children": [], "content_layer": "body", @@ -5349,7 +5378,7 @@ { "self_ref": "#/texts/168", "parent": { - "$ref": "#/groups/8" + "$ref": "#/groups/9" }, "children": [], "content_layer": "body", @@ -5378,7 +5407,7 @@ { "self_ref": "#/texts/169", "parent": { - "$ref": "#/groups/8" + "$ref": "#/groups/9" }, "children": [], "content_layer": "body", @@ -5527,7 +5556,7 @@ { "self_ref": "#/texts/174", "parent": { - "$ref": "#/groups/9" + "$ref": "#/groups/10" }, "children": [], "content_layer": "body", @@ -5638,7 +5667,7 @@ { "self_ref": "#/texts/178", "parent": { - "$ref": "#/groups/10" + "$ref": "#/groups/11" }, "children": [], "content_layer": "body", @@ -5856,7 +5885,7 @@ { "self_ref": "#/texts/186", "parent": { - "$ref": "#/groups/11" + "$ref": "#/groups/12" }, "children": [], "content_layer": "body", @@ -5939,7 +5968,7 @@ { "self_ref": "#/texts/189", "parent": { - "$ref": "#/groups/12" + "$ref": "#/groups/13" }, "children": [], "content_layer": "body", @@ -5995,7 +6024,7 @@ { "self_ref": "#/texts/191", "parent": { - "$ref": "#/groups/12" + "$ref": "#/groups/14" }, "children": [], "content_layer": "body", diff --git a/tests/data/pdf/groundtruth/redp5110_sampled.md b/tests/data/pdf/groundtruth/redp5110_sampled.md index 1a96e31c26..24bedb9450 100644 --- a/tests/data/pdf/groundtruth/redp5110_sampled.md +++ b/tests/data/pdf/groundtruth/redp5110_sampled.md @@ -320,10 +320,10 @@ RETURN CASE WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'HR', 'EMP' ) = 1 THEN E ``` 2. The other column to mask in this example is the TAX\_ID information. In this example, the rules to enforce include the following ones: -- Human Resources can see the unmasked TAX\_ID of the employees. -- Employees can see only their own unmasked TAX\_ID. -- Managers see a masked version of TAX\_ID with the first five characters replaced with the X character (for example, XXX-XX-1234). -- Any other person sees the entire TAX\_ID as masked, for example, XXX-XX-XXXX. + - Human Resources can see the unmasked TAX\_ID of the employees. + - Employees can see only their own unmasked TAX\_ID. + - Managers see a masked version of TAX\_ID with the first five characters replaced with the X character (for example, XXX-XX-1234). + - Any other person sees the entire TAX\_ID as masked, for example, XXX-XX-XXXX. To implement this column mask, run the SQL statement that is shown in Example 3-9. @@ -356,12 +356,13 @@ Figure 3-11 Selecting the EMPLOYEES table from System i Navigator 2. Figure 4-68 shows the Visual Explain of the same SQL statement, but with RCAC enabled. It is clear that the implementation of the SQL statement is more complex because the row permission rule becomes part of the WHERE clause. -3. Compare the advised indexes that are provided by the Optimizer without RCAC and with RCAC enabled. Figure 4-69 shows the index advice for the SQL statement without RCAC enabled. The index being advised is for the ORDER BY clause. Figure 4-68 Visual Explain with RCAC enabled +3. Compare the advised indexes that are provided by the Optimizer without RCAC and with RCAC enabled. Figure 4-69 shows the index advice for the SQL statement without RCAC enabled. The index being advised is for the ORDER BY clause. + Figure 4-69 Index advice with no RCAC diff --git a/tests/data/pdf/groundtruth/table_mislabeled_as_picture.json b/tests/data/pdf/groundtruth/table_mislabeled_as_picture.json index 60762d05a9..5ffdf5c63d 100644 --- a/tests/data/pdf/groundtruth/table_mislabeled_as_picture.json +++ b/tests/data/pdf/groundtruth/table_mislabeled_as_picture.json @@ -29,6 +29,9 @@ { "$ref": "#/groups/1" }, + { + "$ref": "#/groups/2" + }, { "$ref": "#/texts/9" }, @@ -36,7 +39,7 @@ "$ref": "#/texts/10" }, { - "$ref": "#/groups/2" + "$ref": "#/groups/4" }, { "$ref": "#/texts/13" @@ -45,7 +48,7 @@ "$ref": "#/texts/14" }, { - "$ref": "#/groups/3" + "$ref": "#/groups/6" }, { "$ref": "#/texts/16" @@ -54,7 +57,7 @@ "$ref": "#/texts/17" }, { - "$ref": "#/groups/4" + "$ref": "#/groups/7" }, { "$ref": "#/tables/1" @@ -111,10 +114,32 @@ }, { "$ref": "#/texts/4" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/2", + "parent": { + "$ref": "#/body" + }, + "children": [ { "$ref": "#/texts/5" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/3", + "parent": { + "$ref": "#/texts/5" + }, + "children": [ { "$ref": "#/texts/6" }, @@ -130,14 +155,25 @@ "label": "list" }, { - "self_ref": "#/groups/2", + "self_ref": "#/groups/4", "parent": { "$ref": "#/body" }, "children": [ { "$ref": "#/texts/11" - }, + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/5", + "parent": { + "$ref": "#/texts/11" + }, + "children": [ { "$ref": "#/texts/12" } @@ -147,7 +183,7 @@ "label": "list" }, { - "self_ref": "#/groups/3", + "self_ref": "#/groups/6", "parent": { "$ref": "#/body" }, @@ -161,7 +197,7 @@ "label": "list" }, { - "self_ref": "#/groups/4", + "self_ref": "#/groups/7", "parent": { "$ref": "#/body" }, @@ -169,6 +205,20 @@ { "$ref": "#/texts/18" }, + { + "$ref": "#/texts/24" + } + ], + "content_layer": "body", + "name": "list", + "label": "list" + }, + { + "self_ref": "#/groups/8", + "parent": { + "$ref": "#/texts/18" + }, + "children": [ { "$ref": "#/texts/19" }, @@ -183,9 +233,6 @@ }, { "$ref": "#/texts/23" - }, - { - "$ref": "#/texts/24" } ], "content_layer": "body", @@ -340,9 +387,13 @@ { "self_ref": "#/texts/5", "parent": { - "$ref": "#/groups/1" + "$ref": "#/groups/2" }, - "children": [], + "children": [ + { + "$ref": "#/groups/3" + } + ], "content_layer": "body", "label": "list_item", "prov": [ @@ -369,7 +420,7 @@ { "self_ref": "#/texts/6", "parent": { - "$ref": "#/groups/1" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -398,7 +449,7 @@ { "self_ref": "#/texts/7", "parent": { - "$ref": "#/groups/1" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -427,7 +478,7 @@ { "self_ref": "#/texts/8", "parent": { - "$ref": "#/groups/1" + "$ref": "#/groups/3" }, "children": [], "content_layer": "body", @@ -510,9 +561,13 @@ { "self_ref": "#/texts/11", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/4" }, - "children": [], + "children": [ + { + "$ref": "#/groups/5" + } + ], "content_layer": "body", "label": "list_item", "prov": [ @@ -539,7 +594,7 @@ { "self_ref": "#/texts/12", "parent": { - "$ref": "#/groups/2" + "$ref": "#/groups/5" }, "children": [], "content_layer": "body", @@ -622,7 +677,7 @@ { "self_ref": "#/texts/15", "parent": { - "$ref": "#/groups/3" + "$ref": "#/groups/6" }, "children": [], "content_layer": "body", @@ -705,9 +760,13 @@ { "self_ref": "#/texts/18", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/7" }, - "children": [], + "children": [ + { + "$ref": "#/groups/8" + } + ], "content_layer": "body", "label": "list_item", "prov": [ @@ -734,7 +793,7 @@ { "self_ref": "#/texts/19", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/8" }, "children": [], "content_layer": "body", @@ -763,7 +822,7 @@ { "self_ref": "#/texts/20", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/8" }, "children": [], "content_layer": "body", @@ -792,7 +851,7 @@ { "self_ref": "#/texts/21", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/8" }, "children": [], "content_layer": "body", @@ -821,7 +880,7 @@ { "self_ref": "#/texts/22", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/8" }, "children": [], "content_layer": "body", @@ -850,7 +909,7 @@ { "self_ref": "#/texts/23", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/8" }, "children": [], "content_layer": "body", @@ -879,7 +938,7 @@ { "self_ref": "#/texts/24", "parent": { - "$ref": "#/groups/4" + "$ref": "#/groups/7" }, "children": [], "content_layer": "body", diff --git a/tests/data/pdf/groundtruth/table_mislabeled_as_picture.md b/tests/data/pdf/groundtruth/table_mislabeled_as_picture.md index f39a72f8e7..c26cd629e8 100644 --- a/tests/data/pdf/groundtruth/table_mislabeled_as_picture.md +++ b/tests/data/pdf/groundtruth/table_mislabeled_as_picture.md @@ -14,17 +14,18 @@ - \_\_\_\_\_\_\_ at the regional (district) level - \_\_\_\_\_\_\_ at the local (municipal) level - „ „ ☐There is no such limitation + 24. If your country has an institutional legal aid provider (e.g. public defender), do the staff lawyers coordinate to uniformly challenge common violations of national and international due process rights and human rights? -- „ „ Yes, at the national (federal) level -- „ „ Yes, at regional (district) level -- „ „ Yes, at the local (municipal) level + - „ „ Yes, at the national (federal) level + - „ „ Yes, at regional (district) level + - „ „ Yes, at the local (municipal) level „ „ No 25. If your country has an institutional legal aid provider (e.g. public defender), does it have specialized providers and/or units for representing child victims, child witnesses or suspected and accused children? -2. „ „ Yes, at the national (federal) level + - „ „ Yes, at the national (federal) level „ @@ -37,11 +38,11 @@ „ No 26. If your country allows legal aid services through university-based student law clinics, are there national guidelines on how students are supervised in providing legal aid services? (Please select all that apply) -2. „ „ Yes, there are specific guidelines for non-lawyers providing legal aid services -3. „ „ Yes, there are specific guidelines on faculty/student ratios -4. „ „ No, it is up to the discretion of each university -5. „ „ Don't know -6. „ „ There are no university-based student law clinics + - „ „ Yes, there are specific guidelines for non-lawyers providing legal aid services + - „ „ Yes, there are specific guidelines on faculty/student ratios + - „ „ No, it is up to the discretion of each university + - „ „ Don't know + - „ „ There are no university-based student law clinics 28. Are specialized legal aid services provided focusing on specific disadvantaged population groups? If yes, please indicate to whom these services are provided, and whether they are provided by State-funded legal aid, civil society organizations, or both. | If your country allows legal aid services through university-based student law clinics, what type of legal aid services is a student authorized to undertake? (Please select all that apply) | diff --git a/tests/test_listitem_marker_model.py b/tests/test_listitem_marker_model.py index daecec931d..ec127fb1b5 100644 --- a/tests/test_listitem_marker_model.py +++ b/tests/test_listitem_marker_model.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: The Docling Contributors # SPDX-License-Identifier: MIT +import pytest from docling_core.types.doc.base import BoundingBox, CoordOrigin from docling_core.types.doc.document import DoclingDocument, ListItem, ProvenanceItem from docling_core.types.doc.labels import DocItemLabel @@ -10,6 +11,44 @@ ) +@pytest.mark.parametrize("separator", [" ", "\t"]) +def test_word_circle_marker_is_removed_from_list_items(separator: str) -> None: + doc = DoclingDocument(name="Word circle bullets") + group = doc.add_list_group(name="list") + item = doc.add_list_item( + text=f"o{separator}Nested item\ncontinued on another line", parent=group + ) + + ListItemMarkerProcessor().process_document(doc) + + assert item.marker == "o" + assert item.text == "Nested item\ncontinued on another line" + assert item.orig == f"o{separator}Nested item\ncontinued on another line" + assert item.enumerated is False + + +def test_letter_o_in_prose_is_not_a_list_marker() -> None: + doc = DoclingDocument(name="Prose") + item = doc.add_text(label=DocItemLabel.TEXT, text="o rings provide a seal") + + processed = ListItemMarkerProcessor().process_text_item(item) + + assert not isinstance(processed, ListItem) + assert processed.text == "o rings provide a seal" + + +@pytest.mark.parametrize("text", ["other item", "o-ring seal", "O Item", "0 Item"]) +def test_words_and_similar_characters_are_not_circle_markers(text: str) -> None: + doc = DoclingDocument(name="Not circle bullets") + group = doc.add_list_group(name="list") + item = doc.add_list_item(text=text, parent=group) + + ListItemMarkerProcessor().process_document(doc) + + assert item.marker == "" + assert item.text == text + + # Example usage and testing def test_listitem_marker_model(): """Example of how to use the ListItemMarkerProcessor.""" diff --git a/tests/test_readingorder_container_children.py b/tests/test_readingorder_container_children.py index 3d6ae37777..d001517dbd 100644 --- a/tests/test_readingorder_container_children.py +++ b/tests/test_readingorder_container_children.py @@ -127,6 +127,53 @@ def test_container_orders_mixed_children_uniformly() -> None: assert [item.text for item in doc.texts] == ["middle"] +def test_nested_lists_stay_inside_each_form() -> None: + elements = [] + for index, left in enumerate((0, 200)): + container_id = 3 * index + parent_cluster = _cluster( + container_id + 1, DocItemLabel.LIST_ITEM, (left + 10, 20, left + 180, 35) + ) + child_cluster = _cluster( + container_id + 2, DocItemLabel.LIST_ITEM, (left + 40, 60, left + 180, 75) + ) + container_cluster = _cluster( + container_id, DocItemLabel.FORM, (left, 10, left + 190, 100) + ) + container_cluster.children = [parent_cluster, child_cluster] + elements.append( + ContainerElement( + id=container_id, + label=DocItemLabel.FORM, + page_no=1, + cluster=container_cluster, + ) + ) + for cluster, text in [(parent_cluster, "• Parent"), (child_cluster, "o Child")]: + elements.append( + TextElement( + id=cluster.id, + label=cluster.label, + text=text, + page_no=1, + cluster=cluster, + ) + ) + + doc = ReadingOrderModel(ReadingOrderOptions())(_conversion_result(elements)) + + forms = [group for group in doc.groups if group.label == GroupLabel.FORM_AREA] + assert len(forms) == 2 + for form in forms: + root_list = form.children[0].resolve(doc) + parent = root_list.children[0].resolve(doc) + nested_list = parent.children[0].resolve(doc) + child = nested_list.children[0].resolve(doc) + assert root_list.parent == form.get_ref() + assert nested_list.parent == parent.get_ref() + assert child.text == "Child" + + def test_container_table_owns_picture_matched_to_rich_cell() -> None: container_cluster = _cluster(1, DocItemLabel.FORM, (0, 0, 400, 400)) table_cluster = _cluster(2, DocItemLabel.TABLE, (10, 10, 300, 300)) diff --git a/tests/test_readingorder_lists.py b/tests/test_readingorder_lists.py new file mode 100644 index 0000000000..c41378344a --- /dev/null +++ b/tests/test_readingorder_lists.py @@ -0,0 +1,368 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +from pathlib import PurePath + +import pytest +from docling_core.types.doc import BoundingBox, DocItemLabel, GroupLabel, Size +from docling_core.types.doc.document import DoclingDocument, ListItem +from docling_core.types.doc.page import BoundingRectangle, TextCell + +from docling.datamodel.base_models import ( + AssembledUnit, + BasePageElement, + Cluster, + FigureElement, + InputFormat, + Page, + TextElement, +) +from docling.datamodel.document import ConversionResult, InputDocument +from docling.models.stages.reading_order.readingorder_model import ( + ReadingOrderModel, + ReadingOrderOptions, +) + + +def convert_entries( + entries: list[tuple[str, float, int, DocItemLabel]], + *, + boxes: list[BoundingBox] | None = None, + cells: dict[int, list[TextCell]] | None = None, +) -> DoclingDocument: + elements: list[BasePageElement] = [] + page_positions: dict[int, int] = {} + for index, (text, left, page_no, label) in enumerate(entries): + top = 20 + 30 * page_positions.get(page_no, 0) + page_positions[page_no] = page_positions.get(page_no, 0) + 1 + cluster = Cluster( + id=index, + label=label, + bbox=( + boxes[index] + if boxes is not None + else BoundingBox(l=left, t=top, r=450, b=top + 15) + ), + cells=(cells or {}).get(index, []), + ) + if label == DocItemLabel.PICTURE: + elements.append( + FigureElement(id=index, label=label, page_no=page_no, cluster=cluster) + ) + else: + elements.append( + TextElement( + id=index, + label=label, + text=text, + page_no=page_no, + cluster=cluster, + ) + ) + input_doc = InputDocument.model_construct( + file=PurePath("list.pdf"), + document_hash="0" * 64, + valid=True, + format=InputFormat.PDF, + ) + result = ConversionResult( + input=input_doc, + pages=[ + Page(page_no=page_no, size=Size(width=500, height=700)) + for page_no in sorted(page_positions) + ], + assembled=AssembledUnit(elements=elements, body=elements), + ) + return ReadingOrderModel(ReadingOrderOptions())(result) + + +@pytest.mark.parametrize( + ("indents", "expected_levels"), + [ + ([118, 154, 190, 154, 118], [2, 4, 6, 4, 2]), + ([118, 120, 117], [2, 2, 2]), + ], +) +def test_pdf_list_indentation_controls_nesting( + indents: list[float], expected_levels: list[int] +) -> None: + doc = convert_entries( + [ + (f"• Item {index}", left, 1, DocItemLabel.LIST_ITEM) + for index, left in enumerate(indents) + ] + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == expected_levels + assert [item.text for item, _ in items] == [ + f"Item {index}" for index in range(len(indents)) + ] + if len(indents) == 5: + assert items[1][0].parent is not None + nested_group = items[1][0].parent.resolve(doc) + assert nested_group.label == GroupLabel.LIST + assert nested_group.parent == items[0][0].get_ref() + assert items[3][0].parent == items[1][0].parent + assert items[4][0].parent == items[0][0].parent + assert " - Item 1\n - Item 2" in doc.export_to_markdown() + + +def test_nested_pdf_list_continues_across_page_furniture() -> None: + doc = convert_entries( + [ + ("• Parent", 118, 1, DocItemLabel.LIST_ITEM), + ("o First child", 154, 1, DocItemLabel.LIST_ITEM), + ("1", 250, 1, DocItemLabel.PAGE_FOOTER), + ("Page header", 100, 2, DocItemLabel.PAGE_HEADER), + ("o Second child", 154, 2, DocItemLabel.LIST_ITEM), + ("• Next parent", 118, 2, DocItemLabel.LIST_ITEM), + ] + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == [2, 4, 4, 2] + assert items[1][0].parent == items[2][0].parent + assert [item.text for item, _ in items] == [ + "Parent", + "First child", + "Second child", + "Next parent", + ] + + +@pytest.mark.parametrize("next_page", [1, 2]) +def test_a_picture_stays_before_the_next_list(next_page: int) -> None: + doc = convert_entries( + [ + ("1. Parent", 118, 1, DocItemLabel.LIST_ITEM), + ("o First child", 154, 1, DocItemLabel.LIST_ITEM), + ("", 60, 1, DocItemLabel.PICTURE), + ("o Separate item", 154, next_page, DocItemLabel.LIST_ITEM), + ] + ) + labels = [item.label for item, _ in doc.iterate_items()] + assert labels == [ + DocItemLabel.LIST_ITEM, + DocItemLabel.LIST_ITEM, + DocItemLabel.PICTURE, + DocItemLabel.LIST_ITEM, + ] + items = [item for item, _ in doc.iterate_items() if isinstance(item, ListItem)] + assert items[1].parent != items[2].parent + + +def test_numbered_pdf_lists_keep_markers_when_nested() -> None: + doc = convert_entries( + [ + ("1. Parent", 118, 1, DocItemLabel.LIST_ITEM), + ("a) Child", 154, 1, DocItemLabel.LIST_ITEM), + ("2. Next parent", 118, 1, DocItemLabel.LIST_ITEM), + ] + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == [2, 4, 2] + assert [item.marker for item, _ in items] == ["1.", "a)", "2."] + assert all(item.enumerated for item, _ in items) + + +def test_body_paragraph_ends_a_nested_pdf_list() -> None: + doc = convert_entries( + [ + ("• Parent", 118, 1, DocItemLabel.LIST_ITEM), + ("o Child", 154, 1, DocItemLabel.LIST_ITEM), + ("A separate paragraph.", 118, 1, DocItemLabel.TEXT), + ("o New list", 154, 1, DocItemLabel.LIST_ITEM), + ] + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == [2, 4, 2] + assert items[2][0].parent != items[1][0].parent + + +def test_pdf_list_in_next_column_is_not_nested() -> None: + doc = convert_entries( + [ + ("• Left first", 50, 1, DocItemLabel.LIST_ITEM), + ("• Left second", 50, 1, DocItemLabel.LIST_ITEM), + ("• Right first", 300, 1, DocItemLabel.LIST_ITEM), + ("• Right second", 300, 1, DocItemLabel.LIST_ITEM), + ], + boxes=[ + BoundingBox(l=50, t=20, r=220, b=35), + BoundingBox(l=50, t=60, r=220, b=75), + BoundingBox(l=300, t=20, r=470, b=35), + BoundingBox(l=300, t=60, r=470, b=75), + ], + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [item.text for item, _ in items] == [ + "Left first", + "Left second", + "Right first", + "Right second", + ] + assert [level for _, level in items] == [2, 2, 2, 2] + + +@pytest.mark.parametrize("opening_quote", ["«", "\u201c", '"']) +@pytest.mark.parametrize("markers", [("1.", "2.", "3."), ("(1)", "(2)", "(3)")]) +def test_opening_quote_does_not_indent_numbered_siblings( + opening_quote: str, markers: tuple[str, str, str] +) -> None: + doc = convert_entries( + [ + ( + f"{opening_quote}{markers[0]} First item.", + 113.4, + 1, + DocItemLabel.LIST_ITEM, + ), + (f"{markers[1]} Second item.", 118.9, 1, DocItemLabel.LIST_ITEM), + ( + f"{markers[2]} Third item.{opening_quote}", + 118.9, + 1, + DocItemLabel.LIST_ITEM, + ), + ] + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == [2, 2, 2] + assert len({item.parent.cref for item, _ in items if item.parent is not None}) == 1 + assert items[0][0].orig.startswith(opening_quote) + + +def test_quoted_parent_still_has_an_indented_sublist() -> None: + doc = convert_entries( + [ + ("«1. Parent", 113.4, 1, DocItemLabel.LIST_ITEM), + ("a) First child", 154, 1, DocItemLabel.LIST_ITEM), + ("b) Second child", 154, 1, DocItemLabel.LIST_ITEM), + ("2. Next parent»", 118.9, 1, DocItemLabel.LIST_ITEM), + ] + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == [2, 4, 4, 2] + assert items[1][0].parent is not None + assert items[1][0].parent.resolve(doc).parent == items[0][0].get_ref() + assert items[0][0].parent == items[3][0].parent + + +@pytest.mark.parametrize("child_marker", ["a)", "1."]) +def test_small_indent_with_a_new_numbering_style_still_nests(child_marker: str) -> None: + doc = convert_entries( + [ + ("«1. Parent", 113.4, 1, DocItemLabel.LIST_ITEM), + (f"{child_marker} Child", 118.9, 1, DocItemLabel.LIST_ITEM), + ] + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == [2, 4] + + +def test_multiline_quoted_parent_does_not_absorb_a_real_indent() -> None: + first_line = BoundingBox(l=113.4, t=20, r=450, b=30) + doc = convert_entries( + [ + ( + "«1. A parent that spans several lines.", + 113.4, + 1, + DocItemLabel.LIST_ITEM, + ), + ("2. A nested item.", 149.4, 1, DocItemLabel.LIST_ITEM), + ], + boxes=[ + BoundingBox(l=113.4, t=20, r=450, b=75), + BoundingBox(l=149.4, t=90, r=450, b=100), + ], + cells={ + 0: [ + TextCell( + index=0, + text="«1. A parent", + orig="«1. A parent", + from_ocr=False, + rect=BoundingRectangle.from_bounding_box(first_line), + ) + ] + }, + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == [2, 4] + + +def test_nested_list_continues_after_page_footnotes() -> None: + doc = convert_entries( + [ + ("• Parent", 118, 1, DocItemLabel.LIST_ITEM), + ("o First child", 154, 1, DocItemLabel.LIST_ITEM), + ("1 A page footnote.", 100, 1, DocItemLabel.FOOTNOTE), + ("2 Another footnote.", 100, 1, DocItemLabel.FOOTNOTE), + ("1", 250, 1, DocItemLabel.PAGE_FOOTER), + ("Page header", 100, 2, DocItemLabel.PAGE_HEADER), + ("o Second child", 154, 2, DocItemLabel.LIST_ITEM), + ("• Next parent", 118, 2, DocItemLabel.LIST_ITEM), + ("A body paragraph ends the list.", 118, 2, DocItemLabel.TEXT), + ("o A separate list", 154, 2, DocItemLabel.LIST_ITEM), + ] + ) + + items = [ + (item, level) + for item, level in doc.iterate_items() + if isinstance(item, ListItem) + ] + assert [level for _, level in items] == [2, 4, 4, 2, 2] + assert items[1][0].parent == items[2][0].parent + assert items[4][0].parent != items[1][0].parent + footnotes = [item for item in doc.texts if item.label == DocItemLabel.FOOTNOTE] + assert [item.text for item in footnotes] == [ + "1 A page footnote.", + "2 Another footnote.", + ] + assert all(item.prov[0].page_no == 1 for item in footnotes)