diff --git a/docling/models/stages/page_assemble/page_assemble_model.py b/docling/models/stages/page_assemble/page_assemble_model.py index ba4733d205..04eda2ff78 100644 --- a/docling/models/stages/page_assemble/page_assemble_model.py +++ b/docling/models/stages/page_assemble/page_assemble_model.py @@ -109,7 +109,20 @@ def _match_hyperlink( except ValidationError: return Path(best_uri) - def sanitize_text(self, lines): + def sanitize_text(self, lines: List[str]) -> str: + """Join the text lines of a cluster into a single normalized string. + + Lines are joined with a space, except across a hyphen that splits a + word over the line break ("algo-" / "rithms"), which is dropped so + that the two halves close up. A hyphen only splits a word when it is + attached to the word it splits; one that follows whitespace is a + literal character -- a separator dash, a bullet marker, or a wrapped + hyphen-prefixed token -- so it is kept and its lines are joined like + any other line break. + + The joined text is then normalized: a few typographic characters are + replaced by ASCII equivalents and ligatures are expanded. + """ if len(lines) == 0: return "" @@ -120,13 +133,18 @@ def sanitize_text(self, lines): prev_words = re.findall(r"\b[\w]+\b", prev_line) line_words = re.findall(r"\b[\w]+\b", line) + hyphen_attached_to_word = len(prev_line) > 1 and prev_line[-2].isalnum() + if ( - len(prev_words) + hyphen_attached_to_word + and len(prev_words) and len(line_words) and prev_words[-1].isalnum() and line_words[0].isalnum() ): lines[ix] = prev_line[:-1] + elif not hyphen_attached_to_word: + lines[ix] += " " else: lines[ix] += " " diff --git a/tests/data/pdf/groundtruth/2203.01017v2.doctags.txt b/tests/data/pdf/groundtruth/2203.01017v2.doctags.txt index 4d6299e9cf..b6a5e61ba8 100644 --- a/tests/data/pdf/groundtruth/2203.01017v2.doctags.txt +++ b/tests/data/pdf/groundtruth/2203.01017v2.doctags.txt @@ -10,7 +10,7 @@ 1 7 01 2 13 45679 131011128 214151617181920 -Figure 1: Picture of a table with subtle, complex features such as (1) multi-column headers, (2) cell with multi-row text and (3) cells with no content. Image from PubTabNet evaluation set, filename: 'PMC2944238 004 02'. +Figure 1: Picture of a table with subtle, complex features such as (1) multi-column headers, (2) cell with multi-row text and (3) cells with no content. Image from PubTabNet evaluation set, filename: 'PMC2944238 004 02'. Recently, significant progress has been made with vision based approaches to extract tables in documents. For the sake of completeness, the issue of table extraction from documents is typically decomposed into two separate challenges, i.e. (1) finding the location of the table(s) on a document-page and (2) finding the structure of a given table in the document. The first problem is called table-location and has been previously addressed [30, 38, 19, 21, 23, 26, 8] with stateof-the-art object-detection networks (e.g. YOLO and later on Mask-RCNN [9]). For all practical purposes, it can be considered as a solved problem, given enough ground-truth data to train on. 1 @@ -151,7 +151,7 @@ Quanzeng You, Hailin Jin, Zhaowen Wang, Chen Fang, and Jiebo Luo. Image captioning with semantic attention. In Proceedings of the IEEE conference on computer vision and pattern recognition , pages 4651-4659, 2016. 4 Xinyi Zheng, Doug Burdick, Lucian Popa, Peter Zhong, and Nancy Xin Ru Wang. Global table extractor (gte): A framework for joint table identification and cell structure recognition using visual context. Winter Conference for Applications in Computer Vision (WACV) , 2021. 2, 3 Xu Zhong, Elaheh ShafieiBavani, and Antonio Jimeno Yepes. Image-based table recognition: Data, model, -and evaluation. In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan-Michael Frahm, editors, Computer Vision ECCV 2020 , pages 564-580, Cham, 2020. Springer International Publishing. 2, 3, 7 +and evaluation. In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan-Michael Frahm, editors, Computer Vision - ECCV 2020 , pages 564-580, Cham, 2020. Springer International Publishing. 2, 3, 7 Xu Zhong, Jianbin Tang, and Antonio Jimeno Yepes. Publaynet: Largest dataset ever for document layout analysis. In 2019 International Conference on Document Analysis and Recognition (ICDAR) , pages 1015-1022, 2019. 1 10 diff --git a/tests/data/pdf/groundtruth/2203.01017v2.json b/tests/data/pdf/groundtruth/2203.01017v2.json index 1e2067c62c..5eeeaae6e9 100644 --- a/tests/data/pdf/groundtruth/2203.01017v2.json +++ b/tests/data/pdf/groundtruth/2203.01017v2.json @@ -7221,11 +7221,11 @@ }, "charspan": [ 0, - 55 + 56 ] } ], - "orig": "l box = \u03bb iou l iou + \u03bb l 1 l = \u03bbl s +(1 -\u03bb ) l box (1)", + "orig": "l box = \u03bb iou l iou + \u03bb l 1 l = \u03bbl s +(1 - \u03bb ) l box (1)", "text": "" }, { @@ -7671,11 +7671,11 @@ }, "charspan": [ 0, - 74 + 75 ] } ], - "orig": "TEDS( T a , T b ) = 1 -EditDist ( T a , T b ) max( | T a | , | T b | ) (3)", + "orig": "TEDS( T a , T b ) = 1 - EditDist ( T a , T b ) max( | T a | , | T b | ) (3)", "text": "" }, { @@ -13763,12 +13763,12 @@ }, "charspan": [ 0, - 192 + 194 ] } ], - "orig": "and evaluation. In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan-Michael Frahm, editors, Computer Vision ECCV 2020 , pages 564-580, Cham, 2020. Springer International Publishing. 2, 3, 7", - "text": "and evaluation. In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan-Michael Frahm, editors, Computer Vision ECCV 2020 , pages 564-580, Cham, 2020. Springer International Publishing. 2, 3, 7", + "orig": "and evaluation. In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan-Michael Frahm, editors, Computer Vision - ECCV 2020 , pages 564-580, Cham, 2020. Springer International Publishing. 2, 3, 7", + "text": "and evaluation. In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan-Michael Frahm, editors, Computer Vision - ECCV 2020 , pages 564-580, Cham, 2020. Springer International Publishing. 2, 3, 7", "enumerated": false, "marker": "" }, @@ -16295,11 +16295,11 @@ }, "charspan": [ 0, - 64 + 65 ] } ], - "orig": "alignment = arg min c { D c } D c = max { x c } -min { x c } (4)", + "orig": "alignment = arg min c { D c } D c = max { x c } - min { x c } (4)", "text": "" }, { @@ -17294,9 +17294,9 @@ "page_no": 1, "bbox": { "l": 314.77, - "t": 566.19, - "r": 537.93, - "b": 295.62, + "t": 566.39, + "r": 537.95, + "b": 295.61, "coord_origin": "BOTTOMLEFT" }, "charspan": [ @@ -18854,7 +18854,7 @@ "bbox": { "l": 315.46, "t": 561.46, - "r": 537.05, + "r": 537.04, "b": 489.47, "coord_origin": "BOTTOMLEFT" }, @@ -18932,10 +18932,10 @@ { "page_no": 1, "bbox": { - "l": 315.21, - "t": 358.2, + "l": 315.2, + "t": 358.21, "r": 537.11, - "b": 295.44, + "b": 295.43, "coord_origin": "BOTTOMLEFT" }, "charspan": [ diff --git a/tests/data/pdf/groundtruth/2203.01017v2.md b/tests/data/pdf/groundtruth/2203.01017v2.md index bd6fba4d18..32c5996398 100644 --- a/tests/data/pdf/groundtruth/2203.01017v2.md +++ b/tests/data/pdf/groundtruth/2203.01017v2.md @@ -270,7 +270,7 @@ end object detection with transformers. In Andrea Vedaldi, Horst Bischof, Thomas - [35] Quanzeng You, Hailin Jin, Zhaowen Wang, Chen Fang, and Jiebo Luo. Image captioning with semantic attention. In Proceedings of the IEEE conference on computer vision and pattern recognition , pages 4651-4659, 2016. 4 - [36] Xinyi Zheng, Doug Burdick, Lucian Popa, Peter Zhong, and Nancy Xin Ru Wang. Global table extractor (gte): A framework for joint table identification and cell structure recognition using visual context. Winter Conference for Applications in Computer Vision (WACV) , 2021. 2, 3 - [37] Xu Zhong, Elaheh ShafieiBavani, and Antonio Jimeno Yepes. Image-based table recognition: Data, model, -- and evaluation. In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan-Michael Frahm, editors, Computer Vision ECCV 2020 , pages 564-580, Cham, 2020. Springer International Publishing. 2, 3, 7 +- and evaluation. In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan-Michael Frahm, editors, Computer Vision - ECCV 2020 , pages 564-580, Cham, 2020. Springer International Publishing. 2, 3, 7 - [38] Xu Zhong, Jianbin Tang, and Antonio Jimeno Yepes. Publaynet: Largest dataset ever for document layout analysis. In 2019 International Conference on Document Analysis and Recognition (ICDAR) , pages 1015-1022, 2019. 1 ## TableFormer: Table Structure Understanding with Transformers diff --git a/tests/data/pdf/groundtruth/2206.01062.doctags.txt b/tests/data/pdf/groundtruth/2206.01062.doctags.txt index 2e7e94f527..13314d7f9d 100644 --- a/tests/data/pdf/groundtruth/2206.01062.doctags.txt +++ b/tests/data/pdf/groundtruth/2206.01062.doctags.txt @@ -114,7 +114,7 @@ To date, there is still a significant gap between human and ML accuracy on the layout interpretation task, and we hope that this work will inspire the research community to close that gap. REFERENCES Max Göbel, Tamir Hassan, Ermelinda Oro, and Giorgio Orsi. Icdar 2013 table competition. In 2013 12th International Conference on Document Analysis and Recognition , pages 1449-1453, 2013. -Christian Clausner, Apostolos Antonacopoulos, and Stefan Pletschacher. Icdar2017 competition on recognition of documents with complex layouts rdcl2017. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR) , volume 01, pages 1404-1410, 2017. +Christian Clausner, Apostolos Antonacopoulos, and Stefan Pletschacher. Icdar2017 competition on recognition of documents with complex layouts - rdcl2017. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR) , volume 01, pages 1404-1410, 2017. Hervé Déjean, Jean-Luc Meunier, Liangcai Gao, Yilun Huang, Yu Fang, Florian Kleber, and Eva-Maria Lang. ICDAR 2019 Competition on Table Detection and Recognition (cTDaR), April 2019. http://sac.founderit.com/. Antonio Jimeno Yepes, Peter Zhong, and Douglas Burdick. Competition on scientific literature parsing. In Proceedings of the International Conference on Document Analysis and Recognition , ICDAR, pages 605-617. LNCS 12824, SpringerVerlag, sep 2021. Logan Markewich, Hao Zhang, Yubin Xing, Navid Lambert-Shirzad, Jiang Zhexin, Roy Lee, Zhi Li, and Seok-Bum Ko. Segmentation for document layout analysis: not dead yet. International Journal on Document Analysis and Recognition (IJDAR) , pages 1-11, 01 2022. diff --git a/tests/data/pdf/groundtruth/2206.01062.json b/tests/data/pdf/groundtruth/2206.01062.json index eeb39b1c11..a1ca795df0 100644 --- a/tests/data/pdf/groundtruth/2206.01062.json +++ b/tests/data/pdf/groundtruth/2206.01062.json @@ -12874,12 +12874,12 @@ }, "charspan": [ 0, - 279 + 281 ] } ], - "orig": "[2] Christian Clausner, Apostolos Antonacopoulos, and Stefan Pletschacher. Icdar2017 competition on recognition of documents with complex layouts rdcl2017. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR) , volume 01, pages 1404-1410, 2017.", - "text": "Christian Clausner, Apostolos Antonacopoulos, and Stefan Pletschacher. Icdar2017 competition on recognition of documents with complex layouts rdcl2017. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR) , volume 01, pages 1404-1410, 2017.", + "orig": "[2] Christian Clausner, Apostolos Antonacopoulos, and Stefan Pletschacher. Icdar2017 competition on recognition of documents with complex layouts - rdcl2017. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR) , volume 01, pages 1404-1410, 2017.", + "text": "Christian Clausner, Apostolos Antonacopoulos, and Stefan Pletschacher. Icdar2017 competition on recognition of documents with complex layouts - rdcl2017. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR) , volume 01, pages 1404-1410, 2017.", "enumerated": true, "marker": "[2]" }, @@ -15121,10 +15121,10 @@ { "page_no": 1, "bbox": { - "l": 322.79, - "t": 543.26, - "r": 554.34, - "b": 266.09, + "l": 322.78, + "t": 543.27, + "r": 554.33, + "b": 266.1, "coord_origin": "BOTTOMLEFT" }, "charspan": [ diff --git a/tests/data/pdf/groundtruth/2206.01062.md b/tests/data/pdf/groundtruth/2206.01062.md index a686bc8976..cbe6f80b8c 100644 --- a/tests/data/pdf/groundtruth/2206.01062.md +++ b/tests/data/pdf/groundtruth/2206.01062.md @@ -269,7 +269,7 @@ To date, there is still a significant gap between human and ML accuracy on the l ## REFERENCES - [1] Max Göbel, Tamir Hassan, Ermelinda Oro, and Giorgio Orsi. Icdar 2013 table competition. In 2013 12th International Conference on Document Analysis and Recognition , pages 1449-1453, 2013. -- [2] Christian Clausner, Apostolos Antonacopoulos, and Stefan Pletschacher. Icdar2017 competition on recognition of documents with complex layouts rdcl2017. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR) , volume 01, pages 1404-1410, 2017. +- [2] Christian Clausner, Apostolos Antonacopoulos, and Stefan Pletschacher. Icdar2017 competition on recognition of documents with complex layouts - rdcl2017. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR) , volume 01, pages 1404-1410, 2017. - [3] Hervé Déjean, Jean-Luc Meunier, Liangcai Gao, Yilun Huang, Yu Fang, Florian Kleber, and Eva-Maria Lang. ICDAR 2019 Competition on Table Detection and Recognition (cTDaR), April 2019. http://sac.founderit.com/. - [4] Antonio Jimeno Yepes, Peter Zhong, and Douglas Burdick. Competition on scientific literature parsing. In Proceedings of the International Conference on Document Analysis and Recognition , ICDAR, pages 605-617. LNCS 12824, SpringerVerlag, sep 2021. - [5] Logan Markewich, Hao Zhang, Yubin Xing, Navid Lambert-Shirzad, Jiang Zhexin, Roy Lee, Zhi Li, and Seok-Bum Ko. Segmentation for document layout analysis: not dead yet. International Journal on Document Analysis and Recognition (IJDAR) , pages 1-11, 01 2022. diff --git a/tests/data/pdf/groundtruth/2305.03393v1.doctags.txt b/tests/data/pdf/groundtruth/2305.03393v1.doctags.txt index 5a4c41a0f0..2ce7c38b09 100644 --- a/tests/data/pdf/groundtruth/2305.03393v1.doctags.txt +++ b/tests/data/pdf/groundtruth/2305.03393v1.doctags.txt @@ -1,6 +1,6 @@ arXiv:2305.03393v1 [cs.CV] 5 May 2023 Optimized Table Tokenization for Table Structure Recognition -Maksym Lysak [0000 -0002 -3723 -6960] , Ahmed Nassar [0000 -0002 -9468 -0822] , Nikolaos Livathinos [0000 -0001 -8513 -3491] , Christoph Auer [0000 -0001 -5761 -0422] , [0000 -0002 -8088 -0823] +Maksym Lysak [0000 - 0002 - 3723 - 6960] , Ahmed Nassar [0000 - 0002 - 9468 - 0822] , Nikolaos Livathinos [0000 - 0001 - 8513 - 3491] , Christoph Auer [0000 - 0001 - 5761 - 0422] , [0000 - 0002 - 8088 - 0823] and Peter Staar IBM Research {mly,ahn,nli,cau,taa}@zurich.ibm.com @@ -46,11 +46,11 @@ 4.1 Language Definition In Figure 3, we illustrate how the OTSL is defined. In essence, the OTSL defines only 5 tokens that directly describe a tabular structure based on an atomic 2D grid. The OTSL vocabulary is comprised of the following tokens: --"C" cell a new table cell that either has or does not have cell content --"L" cell left-looking cell , merging with the left neighbor cell to create a span --"U" cell up-looking cell , merging with the upper neighbor cell to create a span --"X" cell cross cell , to merge with both left and upper neighbor cells --"NL" new-line , switch to the next row. +"C" cell - a new table cell that either has or does not have cell content +"L" cell - left-looking cell , merging with the left neighbor cell to create a span +"U" cell - up-looking cell , merging with the upper neighbor cell to create a span +"X" cell - cross cell , to merge with both left and upper neighbor cells +"NL" - new-line , switch to the next row. A notable attribute of OTSL is that it has the capability of achieving lossless conversion to HTML. diff --git a/tests/data/pdf/groundtruth/2305.03393v1.json b/tests/data/pdf/groundtruth/2305.03393v1.json index 5b1dfa2082..7701afbc21 100644 --- a/tests/data/pdf/groundtruth/2305.03393v1.json +++ b/tests/data/pdf/groundtruth/2305.03393v1.json @@ -547,12 +547,12 @@ }, "charspan": [ 0, - 193 + 208 ] } ], - "orig": "Maksym Lysak [0000 -0002 -3723 -6960] , Ahmed Nassar [0000 -0002 -9468 -0822] , Nikolaos Livathinos [0000 -0001 -8513 -3491] , Christoph Auer [0000 -0001 -5761 -0422] , [0000 -0002 -8088 -0823]", - "text": "Maksym Lysak [0000 -0002 -3723 -6960] , Ahmed Nassar [0000 -0002 -9468 -0822] , Nikolaos Livathinos [0000 -0001 -8513 -3491] , Christoph Auer [0000 -0001 -5761 -0422] , [0000 -0002 -8088 -0823]" + "orig": "Maksym Lysak [0000 - 0002 - 3723 - 6960] , Ahmed Nassar [0000 - 0002 - 9468 - 0822] , Nikolaos Livathinos [0000 - 0001 - 8513 - 3491] , Christoph Auer [0000 - 0001 - 5761 - 0422] , [0000 - 0002 - 8088 - 0823]", + "text": "Maksym Lysak [0000 - 0002 - 3723 - 6960] , Ahmed Nassar [0000 - 0002 - 9468 - 0822] , Nikolaos Livathinos [0000 - 0001 - 8513 - 3491] , Christoph Auer [0000 - 0001 - 5761 - 0422] , [0000 - 0002 - 8088 - 0823]" }, { "self_ref": "#/texts/3", @@ -3456,14 +3456,14 @@ }, "charspan": [ 0, - 72 + 75 ] } ], - "orig": "-\"C\" cell a new table cell that either has or does not have cell content", - "text": "-\"C\" cell a new table cell that either has or does not have cell content", + "orig": "- \"C\" cell - a new table cell that either has or does not have cell content", + "text": "\"C\" cell - a new table cell that either has or does not have cell content", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/109", @@ -3485,14 +3485,14 @@ }, "charspan": [ 0, - 82 + 85 ] } ], - "orig": "-\"L\" cell left-looking cell , merging with the left neighbor cell to create a span", - "text": "-\"L\" cell left-looking cell , merging with the left neighbor cell to create a span", + "orig": "- \"L\" cell - left-looking cell , merging with the left neighbor cell to create a span", + "text": "\"L\" cell - left-looking cell , merging with the left neighbor cell to create a span", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/110", @@ -3514,14 +3514,14 @@ }, "charspan": [ 0, - 81 + 84 ] } ], - "orig": "-\"U\" cell up-looking cell , merging with the upper neighbor cell to create a span", - "text": "-\"U\" cell up-looking cell , merging with the upper neighbor cell to create a span", + "orig": "- \"U\" cell - up-looking cell , merging with the upper neighbor cell to create a span", + "text": "\"U\" cell - up-looking cell , merging with the upper neighbor cell to create a span", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/111", @@ -3543,14 +3543,14 @@ }, "charspan": [ 0, - 71 + 74 ] } ], - "orig": "-\"X\" cell cross cell , to merge with both left and upper neighbor cells", - "text": "-\"X\" cell cross cell , to merge with both left and upper neighbor cells", + "orig": "- \"X\" cell - cross cell , to merge with both left and upper neighbor cells", + "text": "\"X\" cell - cross cell , to merge with both left and upper neighbor cells", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/112", @@ -3572,14 +3572,14 @@ }, "charspan": [ 0, - 40 + 43 ] } ], - "orig": "-\"NL\" new-line , switch to the next row.", - "text": "-\"NL\" new-line , switch to the next row.", + "orig": "- \"NL\" - new-line , switch to the next row.", + "text": "\"NL\" - new-line , switch to the next row.", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/113", diff --git a/tests/data/pdf/groundtruth/2305.03393v1.md b/tests/data/pdf/groundtruth/2305.03393v1.md index 19ed6f0ec2..5ac9f8a797 100644 --- a/tests/data/pdf/groundtruth/2305.03393v1.md +++ b/tests/data/pdf/groundtruth/2305.03393v1.md @@ -1,6 +1,6 @@ ## Optimized Table Tokenization for Table Structure Recognition -Maksym Lysak [0000 -0002 -3723 -6960] , Ahmed Nassar [0000 -0002 -9468 -0822] , Nikolaos Livathinos [0000 -0001 -8513 -3491] , Christoph Auer [0000 -0001 -5761 -0422] , [0000 -0002 -8088 -0823] +Maksym Lysak [0000 - 0002 - 3723 - 6960] , Ahmed Nassar [0000 - 0002 - 9468 - 0822] , Nikolaos Livathinos [0000 - 0001 - 8513 - 3491] , Christoph Auer [0000 - 0001 - 5761 - 0422] , [0000 - 0002 - 8088 - 0823] and Peter Staar @@ -66,11 +66,11 @@ In Figure 3, we illustrate how the OTSL is defined. In essence, the OTSL defines The OTSL vocabulary is comprised of the following tokens: -- -"C" cell a new table cell that either has or does not have cell content -- -"L" cell left-looking cell , merging with the left neighbor cell to create a span -- -"U" cell up-looking cell , merging with the upper neighbor cell to create a span -- -"X" cell cross cell , to merge with both left and upper neighbor cells -- -"NL" new-line , switch to the next row. +- "C" cell - a new table cell that either has or does not have cell content +- "L" cell - left-looking cell , merging with the left neighbor cell to create a span +- "U" cell - up-looking cell , merging with the upper neighbor cell to create a span +- "X" cell - cross cell , to merge with both left and upper neighbor cells +- "NL" - new-line , switch to the next row. A notable attribute of OTSL is that it has the capability of achieving lossless conversion to HTML. diff --git a/tests/data/pdf/groundtruth/redp5110_sampled.doctags.txt b/tests/data/pdf/groundtruth/redp5110_sampled.doctags.txt index 74348ef82e..d3670419b2 100644 --- a/tests/data/pdf/groundtruth/redp5110_sampled.doctags.txt +++ b/tests/data/pdf/groundtruth/redp5110_sampled.doctags.txt @@ -148,10 +148,10 @@ <_unknown_>RETURN CASE WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'HR', 'EMP' ) = 1 THEN EMPLOYEES . DATE_OF_BIRTH WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'MGR' ) = 1 AND SESSION_USER = EMPLOYEES . USER_ID THEN EMPLOYEES . DATE_OF_BIRTH WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'MGR' ) = 1 AND SESSION_USER <> EMPLOYEES . USER_ID THEN ( 9999 || '-' || MONTH ( EMPLOYEES . DATE_OF_BIRTH ) || '-' || DAY (EMPLOYEES.DATE_OF_BIRTH )) ELSE NULL END ENABLE ; The other column to mask in this example is the TAX_ID information. In this example, the rules to enforce include the following ones: --Human Resources can see the unmasked TAX_ID of the employees. --Employees can see only their own unmasked TAX_ID. --Managers see a masked version of TAX_ID with the first five characters replaced with the X character (for example, XXX-XX-1234). --Any other person sees the entire TAX_ID as masked, for example, XXX-XX-XXXX. +Human Resources can see the unmasked TAX_ID of the employees. +Employees can see only their own unmasked TAX_ID. +Managers see a masked version of TAX_ID with the first five characters replaced with the X character (for example, XXX-XX-1234). +Any other person sees the entire TAX_ID as masked, for example, XXX-XX-XXXX. To implement this column mask, run the SQL statement that is shown in Example 3-9. <_unknown_>CREATE MASK HR_SCHEMA.MASK_TAX_ID_ON_EMPLOYEES ON HR_SCHEMA.EMPLOYEES AS EMPLOYEES FOR COLUMN TAX_ID RETURN CASE WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'HR' ) = 1 THEN EMPLOYEES . TAX_ID WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'MGR' ) = 1 AND SESSION_USER = EMPLOYEES . USER_ID THEN EMPLOYEES . TAX_ID WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'MGR' ) = 1 AND SESSION_USER <> EMPLOYEES . USER_ID THEN ( 'XXX-XX-' CONCAT QSYS2 . SUBSTR ( EMPLOYEES . TAX_ID , 8 , 4 ) ) WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'EMP' ) = 1 THEN EMPLOYEES . TAX_ID ELSE 'XXX-XX-XXXX' END ENABLE ;Example 3-9 Creating a mask on the TAX_ID column diff --git a/tests/data/pdf/groundtruth/redp5110_sampled.json b/tests/data/pdf/groundtruth/redp5110_sampled.json index e2ef7e0da8..5f50ee9cb8 100644 --- a/tests/data/pdf/groundtruth/redp5110_sampled.json +++ b/tests/data/pdf/groundtruth/redp5110_sampled.json @@ -5857,14 +5857,14 @@ }, "charspan": [ 0, - 62 + 63 ] } ], - "orig": "-Human Resources can see the unmasked TAX_ID of the employees.", - "text": "-Human Resources can see the unmasked TAX_ID of the employees.", + "orig": "- Human Resources can see the unmasked TAX_ID of the employees.", + "text": "Human Resources can see the unmasked TAX_ID of the employees.", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/189", @@ -5886,14 +5886,14 @@ }, "charspan": [ 0, - 50 + 51 ] } ], - "orig": "-Employees can see only their own unmasked TAX_ID.", - "text": "-Employees can see only their own unmasked TAX_ID.", + "orig": "- Employees can see only their own unmasked TAX_ID.", + "text": "Employees can see only their own unmasked TAX_ID.", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/190", @@ -5915,14 +5915,14 @@ }, "charspan": [ 0, - 129 + 130 ] } ], - "orig": "-Managers see a masked version of TAX_ID with the first five characters replaced with the X character (for example, XXX-XX-1234).", - "text": "-Managers see a masked version of TAX_ID with the first five characters replaced with the X character (for example, XXX-XX-1234).", + "orig": "- Managers see a masked version of TAX_ID with the first five characters replaced with the X character (for example, XXX-XX-1234).", + "text": "Managers see a masked version of TAX_ID with the first five characters replaced with the X character (for example, XXX-XX-1234).", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/191", @@ -5944,14 +5944,14 @@ }, "charspan": [ 0, - 77 + 78 ] } ], - "orig": "-Any other person sees the entire TAX_ID as masked, for example, XXX-XX-XXXX.", - "text": "-Any other person sees the entire TAX_ID as masked, for example, XXX-XX-XXXX.", + "orig": "- Any other person sees the entire TAX_ID as masked, for example, XXX-XX-XXXX.", + "text": "Any other person sees the entire TAX_ID as masked, for example, XXX-XX-XXXX.", "enumerated": false, - "marker": "" + "marker": "-" }, { "self_ref": "#/texts/192", @@ -8838,7 +8838,7 @@ { "bbox": { "l": 134.71, - "t": 456.71, + "t": 456.7, "r": 502.91, "b": 468.32, "coord_origin": "TOPLEFT" @@ -9237,8 +9237,8 @@ }, { "bbox": { - "l": 135.85, - "t": 605.14, + "l": 135.84, + "t": 605.15, "r": 502.86, "b": 627.85, "coord_origin": "TOPLEFT" @@ -9298,9 +9298,9 @@ { "bbox": { "l": 534.38, - "t": 608.87, + "t": 608.89, "r": 547.1, - "b": 619.7, + "b": 619.72, "coord_origin": "TOPLEFT" }, "row_span": 1, @@ -9357,7 +9357,7 @@ }, { "bbox": { - "l": 147.49, + "l": 147.5, "t": 640.81, "r": 502.86, "b": 652.87, @@ -9418,9 +9418,9 @@ { "bbox": { "l": 533.91, - "t": 650.39, + "t": 650.41, "r": 547.12, - "b": 661.17, + "b": 661.18, "coord_origin": "TOPLEFT" }, "row_span": 1, @@ -9478,7 +9478,7 @@ { "bbox": { "l": 534.15, - "t": 665.39, + "t": 665.38, "r": 547.01, "b": 676.28, "coord_origin": "TOPLEFT" @@ -9518,9 +9518,9 @@ { "bbox": { "l": 534.03, - "t": 675.59, + "t": 675.6, "r": 546.99, - "b": 686.41, + "b": 686.43, "coord_origin": "TOPLEFT" }, "row_span": 1, @@ -10570,7 +10570,7 @@ { "bbox": { "l": 134.71, - "t": 456.71, + "t": 456.7, "r": 502.91, "b": 468.32, "coord_origin": "TOPLEFT" @@ -11031,8 +11031,8 @@ [ { "bbox": { - "l": 135.85, - "t": 605.14, + "l": 135.84, + "t": 605.15, "r": 502.86, "b": 627.85, "coord_origin": "TOPLEFT" @@ -11052,9 +11052,9 @@ { "bbox": { "l": 534.38, - "t": 608.87, + "t": 608.89, "r": 547.1, - "b": 619.7, + "b": 619.72, "coord_origin": "TOPLEFT" }, "row_span": 1, @@ -11115,7 +11115,7 @@ [ { "bbox": { - "l": 147.49, + "l": 147.5, "t": 640.81, "r": 502.86, "b": 652.87, @@ -11178,9 +11178,9 @@ { "bbox": { "l": 533.91, - "t": 650.39, + "t": 650.41, "r": 547.12, - "b": 661.17, + "b": 661.18, "coord_origin": "TOPLEFT" }, "row_span": 1, @@ -11220,7 +11220,7 @@ { "bbox": { "l": 534.15, - "t": 665.39, + "t": 665.38, "r": 547.01, "b": 676.28, "coord_origin": "TOPLEFT" @@ -11262,9 +11262,9 @@ { "bbox": { "l": 534.03, - "t": 675.59, + "t": 675.6, "r": 546.99, - "b": 686.41, + "b": 686.43, "coord_origin": "TOPLEFT" }, "row_span": 1, diff --git a/tests/data/pdf/groundtruth/redp5110_sampled.md b/tests/data/pdf/groundtruth/redp5110_sampled.md index ac9bebb218..2fc954a49a 100644 --- a/tests/data/pdf/groundtruth/redp5110_sampled.md +++ b/tests/data/pdf/groundtruth/redp5110_sampled.md @@ -310,10 +310,10 @@ RETURN CASE WHEN VERIFY_GROUP_FOR_USER ( SESSION_USER , 'HR', 'EMP' ) = 1 THEN E ``` 2. The other column to mask in this example is the TAX\_ID information. In this example, the rules to enforce include the following ones: -2. -Human Resources can see the unmasked TAX\_ID of the employees. -3. -Employees can see only their own unmasked TAX\_ID. -4. -Managers see a masked version of TAX\_ID with the first five characters replaced with the X character (for example, XXX-XX-1234). -5. -Any other person sees the entire TAX\_ID as masked, for example, XXX-XX-XXXX. +- Human Resources can see the unmasked TAX\_ID of the employees. +- Employees can see only their own unmasked TAX\_ID. +- Managers see a masked version of TAX\_ID with the first five characters replaced with the X character (for example, XXX-XX-1234). +- Any other person sees the entire TAX\_ID as masked, for example, XXX-XX-XXXX. To implement this column mask, run the SQL statement that is shown in Example 3-9. diff --git a/tests/test_page_assemble_model.py b/tests/test_page_assemble_model.py index 8df3779f1a..6f293c4bb6 100644 --- a/tests/test_page_assemble_model.py +++ b/tests/test_page_assemble_model.py @@ -141,6 +141,51 @@ def test_regex_matches_new_codepoints(self, model): assert _LIGATURE_RE.search("\uf0a0") is not None, "U+F0A0 not matched by regex" +class TestSanitizeTextHyphenation: + """Tests for end-of-line hyphen handling in sanitize_text().""" + + @pytest.mark.parametrize( + ("lines", "expected"), + [ + (["algo-", "rithms"], "algorithms"), + (["the veri-", "fication step"], "the verification step"), + ], + ) + def test_attached_hyphen_joins_split_word(self, model, lines, expected): + """A hyphen attached to the previous word marks a word split and is dropped.""" + assert model.sanitize_text(list(lines)) == expected + + def test_attached_hyphen_on_non_alnum_token_is_kept(self, model): + """An underscore-bearing token is not treated as a split word (unchanged).""" + assert model.sanitize_text(["foo_bar-", "baz"]) == "foo_bar-baz" + + @pytest.mark.parametrize( + ("lines", "expected"), + [ + # Wrapped CLI flag, as reported: the dash must not be swallowed. + ( + ["gsh create_diameter_peer -ip 10.0.8.72 -pn 1 -", "prio 3 -dh mme"], + "gsh create_diameter_peer -ip 10.0.8.72 -pn 1 - prio 3 -dh mme", + ), + # Separator dash between title parts. + ( + ["editors, Computer Vision -", "ECCV 2020 , pages 564-580"], + "editors, Computer Vision - ECCV 2020 , pages 564-580", + ), + # Bullet marker isolated on its own line. + ( + ["-", "Human Resources can see the unmasked TAX_ID"], + "- Human Resources can see the unmasked TAX_ID", + ), + # Wrapped negative number: the sign must survive. + (["cost is -", "5.00 USD"], "cost is - 5.00 USD"), + ], + ) + def test_detached_hyphen_is_preserved(self, model, lines, expected): + """A hyphen following whitespace is a literal character, not a word split.""" + assert model.sanitize_text(list(lines)) == expected + + def _make_page(hyperlinks: list[PdfHyperlink], page_height: float = 100.0) -> Page: """Create a Page with mocked parsed_page carrying the given hyperlinks.""" page = Page(page_no=0, size=Size(width=200, height=page_height))