diff --git a/docling/models/stages/page_assemble/page_assemble_model.py b/docling/models/stages/page_assemble/page_assemble_model.py index b139d68c47..2e608a2917 100644 --- a/docling/models/stages/page_assemble/page_assemble_model.py +++ b/docling/models/stages/page_assemble/page_assemble_model.py @@ -124,7 +124,10 @@ def sanitize_text(self, lines: List[str]) -> str: attached to the word it splits; one that follows whitespace is a literal character -- a separator dash, a bullet marker, or a wrapped hyphen-prefixed token -- so it is kept and its lines are joined like - any other line break. + any other line break. An attached hyphen is also kept where no + hyphenation rule could have placed it: after a single-character token + ("n-" / "octanal") or before a continuation opening on a digit + ("6-methyl-" / "5-hepten-2-one"). There the hyphen belongs to the word. The joined text is then normalized: a few typographic characters are replaced by ASCII equivalents and ligatures are expanded. @@ -146,7 +149,9 @@ def sanitize_text(self, lines: List[str]) -> str: and len(prev_words) and len(line_words) and prev_words[-1].isalnum() + and len(prev_words[-1]) > 1 and line_words[0].isalnum() + and not line_words[0][0].isdigit() ): lines[ix] = prev_line[:-1] elif not hyphen_attached_to_word: diff --git a/docling/models/stages/reading_order/readingorder_model.py b/docling/models/stages/reading_order/readingorder_model.py index 51479b1631..7aae048e04 100644 --- a/docling/models/stages/reading_order/readingorder_model.py +++ b/docling/models/stages/reading_order/readingorder_model.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MIT import logging +import re import time from collections import defaultdict from collections.abc import Iterator @@ -71,6 +72,7 @@ class ReadingOrderOptions(BaseModel): class ReadingOrderModel: _RICH_CELL_PICTURE_COVERAGE_THRESHOLD = 0.8 + _TOKEN_BEFORE_HYPHEN_RE = re.compile(r"(\w+)-$") def __init__(self, options: ReadingOrderOptions): self.options = options @@ -1066,8 +1068,15 @@ def _merge_elements(self, element, merged_elem, new_item, page_height): bbox=merged_elem.cluster.bbox.to_bottom_left_origin(page_height), ) continuation_text = merged_elem.text.lstrip() + token_before_hyphen = self._TOKEN_BEFORE_HYPHEN_RE.search(new_item.text) + # A hyphen no hyphenation rule could have placed -- after a + # single-character token, or before a digit -- belongs to the word. + hyphen_belongs_to_word = token_before_hyphen is not None and ( + len(token_before_hyphen.group(1)) == 1 or continuation_text[:1].isdigit() + ) if new_item.text.endswith("\u00ad") or ( new_item.text.endswith("-") + and not hyphen_belongs_to_word and continuation_text and continuation_text[0].islower() ): @@ -1076,6 +1085,9 @@ def _merge_elements(self, element, merged_elem, new_item, page_height): new_item.orig = ( new_item.orig[:-1] + merged_elem.text ) # TODO: This is incomplete, we don't have the `orig` field of the merged element. + elif hyphen_belongs_to_word: + new_item.text += continuation_text + new_item.orig += continuation_text else: new_item.text += f" {merged_elem.text}" new_item.orig += f" {merged_elem.text}" # TODO: This is incomplete, we don't have the `orig` field of the merged element. diff --git a/tests/data/pdf/groundtruth/2203.01017v2.json b/tests/data/pdf/groundtruth/2203.01017v2.json index 19f9f98d5f..1402f7e0b0 100644 --- a/tests/data/pdf/groundtruth/2203.01017v2.json +++ b/tests/data/pdf/groundtruth/2203.01017v2.json @@ -10800,12 +10800,12 @@ }, "charspan": [ 0, - 301 + 302 ] } ], - "orig": "[26] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 11621167, 2017. 1", - "text": "Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 11621167, 2017. 1", + "orig": "[26] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 1162-1167, 2017. 1", + "text": "Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 1162-1167, 2017. 1", "enumerated": true, "marker": "[26]" }, @@ -10858,12 +10858,12 @@ }, "charspan": [ 0, - 182 + 183 ] } ], - "orig": "[28] Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 6572, 2010. 2", - "text": "Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 6572, 2010. 2", + "orig": "[28] Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 65-72, 2010. 2", + "text": "Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 65-72, 2010. 2", "enumerated": true, "marker": "[28]" }, @@ -13569,12 +13569,12 @@ }, "charspan": [ 0, - 96 + 97 ] } ], - "orig": "where c is one of {left, centroid, right} and xc is the xcoordinate for the corresponding point.", - "text": "where c is one of {left, centroid, right} and xc is the xcoordinate for the corresponding point." + "orig": "where c is one of {left, centroid, right} and xc is the x-coordinate for the corresponding point.", + "text": "where c is one of {left, centroid, right} and xc is the x-coordinate for the corresponding point." }, { "self_ref": "#/texts/461", diff --git a/tests/data/pdf/groundtruth/2203.01017v2.md b/tests/data/pdf/groundtruth/2203.01017v2.md index 5803ddcc4f..832dda1526 100644 --- a/tests/data/pdf/groundtruth/2203.01017v2.md +++ b/tests/data/pdf/groundtruth/2203.01017v2.md @@ -262,9 +262,9 @@ end object detection with transformers. In Andrea Vedaldi, Horst Bischof, Thomas Computer Vision and Pattern Recognition, pages 658-666, 2019. 6 -- [26] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 11621167, 2017. 1 +- [26] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 1162-1167, 2017. 1 - [27] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR international conference on document analysis and recognition (ICDAR), volume 1, pages 1162-1167. IEEE, 2017. 3 -- [28] Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 6572, 2010. 2 +- [28] Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 65-72, 2010. 2 - [29] Shoaib Ahmed Siddiqui, Imran Ali Fateh, Syed Tahseen Raza Rizvi, Andreas Dengel, and Sheraz Ahmed. Deeptabstr: Deep learning based table structure recognition. In 2019 International Conference on Document Analysis and Recognition (ICDAR), pages 1403-1409. IEEE, 2019. 3 - [30] Peter W J Staar, Michele Dolfi, Christoph Auer, and Costas Bekas. Corpus conversion service: A machine learning platform to ingest documents at scale. In Proceedings of the 24th ACM SIGKDD, KDD '18, pages 774-782, New York, NY, USA, 2018. ACM. 1 - [31] Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Ł ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. V. Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett, editors, Advances in Neural Information Processing Systems 30, pages 5998-6008. Curran Associates, Inc., 2017. 5 @@ -328,7 +328,7 @@ Here is a step-by-step description of the prediction postprocessing: -where c is one of {left, centroid, right} and xc is the xcoordinate for the corresponding point. +where c is one of {left, centroid, right} and xc is the x-coordinate for the corresponding point. 5. Use the alignment computed in step 4, to compute the median x-coordinate for all table columns and the me- diff --git a/tests/data/pdf/groundtruth/2206.01062.json b/tests/data/pdf/groundtruth/2206.01062.json index 10eb5bb5d2..5debf2e10c 100644 --- a/tests/data/pdf/groundtruth/2206.01062.json +++ b/tests/data/pdf/groundtruth/2206.01062.json @@ -13074,12 +13074,12 @@ }, "charspan": [ 0, - 337 + 338 ] } ], - "orig": "[18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 1513715145, feb 2021.", - "text": "Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 1513715145, feb 2021.", + "orig": "[18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 15137-15145, feb 2021.", + "text": "Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 15137-15145, feb 2021.", "enumerated": true, "marker": "[18]" }, diff --git a/tests/data/pdf/groundtruth/2206.01062.md b/tests/data/pdf/groundtruth/2206.01062.md index 84ceda6dd2..940e7b996d 100644 --- a/tests/data/pdf/groundtruth/2206.01062.md +++ b/tests/data/pdf/groundtruth/2206.01062.md @@ -283,7 +283,7 @@ Diaconu, Mai Thanh Minh, Marc, albinxavi, fatih, oleg, and wanghao yang. ultraly - [15] Mingxing Tan, Ruoming Pang, and Quoc V. Le. Efficientdet: Scalable and efficient object detection. CoRR, abs/1911.09070, 2019. - [16] Tsung-Yi Lin, Michael Maire, Serge J. Belongie, Lubomir D. Bourdev, Ross B. Girshick, James Hays, Pietro Perona, Deva Ramanan, Piotr Dollár, and C. Lawrence Zitnick. Microsoft COCO: common objects in context, 2014. - [17] Yuxin Wu, Alexander Kirillov, Francisco Massa, Wan-Yen Lo, and Ross Girshick. Detectron2, 2019. -- [18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 1513715145, feb 2021. +- [18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 15137-15145, feb 2021. - [19] Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, and Ming Zhou. Layoutlm: Pre-training of text and layout for document image understanding. In Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD, pages 1192-1200, New York, USA, 2020. Association for Computing Machinery. - [20] Shoubin Li, Xuyan Ma, Shuaiqun Pan, Jun Hu, Lin Shi, and Qing Wang. Vtlayout: Fusion of visual and text features for document layout analysis, 2021. - [21] Peng Zhang, Can Li, Liang Qiao, Zhanzhan Cheng, Shiliang Pu, Yi Niu, and Fei Wu. Vsr: A unified framework for document layout analysis combining vision, semantics and relations, 2021. diff --git a/tests/data/pdf/groundtruth/2305.03393v1.json b/tests/data/pdf/groundtruth/2305.03393v1.json index dbc9f23896..f98522c4b4 100644 --- a/tests/data/pdf/groundtruth/2305.03393v1.json +++ b/tests/data/pdf/groundtruth/2305.03393v1.json @@ -9717,12 +9717,12 @@ }, "charspan": [ 0, - 301 + 302 ] } ], - "orig": "2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Forn\u00e9s, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545561. Springer International Publishing, Cham (2022)", - "text": "Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Forn\u00e9s, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545561. Springer International Publishing, Cham (2022)", + "orig": "2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Forn\u00e9s, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545-561. Springer International Publishing, Cham (2022)", + "text": "Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Forn\u00e9s, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545-561. Springer International Publishing, Cham (2022)", "enumerated": true, "marker": "2." }, @@ -9887,12 +9887,12 @@ }, "charspan": [ 0, - 264 + 265 ] } ], - "orig": "6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 18681873. IEEE (2022)", - "text": "Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 18681873. IEEE (2022)", + "orig": "6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 1868-1873. IEEE (2022)", + "text": "Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 1868-1873. IEEE (2022)", "enumerated": true, "marker": "6." }, diff --git a/tests/data/pdf/groundtruth/2305.03393v1.md b/tests/data/pdf/groundtruth/2305.03393v1.md index 38906b1dc4..8153564c5e 100644 --- a/tests/data/pdf/groundtruth/2305.03393v1.md +++ b/tests/data/pdf/groundtruth/2305.03393v1.md @@ -166,12 +166,12 @@ Secondly, OTSL has more inherent structure and a significantly restricted vocabu ## References 1. Auer, C., Dolfi, M., Carvalho, A., Ramis, C.B., Staar, P.W.J.: Delivering document conversion as a cloud service with high throughput and responsiveness. CoRR abs/2206.00785 (2022). https://doi.org/10.48550/arXiv.2206.00785, https://doi.org/10.48550/arXiv.2206.00785 -2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Fornés, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545561. Springer International Publishing, Cham (2022) +2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Fornés, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545-561. Springer International Publishing, Cham (2022) 3. Chi, Z., Huang, H., Xu, H.D., Yu, H., Yin, W., Mao, X.L.: Complicated table structure recognition. arXiv preprint arXiv:1908.04729 (2019) 4. Deng, Y., Rosenberg, D., Mann, G.: Challenges in end-to-end neural scientific table recognition. In: 2019 International Conference on Document Analysis and Recognition (ICDAR). pp. 894-901. IEEE (2019) 5. Kayal, P., Anand, M., Desai, H., Singh, M.: Tables to latex: structure and content extraction from scientific tables. International Journal on Document Analysis and Recognition (IJDAR) pp. 1-10 (2022) -6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 18681873. IEEE (2022) +6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 1868-1873. IEEE (2022) 7. Li, M., Cui, L., Huang, S., Wei, F., Zhou, M., Li, Z.: Tablebank: A benchmark dataset for table detection and recognition (2019) 8. Livathinos, N., Berrospi, C., Lysak, M., Kuropiatnyk, V., Nassar, A., Carvalho, A., Dolfi, M., Auer, C., Dinkla, K., Staar, P.: Robust pdf document conversion using recurrent neural networks. Proceedings of the AAAI Conference on Artificial Intelligence 35(17), 15137-15145 (May 2021), https://ojs.aaai.org/index.php/ AAAI/article/view/17777 9. Nassar, A., Livathinos, N., Lysak, M., Staar, P.: Tableformer: Table structure understanding with transformers. In: Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR). pp. 4614-4623 (June 2022) diff --git a/tests/data/pdf/groundtruth/newspaper-00.json b/tests/data/pdf/groundtruth/newspaper-00.json index ad65249dbe..33e5aab0f4 100644 --- a/tests/data/pdf/groundtruth/newspaper-00.json +++ b/tests/data/pdf/groundtruth/newspaper-00.json @@ -1668,12 +1668,12 @@ }, "charspan": [ 0, - 218 + 219 ] } ], - "orig": "\u00abIn unseren Pausen r\u00e4umen wird sehr oft 20 Minuten gelesen pr\u00fcfen wir gerade, o den QR-Code f\u00fcr die lineausgabe an den W\u00e4nden aufh\u00e4ngen sollen, damit alle Mitarbeitenden Zugang haben\u00bb. Anna Hug, nft . Hier ob wir e Onn", - "text": "\u00abIn unseren Pausen r\u00e4umen wird sehr oft 20 Minuten gelesen pr\u00fcfen wir gerade, o den QR-Code f\u00fcr die lineausgabe an den W\u00e4nden aufh\u00e4ngen sollen, damit alle Mitarbeitenden Zugang haben\u00bb. Anna Hug, nft . Hier ob wir e Onn" + "orig": "\u00abIn unseren Pausen r\u00e4umen wird sehr oft 20 Minuten gelesen pr\u00fcfen wir gerade, o den QR-Code f\u00fcr die lineausgabe an den W\u00e4nden aufh\u00e4ngen sollen, damit alle Mitarbeitenden Zugang haben\u00bb. Anna Hug, n-ft . Hier ob wir e Onn", + "text": "\u00abIn unseren Pausen r\u00e4umen wird sehr oft 20 Minuten gelesen pr\u00fcfen wir gerade, o den QR-Code f\u00fcr die lineausgabe an den W\u00e4nden aufh\u00e4ngen sollen, damit alle Mitarbeitenden Zugang haben\u00bb. Anna Hug, n-ft . Hier ob wir e Onn" }, { "self_ref": "#/texts/52", diff --git a/tests/data/pdf/groundtruth/newspaper-00.md b/tests/data/pdf/groundtruth/newspaper-00.md index 67e8b7d87d..5c637fea85 100644 --- a/tests/data/pdf/groundtruth/newspaper-00.md +++ b/tests/data/pdf/groundtruth/newspaper-00.md @@ -100,7 +100,7 @@ Dass 20 Minuten einen gesellschaftlichen Mehrwert schafft: das Publikum gut info DÉSIRÉE POMPER -«In unseren Pausen räumen wird sehr oft 20 Minuten gelesen prüfen wir gerade, o den QR-Code für die lineausgabe an den Wänden aufhängen sollen, damit alle Mitarbeitenden Zugang haben». Anna Hug, nft . Hier ob wir e Onn +«In unseren Pausen räumen wird sehr oft 20 Minuten gelesen prüfen wir gerade, o den QR-Code für die lineausgabe an den Wänden aufhängen sollen, damit alle Mitarbeitenden Zugang haben». Anna Hug, n-ft . Hier ob wir e Onn Co-Geschäftsleiterin Hug AG diff --git a/tests/test_page_assemble_model.py b/tests/test_page_assemble_model.py index 93ef7bb9a8..d99731f102 100644 --- a/tests/test_page_assemble_model.py +++ b/tests/test_page_assemble_model.py @@ -162,6 +162,24 @@ def test_attached_hyphen_on_non_alnum_token_is_kept(self, model): """An underscore-bearing token is not treated as a split word (unchanged).""" assert model.sanitize_text(["foo_bar-", "baz"]) == "foo_bar-baz" + @pytest.mark.parametrize( + ("lines", "expected"), + [ + # Chemical nomenclature, as reported: the prefix is one character. + ( + ["undecanal or n-", "tridecanal, from each blend"], + "undecanal or n-tridecanal, from each blend", + ), + (["i-", "butyric acid"], "i-butyric acid"), + # A locant opens the continuation. + (["6-methyl-", "5-hepten-2-one"], "6-methyl-5-hepten-2-one"), + (["COVID-", "19 cases"], "COVID-19 cases"), + ], + ) + def test_attached_hyphen_belonging_to_word_is_kept(self, model, lines, expected): + """A hyphen no hyphenation rule could have placed is part of the word.""" + assert model.sanitize_text(list(lines)) == expected + @pytest.mark.parametrize( ("lines", "expected"), [ diff --git a/tests/test_readingorder_hyphenated_merges.py b/tests/test_readingorder_hyphenated_merges.py index cde8eadc62..ae9addb420 100644 --- a/tests/test_readingorder_hyphenated_merges.py +++ b/tests/test_readingorder_hyphenated_merges.py @@ -57,6 +57,10 @@ def _element(cluster_id: int, text: str) -> TextElement: ("algo-", "rithms", "algorithms"), ("algo\u00ad", "rithms", "algorithms"), ("algo-", "Rithms", "algo- Rithms"), + ("undecanal or n-", "tridecanal", "undecanal or n-tridecanal"), + ("6-methyl-", "5-hepten-2-one", "6-methyl-5-hepten-2-one"), + ("6-methyl-", " 5-hepten-2-one", "6-methyl-5-hepten-2-one"), + ("cost is -", "5.00 USD", "cost is - 5.00 USD"), ], ) def test_merge_elements_dehyphenates_lowercase_continuations(