fix(pdf): keep a line-final hyphen that belongs to the word

docling/models/stages/page_assemble/page_assemble_model.py

@@ -124,7 +124,10 @@ def sanitize_text(self, lines: List[str]) -> str:
         attached to the word it splits; one that follows whitespace is a
         literal character -- a separator dash, a bullet marker, or a wrapped
         hyphen-prefixed token -- so it is kept and its lines are joined like
-        any other line break.
+        any other line break. An attached hyphen is also kept where no
+        hyphenation rule could have placed it: after a single-character token
+        ("n-" / "octanal") or before a continuation opening on a digit
+        ("6-methyl-" / "5-hepten-2-one"). There the hyphen belongs to the word.
 
         The joined text is then normalized: a few typographic characters are
         replaced by ASCII equivalents and ligatures are expanded.
@@ -146,7 +149,9 @@ def sanitize_text(self, lines: List[str]) -> str:
                     and len(prev_words)
                     and len(line_words)
                     and prev_words[-1].isalnum()
+                    and len(prev_words[-1]) > 1
                     and line_words[0].isalnum()
+                    and not line_words[0][0].isdigit()
                 ):
                     lines[ix] = prev_line[:-1]
                 elif not hyphen_attached_to_word:

docling/models/stages/reading_order/readingorder_model.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: MIT
 
 import logging
+import re
 import time
 from collections import defaultdict
 from collections.abc import Iterator
@@ -71,6 +72,7 @@ class ReadingOrderOptions(BaseModel):
 
 class ReadingOrderModel:
     _RICH_CELL_PICTURE_COVERAGE_THRESHOLD = 0.8
+    _TOKEN_BEFORE_HYPHEN_RE = re.compile(r"(\w+)-$")
 
     def __init__(self, options: ReadingOrderOptions):
         self.options = options
@@ -1066,8 +1068,15 @@ def _merge_elements(self, element, merged_elem, new_item, page_height):
             bbox=merged_elem.cluster.bbox.to_bottom_left_origin(page_height),
         )
         continuation_text = merged_elem.text.lstrip()
+        token_before_hyphen = self._TOKEN_BEFORE_HYPHEN_RE.search(new_item.text)
+        # A hyphen no hyphenation rule could have placed -- after a
+        # single-character token, or before a digit -- belongs to the word.
+        hyphen_belongs_to_word = token_before_hyphen is not None and (
+            len(token_before_hyphen.group(1)) == 1 or continuation_text[:1].isdigit()
+        )
         if new_item.text.endswith("\u00ad") or (
             new_item.text.endswith("-")
+            and not hyphen_belongs_to_word
             and continuation_text
             and continuation_text[0].islower()
         ):
@@ -1076,6 +1085,9 @@ def _merge_elements(self, element, merged_elem, new_item, page_height):
             new_item.orig = (
                 new_item.orig[:-1] + merged_elem.text
             )  # TODO: This is incomplete, we don't have the `orig` field of the merged element.
+        elif hyphen_belongs_to_word:
+            new_item.text += continuation_text
+            new_item.orig += continuation_text
         else:
             new_item.text += f" {merged_elem.text}"
             new_item.orig += f" {merged_elem.text}"  # TODO: This is incomplete, we don't have the `orig` field of the merged element.

tests/data/pdf/groundtruth/2203.01017v2.json

@@ -10800,12 +10800,12 @@
           },
           "charspan": [
             0,
-            301
+            302
           ]
         }
       ],
-      "orig": "[26] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 11621167, 2017. 1",
-      "text": "Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 11621167, 2017. 1",
+      "orig": "[26] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 1162-1167, 2017. 1",
+      "text": "Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 1162-1167, 2017. 1",
       "enumerated": true,
       "marker": "[26]"
     },
@@ -10858,12 +10858,12 @@
           },
           "charspan": [
             0,
-            182
+            183
           ]
         }
       ],
-      "orig": "[28] Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 6572, 2010. 2",
-      "text": "Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 6572, 2010. 2",
+      "orig": "[28] Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 65-72, 2010. 2",
+      "text": "Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 65-72, 2010. 2",
       "enumerated": true,
       "marker": "[28]"
     },
@@ -13569,12 +13569,12 @@
           },
           "charspan": [
             0,
-            96
+            97
           ]
         }
       ],
-      "orig": "where c is one of {left, centroid, right} and xc is the xcoordinate for the corresponding point.",
-      "text": "where c is one of {left, centroid, right} and xc is the xcoordinate for the corresponding point."
+      "orig": "where c is one of {left, centroid, right} and xc is the x-coordinate for the corresponding point.",
+      "text": "where c is one of {left, centroid, right} and xc is the x-coordinate for the corresponding point."
     },
     {
       "self_ref": "#/texts/461",

tests/data/pdf/groundtruth/2203.01017v2.md

@@ -262,9 +262,9 @@ end object detection with transformers. In Andrea Vedaldi, Horst Bischof, Thomas
 
 Computer Vision and Pattern Recognition, pages 658-666, 2019. 6
 
-- [26] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 11621167, 2017. 1
+- [26] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR International Conference on Document Analysis and Recognition (ICDAR), volume 01, pages 1162-1167, 2017. 1
 - [27] Sebastian Schreiber, Stefan Agne, Ivo Wolf, Andreas Dengel, and Sheraz Ahmed. Deepdesrt: Deep learning for detection and structure recognition of tables in document images. In 2017 14th IAPR international conference on document analysis and recognition (ICDAR), volume 1, pages 1162-1167. IEEE, 2017. 3
-- [28] Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 6572, 2010. 2
+- [28] Faisal Shafait and Ray Smith. Table detection in heterogeneous documents. In Proceedings of the 9th IAPR International Workshop on Document Analysis Systems, pages 65-72, 2010. 2
 - [29] Shoaib Ahmed Siddiqui, Imran Ali Fateh, Syed Tahseen Raza Rizvi, Andreas Dengel, and Sheraz Ahmed. Deeptabstr: Deep learning based table structure recognition. In 2019 International Conference on Document Analysis and Recognition (ICDAR), pages 1403-1409. IEEE, 2019. 3
 - [30] Peter W J Staar, Michele Dolfi, Christoph Auer, and Costas Bekas. Corpus conversion service: A machine learning platform to ingest documents at scale. In Proceedings of the 24th ACM SIGKDD, KDD '18, pages 774-782, New York, NY, USA, 2018. ACM. 1
 - [31] Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Ł ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. V. Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett, editors, Advances in Neural Information Processing Systems 30, pages 5998-6008. Curran Associates, Inc., 2017. 5
@@ -328,7 +328,7 @@ Here is a step-by-step description of the prediction postprocessing:
 
 <!-- formula-not-decoded -->
 
-where c is one of {left, centroid, right} and xc is the xcoordinate for the corresponding point.
+where c is one of {left, centroid, right} and xc is the x-coordinate for the corresponding point.
 
 5. Use the alignment computed in step 4, to compute the median x-coordinate for all table columns and the me-
 

tests/data/pdf/groundtruth/2206.01062.json

@@ -13074,12 +13074,12 @@
           },
           "charspan": [
             0,
-            337
+            338
           ]
         }
       ],
-      "orig": "[18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 1513715145, feb 2021.",
-      "text": "Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 1513715145, feb 2021.",
+      "orig": "[18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 15137-15145, feb 2021.",
+      "text": "Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 15137-15145, feb 2021.",
       "enumerated": true,
       "marker": "[18]"
     },

tests/data/pdf/groundtruth/2206.01062.md

@@ -283,7 +283,7 @@ Diaconu, Mai Thanh Minh, Marc, albinxavi, fatih, oleg, and wanghao yang. ultraly
 - [15] Mingxing Tan, Ruoming Pang, and Quoc V. Le. Efficientdet: Scalable and efficient object detection. CoRR, abs/1911.09070, 2019.
 - [16] Tsung-Yi Lin, Michael Maire, Serge J. Belongie, Lubomir D. Bourdev, Ross B. Girshick, James Hays, Pietro Perona, Deva Ramanan, Piotr Dollár, and C. Lawrence Zitnick. Microsoft COCO: common objects in context, 2014.
 - [17] Yuxin Wu, Alexander Kirillov, Francisco Massa, Wan-Yen Lo, and Ross Girshick. Detectron2, 2019.
-- [18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 1513715145, feb 2021.
+- [18] Nikolaos Livathinos, Cesar Berrospi, MaksymLysak, Viktor Kuropiatnyk, Ahmed Nassar, Andre Carvalho, Michele Dolfi, Christoph Auer, Kasper Dinkla, and Peter W. J. Staar. Robust pdf document conversion using recurrent neural networks. In Proceedings of the 35th Conference on Artificial Intelligence, AAAI, pages 15137-15145, feb 2021.
 - [19] Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, and Ming Zhou. Layoutlm: Pre-training of text and layout for document image understanding. In Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD, pages 1192-1200, New York, USA, 2020. Association for Computing Machinery.
 - [20] Shoubin Li, Xuyan Ma, Shuaiqun Pan, Jun Hu, Lin Shi, and Qing Wang. Vtlayout: Fusion of visual and text features for document layout analysis, 2021.
 - [21] Peng Zhang, Can Li, Liang Qiao, Zhanzhan Cheng, Shiliang Pu, Yi Niu, and Fei Wu. Vsr: A unified framework for document layout analysis combining vision, semantics and relations, 2021.

tests/data/pdf/groundtruth/2305.03393v1.json

@@ -9717,12 +9717,12 @@
           },
           "charspan": [
             0,
-            301
+            302
           ]
         }
       ],
-      "orig": "2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Forn\u00e9s, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545561. Springer International Publishing, Cham (2022)",
-      "text": "Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Forn\u00e9s, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545561. Springer International Publishing, Cham (2022)",
+      "orig": "2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Forn\u00e9s, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545-561. Springer International Publishing, Cham (2022)",
+      "text": "Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Forn\u00e9s, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545-561. Springer International Publishing, Cham (2022)",
       "enumerated": true,
       "marker": "2."
     },
@@ -9887,12 +9887,12 @@
           },
           "charspan": [
             0,
-            264
+            265
           ]
         }
       ],
-      "orig": "6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 18681873. IEEE (2022)",
-      "text": "Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 18681873. IEEE (2022)",
+      "orig": "6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 1868-1873. IEEE (2022)",
+      "text": "Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 1868-1873. IEEE (2022)",
       "enumerated": true,
       "marker": "6."
     },

tests/data/pdf/groundtruth/2305.03393v1.md

@@ -166,12 +166,12 @@ Secondly, OTSL has more inherent structure and a significantly restricted vocabu
 ## References
 
 1. Auer, C., Dolfi, M., Carvalho, A., Ramis, C.B., Staar, P.W.J.: Delivering document conversion as a cloud service with high throughput and responsiveness. CoRR abs/2206.00785 (2022). https://doi.org/10.48550/arXiv.2206.00785, https://doi.org/10.48550/arXiv.2206.00785
-2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Fornés, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545561. Springer International Publishing, Cham (2022)
+2. Chen, B., Peng, D., Zhang, J., Ren, Y., Jin, L.: Complex table structure recognition in the wild using transformer and identity matrix-based augmentation. In: Porwal, U., Fornés, A., Shafait, F. (eds.) Frontiers in Handwriting Recognition. pp. 545-561. Springer International Publishing, Cham (2022)
 3. Chi, Z., Huang, H., Xu, H.D., Yu, H., Yin, W., Mao, X.L.: Complicated table structure recognition. arXiv preprint arXiv:1908.04729 (2019)
 4. Deng, Y., Rosenberg, D., Mann, G.: Challenges in end-to-end neural scientific table recognition. In: 2019 International Conference on Document Analysis and Recognition (ICDAR). pp. 894-901. IEEE (2019)
 
 5. Kayal, P., Anand, M., Desai, H., Singh, M.: Tables to latex: structure and content extraction from scientific tables. International Journal on Document Analysis and Recognition (IJDAR) pp. 1-10 (2022)
-6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 18681873. IEEE (2022)
+6. Lee, E., Kwon, J., Yang, H., Park, J., Lee, S., Koo, H.I., Cho, N.I.: Table structure recognition based on grid shape graph. In: 2022 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). pp. 1868-1873. IEEE (2022)
 7. Li, M., Cui, L., Huang, S., Wei, F., Zhou, M., Li, Z.: Tablebank: A benchmark dataset for table detection and recognition (2019)
 8. Livathinos, N., Berrospi, C., Lysak, M., Kuropiatnyk, V., Nassar, A., Carvalho, A., Dolfi, M., Auer, C., Dinkla, K., Staar, P.: Robust pdf document conversion using recurrent neural networks. Proceedings of the AAAI Conference on Artificial Intelligence 35(17), 15137-15145 (May 2021), https://ojs.aaai.org/index.php/ AAAI/article/view/17777
 9. Nassar, A., Livathinos, N., Lysak, M., Staar, P.: Tableformer: Table structure understanding with transformers. In: Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR). pp. 4614-4623 (June 2022)

tests/data/pdf/groundtruth/newspaper-00.json

@@ -1668,12 +1668,12 @@
           },
           "charspan": [
             0,
-            218
+            219
           ]
         }
       ],
-      "orig": "\u00abIn unseren Pausen r\u00e4umen wird sehr oft 20 Minuten gelesen pr\u00fcfen wir gerade, o den QR-Code f\u00fcr die lineausgabe an den W\u00e4nden aufh\u00e4ngen sollen, damit alle Mitarbeitenden Zugang haben\u00bb. Anna Hug, nft . Hier ob wir e Onn",
-      "text": "\u00abIn unseren Pausen r\u00e4umen wird sehr oft 20 Minuten gelesen pr\u00fcfen wir gerade, o den QR-Code f\u00fcr die lineausgabe an den W\u00e4nden aufh\u00e4ngen sollen, damit alle Mitarbeitenden Zugang haben\u00bb. Anna Hug, nft . Hier ob wir e Onn"
+      "orig": "\u00abIn unseren Pausen r\u00e4umen wird sehr oft 20 Minuten gelesen pr\u00fcfen wir gerade, o den QR-Code f\u00fcr die lineausgabe an den W\u00e4nden aufh\u00e4ngen sollen, damit alle Mitarbeitenden Zugang haben\u00bb. Anna Hug, n-ft . Hier ob wir e Onn",
+      "text": "\u00abIn unseren Pausen r\u00e4umen wird sehr oft 20 Minuten gelesen pr\u00fcfen wir gerade, o den QR-Code f\u00fcr die lineausgabe an den W\u00e4nden aufh\u00e4ngen sollen, damit alle Mitarbeitenden Zugang haben\u00bb. Anna Hug, n-ft . Hier ob wir e Onn"
     },
     {
       "self_ref": "#/texts/52",

tests/data/pdf/groundtruth/newspaper-00.md

@@ -100,7 +100,7 @@ Dass 20 Minuten einen gesellschaftlichen Mehrwert schafft: das Publikum gut info
 
 DÉSIRÉE POMPER
 
-«In unseren Pausen räumen wird sehr oft 20 Minuten gelesen prüfen wir gerade, o den QR-Code für die lineausgabe an den Wänden aufhängen sollen, damit alle Mitarbeitenden Zugang haben». Anna Hug, nft . Hier ob wir e Onn
+«In unseren Pausen räumen wird sehr oft 20 Minuten gelesen prüfen wir gerade, o den QR-Code für die lineausgabe an den Wänden aufhängen sollen, damit alle Mitarbeitenden Zugang haben». Anna Hug, n-ft . Hier ob wir e Onn
 
 Co-Geschäftsleiterin Hug AG