feat(pdf): join CJK text lines without a space

docling/models/stages/page_assemble/page_assemble_model.py

@@ -5,7 +5,7 @@
 import re
 from collections.abc import Iterable
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Dict, Final, List, Optional, Union
 
 import numpy as np
 from docling_core.types.doc import BoundingBox
@@ -33,6 +33,33 @@
 
 _log = logging.getLogger(__name__)
 
+# Scripts written without spaces between words: Han ideographs, kana and
+# CJK / full-width punctuation. A line break between two such characters is
+# not a word boundary, so the lines are joined without a space. Hangul is
+# deliberately absent: Korean separates words with spaces.
+_CJK_NO_SPACE_RE = re.compile(
+    "["
+    "\u2e80-\u2fdf"  # CJK radicals, Kangxi radicals
+    "\u3000-\u303f"  # CJK symbols and punctuation
+    "\u3040-\u309f"  # Hiragana
+    "\u30a0-\u30ff"  # Katakana
+    "\u31c0-\u31ef"  # CJK strokes
+    "\u31f0-\u31ff"  # Katakana phonetic extensions
+    "\u3400-\u4dbf"  # CJK Unified Ideographs Extension A
+    "\u4e00-\u9fff"  # CJK Unified Ideographs
+    "\uf900-\ufaff"  # CJK Compatibility Ideographs
+    "\ufe30-\ufe4f"  # CJK compatibility forms (vertical punctuation)
+    "\uff01-\uff60"  # Full-width ASCII variants and punctuation
+    "\uff61-\uff9f"  # Half-width CJK punctuation and katakana
+    "\U00020000-\U0003134f"  # CJK Unified Ideographs Extensions B-H
+    "]"
+)
+# Lowest code point in _CJK_NO_SPACE_RE: a cheap pre-check for Latin text.
+_CJK_MIN_CHAR: Final = "\u2e80"
+_HANGUL_RE = re.compile(
+    "[\u1100-\u11ff\u3130-\u318f\ua960-\ua97f\uac00-\ud7af\ud7b0-\ud7ff]"
+)
+
 # Ligature normalization map.
 # Covers the Unicode Alphabetic Presentation Forms block (U+FB00-U+FB06),
 # the Latin capital/small ligature IJ (U+0132/U+0133), and the Private-Use
@@ -123,12 +150,23 @@ def sanitize_text(self, lines: List[str]) -> str:
         hyphen-prefixed token -- so it is kept and its lines are joined like
         any other line break.
 
+        A break between two CJK characters (Han, kana or CJK / full-width
+        punctuation) is not a word boundary either, since those scripts put
+        no spaces between words, so it is closed up as well -- unless the
+        cluster contains Hangul: Korean separates words with spaces, Hanja
+        included.
+
         The joined text is then normalized: a few typographic characters are
         replaced by ASCII equivalents and ligatures are expanded.
         """
         if len(lines) == 0:
             return ""
 
+        # Korean text mixes Hanja into space-separated words, so a cluster
+        # that contains Hangul keeps the space at every line break. Only
+        # checked once a CJK/CJK break is found, so other text pays nothing.
+        has_hangul: Optional[bool] = None
+
         for ix, line in enumerate(lines[1:]):
             prev_line = lines[ix]
 
@@ -148,6 +186,18 @@ def sanitize_text(self, lines: List[str]) -> str:
                     lines[ix] = prev_line[:-1]
                 elif not hyphen_attached_to_word:
                     lines[ix] += " "
+            elif (
+                prev_line
+                and line
+                and prev_line[-1] >= _CJK_MIN_CHAR
+                and line[0] >= _CJK_MIN_CHAR
+                and _CJK_NO_SPACE_RE.match(prev_line[-1])
+                and _CJK_NO_SPACE_RE.match(line[0])
+            ):
+                if has_hangul is None:
+                    has_hangul = any(_HANGUL_RE.search(ln) for ln in lines)
+                if has_hangul:
+                    lines[ix] += " "
             else:
                 lines[ix] += " "
 

tests/test_page_assemble_model.py

@@ -308,3 +308,67 @@ def test_no_page_size_returns_none(self):
         page.parsed_page = pp
         bbox = BoundingBox(l=10, t=10, r=90, b=20)
         assert PageAssembleModel._match_hyperlink(bbox, page) is None
+
+
+class TestSanitizeTextCjkLineJoin:
+    """Line breaks inside CJK text are not word boundaries (issue #4672)."""
+
+    @pytest.mark.parametrize(
+        ("lines", "expected"),
+        [
+            # Japanese: kanji / kana on both sides of the break
+            (
+                ["衆議院議員早稲田夕季君提出", "インクルーシブ防災を実現する"],
+                "衆議院議員早稲田夕季君提出インクルーシブ防災を実現する",
+            ),
+            # Chinese
+            (
+                ["我们提出了一种新的文档", "转换方法。"],
+                "我们提出了一种新的文档转换方法。",
+            ),
+            # full-width punctuation ends or starts a line
+            (["避難所における、", "良好な生活環境"], "避難所における、良好な生活環境"),
+            (["政府としては", "「避難所」"], "政府としては「避難所」"),
+            # half-width katakana and an ideograph from Extension B
+            (["ｶﾀｶﾅ", "𠀋"], "ｶﾀｶﾅ𠀋"),
+            # more than two lines
+            (["令和", "元年", "十月"], "令和元年十月"),
+        ],
+    )
+    def test_cjk_lines_join_without_space(self, model, lines, expected):
+        assert model.sanitize_text(lines) == expected
+
+    @pytest.mark.parametrize(
+        ("lines", "expected"),
+        [
+            # Latin on one side of the break keeps the space
+            (["docling は", "PDF を変換する"], "docling は PDF を変換する"),
+            (["変換ツール", "docling"], "変換ツール docling"),
+            (["Docling", "文書変換"], "Docling 文書変換"),
+            # digits are not CJK
+            (["令和元年", "10月"], "令和元年 10月"),
+            # plain Latin text is unchanged
+            (["hello", "world"], "hello world"),
+        ],
+    )
+    def test_mixed_script_keeps_space(self, model, lines, expected):
+        assert model.sanitize_text(lines) == expected
+
+    def test_korean_keeps_space(self, model):
+        """Korean separates words with spaces; a line break is a word boundary."""
+        assert (
+            model.sanitize_text(["문서 변환", "도구입니다"]) == "문서 변환 도구입니다"
+        )
+
+    def test_korean_with_hanja_keeps_space(self, model):
+        """Hanja inside a Korean cluster are space-separated words too."""
+        assert (
+            model.sanitize_text(["대한민국 憲法", "第一條 대한민국은"])
+            == "대한민국 憲法 第一條 대한민국은"
+        )
+
+    def test_hyphenation_still_applies(self, model):
+        assert model.sanitize_text(["文書 algo-", "rithms"]) == "文書 algorithms"
+
+    def test_empty_line_between_cjk_lines(self, model):
+        assert model.sanitize_text(["日本", "", "語"]) == "日本  語"