diff --git a/docling/models/stages/page_assemble/page_assemble_model.py b/docling/models/stages/page_assemble/page_assemble_model.py index a7bd223f63..5ca9fdb852 100644 --- a/docling/models/stages/page_assemble/page_assemble_model.py +++ b/docling/models/stages/page_assemble/page_assemble_model.py @@ -5,7 +5,7 @@ import re from collections.abc import Iterable from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, Final, List, Optional, Union import numpy as np from docling_core.types.doc import BoundingBox @@ -33,6 +33,33 @@ _log = logging.getLogger(__name__) +# Scripts written without spaces between words: Han ideographs, kana and +# CJK / full-width punctuation. A line break between two such characters is +# not a word boundary, so the lines are joined without a space. Hangul is +# deliberately absent: Korean separates words with spaces. +_CJK_NO_SPACE_RE = re.compile( + "[" + "\u2e80-\u2fdf" # CJK radicals, Kangxi radicals + "\u3000-\u303f" # CJK symbols and punctuation + "\u3040-\u309f" # Hiragana + "\u30a0-\u30ff" # Katakana + "\u31c0-\u31ef" # CJK strokes + "\u31f0-\u31ff" # Katakana phonetic extensions + "\u3400-\u4dbf" # CJK Unified Ideographs Extension A + "\u4e00-\u9fff" # CJK Unified Ideographs + "\uf900-\ufaff" # CJK Compatibility Ideographs + "\ufe30-\ufe4f" # CJK compatibility forms (vertical punctuation) + "\uff01-\uff60" # Full-width ASCII variants and punctuation + "\uff61-\uff9f" # Half-width CJK punctuation and katakana + "\U00020000-\U0003134f" # CJK Unified Ideographs Extensions B-H + "]" +) +# Lowest code point in _CJK_NO_SPACE_RE: a cheap pre-check for Latin text. +_CJK_MIN_CHAR: Final = "\u2e80" +_HANGUL_RE = re.compile( + "[\u1100-\u11ff\u3130-\u318f\ua960-\ua97f\uac00-\ud7af\ud7b0-\ud7ff]" +) + # Ligature normalization map. # Covers the Unicode Alphabetic Presentation Forms block (U+FB00-U+FB06), # the Latin capital/small ligature IJ (U+0132/U+0133), and the Private-Use @@ -123,12 +150,23 @@ def sanitize_text(self, lines: List[str]) -> str: hyphen-prefixed token -- so it is kept and its lines are joined like any other line break. + A break between two CJK characters (Han, kana or CJK / full-width + punctuation) is not a word boundary either, since those scripts put + no spaces between words, so it is closed up as well -- unless the + cluster contains Hangul: Korean separates words with spaces, Hanja + included. + The joined text is then normalized: a few typographic characters are replaced by ASCII equivalents and ligatures are expanded. """ if len(lines) == 0: return "" + # Korean text mixes Hanja into space-separated words, so a cluster + # that contains Hangul keeps the space at every line break. Only + # checked once a CJK/CJK break is found, so other text pays nothing. + has_hangul: Optional[bool] = None + for ix, line in enumerate(lines[1:]): prev_line = lines[ix] @@ -148,6 +186,18 @@ def sanitize_text(self, lines: List[str]) -> str: lines[ix] = prev_line[:-1] elif not hyphen_attached_to_word: lines[ix] += " " + elif ( + prev_line + and line + and prev_line[-1] >= _CJK_MIN_CHAR + and line[0] >= _CJK_MIN_CHAR + and _CJK_NO_SPACE_RE.match(prev_line[-1]) + and _CJK_NO_SPACE_RE.match(line[0]) + ): + if has_hangul is None: + has_hangul = any(_HANGUL_RE.search(ln) for ln in lines) + if has_hangul: + lines[ix] += " " else: lines[ix] += " " diff --git a/tests/test_page_assemble_model.py b/tests/test_page_assemble_model.py index 93ef7bb9a8..861657080f 100644 --- a/tests/test_page_assemble_model.py +++ b/tests/test_page_assemble_model.py @@ -308,3 +308,67 @@ def test_no_page_size_returns_none(self): page.parsed_page = pp bbox = BoundingBox(l=10, t=10, r=90, b=20) assert PageAssembleModel._match_hyperlink(bbox, page) is None + + +class TestSanitizeTextCjkLineJoin: + """Line breaks inside CJK text are not word boundaries (issue #4672).""" + + @pytest.mark.parametrize( + ("lines", "expected"), + [ + # Japanese: kanji / kana on both sides of the break + ( + ["衆議院議員早稲田夕季君提出", "インクルーシブ防災を実現する"], + "衆議院議員早稲田夕季君提出インクルーシブ防災を実現する", + ), + # Chinese + ( + ["我们提出了一种新的文档", "转换方法。"], + "我们提出了一种新的文档转换方法。", + ), + # full-width punctuation ends or starts a line + (["避難所における、", "良好な生活環境"], "避難所における、良好な生活環境"), + (["政府としては", "「避難所」"], "政府としては「避難所」"), + # half-width katakana and an ideograph from Extension B + (["ｶﾀｶﾅ", "𠀋"], "ｶﾀｶﾅ𠀋"), + # more than two lines + (["令和", "元年", "十月"], "令和元年十月"), + ], + ) + def test_cjk_lines_join_without_space(self, model, lines, expected): + assert model.sanitize_text(lines) == expected + + @pytest.mark.parametrize( + ("lines", "expected"), + [ + # Latin on one side of the break keeps the space + (["docling は", "PDF を変換する"], "docling は PDF を変換する"), + (["変換ツール", "docling"], "変換ツール docling"), + (["Docling", "文書変換"], "Docling 文書変換"), + # digits are not CJK + (["令和元年", "10月"], "令和元年 10月"), + # plain Latin text is unchanged + (["hello", "world"], "hello world"), + ], + ) + def test_mixed_script_keeps_space(self, model, lines, expected): + assert model.sanitize_text(lines) == expected + + def test_korean_keeps_space(self, model): + """Korean separates words with spaces; a line break is a word boundary.""" + assert ( + model.sanitize_text(["문서 변환", "도구입니다"]) == "문서 변환 도구입니다" + ) + + def test_korean_with_hanja_keeps_space(self, model): + """Hanja inside a Korean cluster are space-separated words too.""" + assert ( + model.sanitize_text(["대한민국 憲法", "第一條 대한민국은"]) + == "대한민국 憲法 第一條 대한민국은" + ) + + def test_hyphenation_still_applies(self, model): + assert model.sanitize_text(["文書 algo-", "rithms"]) == "文書 algorithms" + + def test_empty_line_between_cjk_lines(self, model): + assert model.sanitize_text(["日本", "", "語"]) == "日本 語"