From 1a902ea5c8b68bc65bff22616fa4b3c4fade2eff Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:44:26 +0900 Subject: [PATCH 1/2] feat(pdf): join CJK text lines without a space A line break between two Han, kana or CJK/full-width punctuation characters is not a word boundary, so sanitize_text no longer inserts a space there. Clusters that contain Hangul keep the space, since Korean separates words with spaces. Part B of #4672. Signed-off-by: kevin9327 <5299031+kevin9327@users.noreply.github.com> --- .../skills/docling/references/extraction.md | 23 ++++--- .../skills/docling/references/python-sdk.md | 66 +++++++++++-------- .../.agents/skills/docling/references/rag.md | 4 +- .../docling/references/service-client.md | 4 +- .../page_assemble/page_assemble_model.py | 52 ++++++++++++++- tests/test_page_assemble_model.py | 64 ++++++++++++++++++ 6 files changed, 172 insertions(+), 41 deletions(-) diff --git a/docling/.agents/skills/docling/references/extraction.md b/docling/.agents/skills/docling/references/extraction.md index c7a53655c8..10cdb30681 100644 --- a/docling/.agents/skills/docling/references/extraction.md +++ b/docling/.agents/skills/docling/references/extraction.md @@ -33,25 +33,30 @@ a Pydantic model **instance** (`Union[str, dict, BaseModel, Type[BaseModel]]`). ```python # 1. JSON-ish string -result = extractor.extract(source="invoice.pdf", - template='{"bill_no": "string", "total": "float"}') +result = extractor.extract( + source="invoice.pdf", template='{"bill_no": "string", "total": "float"}' +) # 2. dict template -result = extractor.extract(source="invoice.pdf", - template={"bill_no": "string", "total": "float"}) +result = extractor.extract( + source="invoice.pdf", template={"bill_no": "string", "total": "float"} +) # 3. Pydantic model class (recommended — typed, self-documenting) from pydantic import BaseModel + class Invoice(BaseModel): bill_no: str total: float + result = extractor.extract(source="invoice.pdf", template=Invoice) # 4. Pydantic instance (fields double as examples / defaults) -result = extractor.extract(source="invoice.pdf", - template=Invoice(bill_no="INV-0001", total=0.0)) +result = extractor.extract( + source="invoice.pdf", template=Invoice(bill_no="INV-0001", total=0.0) +) ``` Prefer a **Pydantic model class** for durable schemas — it documents intent and @@ -70,7 +75,9 @@ result = extractor.extract(source="invoice.pdf", template=Invoice) if result.status in (ConversionStatus.SUCCESS, ConversionStatus.PARTIAL_SUCCESS): for page in result.pages: - print(page.page_no, page.extracted_data) # e.g. {"bill_no": "...", "total": 42.0} + print( + page.page_no, page.extracted_data + ) # e.g. {"bill_no": "...", "total": 42.0} else: print("extraction failed:", result.errors) ``` @@ -81,7 +88,7 @@ else: for result in extractor.extract_all( source=["a.pdf", "b.pdf", "https://example.com/c.pdf"], template=Invoice, - raises_on_error=False, # keep going past individual failures + raises_on_error=False, # keep going past individual failures ): print(result.input.file.name, result.status) ``` diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md index 6b764830bb..ab6928e65c 100644 --- a/docling/.agents/skills/docling/references/python-sdk.md +++ b/docling/.agents/skills/docling/references/python-sdk.md @@ -11,11 +11,11 @@ controlled by **`PipelineOptions`** subclasses passed per input format. from docling.document_converter import DocumentConverter converter = DocumentConverter() -result = converter.convert("report.pdf") # local path or http(s) URL +result = converter.convert("report.pdf") # local path or http(s) URL -doc = result.document # a DoclingDocument +doc = result.document # a DoclingDocument print(doc.export_to_markdown()) -data = doc.export_to_dict() # structured / lossless +data = doc.export_to_dict() # structured / lossless ``` `result.status` reports success/failure; `result.document` is always a @@ -40,11 +40,11 @@ from docling.datamodel.base_models import InputFormat from docling.datamodel.pipeline_options import PdfPipelineOptions pipeline_options = PdfPipelineOptions( - do_ocr=True, # OCR scanned/image regions (False to skip) - do_table_structure=True, # detect table structure (False = faster) - do_code_enrichment=False, # code understanding + do_ocr=True, # OCR scanned/image regions (False to skip) + do_table_structure=True, # detect table structure (False = faster) + do_code_enrichment=False, # code understanding do_formula_enrichment=False, # formula understanding - generate_page_images=False, # keep page raster images on the document + generate_page_images=False, # keep page raster images on the document ) converter = DocumentConverter( @@ -76,15 +76,20 @@ Useful `PdfPipelineOptions` / base fields: ```python from docling.datamodel.pipeline_options import ( - PdfPipelineOptions, RapidOcrOptions, TesseractOcrOptions, OcrMacOptions, + PdfPipelineOptions, + RapidOcrOptions, + TesseractOcrOptions, + OcrMacOptions, ) # Default is EasyOCR: opts = PdfPipelineOptions(do_ocr=True) # Alternatives: -opts = PdfPipelineOptions(do_ocr=True, ocr_options=RapidOcrOptions()) # lightweight -opts = PdfPipelineOptions(do_ocr=True, ocr_options=TesseractOcrOptions())# system Tesseract -opts = PdfPipelineOptions(do_ocr=True, ocr_options=OcrMacOptions()) # macOS only +opts = PdfPipelineOptions(do_ocr=True, ocr_options=RapidOcrOptions()) # lightweight +opts = PdfPipelineOptions( + do_ocr=True, ocr_options=TesseractOcrOptions() +) # system Tesseract +opts = PdfPipelineOptions(do_ocr=True, ocr_options=OcrMacOptions()) # macOS only ``` Each engine is an optional dependency — see [slim-packaging.md](slim-packaging.md). @@ -97,7 +102,7 @@ from docling.datamodel.pipeline_options import KserveV2OcrOptions opts = PdfPipelineOptions( do_ocr=True, - enable_remote_services=True, # REQUIRED for KserveV2OcrOptions + enable_remote_services=True, # REQUIRED for KserveV2OcrOptions ocr_options=KserveV2OcrOptions(url="localhost:8001", model_name="rapidocr"), ) ``` @@ -129,7 +134,9 @@ from docling.datamodel.accelerator_options import AcceleratorDevice, Accelerator from docling.datamodel.pipeline_options import PdfPipelineOptions opts = PdfPipelineOptions( - accelerator_options=AcceleratorOptions(device=AcceleratorDevice.CUDA, num_threads=8), + accelerator_options=AcceleratorOptions( + device=AcceleratorDevice.CUDA, num_threads=8 + ), ) ``` @@ -208,7 +215,7 @@ tables through the VLM. Reduces hallucination on text-heavy pages. ```python pipeline_options = VlmPipelineOptions( vlm_options=vlm_model_specs.GRANITEDOCLING_TRANSFORMERS, - force_backend_text=True, # hybrid + force_backend_text=True, # hybrid generate_page_images=True, ) ``` @@ -237,7 +244,7 @@ vlm_opts = ApiVlmOptions( pipeline_options = VlmPipelineOptions( vlm_options=vlm_opts, generate_page_images=True, - enable_remote_services=True, # REQUIRED — gates all outbound HTTP + enable_remote_services=True, # REQUIRED — gates all outbound HTTP ) converter = DocumentConverter( format_options={ @@ -272,9 +279,9 @@ tokenizer = HuggingFaceTokenizer.from_pretrained( chunker = HybridChunker(tokenizer=tokenizer, merge_peers=True) for chunk in chunker.chunk(result.document): - embed_text = chunker.contextualize(chunk) # heading-prefixed text to embed - print(chunk.meta.headings) # heading breadcrumb - print(chunk.meta.origin.page_no) # source page + embed_text = chunker.contextualize(chunk) # heading-prefixed text to embed + print(chunk.meta.headings) # heading breadcrumb + print(chunk.meta.origin.page_no) # source page ``` OpenAI tokenizer (requires `pip install 'docling-core[chunking-openai]'`): @@ -301,11 +308,11 @@ for item, level in doc.iterate_items(): print(f"{'#' * level} {item.text}") for table in doc.tables: - print(table.export_to_dataframe()) # pandas DataFrame + print(table.export_to_dataframe()) # pandas DataFrame print(table.export_to_markdown()) for picture in doc.pictures: - print(picture.caption_text(doc)) # caption if present + print(picture.caption_text(doc)) # caption if present ``` ## Batch conversion & error handling @@ -318,7 +325,7 @@ from docling.document_converter import DocumentConverter from docling.datamodel.base_models import ConversionStatus, InputFormat converter = DocumentConverter( - allowed_formats=[InputFormat.PDF, InputFormat.DOCX], # reject everything else + allowed_formats=[InputFormat.PDF, InputFormat.DOCX], # reject everything else ) sources = ["a.pdf", "b.docx", "https://example.com/c.pdf"] @@ -376,7 +383,7 @@ converter = DocumentConverter( } ) result = converter.convert("interview.wav") -print(result.document.export_to_markdown()) # timestamped transcript +print(result.document.export_to_markdown()) # timestamped transcript ``` The CLI can also transcribe: `docling interview.wav --to md --output /tmp/`. @@ -398,7 +405,7 @@ from docling.datamodel.pipeline_options import PdfPipelineOptions opts = PdfPipelineOptions( generate_page_images=True, generate_picture_images=True, - images_scale=2.0, # higher-resolution rasters + images_scale=2.0, # higher-resolution rasters ) ``` @@ -437,11 +444,13 @@ from docling.datamodel.backend_options import DeclarativeBackendOptions from docling.datamodel.base_models import InputFormat from docling.document_converter import DocumentConverter, DoclingJSONFormatOption -converter = DocumentConverter(format_options={ - InputFormat.JSON_DOCLING: DoclingJSONFormatOption( - backend_options=DeclarativeBackendOptions(enable_local_fetch=True) - ), -}) +converter = DocumentConverter( + format_options={ + InputFormat.JSON_DOCLING: DoclingJSONFormatOption( + backend_options=DeclarativeBackendOptions(enable_local_fetch=True) + ), + } +) ``` Or save with `ImageRefMode.EMBEDDED` so the JSON carries its images. @@ -493,5 +502,6 @@ equivalent is `docling report.pdf --artifacts-path /models`. ```python from importlib.metadata import version + print(version("docling"), version("docling-core")) ``` diff --git a/docling/.agents/skills/docling/references/rag.md b/docling/.agents/skills/docling/references/rag.md index b549283125..915e05154f 100644 --- a/docling/.agents/skills/docling/references/rag.md +++ b/docling/.agents/skills/docling/references/rag.md @@ -27,9 +27,9 @@ from langchain_docling.loader import ExportType loader = DoclingLoader( file_path=["report.pdf", "https://example.com/paper.pdf"], - export_type=ExportType.DOC_CHUNKS, # chunked; or ExportType.MARKDOWN + export_type=ExportType.DOC_CHUNKS, # chunked; or ExportType.MARKDOWN ) -docs = loader.load() # list[langchain_core.documents.Document] +docs = loader.load() # list[langchain_core.documents.Document] # each chunk keeps metadata: source, headings, page numbers ``` diff --git a/docling/.agents/skills/docling/references/service-client.md b/docling/.agents/skills/docling/references/service-client.md index 37c1b8c450..dc59a709e8 100644 --- a/docling/.agents/skills/docling/references/service-client.md +++ b/docling/.agents/skills/docling/references/service-client.md @@ -54,7 +54,7 @@ DOCLING_SERVICE_API_KEY=your-api-key # omit if the service is unauthenticated ```python from docling.service_client import DoclingServiceClient -client = DoclingServiceClient(url=..., api_key=...) # or rely on env / .env +client = DoclingServiceClient(url=..., api_key=...) # or rely on env / .env result = client.convert(source="path/to/report.pdf") # local path or http(s) URL print(result.document.export_to_markdown()) ``` @@ -67,7 +67,7 @@ from docling.datamodel.service.options import ConvertDocumentsOptions result = client.convert( source="report.pdf", - options=ConvertDocumentsOptions(...), # per-request pipeline overrides + options=ConvertDocumentsOptions(...), # per-request pipeline overrides ) ``` diff --git a/docling/models/stages/page_assemble/page_assemble_model.py b/docling/models/stages/page_assemble/page_assemble_model.py index a7bd223f63..5ca9fdb852 100644 --- a/docling/models/stages/page_assemble/page_assemble_model.py +++ b/docling/models/stages/page_assemble/page_assemble_model.py @@ -5,7 +5,7 @@ import re from collections.abc import Iterable from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, Final, List, Optional, Union import numpy as np from docling_core.types.doc import BoundingBox @@ -33,6 +33,33 @@ _log = logging.getLogger(__name__) +# Scripts written without spaces between words: Han ideographs, kana and +# CJK / full-width punctuation. A line break between two such characters is +# not a word boundary, so the lines are joined without a space. Hangul is +# deliberately absent: Korean separates words with spaces. +_CJK_NO_SPACE_RE = re.compile( + "[" + "\u2e80-\u2fdf" # CJK radicals, Kangxi radicals + "\u3000-\u303f" # CJK symbols and punctuation + "\u3040-\u309f" # Hiragana + "\u30a0-\u30ff" # Katakana + "\u31c0-\u31ef" # CJK strokes + "\u31f0-\u31ff" # Katakana phonetic extensions + "\u3400-\u4dbf" # CJK Unified Ideographs Extension A + "\u4e00-\u9fff" # CJK Unified Ideographs + "\uf900-\ufaff" # CJK Compatibility Ideographs + "\ufe30-\ufe4f" # CJK compatibility forms (vertical punctuation) + "\uff01-\uff60" # Full-width ASCII variants and punctuation + "\uff61-\uff9f" # Half-width CJK punctuation and katakana + "\U00020000-\U0003134f" # CJK Unified Ideographs Extensions B-H + "]" +) +# Lowest code point in _CJK_NO_SPACE_RE: a cheap pre-check for Latin text. +_CJK_MIN_CHAR: Final = "\u2e80" +_HANGUL_RE = re.compile( + "[\u1100-\u11ff\u3130-\u318f\ua960-\ua97f\uac00-\ud7af\ud7b0-\ud7ff]" +) + # Ligature normalization map. # Covers the Unicode Alphabetic Presentation Forms block (U+FB00-U+FB06), # the Latin capital/small ligature IJ (U+0132/U+0133), and the Private-Use @@ -123,12 +150,23 @@ def sanitize_text(self, lines: List[str]) -> str: hyphen-prefixed token -- so it is kept and its lines are joined like any other line break. + A break between two CJK characters (Han, kana or CJK / full-width + punctuation) is not a word boundary either, since those scripts put + no spaces between words, so it is closed up as well -- unless the + cluster contains Hangul: Korean separates words with spaces, Hanja + included. + The joined text is then normalized: a few typographic characters are replaced by ASCII equivalents and ligatures are expanded. """ if len(lines) == 0: return "" + # Korean text mixes Hanja into space-separated words, so a cluster + # that contains Hangul keeps the space at every line break. Only + # checked once a CJK/CJK break is found, so other text pays nothing. + has_hangul: Optional[bool] = None + for ix, line in enumerate(lines[1:]): prev_line = lines[ix] @@ -148,6 +186,18 @@ def sanitize_text(self, lines: List[str]) -> str: lines[ix] = prev_line[:-1] elif not hyphen_attached_to_word: lines[ix] += " " + elif ( + prev_line + and line + and prev_line[-1] >= _CJK_MIN_CHAR + and line[0] >= _CJK_MIN_CHAR + and _CJK_NO_SPACE_RE.match(prev_line[-1]) + and _CJK_NO_SPACE_RE.match(line[0]) + ): + if has_hangul is None: + has_hangul = any(_HANGUL_RE.search(ln) for ln in lines) + if has_hangul: + lines[ix] += " " else: lines[ix] += " " diff --git a/tests/test_page_assemble_model.py b/tests/test_page_assemble_model.py index 93ef7bb9a8..861657080f 100644 --- a/tests/test_page_assemble_model.py +++ b/tests/test_page_assemble_model.py @@ -308,3 +308,67 @@ def test_no_page_size_returns_none(self): page.parsed_page = pp bbox = BoundingBox(l=10, t=10, r=90, b=20) assert PageAssembleModel._match_hyperlink(bbox, page) is None + + +class TestSanitizeTextCjkLineJoin: + """Line breaks inside CJK text are not word boundaries (issue #4672).""" + + @pytest.mark.parametrize( + ("lines", "expected"), + [ + # Japanese: kanji / kana on both sides of the break + ( + ["衆議院議員早稲田夕季君提出", "インクルーシブ防災を実現する"], + "衆議院議員早稲田夕季君提出インクルーシブ防災を実現する", + ), + # Chinese + ( + ["我们提出了一种新的文档", "转换方法。"], + "我们提出了一种新的文档转换方法。", + ), + # full-width punctuation ends or starts a line + (["避難所における、", "良好な生活環境"], "避難所における、良好な生活環境"), + (["政府としては", "「避難所」"], "政府としては「避難所」"), + # half-width katakana and an ideograph from Extension B + (["カタカナ", "𠀋"], "カタカナ𠀋"), + # more than two lines + (["令和", "元年", "十月"], "令和元年十月"), + ], + ) + def test_cjk_lines_join_without_space(self, model, lines, expected): + assert model.sanitize_text(lines) == expected + + @pytest.mark.parametrize( + ("lines", "expected"), + [ + # Latin on one side of the break keeps the space + (["docling は", "PDF を変換する"], "docling は PDF を変換する"), + (["変換ツール", "docling"], "変換ツール docling"), + (["Docling", "文書変換"], "Docling 文書変換"), + # digits are not CJK + (["令和元年", "10月"], "令和元年 10月"), + # plain Latin text is unchanged + (["hello", "world"], "hello world"), + ], + ) + def test_mixed_script_keeps_space(self, model, lines, expected): + assert model.sanitize_text(lines) == expected + + def test_korean_keeps_space(self, model): + """Korean separates words with spaces; a line break is a word boundary.""" + assert ( + model.sanitize_text(["문서 변환", "도구입니다"]) == "문서 변환 도구입니다" + ) + + def test_korean_with_hanja_keeps_space(self, model): + """Hanja inside a Korean cluster are space-separated words too.""" + assert ( + model.sanitize_text(["대한민국 憲法", "第一條 대한민국은"]) + == "대한민국 憲法 第一條 대한민국은" + ) + + def test_hyphenation_still_applies(self, model): + assert model.sanitize_text(["文書 algo-", "rithms"]) == "文書 algorithms" + + def test_empty_line_between_cjk_lines(self, model): + assert model.sanitize_text(["日本", "", "語"]) == "日本 語" From 130bb27ffc00f230c0a40f34302482e14165c77e Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sun, 18 Oct 2026 09:21:34 +0900 Subject: [PATCH 2/2] chore: drop unrelated reformatting of the docling skill references Keep this PR to the CJK line-join change in sanitize_text. Signed-off-by: kevin9327 <5299031+kevin9327@users.noreply.github.com> --- .../skills/docling/references/extraction.md | 23 +++---- .../skills/docling/references/python-sdk.md | 66 ++++++++----------- .../.agents/skills/docling/references/rag.md | 4 +- .../docling/references/service-client.md | 4 +- 4 files changed, 40 insertions(+), 57 deletions(-) diff --git a/docling/.agents/skills/docling/references/extraction.md b/docling/.agents/skills/docling/references/extraction.md index 10cdb30681..c7a53655c8 100644 --- a/docling/.agents/skills/docling/references/extraction.md +++ b/docling/.agents/skills/docling/references/extraction.md @@ -33,30 +33,25 @@ a Pydantic model **instance** (`Union[str, dict, BaseModel, Type[BaseModel]]`). ```python # 1. JSON-ish string -result = extractor.extract( - source="invoice.pdf", template='{"bill_no": "string", "total": "float"}' -) +result = extractor.extract(source="invoice.pdf", + template='{"bill_no": "string", "total": "float"}') # 2. dict template -result = extractor.extract( - source="invoice.pdf", template={"bill_no": "string", "total": "float"} -) +result = extractor.extract(source="invoice.pdf", + template={"bill_no": "string", "total": "float"}) # 3. Pydantic model class (recommended — typed, self-documenting) from pydantic import BaseModel - class Invoice(BaseModel): bill_no: str total: float - result = extractor.extract(source="invoice.pdf", template=Invoice) # 4. Pydantic instance (fields double as examples / defaults) -result = extractor.extract( - source="invoice.pdf", template=Invoice(bill_no="INV-0001", total=0.0) -) +result = extractor.extract(source="invoice.pdf", + template=Invoice(bill_no="INV-0001", total=0.0)) ``` Prefer a **Pydantic model class** for durable schemas — it documents intent and @@ -75,9 +70,7 @@ result = extractor.extract(source="invoice.pdf", template=Invoice) if result.status in (ConversionStatus.SUCCESS, ConversionStatus.PARTIAL_SUCCESS): for page in result.pages: - print( - page.page_no, page.extracted_data - ) # e.g. {"bill_no": "...", "total": 42.0} + print(page.page_no, page.extracted_data) # e.g. {"bill_no": "...", "total": 42.0} else: print("extraction failed:", result.errors) ``` @@ -88,7 +81,7 @@ else: for result in extractor.extract_all( source=["a.pdf", "b.pdf", "https://example.com/c.pdf"], template=Invoice, - raises_on_error=False, # keep going past individual failures + raises_on_error=False, # keep going past individual failures ): print(result.input.file.name, result.status) ``` diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md index ab6928e65c..6b764830bb 100644 --- a/docling/.agents/skills/docling/references/python-sdk.md +++ b/docling/.agents/skills/docling/references/python-sdk.md @@ -11,11 +11,11 @@ controlled by **`PipelineOptions`** subclasses passed per input format. from docling.document_converter import DocumentConverter converter = DocumentConverter() -result = converter.convert("report.pdf") # local path or http(s) URL +result = converter.convert("report.pdf") # local path or http(s) URL -doc = result.document # a DoclingDocument +doc = result.document # a DoclingDocument print(doc.export_to_markdown()) -data = doc.export_to_dict() # structured / lossless +data = doc.export_to_dict() # structured / lossless ``` `result.status` reports success/failure; `result.document` is always a @@ -40,11 +40,11 @@ from docling.datamodel.base_models import InputFormat from docling.datamodel.pipeline_options import PdfPipelineOptions pipeline_options = PdfPipelineOptions( - do_ocr=True, # OCR scanned/image regions (False to skip) - do_table_structure=True, # detect table structure (False = faster) - do_code_enrichment=False, # code understanding + do_ocr=True, # OCR scanned/image regions (False to skip) + do_table_structure=True, # detect table structure (False = faster) + do_code_enrichment=False, # code understanding do_formula_enrichment=False, # formula understanding - generate_page_images=False, # keep page raster images on the document + generate_page_images=False, # keep page raster images on the document ) converter = DocumentConverter( @@ -76,20 +76,15 @@ Useful `PdfPipelineOptions` / base fields: ```python from docling.datamodel.pipeline_options import ( - PdfPipelineOptions, - RapidOcrOptions, - TesseractOcrOptions, - OcrMacOptions, + PdfPipelineOptions, RapidOcrOptions, TesseractOcrOptions, OcrMacOptions, ) # Default is EasyOCR: opts = PdfPipelineOptions(do_ocr=True) # Alternatives: -opts = PdfPipelineOptions(do_ocr=True, ocr_options=RapidOcrOptions()) # lightweight -opts = PdfPipelineOptions( - do_ocr=True, ocr_options=TesseractOcrOptions() -) # system Tesseract -opts = PdfPipelineOptions(do_ocr=True, ocr_options=OcrMacOptions()) # macOS only +opts = PdfPipelineOptions(do_ocr=True, ocr_options=RapidOcrOptions()) # lightweight +opts = PdfPipelineOptions(do_ocr=True, ocr_options=TesseractOcrOptions())# system Tesseract +opts = PdfPipelineOptions(do_ocr=True, ocr_options=OcrMacOptions()) # macOS only ``` Each engine is an optional dependency — see [slim-packaging.md](slim-packaging.md). @@ -102,7 +97,7 @@ from docling.datamodel.pipeline_options import KserveV2OcrOptions opts = PdfPipelineOptions( do_ocr=True, - enable_remote_services=True, # REQUIRED for KserveV2OcrOptions + enable_remote_services=True, # REQUIRED for KserveV2OcrOptions ocr_options=KserveV2OcrOptions(url="localhost:8001", model_name="rapidocr"), ) ``` @@ -134,9 +129,7 @@ from docling.datamodel.accelerator_options import AcceleratorDevice, Accelerator from docling.datamodel.pipeline_options import PdfPipelineOptions opts = PdfPipelineOptions( - accelerator_options=AcceleratorOptions( - device=AcceleratorDevice.CUDA, num_threads=8 - ), + accelerator_options=AcceleratorOptions(device=AcceleratorDevice.CUDA, num_threads=8), ) ``` @@ -215,7 +208,7 @@ tables through the VLM. Reduces hallucination on text-heavy pages. ```python pipeline_options = VlmPipelineOptions( vlm_options=vlm_model_specs.GRANITEDOCLING_TRANSFORMERS, - force_backend_text=True, # hybrid + force_backend_text=True, # hybrid generate_page_images=True, ) ``` @@ -244,7 +237,7 @@ vlm_opts = ApiVlmOptions( pipeline_options = VlmPipelineOptions( vlm_options=vlm_opts, generate_page_images=True, - enable_remote_services=True, # REQUIRED — gates all outbound HTTP + enable_remote_services=True, # REQUIRED — gates all outbound HTTP ) converter = DocumentConverter( format_options={ @@ -279,9 +272,9 @@ tokenizer = HuggingFaceTokenizer.from_pretrained( chunker = HybridChunker(tokenizer=tokenizer, merge_peers=True) for chunk in chunker.chunk(result.document): - embed_text = chunker.contextualize(chunk) # heading-prefixed text to embed - print(chunk.meta.headings) # heading breadcrumb - print(chunk.meta.origin.page_no) # source page + embed_text = chunker.contextualize(chunk) # heading-prefixed text to embed + print(chunk.meta.headings) # heading breadcrumb + print(chunk.meta.origin.page_no) # source page ``` OpenAI tokenizer (requires `pip install 'docling-core[chunking-openai]'`): @@ -308,11 +301,11 @@ for item, level in doc.iterate_items(): print(f"{'#' * level} {item.text}") for table in doc.tables: - print(table.export_to_dataframe()) # pandas DataFrame + print(table.export_to_dataframe()) # pandas DataFrame print(table.export_to_markdown()) for picture in doc.pictures: - print(picture.caption_text(doc)) # caption if present + print(picture.caption_text(doc)) # caption if present ``` ## Batch conversion & error handling @@ -325,7 +318,7 @@ from docling.document_converter import DocumentConverter from docling.datamodel.base_models import ConversionStatus, InputFormat converter = DocumentConverter( - allowed_formats=[InputFormat.PDF, InputFormat.DOCX], # reject everything else + allowed_formats=[InputFormat.PDF, InputFormat.DOCX], # reject everything else ) sources = ["a.pdf", "b.docx", "https://example.com/c.pdf"] @@ -383,7 +376,7 @@ converter = DocumentConverter( } ) result = converter.convert("interview.wav") -print(result.document.export_to_markdown()) # timestamped transcript +print(result.document.export_to_markdown()) # timestamped transcript ``` The CLI can also transcribe: `docling interview.wav --to md --output /tmp/`. @@ -405,7 +398,7 @@ from docling.datamodel.pipeline_options import PdfPipelineOptions opts = PdfPipelineOptions( generate_page_images=True, generate_picture_images=True, - images_scale=2.0, # higher-resolution rasters + images_scale=2.0, # higher-resolution rasters ) ``` @@ -444,13 +437,11 @@ from docling.datamodel.backend_options import DeclarativeBackendOptions from docling.datamodel.base_models import InputFormat from docling.document_converter import DocumentConverter, DoclingJSONFormatOption -converter = DocumentConverter( - format_options={ - InputFormat.JSON_DOCLING: DoclingJSONFormatOption( - backend_options=DeclarativeBackendOptions(enable_local_fetch=True) - ), - } -) +converter = DocumentConverter(format_options={ + InputFormat.JSON_DOCLING: DoclingJSONFormatOption( + backend_options=DeclarativeBackendOptions(enable_local_fetch=True) + ), +}) ``` Or save with `ImageRefMode.EMBEDDED` so the JSON carries its images. @@ -502,6 +493,5 @@ equivalent is `docling report.pdf --artifacts-path /models`. ```python from importlib.metadata import version - print(version("docling"), version("docling-core")) ``` diff --git a/docling/.agents/skills/docling/references/rag.md b/docling/.agents/skills/docling/references/rag.md index 915e05154f..b549283125 100644 --- a/docling/.agents/skills/docling/references/rag.md +++ b/docling/.agents/skills/docling/references/rag.md @@ -27,9 +27,9 @@ from langchain_docling.loader import ExportType loader = DoclingLoader( file_path=["report.pdf", "https://example.com/paper.pdf"], - export_type=ExportType.DOC_CHUNKS, # chunked; or ExportType.MARKDOWN + export_type=ExportType.DOC_CHUNKS, # chunked; or ExportType.MARKDOWN ) -docs = loader.load() # list[langchain_core.documents.Document] +docs = loader.load() # list[langchain_core.documents.Document] # each chunk keeps metadata: source, headings, page numbers ``` diff --git a/docling/.agents/skills/docling/references/service-client.md b/docling/.agents/skills/docling/references/service-client.md index dc59a709e8..37c1b8c450 100644 --- a/docling/.agents/skills/docling/references/service-client.md +++ b/docling/.agents/skills/docling/references/service-client.md @@ -54,7 +54,7 @@ DOCLING_SERVICE_API_KEY=your-api-key # omit if the service is unauthenticated ```python from docling.service_client import DoclingServiceClient -client = DoclingServiceClient(url=..., api_key=...) # or rely on env / .env +client = DoclingServiceClient(url=..., api_key=...) # or rely on env / .env result = client.convert(source="path/to/report.pdf") # local path or http(s) URL print(result.document.export_to_markdown()) ``` @@ -67,7 +67,7 @@ from docling.datamodel.service.options import ConvertDocumentsOptions result = client.convert( source="report.pdf", - options=ConvertDocumentsOptions(...), # per-request pipeline overrides + options=ConvertDocumentsOptions(...), # per-request pipeline overrides ) ```