diff --git a/docling/.agents/skills/docling/references/cli.md b/docling/.agents/skills/docling/references/cli.md index c432f5cd3b..bff04bad0a 100644 --- a/docling/.agents/skills/docling/references/cli.md +++ b/docling/.agents/skills/docling/references/cli.md @@ -99,8 +99,16 @@ docling scan.pdf --force-ocr --output /tmp/ # re-OCR even extractab docling report.pdf --no-ocr --output /tmp/ # skip OCR (faster) docling scan.pdf --ocr-lang eng,deu --output /tmp/ # the engine's own codes docling scan.pdf --ocr-lang iso:en,iso:de --output /tmp/ # BCP-47 tags behind `iso:` +docling scan.pdf --ocr-engine rapidocr --ocr-option model_size=tiny --output /tmp/ # engine-specific field ``` +`--ocr-option KEY=VALUE` (repeatable) sets any field of the selected OCR engine. +VALUE is read as JSON when possible (`text_score=0.6`, `lang=["en"]`), else as a +string. List the keys of an engine with `docling options ocr `. The +same works for `--layout-option` / `docling options layout ` and +`--table-structure-option` / `docling options table-structure `. These +options are for local `convert` only; `convert-remote` does not accept them. + OCR engines are optional dependencies ā€” see [slim-packaging.md](slim-packaging.md) for the `feat-ocr-*` extras. diff --git a/docling/cli/engine_options.py b/docling/cli/engine_options.py new file mode 100644 index 0000000000..e364de0fcb --- /dev/null +++ b/docling/cli/engine_options.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Generic ``KEY=VALUE`` options for the engines that the CLI selects by name.""" + +import json +from collections.abc import Mapping +from enum import Enum +from typing import TYPE_CHECKING, Any, Literal, get_args, get_origin + +import typer +from pydantic import ValidationError +from pydantic_core import PydanticUndefined + +if TYPE_CHECKING: + from docling.datamodel.pipeline_options import BaseOptions + from docling.models.factories.base_factory import BaseFactory + + +def parse_engine_option_pairs( + values: list[str] | None, param_hint: str +) -> dict[str, Any]: + """Parse repeated ``KEY=VALUE`` items into keyword arguments. + + The value is decoded as JSON when possible (``0.6``, ``false``, + ``["en","de"]``); otherwise the raw string is kept. + """ + parsed: dict[str, Any] = {} + for item in values or []: + key, separator, raw_value = item.partition("=") + key = key.strip() + if not separator or not key: + raise typer.BadParameter( + f"expected KEY=VALUE, got {item!r}", param_hint=param_hint + ) + if key in parsed: + raise typer.BadParameter( + f"key {key!r} is given more than once", param_hint=param_hint + ) + try: + parsed[key] = json.loads(raw_value) + except json.JSONDecodeError: + parsed[key] = raw_value + return parsed + + +def find_options_class(factory: "BaseFactory", kind: str) -> type["BaseOptions"]: + for options_cls in factory.classes: + if options_cls.kind == kind: + return options_cls + known = ", ".join(sorted(factory.registered_kind)) + raise typer.BadParameter(f"unknown engine {kind!r}; known engines: {known}") + + +def create_engine_options( + factory: "BaseFactory", + kind: str, + *, + flag_kwargs: Mapping[str, Any], + flag_names: Mapping[str, str], + explicit_flag_fields: set[str], + option_values: list[str] | None, + param_hint: str, +) -> "BaseOptions": + """Create the options of engine ``kind`` from dedicated flags and ``KEY=VALUE`` items. + + Args: + factory: The factory that registers the engine. + kind: The selected engine. + flag_kwargs: Fields set from dedicated CLI flags, such as ``--ocr-lang``. + flag_names: The CLI flag for each field in ``flag_kwargs``. + explicit_flag_fields: Fields in ``flag_kwargs`` that the user set + explicitly. A ``KEY=VALUE`` item for one of these fields is an error. + option_values: The raw ``KEY=VALUE`` items. + param_hint: The repeatable option that takes the items, for error messages. + """ + options_cls = find_options_class(factory, kind) + extra_kwargs = parse_engine_option_pairs(option_values, param_hint) + + valid_keys = sorted(options_cls.model_fields) + for key in extra_kwargs: + if key not in options_cls.model_fields: + raise typer.BadParameter( + f"unknown key {key!r} for engine {kind!r}; " + f"valid keys: {', '.join(valid_keys)}", + param_hint=param_hint, + ) + if key in explicit_flag_fields: + raise typer.BadParameter( + f"key {key!r} is also set by {flag_names[key]}; use only one of them", + param_hint=param_hint, + ) + + kwargs = {**flag_kwargs, **extra_kwargs} + try: + return factory.create_options(kind=kind, **kwargs) + except ValidationError as err: + error = err.errors()[0] + field = str(error["loc"][0]) if error["loc"] else "" + message = error["msg"].removeprefix("Value error, ") + if field in extra_kwargs: + raise typer.BadParameter( + f"{field}: {message}", param_hint=param_hint + ) from err + raise typer.BadParameter( + message, param_hint=flag_names.get(field, param_hint) + ) from err + + +def describe_options_class( + options_cls: type["BaseOptions"], +) -> list[tuple[str, str, str, str]]: + """Return ``(name, type, default, description)`` for each field of ``options_cls``.""" + rows = [] + for name, field in options_cls.model_fields.items(): + annotation = field.annotation + if isinstance(annotation, type) and issubclass(annotation, Enum): + type_name = f"[{'|'.join(str(member.value) for member in annotation)}]" + elif get_origin(annotation) is Literal: + type_name = f"[{'|'.join(str(value) for value in get_args(annotation))}]" + elif isinstance(annotation, type) and get_origin(annotation) is None: + type_name = annotation.__name__ + else: + type_name = str(annotation).replace("typing.", "") + if field.is_required(): + default = "required" + else: + default_value = field.get_default(call_default_factory=True) + default = ( + "" + if default_value is PydanticUndefined + else _format_default(default_value) + ) + rows.append((name, type_name, default, field.description or "")) + return rows + + +def _format_default(value: Any) -> str: + """Show a default as the JSON that ``KEY=VALUE`` accepts, when it can.""" + try: + return json.dumps(value, default=lambda obj: obj.model_dump(mode="json")) + except (TypeError, ValueError, AttributeError): + return repr(value) diff --git a/docling/cli/main.py b/docling/cli/main.py index ec6d123ccb..4269e6412a 100644 --- a/docling/cli/main.py +++ b/docling/cli/main.py @@ -20,6 +20,7 @@ # Check for CLI dependencies try: + import rich.markup import rich.table import typer except ImportError as e: @@ -50,6 +51,11 @@ from rich.console import Console from tqdm.contrib.logging import logging_redirect_tqdm +from docling.cli.engine_options import ( + create_engine_options, + describe_options_class, + find_options_class, +) from docling.cli.export_utils import ( _export_flags_from_formats, _is_empty_output, @@ -178,16 +184,6 @@ from docling.models.factories.base_factory import BaseFactory -def _first_error_message(err: ValidationError) -> str: - """The most useful line of a pydantic error, for a typer.BadParameter.""" - errors = err.errors() - if not errors: - return str(err) - message = errors[0].get("msg", "") - # Pydantic prefixes messages raised from a validator with "Value error, ". - return message.removeprefix("Value error, ") or str(err) - - warnings.filterwarnings(action="ignore", category=UserWarning, module="pydantic|torch") warnings.filterwarnings(action="ignore", category=FutureWarning, module="easyocr") @@ -279,6 +275,11 @@ def _iter_input_paths_from_directory( yield path +def _choices_metavar(choices: Iterable[str]) -> str: + values = list(choices) + return f"[{'|'.join(values)}]" if values else "TEXT" + + def _expand_from_formats(from_formats: list[str] | None) -> list[InputFormat]: if from_formats is None: return list(InputFormat) @@ -822,6 +823,7 @@ def convert( # noqa: C901 from_formats: list[str] = typer.Option( None, "--from", + metavar=_choices_metavar([*(fmt.value for fmt in InputFormat), "odf"]), help="Input formats to accept. Use 'odf' for odt, ods, and odp. Defaults to all.", ), to_formats: list[OutputFormat] = typer.Option( @@ -887,6 +889,7 @@ def convert( # noqa: C901 str, typer.Option( ..., + metavar=_choices_metavar(vlm_preset_ids), help=f"Choose the VLM preset to use with PDF or image files. Available presets: {', '.join(vlm_preset_ids)}", ), ] = "granite_docling", @@ -983,6 +986,7 @@ def convert( # noqa: C901 str, typer.Option( ..., + metavar=_choices_metavar(o.value for o in layout_engines_enum_internal), help=( f"The layout engine to use. When --allow-external-plugins is *not* set, the available values are: " f"{', '.join(o.value for o in layout_engines_enum_internal)}. " @@ -994,6 +998,9 @@ def convert( # noqa: C901 str, typer.Option( ..., + metavar=_choices_metavar( + o.value for o in table_structure_engines_enum_internal + ), help=( f"The table structure engine to use. When --allow-external-plugins is *not* set, the available values are: " f"{', '.join(o.value for o in table_structure_engines_enum_internal)}. " @@ -1005,6 +1012,7 @@ def convert( # noqa: C901 str, typer.Option( ..., + metavar=_choices_metavar(o.value for o in ocr_engines_enum_internal), help=( f"The OCR engine to use. When --allow-external-plugins is *not* set, the available values are: " f"{', '.join(o.value for o in ocr_engines_enum_internal)}. " @@ -1012,6 +1020,44 @@ def convert( # noqa: C901 ), ), ] = OcrAutoOptions.kind, + ocr_option: Annotated[ + list[str] | None, + typer.Option( + "--ocr-option", + metavar="KEY=VALUE", + help=( + "Set an option of the selected OCR engine, e.g. " + "'--ocr-engine rapidocr --ocr-option model_size=tiny'. Repeatable. " + 'VALUE is read as JSON when possible (0.6, false, ["en","de"]), ' + "else as a string. Run 'docling options ocr ' to list the keys." + ), + ), + ] = None, + layout_option: Annotated[ + list[str] | None, + typer.Option( + "--layout-option", + metavar="KEY=VALUE", + help=( + "Set an option of the selected layout engine. Repeatable, same " + "format as --ocr-option. Run 'docling options layout ' to " + "list the keys." + ), + ), + ] = None, + table_structure_option: Annotated[ + list[str] | None, + typer.Option( + "--table-structure-option", + metavar="KEY=VALUE", + help=( + "Set an option of the selected table structure engine, e.g. " + "'--table-structure-option mode=fast'. Repeatable, same format as " + "--ocr-option. Run 'docling options table-structure ' to " + "list the keys." + ), + ), + ] = None, ocr_lang: Annotated[ str | None, typer.Option( @@ -1038,7 +1084,10 @@ def convert( # noqa: C901 int | None, typer.Option( ..., - help="Page Segmentation Mode for the OCR engine (0-13).", + help=( + "DEPRECATED: use `--ocr-option psm=N` instead. " + "Page Segmentation Mode for the Tesseract OCR engines (0-13)." + ), ), ] = None, pdf_backend: Annotated[ @@ -1504,24 +1553,44 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] else: resolved_ocr_mode = ocr_mode ocr_kwargs: dict[str, Any] = {"mode": resolved_ocr_mode} + ocr_flag_names = { + "mode": "--ocr-mode", + "lang": "--ocr-lang", + "psm": "--psm", + } + # `--ocr-mode` always has a value; count it as set only when it differs + # from the default, so `--ocr-option mode=...` works on its own. + explicit_ocr_fields = ( + {"mode"} if force_ocr or ocr_mode != OcrMode.DEFAULT else set() + ) ocr_lang_list = _split_list(ocr_lang) # `_split_list` returns None only when the option was not given, so an # explicitly empty value reaches the engine as `lang=[]`: "your default". if ocr_lang_list is not None: ocr_kwargs["lang"] = ocr_lang_list - try: - ocr_options: OcrOptions = ocr_factory.create_options( # type: ignore - kind=ocr_engine, - **ocr_kwargs, + explicit_ocr_fields.add("lang") + if psm is not None: + warnings.warn( + "`--psm` is deprecated; use `--ocr-option psm=N` instead.", + DeprecationWarning, + stacklevel=2, ) - except ValidationError as err: - raise typer.BadParameter( - _first_error_message(err), param_hint="--ocr-lang" - ) from err - if psm is not None and isinstance( - ocr_options, TesseractOcrOptions | TesseractCliOcrOptions - ): - ocr_options.psm = psm + # Only the Tesseract engines know `psm`; the other engines ignore it. + if ocr_engine in (TesseractOcrOptions.kind, TesseractCliOcrOptions.kind): + ocr_kwargs["psm"] = psm + explicit_ocr_fields.add("psm") + ocr_options = cast( + OcrOptions, + create_engine_options( + ocr_factory, + ocr_engine, + flag_kwargs=ocr_kwargs, + flag_names=ocr_flag_names, + explicit_flag_fields=explicit_ocr_fields, + option_values=ocr_option, + param_hint="--ocr-option", + ), + ) accelerator_options = AcceleratorOptions(num_threads=num_threads, device=device) pipeline_options: PipelineOptions format_options: dict[InputFormat, FormatOption] = {} @@ -1530,15 +1599,41 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] layout_factory = get_layout_factory( allow_external_plugins=allow_external_plugins ) - layout_options: BaseLayoutOptions = layout_factory.create_options( # type: ignore - kind=layout_engine + layout_options = cast( + BaseLayoutOptions, + create_engine_options( + layout_factory, + layout_engine, + flag_kwargs={}, + flag_names={}, + explicit_flag_fields=set(), + option_values=layout_option, + param_hint="--layout-option", + ), ) table_structure_factory = get_table_structure_factory( allow_external_plugins=allow_external_plugins ) - table_structure_options: BaseTableStructureOptions = ( # type: ignore - table_structure_factory.create_options(kind=table_structure_engine) + # Only TableFormer V1 has a `mode`; the other engines ignore --table-mode. + table_kwargs: dict[str, Any] = ( + {"mode": table_mode} + if table_structure_engine == TableStructureOptions.kind + else {} + ) + table_structure_options = cast( + BaseTableStructureOptions, + create_engine_options( + table_structure_factory, + table_structure_engine, + flag_kwargs=table_kwargs, + flag_names={"mode": "--table-mode"}, + explicit_flag_fields=( + {"mode"} if table_mode != TableFormerMode.ACCURATE else set() + ), + option_values=table_structure_option, + param_hint="--table-structure-option", + ), ) try: @@ -1576,11 +1671,6 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] chart_extraction_options=chart_extraction_options, document_timeout=document_timeout, ) - if isinstance( - pipeline_options.table_structure_options, TableStructureOptions - ): - pipeline_options.table_structure_options.do_cell_matching = True - pipeline_options.table_structure_options.mode = table_mode if picture_description_max_new_tokens is not None: pipeline_options.picture_description_options.generation_config[ "max_new_tokens" @@ -1911,6 +2001,66 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] _log.info(f"All documents were converted in {end_time:.2f} seconds.") +class EngineStage(str, Enum): + OCR = "ocr" + LAYOUT = "layout" + TABLE_STRUCTURE = "table-structure" + + +@app.command("options") +def show_engine_options( + stage: Annotated[ + EngineStage, + typer.Argument(help="The pipeline stage of the engine."), + ], + engine: Annotated[ + str | None, + typer.Argument( + help="The engine to describe. Omit it to list the engines of the stage." + ), + ] = None, + allow_external_plugins: Annotated[ + bool, + typer.Option( + ..., help="Must be enabled for loading modules from third-party plugins." + ), + ] = False, +) -> None: + """Show the KEY=VALUE options that `docling convert` accepts for an engine. + + The keys go to --ocr-option, --layout-option or --table-structure-option. + """ + _require_local_model_stack("docling options") + factory: BaseFactory + if stage == EngineStage.OCR: + factory = get_ocr_factory(allow_external_plugins=allow_external_plugins) + elif stage == EngineStage.LAYOUT: + factory = get_layout_factory(allow_external_plugins=allow_external_plugins) + else: + factory = get_table_structure_factory( + allow_external_plugins=allow_external_plugins + ) + + if engine is None: + for kind in factory.registered_kind: + console.print(kind, highlight=False) + return + + try: + options_cls = find_options_class(factory, engine) + except typer.BadParameter as err: + err.param_hint = "ENGINE" + raise + table = rich.table.Table(title=f"Options of the {stage.value} engine {engine!r}") + table.add_column("Key", style="bold", no_wrap=True) + table.add_column("Type") + table.add_column("Default") + table.add_column("Description") + for row in describe_options_class(options_cls): + table.add_row(*(rich.markup.escape(cell) for cell in row)) + console.print(table) + + # Register `convert-remote` only when the service-client extra is installed. # Imported here (after `app`, `export_documents`, and the source-collection # helpers are defined) so the command is attached before the click app is built diff --git a/docling/models/stages/ocr/rapid_ocr_model.py b/docling/models/stages/ocr/rapid_ocr_model.py index b2f660dc54..39c744adea 100644 --- a/docling/models/stages/ocr/rapid_ocr_model.py +++ b/docling/models/stages/ocr/rapid_ocr_model.py @@ -624,13 +624,13 @@ def __init__( # Global settings (these are still correct) "Global.text_score": self.options.text_score, "Global.font_path": font_path, + "Global.log_level": "info" if self.options.print_verbose else "warning", # Engine-level ONNXRuntime settings "EngineConfig.onnxruntime.intra_op_num_threads": intra_op_num_threads, "EngineConfig.onnxruntime.use_cuda": use_cuda, "EngineConfig.onnxruntime.cuda_ep_cfg.device_id": gpu_id, # Engine-level OpenVINO settings "EngineConfig.openvino.inference_num_threads": intra_op_num_threads, - # "Global.verbose": self.options.print_verbose, # Detection model settings "Det.model_path": det_model_path, "Det.use_cuda": use_cuda, diff --git a/docs/concepts/OCR.md b/docs/concepts/OCR.md index 7cfd981785..8ddcdfad60 100644 --- a/docs/concepts/OCR.md +++ b/docs/concepts/OCR.md @@ -223,6 +223,19 @@ from docling.datamodel.pipeline_options import RapidOcrOptions options = RapidOcrOptions(lang=["en"], model_size="tiny") ``` +CLI conversion: + +```sh +docling convert --from pdf --to dclx --ocr-engine rapidocr --ocr-lang en --ocr-option model_size=tiny document.pdf +``` + +`--ocr-option KEY=VALUE` sets any field of the selected engine's options class and can be +repeated. The value is read as JSON when possible (`text_score=0.6`, `use_det=false`, +`lang=["en"]`), otherwise as a string. An unknown key is an error. Run +`docling options ocr rapidocr` to list the keys, types, and defaults of an engine. +`--layout-option` and `--table-structure-option` do the same for the layout and table +structure engines. + CLI prefetch (for offline/`artifacts_path` use). `--rapidocr-model-size` is a single value applied to every `--rapidocr-backend-lang` pair given (or to the default pair, if none are given) -- unlike `--rapidocr-backend-lang`, which is repeatable: diff --git a/tests/test_cli.py b/tests/test_cli.py index 8f3fbe6f36..5e7d4d6cd3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -30,7 +30,12 @@ from docling.datamodel.accelerator_options import AcceleratorDevice from docling.datamodel.backend_options import ThreadedDoclingParseBackendOptions from docling.datamodel.base_models import InputFormat, OutputFormat -from docling.datamodel.pipeline_options import OcrMode, PdfBackend, VlmPipelineOptions +from docling.datamodel.pipeline_options import ( + OcrMode, + PdfBackend, + TableFormerMode, + VlmPipelineOptions, +) from docling.datamodel.settings import DEFAULT_PAGE_RANGE, PageRange from docling.document_converter import PdfFormatOption from tests.fakes.image_server import SERVER_IP, local_server, use_test_network @@ -134,6 +139,22 @@ def test_cli_convert_help(): ) assert separator_option.opts == ["--reading-order-separators"] assert separator_option.secondary_opts == ["--no-reading-order-separators"] + metavar_by_name = { + parameter.name: parameter.metavar for parameter in convert_command.params + } + for name in ( + "from_formats", + "vlm_model", + "layout_engine", + "table_structure_engine", + "ocr_engine", + ): + assert metavar_by_name[name].startswith("[") + assert "odf" in metavar_by_name["from_formats"] + assert "granite_docling" in metavar_by_name["vlm_model"] + assert metavar_by_name["ocr_engine"] == ( + "[auto|easyocr|kserve_v2_ocr|nemotron-ocr|ocrmac|rapidocr|tesserocr|tesseract]" + ) def test_cli_version(): @@ -1606,6 +1627,123 @@ def test_cli_ocr_engine_can_be_set(tmp_path, monkeypatch): assert ocr_options.kind == "easyocr" +def _flatten_error_box(output: str) -> str: + """Join the lines of a rich error box into one line of text.""" + return re.sub(r"[\sā”‚]+", " ", strip_ansi(output)) + + +def test_cli_ocr_option_sets_engine_fields(tmp_path, monkeypatch): + result, ocr_options = _capture_cli_engine_options( + monkeypatch, + [ + "--ocr-engine", + "rapidocr", + "--ocr-option", + "model_size=tiny", + "--ocr-option", + "text_score=0.6", + "--ocr-option", + 'lang=["en"]', + "--ocr-option", + "use_det=false", + ], + tmp_path, + "ocr_options", + ) + assert result.exit_code == 0, result.output + assert ocr_options.model_size == "tiny" + assert ocr_options.text_score == 0.6 + assert ocr_options.lang == ["en"] + assert ocr_options.use_det is False + + +@pytest.mark.parametrize( + ("extra_args", "expected"), + [ + ( + ["--ocr-engine", "rapidocr", "--ocr-option", "model_sise=tiny"], + "unknown key 'model_sise' for engine 'rapidocr'", + ), + ( + ["--ocr-engine", "rapidocr", "--ocr-option", "model_size=huge"], + "model_size:", + ), + ( + ["--ocr-engine", "rapidocr", "--ocr-option", "model_size"], + "expected KEY=VALUE", + ), + ( + [ + "--ocr-engine", + "rapidocr", + "--ocr-lang", + "en", + "--ocr-option", + 'lang=["de"]', + ], + "also set by --ocr-lang", + ), + ( + ["--table-mode", "fast", "--table-structure-option", "mode=accurate"], + "also set by --table-mode", + ), + ( + ["--layout-option", "no_such_key=1"], + "unknown key 'no_such_key'", + ), + ], +) +def test_cli_engine_option_rejects_invalid_input( + tmp_path, monkeypatch, extra_args, expected +): + result, _ = _capture_cli_engine_options( + monkeypatch, extra_args, tmp_path, "ocr_options" + ) + assert result.exit_code == 2 + assert expected in _flatten_error_box(result.output) + + +def test_cli_table_structure_option_sets_engine_fields(tmp_path, monkeypatch): + result, table_options = _capture_cli_engine_options( + monkeypatch, + [ + "--table-structure-option", + "mode=fast", + "--table-structure-option", + "do_cell_matching=false", + ], + tmp_path, + "table_structure_options", + ) + assert result.exit_code == 0, result.output + assert table_options.mode == TableFormerMode.FAST + assert table_options.do_cell_matching is False + + +def test_cli_deprecated_psm_still_sets_tesseract_psm(tmp_path, monkeypatch): + result, ocr_options = _capture_cli_engine_options( + monkeypatch, + ["--ocr-engine", "tesseract", "--psm", "4"], + tmp_path, + "ocr_options", + ) + assert result.exit_code == 0, result.output + assert ocr_options.psm == 4 + + +def test_cli_options_lists_engine_fields(): + result = runner.invoke(app, ["options", "ocr", "rapidocr"], terminal_width=200) + assert result.exit_code == 0, result.output + assert "model_size" in result.output + assert "[tiny|small|medium]" in result.output + + +def test_cli_options_rejects_unknown_engine(): + result = runner.invoke(app, ["options", "layout", "no_such_engine"]) + assert result.exit_code == 2 + assert "unknown engine 'no_such_engine'" in _flatten_error_box(result.output) + + @pytest.mark.parametrize( ("extra_args", "expected"), [