diff --git a/docs/changelog.rst b/docs/changelog.rst index 3be7179b..da309469 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,22 @@ NEXT RELEASE .. THANKS FOR CONTRIBUTING; ADD YOUR UNRELEASED CHANGES HERE! +* Publish a wheel alongside the sdist on PyPI (`#560`_ by `@KyleKing`_) +* **Drop support for Python 3.9, now requiring 3.10+** (`#585`_ by `@KyleKing`_) +* Publish a ``-full`` Docker image to GHCR with all system dependencies + preinstalled except LibreOffice, which most users don't need; includes + pocketsphinx for offline audio transcription (`#600`_ by `@KyleKing`_) +* Minor nitpick to remove unused line (`#559`_ by `@KyleKing`_) +* Utilize openpyxl for xlsx after xlrd dropped support (`#572`_ by `@KyleKing`_) +* Prevent opening a cmd window with ``Popen()`` on Windows (`#574`_ by + `@KyleKing`_) +* Replace unmaintained ``antiword`` with LibreOffice for ``.doc`` extraction + (`#582`_ by `@KyleKing`_) +* Add ``.ods`` spreadsheet support (`#583`_ by `@KyleKing`_) +* Report which file extension triggered a missing module or missing + executable error (`#575`_ by `@KyleKing`_) +* Standardize PDF text extraction to UTF-8 to avoid mangled non-Latin text + (`#596`_ by `@KyleKing`_) * Honor ``input_encoding`` in the csv, json, eml, and html parsers and add the ``--input-encoding`` CLI flag; introduces ``DecodedParser`` base class for parsers that operate on decoded text. A valid but incorrect @@ -25,17 +41,6 @@ NEXT RELEASE ``process_source`` (or ``extract``/``extract_from_text``/ ``extract_from_bytes``, which are still honored)** (`#588`_ by `@KyleKing`_) -TBD -------------------- - -* Publish a ``-full`` Docker image to GHCR with all system dependencies preinstalled except LibreOffice, which most users don't need; includes pocketsphinx for offline audio transcription (`#526`_ by `@KyleKing`_) -* Minor nitpick to remove unused line (`#562`_ by `@KyleKing`_) -* Utilize openpyxl for xlsx after xlrd dropped support (`#544`_ by `@KyleKing`_) -* Prevent opening a cmd window with ``Popen()`` on Windows (`#574`_ by `@KyleKing`_) -* Replace unmaintained ``antiword`` with LibreOffice for ``.doc`` extraction (`#582`_ by `@KyleKing`_) -* Add ``.ods`` spreadsheet support (`#583`_ by `@KyleKing`_) -* Report which file extension triggered a missing module or missing executable error (`#575`_ by `@KyleKing`_) - 2.0.0 ------------------- @@ -395,8 +400,8 @@ TBD .. _#114: https://github.com/deanmalmgren/textract/issues/114 .. _#116: https://github.com/deanmalmgren/textract/issues/116 .. _#119: https://github.com/deanmalmgren/textract/issues/119 -.. _#126: https://github.com/deanmalmgren/textract/issues/126 .. _#122: https://github.com/deanmalmgren/textract/issues/122 +.. _#126: https://github.com/deanmalmgren/textract/issues/126 .. _#127: https://github.com/deanmalmgren/textract/issues/127 .. _#136: https://github.com/deanmalmgren/textract/issues/136 .. _#139: https://github.com/deanmalmgren/textract/issues/139 @@ -414,14 +419,16 @@ TBD .. _#495: https://github.com/deanmalmgren/textract/pull/495 .. _#502: https://github.com/deanmalmgren/textract/pull/502 .. _#520: https://github.com/deanmalmgren/textract/pull/520 -.. _#526: https://github.com/deanmalmgren/textract/issues/526 .. _#543: https://github.com/deanmalmgren/textract/pull/543 -.. _#544: https://github.com/deanmalmgren/textract/issues/544 .. _#559: https://github.com/deanmalmgren/textract/pull/559 -.. _#562: https://github.com/deanmalmgren/textract/pull/562 +.. _#560: https://github.com/deanmalmgren/textract/pull/560 +.. _#572: https://github.com/deanmalmgren/textract/pull/572 .. _#573: https://github.com/deanmalmgren/textract/pull/573 .. _#574: https://github.com/deanmalmgren/textract/pull/574 .. _#575: https://github.com/deanmalmgren/textract/pull/575 .. _#582: https://github.com/deanmalmgren/textract/pull/582 .. _#583: https://github.com/deanmalmgren/textract/pull/583 +.. _#585: https://github.com/deanmalmgren/textract/pull/585 .. _#588: https://github.com/deanmalmgren/textract/pull/588 +.. _#596: https://github.com/deanmalmgren/textract/pull/596 +.. _#600: https://github.com/deanmalmgren/textract/pull/600 \ No newline at end of file diff --git a/tests/pdf/unicode_text.pdf b/tests/pdf/unicode_text.pdf new file mode 100644 index 00000000..e3d307fb Binary files /dev/null and b/tests/pdf/unicode_text.pdf differ diff --git a/tests/test_pdf.py b/tests/test_pdf.py index 1c1c2fdd..26036329 100644 --- a/tests/test_pdf.py +++ b/tests/test_pdf.py @@ -157,3 +157,33 @@ def test_pdftotext_fallback_to_pdfminer(self): ): result = textract.process(self.raw_text_filename) assert result + + @pytest.mark.skipif(not _HAS_PDFTOTEXT, reason=_NO_PDFTOTEXT_REASON) + def test_pdftotext_preserves_non_latin_text(self): + """pdftotext extraction keeps Czech diacritics and CJK characters. + + Guards the ``-enc UTF-8`` flag: without it, poppler builds that + default to Latin-1 mangle non-Latin scripts (issues #304 and #107). + """ + d = Path(self.get_extension_directory()) + result = textract.process(str(d / "unicode_text.pdf"), method="pdftotext") + text = result.decode("utf-8") + assert "žluťoučký" in text + assert "ďábelské" in text + assert "안녕하세요" in text + + @pytest.mark.skipif(not _HAS_PDFTOTEXT, reason=_NO_PDFTOTEXT_REASON) + def test_pdftotext_ignores_input_encoding(self): + """pdftotext output is always UTF-8, so a conflicting input_encoding + must not corrupt it, and passing one warns that it's ignored. + """ + d = Path(self.get_extension_directory()) + with pytest.warns(UserWarning, match="input_encoding is ignored"): + result = textract.process( + str(d / "unicode_text.pdf"), + method="pdftotext", + input_encoding="latin-1", + ) + text = result.decode("utf-8") + assert "žluťoučký" in text + assert "안녕하세요" in text diff --git a/textract/parsers/pdf_parser.py b/textract/parsers/pdf_parser.py index f2d6beca..3eccf7fd 100644 --- a/textract/parsers/pdf_parser.py +++ b/textract/parsers/pdf_parser.py @@ -1,5 +1,6 @@ import os import shutil +import warnings from tempfile import mkdtemp from pdfminer.high_level import extract_text @@ -9,12 +10,27 @@ from .image import Parser as TesseractParser from .utils import ShellParser +PDFTOTEXT_ENC = "UTF-8" + class Parser(ShellParser): """Extract text from pdf files using either the ``pdftotext`` method (default) or the ``pdfminer`` method. """ + def process_source(self, source, input_encoding, output_encoding="utf8", **kwargs): + method = kwargs.get("method", "") + if input_encoding is not None and method in {"", "pdftotext"}: + warnings.warn( + "input_encoding is ignored by the pdftotext method: its output " + "is always forced to UTF-8 via -enc UTF-8, so this argument has " + "no effect unless pdftotext is unavailable and pdfminer is used " + "instead.", + UserWarning, + stacklevel=2, + ) + return super().process_source(source, input_encoding, output_encoding, **kwargs) + def extract(self, filename, method="", **kwargs): if method in {"", "pdftotext"}: try: @@ -36,12 +52,14 @@ def extract(self, filename, method="", **kwargs): def extract_pdftotext(self, filename, **kwargs): """Extract text from pdfs using the pdftotext command line utility.""" + # Force UTF-8 output because some OSes default to Latin-1, which + # mangles non-Latin text (e.g. Czech diacritics, CJK) + args = ["pdftotext", "-enc", PDFTOTEXT_ENC] if "layout" in kwargs: - args = ["pdftotext", "-layout", filename, "-"] - else: - args = ["pdftotext", filename, "-"] + args.append("-layout") + args += [filename, "-"] stdout, _ = self.run(args) - return stdout + return stdout.decode(PDFTOTEXT_ENC, errors="replace") def extract_pdfminer(self, filename, **kwargs): """Extract text from pdfs using pdfminer."""