fix(#304): Standardize encoding from pdftotext

docs/changelog.rst

@@ -10,6 +10,22 @@ NEXT RELEASE
 
 .. THANKS FOR CONTRIBUTING; ADD YOUR UNRELEASED CHANGES HERE!
 
+* Publish a wheel alongside the sdist on PyPI (`#560`_ by `@KyleKing`_)
+* **Drop support for Python 3.9, now requiring 3.10+** (`#585`_ by `@KyleKing`_)
+* Publish a ``-full`` Docker image to GHCR with all system dependencies
+  preinstalled except LibreOffice, which most users don't need; includes
+  pocketsphinx for offline audio transcription (`#600`_ by `@KyleKing`_)
+* Minor nitpick to remove unused line (`#559`_ by `@KyleKing`_)
+* Utilize openpyxl for xlsx after xlrd dropped support (`#572`_ by `@KyleKing`_)
+* Prevent opening a cmd window with ``Popen()`` on Windows (`#574`_ by
+  `@KyleKing`_)
+* Replace unmaintained ``antiword`` with LibreOffice for ``.doc`` extraction
+  (`#582`_ by `@KyleKing`_)
+* Add ``.ods`` spreadsheet support (`#583`_ by `@KyleKing`_)
+* Report which file extension triggered a missing module or missing
+  executable error (`#575`_ by `@KyleKing`_)
+* Standardize PDF text extraction to UTF-8 to avoid mangled non-Latin text
+  (`#596`_ by `@KyleKing`_)
 * Honor ``input_encoding`` in the csv, json, eml, and html parsers and add
   the ``--input-encoding`` CLI flag; introduces ``DecodedParser`` base class
   for parsers that operate on decoded text. A valid but incorrect
@@ -25,17 +41,6 @@ NEXT RELEASE
   ``process_source`` (or ``extract``/``extract_from_text``/
   ``extract_from_bytes``, which are still honored)** (`#588`_ by `@KyleKing`_)
 
-TBD
--------------------
-
-* Publish a ``-full`` Docker image to GHCR with all system dependencies preinstalled except LibreOffice, which most users don't need; includes pocketsphinx for offline audio transcription (`#526`_ by `@KyleKing`_)
-* Minor nitpick to remove unused line (`#562`_ by `@KyleKing`_)
-* Utilize openpyxl for xlsx after xlrd dropped support (`#544`_ by `@KyleKing`_)
-* Prevent opening a cmd window with ``Popen()`` on Windows (`#574`_ by `@KyleKing`_)
-* Replace unmaintained ``antiword`` with LibreOffice for ``.doc`` extraction (`#582`_ by `@KyleKing`_)
-* Add ``.ods`` spreadsheet support (`#583`_ by `@KyleKing`_)
-* Report which file extension triggered a missing module or missing executable error (`#575`_ by `@KyleKing`_)
-
 2.0.0
 -------------------
 
@@ -395,8 +400,8 @@ TBD
 .. _#114: https://github.com/deanmalmgren/textract/issues/114
 .. _#116: https://github.com/deanmalmgren/textract/issues/116
 .. _#119: https://github.com/deanmalmgren/textract/issues/119
-.. _#126: https://github.com/deanmalmgren/textract/issues/126
 .. _#122: https://github.com/deanmalmgren/textract/issues/122
+.. _#126: https://github.com/deanmalmgren/textract/issues/126
 .. _#127: https://github.com/deanmalmgren/textract/issues/127
 .. _#136: https://github.com/deanmalmgren/textract/issues/136
 .. _#139: https://github.com/deanmalmgren/textract/issues/139
@@ -414,14 +419,16 @@ TBD
 .. _#495: https://github.com/deanmalmgren/textract/pull/495
 .. _#502: https://github.com/deanmalmgren/textract/pull/502
 .. _#520: https://github.com/deanmalmgren/textract/pull/520
-.. _#526: https://github.com/deanmalmgren/textract/issues/526
 .. _#543: https://github.com/deanmalmgren/textract/pull/543
-.. _#544: https://github.com/deanmalmgren/textract/issues/544
 .. _#559: https://github.com/deanmalmgren/textract/pull/559
-.. _#562: https://github.com/deanmalmgren/textract/pull/562
+.. _#560: https://github.com/deanmalmgren/textract/pull/560
+.. _#572: https://github.com/deanmalmgren/textract/pull/572
 .. _#573: https://github.com/deanmalmgren/textract/pull/573
 .. _#574: https://github.com/deanmalmgren/textract/pull/574
 .. _#575: https://github.com/deanmalmgren/textract/pull/575
 .. _#582: https://github.com/deanmalmgren/textract/pull/582
 .. _#583: https://github.com/deanmalmgren/textract/pull/583
+.. _#585: https://github.com/deanmalmgren/textract/pull/585
 .. _#588: https://github.com/deanmalmgren/textract/pull/588
+.. _#596: https://github.com/deanmalmgren/textract/pull/596
+.. _#600: https://github.com/deanmalmgren/textract/pull/600

tests/test_pdf.py

@@ -157,3 +157,33 @@ def test_pdftotext_fallback_to_pdfminer(self):
         ):
             result = textract.process(self.raw_text_filename)
         assert result
+
+    @pytest.mark.skipif(not _HAS_PDFTOTEXT, reason=_NO_PDFTOTEXT_REASON)
+    def test_pdftotext_preserves_non_latin_text(self):
+        """pdftotext extraction keeps Czech diacritics and CJK characters.
+
+        Guards the ``-enc UTF-8`` flag: without it, poppler builds that
+        default to Latin-1 mangle non-Latin scripts (issues #304 and #107).
+        """
+        d = Path(self.get_extension_directory())
+        result = textract.process(str(d / "unicode_text.pdf"), method="pdftotext")
+        text = result.decode("utf-8")
+        assert "žluťoučký" in text
+        assert "ďábelské" in text
+        assert "안녕하세요" in text
+
+    @pytest.mark.skipif(not _HAS_PDFTOTEXT, reason=_NO_PDFTOTEXT_REASON)
+    def test_pdftotext_ignores_input_encoding(self):
+        """pdftotext output is always UTF-8, so a conflicting input_encoding
+        must not corrupt it, and passing one warns that it's ignored.
+        """
+        d = Path(self.get_extension_directory())
+        with pytest.warns(UserWarning, match="input_encoding is ignored"):
+            result = textract.process(
+                str(d / "unicode_text.pdf"),
+                method="pdftotext",
+                input_encoding="latin-1",
+            )
+        text = result.decode("utf-8")
+        assert "žluťoučký" in text
+        assert "안녕하세요" in text

textract/parsers/pdf_parser.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import warnings
 from tempfile import mkdtemp
 
 from pdfminer.high_level import extract_text
@@ -9,12 +10,27 @@
 from .image import Parser as TesseractParser
 from .utils import ShellParser
 
+PDFTOTEXT_ENC = "UTF-8"
+
 
 class Parser(ShellParser):
     """Extract text from pdf files using either the ``pdftotext`` method
     (default) or the ``pdfminer`` method.
     """
 
+    def process_source(self, source, input_encoding, output_encoding="utf8", **kwargs):
+        method = kwargs.get("method", "")
+        if input_encoding is not None and method in {"", "pdftotext"}:
+            warnings.warn(
+                "input_encoding is ignored by the pdftotext method: its output "
+                "is always forced to UTF-8 via -enc UTF-8, so this argument has "
+                "no effect unless pdftotext is unavailable and pdfminer is used "
+                "instead.",
+                UserWarning,
+                stacklevel=2,
+            )
+        return super().process_source(source, input_encoding, output_encoding, **kwargs)
+
     def extract(self, filename, method="", **kwargs):
         if method in {"", "pdftotext"}:
             try:
@@ -36,12 +52,14 @@ def extract(self, filename, method="", **kwargs):
 
     def extract_pdftotext(self, filename, **kwargs):
         """Extract text from pdfs using the pdftotext command line utility."""
+        # Force UTF-8 output because some OSes default to Latin-1, which
+        # mangles non-Latin text (e.g. Czech diacritics, CJK)
+        args = ["pdftotext", "-enc", PDFTOTEXT_ENC]
         if "layout" in kwargs:
-            args = ["pdftotext", "-layout", filename, "-"]
-        else:
-            args = ["pdftotext", filename, "-"]
+            args.append("-layout")
+        args += [filename, "-"]
         stdout, _ = self.run(args)
-        return stdout
+        return stdout.decode(PDFTOTEXT_ENC, errors="replace")
 
     def extract_pdfminer(self, filename, **kwargs):
         """Extract text from pdfs using pdfminer."""