From 70e24263968628a164a5ec3bb189dff715fb7e33 Mon Sep 17 00:00:00 2001 From: Kyle King Date: Sat, 1 Aug 2026 21:32:48 -0600 Subject: [PATCH 1/5] fix(#304): Standardize encoding from PDF --- tests/pdf/unicode_text.pdf | Bin 0 -> 19248 bytes tests/test_pdf.py | 14 ++++++++++++++ textract/parsers/pdf_parser.py | 8 +++++--- 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 tests/pdf/unicode_text.pdf diff --git a/tests/pdf/unicode_text.pdf b/tests/pdf/unicode_text.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e3d307fbd7e60507d368cbbbb68cee2fe780a5cb GIT binary patch literal 19248 zcmeIa2T;^W^FItqP?F?~tROjT1ePp0=cFJgVac#W$tWmEkSIxVmYj2vEICLLB#I=7 zq5?{Ich$S|ynE{T)l+pf> z9s~daAWf|CL_~nf2zv_`O8^u#q6w6>vUNc?0cCBCT@ca;Q=}O}ObpN2#R*|-hv%Lg zFtVW30V2lTJ3OJlZMmF#*vw;vpEHt{4sxL0lklc4Aq!C9!9G1`z8BM$5El6+ia!2I z59LR@%uL34{=#^_J{d$ya0ri*^<0F{+q6l{p zmx_}K-K=39I;aqXjZL&ZgHW&8NtU}O3E|m1)+bJAXEiO`3tMYo@CxTsg1?g-j^--) zR5I2yn&5yx=@uXXN^N+4F~aCFkT1W^vO6Yif{ErCi>9*mo^4KrJ+vaiy*_ zZ3D4iR%V6OlWf(wEl1MM@=T{vY^co(ncXgGyGJ@(_{BuDP2BZ!N#|#L%|>x|n-day zKK#hEAUj{cG^6isteyNS5NqU0mubif=g~d9^pynIRYrmyaiODIR5c4&>UN!C{&zM^ zZ)oyV)5K}z^XVHtn~zL2h9yFIZcT38mNW8;0lExxejty3aX!U3(O(e1BwQd{CP~$N z=fe`^4ay?i1QB30>#*9rK9|jgao2NT@kn2U=^g>SKkg!Sb`7VL@M7a0D7od)DS6YS zh<1B8=3)EQgoA36h^*vUc2{DdRwLnK!Olh9qsyDk7vG(d+FxgW6u>Lrz6}a*FsK9X zyw-+#-w1zEDUG|pnloT%*O!6SHlyV4Ee|j^FN*o1m{0F;i{F>E40%byvnW zO0){hTS`7+#{$#OtD(IBY!;Cb$80dB}M7<&!1n=VpR#AU>u5e#V`9vU69q4gb7?3JzNFs(?z04r6KWvX^bg@7cgJ5~+{Z_PgS z5oI63>Mk}NUCW8rdUh}(E5v`BH(8=My67F9%(=Q*b~(P?b<;0Zo^C$y;wFObv5)eb zQKU+5_Q9GJ3F`hFR<}Xj5BLo=LsR*2q)htz!n<*T`YG6f2At+~bnC?9T2Yw|M&s&H z-r5j`d;6w{Jh^)jN$HJ+HF2Nh$#8bvsByR(;))4dO&5 zamLFgSI#CH3=ime4`6EjQ-l}W{TBi4yzy8aC61U4<=2Qxw<2dnT!h0PEbuz+>wDA($tn$Or`q=Q+B(Bk3s3H2+lhGO?A(=#0^NG-r ziL3$A<|kDpN8M2+IIpo3;=B+&j(aMpbV(h;;a?oCU4h9D)OZd+)ZWU=J=b%&Cn$N- zt=T)Mg9Q&W%;Cix36Z>2(4|4Kwe((+^!kbd(_2Yiulpt#4#=feFcs+a#@!RC-GZbj zA6{m+4z2RWeS7!fOUrhm`G`l2_SfcRRH7(>WEWJuayp9*Ub+$EDm!M;EZeq#-=yy; zY%4t~$aX0}K8wzcd&ZvE(z}FLx1#1j^RS~SliI?PMLj7u&>^uMlV{b!hip5#tV&5? z*sZQ(=Yep0sH9TRN)px;nOm3-O-eegCs>MU?p{+~Y+uR`#L8UfKIAaTb8rtph1c}~6jgLnM)QwYRPa5`l!>!V!nq?K)x`(O zzOiG#=MX4iTX3{}>Vjy8RM_emdeHFNxtiKnn7&K6aj&X)2X5aET?p4ePJ_VhA-!7aNgW#1$_`Q*P!*I-*yp&Dm5$MoMAJ$t@ zL7b!y+DKHdlH}G2ZoTUB>8OhE?Y`l!EU$H0UxJM~!KE84n_a>@eE>_5mgQBaq*X0rIsa&0UkvA&LSr8+Z!29_U4qa`mtk`vyK0Cjc z@_IKo`OAzg_acHjo2ffijKTt+6Pi2|zRk8d=c;+Xc>E66VPTFhVJ>?)NwQd2Ig%vg zDW;9k?VHiwJE7quuYxqQFVsB`OE}A-e=_^1@oalQ?uatO??PKT4k3P#ZeBFe(!5?% zX#%wOYP>O1mt0yYJXgIOcBBIy~H_lJ-)oew@yhu6VCNR zpYER-ZHlW!#fvj-ZHynS%6=5ExtbnMO zc3bxkmEBC3e9)CGP*D7wEY=?x2&6dFA^8BJB&J~Wh)g-&uHCxMD%d|DA zE7`KhL{n2Oi(Z3H`;^wnk8dz1*yG;;r@MLRPuq3uE7yg@=~c$0_)ly8iK~vK=$&lWVc`A3TJSO@7@j zVhwS{(%RwqGluINEfo)rj%cwwb+Pjkp;LNx{gc|nZW(<2tJkp!1l}V{TfX2aG#FEpH0*1Vq*ud0n5KUv({ZcLskao4^XRk^ZxbJj1-e(vdn z%cnW~B-^2Ao5@~Z`WxPTdxP@0e3B)BEc&u4<5r%0{pCEEL-HWT(Wub2x&#tU5U9=O zQoAJ!?|aPEs1Oy~gU;#!8{u2SB`J5#tf>0L!{LG&zC`W;ww1bt%=a33r;Ka#!7p_U z8gxQ4@{3!zr`U;Cx#?D&32Q1U<|vfQ>0+8kv`kq~=N6hzOV}oY_I%=vsl=<6UUt0k z51IVRAAAKA0%W|uil_7Xgkqnr+sOP(<@5f^O3#+KPT@#sy}wk>EAXvbvIaG;M@T`l zRq%OIb&@z1Sq`}Q!qco*M!Gus*4~swW`&{8Tyk_&w&5O@ZN<{c)m6B)q90$XF?vV( z*gjp!oo7suFR|o(bF2IS;M|&gm_+h0=cbM4%Rtg=Tv~P*eACV@-Ej|wI`zS?!NA;C z-UZXjVU3aCB#6&M;ugKW2l*BI4JyX7PmE;uPd{c^+?#jaZSncK!-&AsqypJkL=i+k zxpi_U|L}Zom2u%{E^#3X8NK82T$qYUQbNdb+**m!Oz^`p>Rc*}~Fg9FyVS_yPMJ1J14 z8eLp|2PI1jFZ-(XwU%C3cBm;zU(tx={;q15Ht;@;*_hURNTY^@;iiy1+i66bcUx3` z|6RzKvbm4`v)9)BBOBVeFy`lFAbO`o^_D}E&*qw4)a~udw7p;TCA~Z2d|2k8ml9oF ztMg2{O8N%IDe^+@Ju&A`^n&mK9XMT6d>cL2KXVQV@6S17U$kAN8ouuk{pCLS>2MqM zOrG_nEg>(}5=wa9MlvB1SIjDIsdA z5{j^@I)%IA%cg-8uz16OF8kRk8r=mGClCFp9Q)HI!!FT$F=aS~p3kmZ>iMTvbh~w4 zvo*rC_GWl&Mzm9B>=f$dL90$JSG}sppl86{v&{ivr06+mad~mVnr|mNKGY{b-5gh5 z&&G+HQbp+f%SzLb*=;LUvdL$?PXoDTZ+*h%gN7-k2t6t36=3HOZSFN{xXvZ0oXa(4 z%MxJyB=nlA_Zjf??hDNq}RJ7k@5QArdbi_u*mN%DLIG~VV z2UhVx_4)KlyZ-?Nml8{U6i-RUQH6l6Xx&myj9E`$PyzL7!WXcr=>G2e+HKSUAKI=J zBiZiEibzTHvOAnx$pCUk4DhikDJ%%L`n)|A7dhmd5e%!_xp5o64gZ?4L_L^IK+E9v zz(KuiO$BGA83F)(6=^XL@-BI>J`$RkHC8Do=;2WlcO$iTu{)MRcUeQ$CNlX&?Azsb zzxRPOg{w@Hc+)a{UxcdPBt0&9dOAXG(UARSu!2Gd$Ql~V`mk+$DQ>NSwgj(OqcA?q z?rikp%JG2!OAwP48|TYFDx)ToWi>-w12v+#N$Oy$u!JKWXXJiErUMWSb8PtUiZX(i^c0kTa~`r6f9v#+lfTHX#Z80^fq_;DyV z$9vvyf|w+Hy>w{IS+??piq%N$a5>TKY@5YHjhQhk!$O`TRiZKgrYj_iS?@-eDju5^ zt^Kub%;yA?L#3HdkR$C-y9Xk=&X_Q9x8q^tC+hM+dX z?a}Tj8^g!0j6=NoNE`0ZI7R%)@&Ww|9i(+}2KI8d^Ua!FMUha^gBb0V(t$*FfqU0% z%&=n9?T_EEZPaMZ36fxhrn6Z+_tw!DQp)8cON>vI>%&U&78%d;8EkX-q7FZhh9AJ# z)3Ec!zmBG1uSei*`Cr}Fw9y{e;p6O$u>x9;U{bt3k6*~1U+J#kU1^P|dF6by+K}aK zZnPaxc2=jddDCh!1hF_EJ#7%GAxa3>=NpkhuypTe6 z*YNHKsTZHydA_azB4+8M8ISE>J~O79`^t`Npf62A7;$}=x2?`gwAdRvNYRgI((ijO z33T#pi}bDxY8&`+r|0N)qr`GN>&P1u`RK+c+v@K(JS?lrrZFP(b(PjKMjvRbje}Wr z+13eEgyO2hahkU9_`6dPg3Rv`Hddx&UgBPlCo24T@zQ z4`V7HbTfRrj+t_ysapJAPaw-dNl|I5(>$q8himN{C&RmdNeyicjg|c8=?SAg@yn6d zOmQGYg25Nzw*$BO9n+gookGO?ZH^x20|WuNQG=d_Q4IHCnkEl~8c$Y|dm+U>Q6rQa z?pW$h_!hX`R=Fh9#_u)Bt>);MZ)k`io26jakO}MGOnh<-xOw4f1c^%>W8T}*vHjR> zDc`RN^+DbUuHKJ6r$$T`RL*V_lZzR7sv>Wi5!P}FJT2E~XX{-Z?5GZSb5pB|>l~+) z*eZc@H_v+BGqxTL2$#m3YUOh*NgPr?mdg#Bb9&6674pEH%SU#VuB<<#w~5ppEY~Kp z@-+F5qQ@(<`LY2Ux%sj=$MSm{yOzpBqB(_uRIapCzBsDK0?AaH0%u#xr5HCUN{uHK z#rHspR8hY76Y4Cw)v#A8(ugh{;@P$foT}XgwuCB<5xsDPUryThy1EJK32)b>_A`b* zw)w)=>BqHbRKX~Cf>lE z#EjRJ{jfn@RIOQ`K@X`Vo_?ksle@%s027}| z9&gG&23jB6KJU~q=Ut_Lc;va{soeUuh-P-o5|bw3^X`n*oy#pR3=2mHO1zB&%=fQP z=8e#Qu+_4pDB*PDtfJ+5FA*^LLYA09AOF@0UcjUT&aI*mtLqZaDDZNy(5+FiLFdrb>XS>Aap#(THb4-f6QW-J+*qpk!W$N<$L4(8bmDdE4NU zC#+d?9oNLjm2r+!Z+ds!b+NnN8~<@&wUDcj^SIEYyxPaOMU0<1f)FQWboz#j{^>j0 z32&~)mUyiFM>!WoDF#P($_Dy@5rm!k9X3;+uyNzBd$wQ9*aO@dy3_4&>tJT@u4HB* zR{^IrZ@6JMjmn;cm>@Mlz}JTl1rEyRmFyOAJ8ZH-%-o5Evo3mj$rxZSkSYh_MteH| zI!m9W)@9ZT%#h`#h%U965ht4VUV7&7s-ex2iF`Y%iiI<`IL%Mm03t%Z8jl;(>%hdX zPNkgATU8`h!%M<&DSrPw^Jwa&RNbZ44qe#ZemGYxexVaCX{WPa)$ z1_CywQNL<$CV1o`*;1QjDU-jNtiM9<;hBRIXohU-{phpE>Aj&IQ@3L@2%mt41IRF&-VDW4vUR%$Mn zfI**zJsmD<6yT5S%m<56<7Mn#x>;ml*hpt_Vg(n@j>rC(qtV>whL7;1=A}5{G8k2q?u56_MZ1iEZzs})h^e; zb9bIl>MZFT0M~pf2iW_TINBA2m|jn-ZMV>TH5vK(1itr@Dpx~kQ9hwYQoG=KBZJ@! zyB>ZDhjC=~3=aEL!Tk(JZ0oJAV^NsDR&L8xqzM!9z(Oc>JEV zK0a+73gNe&Bc7;F{;af(#ZcIBPjX+WTRFNsd|CX#De2YsqqGWckH#b(TLmjdez3Wb zIQN96=YAHiQ6inT3G@cDK=w_cY|9w+ot#6g3E>Aq?%5B=ZBq5Qu8nymQi#a#$eRLs zni=I{p4Q@i{kqpEkzeiGDL*+=to_PUKPk%UJm(f9cWTkOxiC`pi>B=g!KTg!oiWZ{ zhHNoI>8fN|g%51X#ZR9Dicjs6N98{C-IK9Q#j<+Zp*S8h{)J;)@^O6h4t$#b=#7A@UeDXvwYotlw7N)gb@aL}FsJOZ>3Alw@4LEw8p@9P_m<@E`l1hnjwx3B$rlnJKvlimY!fB%;-{Ae zJK)?eDs!lcV_GO+bQY6p5t^bDj|b<^-+M$GIzC<;;G{{e$zim!)t&V7+xTK2kQJTB zG#Xf_0W}m{Ry;eRSUKvyE?@0@JabECYDlR$vJ(7w`+7B-M0ruVSW(wL#q*5B)U0>4 z5ufU!4bSfRrLkNc!+FFVbZ@Lcq?Dx)fS;Vrkf!Qgg27mi^EGLo;|b7T#_XCH71Q|z zZ(sb`vC6;!LCDr4778<>ggujW9Y1CJMCaWHCRnrxT^QDmGHih6Yq`gCxXk zI2?VKfsSXj=Xh&9XS8ec5j*GWdvdm0?eq8p`S>e-_?;dZ2a;PC@YezklxJ|y6xQyy ztU0!v&oiyvZ_-xQ~6yP)~Y4Tu2BCTc9q*-q}N@|YpzC^V>hKJOLe-XWvW5aDUzt(fnq4obwmO2ufQDt`9mZP;P?Oi#jBMwYxQ&A!36 zqE*AxRUji*UNl|?rpg1EWlcznvfs6od|f^)KYa7f(RX3jQFor_&0spA+-}9YV-^D! zH$=jzxLDoD@e5K0t49F|rg|J|;#TB}l0(7_F{*FdI&jq(%mN-B>-a9E5Bdo_97T#O zimH3vedULbtKFR!f@#tT*c&(#y}EWd!7~)v#T!Z51u zoKJmQS!?5M+ephi8(^B^&X6V|2a`*_D$atH$1;5X?I!Emr!&qqTJMtXyd0rQLz(G& zCC|E)r^GgHuEQ!;;Jw#AGxMn$(=)ziIdhC=CC_*;%Y68}nzT zhwkQ(6qSm|z1)CJ(h5IG3QHbc=PLnqhj~4*coHM4!t>b>Q~`b=<1ccTnG>S=`AM-g z7mp!-NusSG>!$u>uvyT}an`wj*g~04L?knMr5n`DX0S5X;j7R4@389Dx&|8_`yos& zU6x=9i~_xjc%9B0DbE`HUh&-(jh9$Y<5**kvI&L!sh+C(1dju9Ld+l;9~TL1S(K(m zy{wqPd>YF#7t{$Kt1X&5j)~5>>_1B^<`9=nZV_{R#OiXBO1`v$0IT9sHreS^Z4U!d;mX6B6BOripxnfN`RO0vd?D20Y<)2daaM*uI*c(O<+CKyl zJlGYH37^YgA_^!Wm9*xN7u}LG^}@i*m7>MO_FSI@)-Ub34-|5>e1Kbm&o#x90gGvT!nQ5_#Bfx2<$WI*%md z?64t@HMez4-NZ4p15#(}TX6F9bKjUfn+-ML5Of=~*!Jc;(k?bl@!%gYkfH~(2`@i! zP^DK=l}-lHmCPx6tDUgZlkGkd==9pqB4vD3ug2>Q5K|OOGmvir@Uo~{_{G6L;qI$F z7Z1LFq0{Yt1AcI8hS-ZUVy*kc=gaZ~XMpOhD+WUg3dzGJ_eYX}XDrUVf-bk`UoI2Jm>ad+kUEw|x| z>MZ<(OlNXO+8O?%KUz6CyGU6Y zJE0v-+4x5rigHjrD>D~MX9Ey`ALW|=;~VdP0RC2h@+Xhy`$Jbhid4Y-|E*qvA1v^rb^>*X1VPXr9Vmbg3i-K=Vms0UHtxmP zQ_4a>Qc-xqZTRT*0LV07R=ZC*q$sTf{5?R%o440nm9}YMtPf5niTECT@>0UE_QFVM(X40_ zkF~vY!ejUP!k6}si{~$~z|ZgFvI_^Y#o$4kFu3OlFa+sAH))$i^ti8HaR6fsq)SR* zkgseIrbiOwo>ySs#XbtU0ArQ=VslK5@$Rx#Ih=l#>yE@U#iO=F(lhB?u9xIWou?rV z3t`ju!ON}Jni4nCUTzss;vif6&vY9sFoG_EPxM&p-%K$gX_bR8F5&r$b;;8~=*-o_&<^`*(!H?*exDC(Z|(E& z(@xY4pf1N>Bet0`I=QD>Hda{LyK;IjBb=uaF0eWE;~#E+RcDHe*|IVz^@N-!i*@k~_J&HQ ziIGETKKr=}v!t1OSENhc3ry#zeboB!$Z3B`09L^fu%zeOcKwZ36+q9g$8>>=gr6p6 zp?=mZH@WYYfMDqj=pdum>|P6}$9egN$~vacQ{DDieYKgq#ypXZT#9s-Iq5|ciO=p9 zxJD~f6P@K7T3{V@qQn;pc6|knqqg~90x82=Kq|zW=LIpI>)BiPsO~m*YG;%i;qt2ui@Okhj@TI}m5j~Bdoo<=+%D=fQ684nH6lS=YZ=AC2~XrfA^`|8-h zIQu$*&iUgz9!|LrL(Y#h6m2fEI7;C2jQf_zU zxJM2q;v>dF=6(@S#O4=DNWCsbUV2AdOP5^OKKqdeTRt;$jrGS@p`n6gA}yKkhT>B` z&!iNXkiWMAlE}2Go)0sI;@{vj-XKcE!>5D3k;8dME*5-6*0wFR-_%s{ZL;zj@o{tA zbO>c9qQ^+Lv{`a;e=KH6_FC3{ib(68#Hy`+l-q>bd{;5YUErZV=5t+}!D0`_P`HX% zD5G^$cuYr;)ypknU2UW$H$_|!qk!syGc(Aef6t0r4zrPwF?ON2AeZNX*XxHXMeRT= zg-Y+`=Z5t@8DlL@5JM?o)>h&|&!ZU0LYHQ)O=iEhNneW^$QbykDD5GZ%4L1FyEY7) zefQ?zIT^dtmlW0>&lLKT%t|WWl>NYhRM0AkTO)NZw1OAj7=8ab@NtsvlP#({&qTRnE=BScjjl#OMqTgh~VF8vRi z84gydKZmj+UT7X$c4uaJdf&Np?mK&TK8hhTRr#^a^<^hs&(bS| z-Y2})MUd?mmwO^*qsS5jN2W@P-tSxjHfnefZ+(j4v5l#Q-lm=aDtJX&ly7it`F_-h582~+9yoTbYq-NWX$$+aQoD*?I>x8XjDFLv^sa?elQ;6- zVUuzWcJ}aO6}8H(vH0YT!Oo>^B2Jc|Jg~#Ww<#x75B$Vp>(lG$cr$=R|88mG=c8QF z8tY486PKuK2Ra$Ucw0q=sdBPs7cpj!p?l8c#`?U~{;bIZ5sjy+!p+auPs2Y?<;gYNX??D$$R1!6AcI?K zAE^+=aie#*Pk6&Dgn3HVo`iVly7#4-C6cVmaVeKuG}?_|^Dnvf>_T4yM=EEdNZ*BN zxG53bP*2Y4CajYk%8u;H9Q-h$GV5;L*!|l0D~V>Vgc-YdcA?@8==j-0F_z%WSCndIfV}^0a z>3pAU38uLoPDkc<#q~0N?xsr37$302O z-X*G?98nV`HjPR#+L-t_4fDHUEo6>U=RU8xxg$oc!4~Jds7huN3oK9{djF`#t=@a< zIAf|#9pcT-TrJ~_%sf@^%)k^f_pjBy3+%l94i9)mW?;aS{2>3jLM#>SbDAz+fvBs5 zlp1xtF4ttLi*%M&tFd0h8taPA8rSld`cf>%xn&n&&Ay(q(|ly<-koq_X-0<`s9Zy# z->+7bUzEucCZi)Z3?5|*r|{A-Cp>p-N%XN3FZ8u)cy!8b!@AvrEBw{~ahJyv;tuUr_IW+wmH`~?9wZ1} zcLmw59J+ez@d61aX9a$}&pAec(@o9{8WA%$Jek&lL0cTW%>fEbu-# z15TY^JQKJevO)ZE-{OluecA#>+Tf(|-#x{mAH%+tw?bh*1z7@rJg@!!Z2QME-ycDi z)b}7u!ok57p(QPcbg{HDRYf3qG!YiAw#H7sgCC@m6G;eFuOba90GoXGc9M``S!4 zyJmcWCed6j!VBYqYG4%x(@JD*X~rDuHpZ8*^96o$JGad+qOC!L*HYnbQN`Ni0A_;b(;BPlTBv(jMI$LoCChUzuL%3mfR@oW-o^o6;el z4g%JZFc_f@$Rs7y0ce5~cl)cVv3`(UM4WhA&*}q8jv-{0eBoV3Nq;Ky3z`El^;O4@Lg&7Jr01sJI6V{5{_J z@2>H;kq`e*fe)OY5Ab*IzX?Ln4*TaI1a0&`xdG&-qHS4JIW{_eLzT&+gNwhAqKmoF zrveqBqk}iVH(pJkx)ah=3*lk_M3vzKwGr+vhJT83{$Tp19d%<11W?`B2@Mx~n*ykb za7Ma1nIfD4kniQAeXUK>9uloAkx7y`A8O`L&RXzzA%HFfze!5JMNN}<+* zl7?bOulOfO0rD#|5@23DtMBLrM*whB$0x&^XSlNp* z?KZYD0j$hKne_NnKq?MW2umw@PbY+yr>eH8r;Vwg8I!meo`}1UyPbm_Y8HUIovppI zkh>_8F{(HoA%tq9hk;CDcmNS6GjkzLY1tn%sGca3rHhM$5D@6*=Emy=|KoAdF-8;(IURf`0egN!8ic4Ok)$ItBWWT(+|N8KMA&X=KZ!1 zUQ?tU5cLHR%nSXt7?dbfR|sW8GgniDlNbsaazdz?n_HP801$p&2rrBm0$`Vr(3Fzr zhjECYdH*0rE#wEgw5%?|$r-f}G1Tuceo*}}hF&lby~3YX{D0jAN}<22=TEzEH+Den z0`%K1P;^34sNZp*;4W<>(pF5*$qMyL4-w$6XVTUjJ_7|8I8u*CPK~#d*XVv{fjQ?8XA5!f9EcRcE{LiZU zhZz60$Umgm|5@z67Wto5_YX1tYmt9QvH!E!|933%2Zn~K@eyTmLnCB=gP;CX1@R3a z{EqL**dpwJS}1H0VUBdNGj?%7m@z{@Xmk+|s1EpsQlc6bfNzK)su&4?0wvW@r6L|E z2n?uhYix?JLt%nINn50;4H}LFDxuKCKLL}!Vf-k@e|BE*pXdDvcmM0WS`Nnc=neir zZvWVzva!7dyQd|Olqv^$FOWaW{53Q2kLmCmehm2Y~66j4qzL&uK zWHu#(Z+Qs-;9K1b;QO|J5&wX4N@q$+7@Yn+9XQkcfcIIV+#ZdRzv`GoDpiSF1A+o z=;^-e{Wrb*uI+E0gRtk((L#BSim{W8g1tEsr5bb-VFo~<>FED~1N;pe2Yf@#{{*A| zbW(mGtRk{9va%o$p8yCX00DvcQAsHQ80wP`-4{b=1iqokKgeKG-%0pUq~FLN|+c_x86>l=$EKC>jVHB?J8Z&4;3gq4;5_F*F|( zO)Dq{{I*yWCXGf}*` zP(FY;;QJUD0*0fkjeY^_e~h8p=w!(+V^9Iq)%2$^uz-NzU&o-R^2%T7VElg_-;hY1d1-d`}4d4aQLtL1q%oY{)HC~ zM&CDo=7mFkwxx@cF{*^s37rMgvhqaTIRZfRJp}l6bWzzh^zlNaPSCk1R7V)XRu&=!;uDmHOQY&{B_aG!Spi9utK$8467+q6x=f`k5vDfIu66*Rv;<5* z3L*>PgGj(7ATVh@Nid8L0)`4erKM1ogrRRaXBT59mv3&3`Zr4;7#<6YjG8Ro{|A1V BulN7} literal 0 HcmV?d00001 diff --git a/tests/test_pdf.py b/tests/test_pdf.py index 4f3c8edd..f4e80883 100644 --- a/tests/test_pdf.py +++ b/tests/test_pdf.py @@ -140,3 +140,17 @@ def test_pdftotext_fallback_to_pdfminer(self): ): result = textract.process(self.raw_text_filename) assert result + + @pytest.mark.skipif(not _HAS_PDFTOTEXT, reason=_NO_PDFTOTEXT_REASON) + def test_pdftotext_preserves_non_latin_text(self): + """pdftotext extraction keeps Czech diacritics and CJK characters. + + Guards the ``-enc UTF-8`` flag: without it, poppler builds that + default to Latin-1 mangle non-Latin scripts (issues #304 and #107). + """ + d = Path(self.get_extension_directory()) + result = textract.process(str(d / "unicode_text.pdf"), method="pdftotext") + text = result.decode("utf-8") + assert "žluťoučký" in text + assert "ďábelské" in text + assert "안녕하세요" in text diff --git a/textract/parsers/pdf_parser.py b/textract/parsers/pdf_parser.py index f2d6beca..f987b5e6 100644 --- a/textract/parsers/pdf_parser.py +++ b/textract/parsers/pdf_parser.py @@ -36,10 +36,12 @@ def extract(self, filename, method="", **kwargs): def extract_pdftotext(self, filename, **kwargs): """Extract text from pdfs using the pdftotext command line utility.""" + # Force UTF-8 output: some poppler builds default to Latin-1, which + # mangles non-Latin text (e.g. Czech diacritics, CJK). + args = ["pdftotext", "-enc", "UTF-8"] if "layout" in kwargs: - args = ["pdftotext", "-layout", filename, "-"] - else: - args = ["pdftotext", filename, "-"] + args.append("-layout") + args += [filename, "-"] stdout, _ = self.run(args) return stdout From 58d0885d832d3ffef3b3ed3b59abde360b858535 Mon Sep 17 00:00:00 2001 From: Kyle King Date: Sat, 1 Aug 2026 22:06:20 -0600 Subject: [PATCH 2/5] fix: synchronize UTF8 --- tests/test_pdf.py | 16 ++++++++++++++++ textract/parsers/pdf_parser.py | 21 +++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/test_pdf.py b/tests/test_pdf.py index f4e80883..487a6be6 100644 --- a/tests/test_pdf.py +++ b/tests/test_pdf.py @@ -154,3 +154,19 @@ def test_pdftotext_preserves_non_latin_text(self): assert "žluťoučký" in text assert "ďábelské" in text assert "안녕하세요" in text + + @pytest.mark.skipif(not _HAS_PDFTOTEXT, reason=_NO_PDFTOTEXT_REASON) + def test_pdftotext_ignores_input_encoding(self): + """pdftotext output is always UTF-8, so a conflicting input_encoding + must not corrupt it, and passing one warns that it's ignored. + """ + d = Path(self.get_extension_directory()) + with pytest.warns(UserWarning, match="input_encoding is ignored"): + result = textract.process( + str(d / "unicode_text.pdf"), + method="pdftotext", + input_encoding="latin-1", + ) + text = result.decode("utf-8") + assert "žluťoučký" in text + assert "안녕하세요" in text diff --git a/textract/parsers/pdf_parser.py b/textract/parsers/pdf_parser.py index f987b5e6..bbe1c898 100644 --- a/textract/parsers/pdf_parser.py +++ b/textract/parsers/pdf_parser.py @@ -1,5 +1,6 @@ import os import shutil +import warnings from tempfile import mkdtemp from pdfminer.high_level import extract_text @@ -15,6 +16,19 @@ class Parser(ShellParser): (default) or the ``pdfminer`` method. """ + def process(self, filename, input_encoding, output_encoding="utf8", **kwargs): + method = kwargs.get("method", "") + if input_encoding is not None and method in {"", "pdftotext"}: + warnings.warn( + "input_encoding is ignored by the pdftotext method: its output " + "is always forced to UTF-8 via -enc UTF-8, so this argument has " + "no effect unless pdftotext is unavailable and pdfminer is used " + "instead.", + UserWarning, + stacklevel=2, + ) + return super().process(filename, input_encoding, output_encoding, **kwargs) + def extract(self, filename, method="", **kwargs): if method in {"", "pdftotext"}: try: @@ -37,13 +51,16 @@ def extract(self, filename, method="", **kwargs): def extract_pdftotext(self, filename, **kwargs): """Extract text from pdfs using the pdftotext command line utility.""" # Force UTF-8 output: some poppler builds default to Latin-1, which - # mangles non-Latin text (e.g. Czech diacritics, CJK). + # mangles non-Latin text (e.g. Czech diacritics, CJK). Decode here + # (instead of returning bytes) so the "unicode sandwich" passes this + # through unchanged rather than second-guessing it via chardet or a + # caller-supplied input_encoding. args = ["pdftotext", "-enc", "UTF-8"] if "layout" in kwargs: args.append("-layout") args += [filename, "-"] stdout, _ = self.run(args) - return stdout + return stdout.decode("utf-8", errors="replace") def extract_pdfminer(self, filename, **kwargs): """Extract text from pdfs using pdfminer.""" From ac29024a05e8f05d393ca69861587ed98982fa6c Mon Sep 17 00:00:00 2001 From: Kyle King Date: Sat, 1 Aug 2026 22:13:34 -0600 Subject: [PATCH 3/5] docs: add changelog entry and refactor --- docs/changelog.rst | 2 ++ textract/parsers/pdf_parser.py | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 29540552..28caa5fe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,7 @@ TBD * Prevent opening a cmd window with ``Popen()`` on Windows (`#574`_ by `@KyleKing`_) * Replace unmaintained ``antiword`` with LibreOffice for ``.doc`` extraction (`#582`_ by `@KyleKing`_) * Report which file extension triggered a missing module or missing executable error (`#575`_ by `@KyleKing`_) +* Standardize PDF text extraction to UTF-8 to avoid mangled non-Latin text (`#596`_ by `@KyleKing`_) 2.0.0 ------------------- @@ -404,3 +405,4 @@ TBD .. _#574: https://github.com/deanmalmgren/textract/pull/574 .. _#575: https://github.com/deanmalmgren/textract/pull/575 .. _#582: https://github.com/deanmalmgren/textract/pull/582 +.. _#596: https://github.com/deanmalmgren/textract/pull/596 diff --git a/textract/parsers/pdf_parser.py b/textract/parsers/pdf_parser.py index bbe1c898..8c8d1d09 100644 --- a/textract/parsers/pdf_parser.py +++ b/textract/parsers/pdf_parser.py @@ -10,6 +10,8 @@ from .image import Parser as TesseractParser from .utils import ShellParser +PDFTOTEXT_ENC = "UTF-8" + class Parser(ShellParser): """Extract text from pdf files using either the ``pdftotext`` method @@ -50,17 +52,14 @@ def extract(self, filename, method="", **kwargs): def extract_pdftotext(self, filename, **kwargs): """Extract text from pdfs using the pdftotext command line utility.""" - # Force UTF-8 output: some poppler builds default to Latin-1, which - # mangles non-Latin text (e.g. Czech diacritics, CJK). Decode here - # (instead of returning bytes) so the "unicode sandwich" passes this - # through unchanged rather than second-guessing it via chardet or a - # caller-supplied input_encoding. - args = ["pdftotext", "-enc", "UTF-8"] + # Force UTF-8 output because some OSes default to Latin-1, which + # mangles non-Latin text (e.g. Czech diacritics, CJK) + args = ["pdftotext", "-enc", PDFTOTEXT_ENC] if "layout" in kwargs: args.append("-layout") args += [filename, "-"] stdout, _ = self.run(args) - return stdout.decode("utf-8", errors="replace") + return stdout.decode(PDFTOTEXT_ENC, errors="replace") def extract_pdfminer(self, filename, **kwargs): """Extract text from pdfs using pdfminer.""" From ea099f07254f9613e031703da2eaec08a4eda40e Mon Sep 17 00:00:00 2001 From: Kyle King Date: Thu, 27 Aug 2026 06:34:24 -0600 Subject: [PATCH 4/5] fix: use process_source --- textract/parsers/pdf_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/textract/parsers/pdf_parser.py b/textract/parsers/pdf_parser.py index 8c8d1d09..3eccf7fd 100644 --- a/textract/parsers/pdf_parser.py +++ b/textract/parsers/pdf_parser.py @@ -18,7 +18,7 @@ class Parser(ShellParser): (default) or the ``pdfminer`` method. """ - def process(self, filename, input_encoding, output_encoding="utf8", **kwargs): + def process_source(self, source, input_encoding, output_encoding="utf8", **kwargs): method = kwargs.get("method", "") if input_encoding is not None and method in {"", "pdftotext"}: warnings.warn( @@ -29,7 +29,7 @@ def process(self, filename, input_encoding, output_encoding="utf8", **kwargs): UserWarning, stacklevel=2, ) - return super().process(filename, input_encoding, output_encoding, **kwargs) + return super().process_source(source, input_encoding, output_encoding, **kwargs) def extract(self, filename, method="", **kwargs): if method in {"", "pdftotext"}: From 611a40b8206cb3c16e95131799bce0b8cb570d5e Mon Sep 17 00:00:00 2001 From: Kyle King Date: Thu, 27 Aug 2026 06:51:07 -0600 Subject: [PATCH 5/5] docs: update changelog to capture recent releases --- docs/changelog.rst | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1652bcec..da309469 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,22 @@ NEXT RELEASE .. THANKS FOR CONTRIBUTING; ADD YOUR UNRELEASED CHANGES HERE! +* Publish a wheel alongside the sdist on PyPI (`#560`_ by `@KyleKing`_) +* **Drop support for Python 3.9, now requiring 3.10+** (`#585`_ by `@KyleKing`_) +* Publish a ``-full`` Docker image to GHCR with all system dependencies + preinstalled except LibreOffice, which most users don't need; includes + pocketsphinx for offline audio transcription (`#600`_ by `@KyleKing`_) +* Minor nitpick to remove unused line (`#559`_ by `@KyleKing`_) +* Utilize openpyxl for xlsx after xlrd dropped support (`#572`_ by `@KyleKing`_) +* Prevent opening a cmd window with ``Popen()`` on Windows (`#574`_ by + `@KyleKing`_) +* Replace unmaintained ``antiword`` with LibreOffice for ``.doc`` extraction + (`#582`_ by `@KyleKing`_) +* Add ``.ods`` spreadsheet support (`#583`_ by `@KyleKing`_) +* Report which file extension triggered a missing module or missing + executable error (`#575`_ by `@KyleKing`_) +* Standardize PDF text extraction to UTF-8 to avoid mangled non-Latin text + (`#596`_ by `@KyleKing`_) * Honor ``input_encoding`` in the csv, json, eml, and html parsers and add the ``--input-encoding`` CLI flag; introduces ``DecodedParser`` base class for parsers that operate on decoded text. A valid but incorrect @@ -25,18 +41,6 @@ NEXT RELEASE ``process_source`` (or ``extract``/``extract_from_text``/ ``extract_from_bytes``, which are still honored)** (`#588`_ by `@KyleKing`_) -TBD -------------------- - -* Publish a ``-full`` Docker image to GHCR with all system dependencies preinstalled except LibreOffice, which most users don't need; includes pocketsphinx for offline audio transcription (`#526`_ by `@KyleKing`_) -* Minor nitpick to remove unused line (`#562`_ by `@KyleKing`_) -* Utilize openpyxl for xlsx after xlrd dropped support (`#544`_ by `@KyleKing`_) -* Prevent opening a cmd window with ``Popen()`` on Windows (`#574`_ by `@KyleKing`_) -* Replace unmaintained ``antiword`` with LibreOffice for ``.doc`` extraction (`#582`_ by `@KyleKing`_) -* Add ``.ods`` spreadsheet support (`#583`_ by `@KyleKing`_) -* Report which file extension triggered a missing module or missing executable error (`#575`_ by `@KyleKing`_) -* Standardize PDF text extraction to UTF-8 to avoid mangled non-Latin text (`#596`_ by `@KyleKing`_) - 2.0.0 ------------------- @@ -396,8 +400,8 @@ TBD .. _#114: https://github.com/deanmalmgren/textract/issues/114 .. _#116: https://github.com/deanmalmgren/textract/issues/116 .. _#119: https://github.com/deanmalmgren/textract/issues/119 -.. _#126: https://github.com/deanmalmgren/textract/issues/126 .. _#122: https://github.com/deanmalmgren/textract/issues/122 +.. _#126: https://github.com/deanmalmgren/textract/issues/126 .. _#127: https://github.com/deanmalmgren/textract/issues/127 .. _#136: https://github.com/deanmalmgren/textract/issues/136 .. _#139: https://github.com/deanmalmgren/textract/issues/139 @@ -415,15 +419,16 @@ TBD .. _#495: https://github.com/deanmalmgren/textract/pull/495 .. _#502: https://github.com/deanmalmgren/textract/pull/502 .. _#520: https://github.com/deanmalmgren/textract/pull/520 -.. _#526: https://github.com/deanmalmgren/textract/issues/526 .. _#543: https://github.com/deanmalmgren/textract/pull/543 -.. _#544: https://github.com/deanmalmgren/textract/issues/544 .. _#559: https://github.com/deanmalmgren/textract/pull/559 -.. _#562: https://github.com/deanmalmgren/textract/pull/562 +.. _#560: https://github.com/deanmalmgren/textract/pull/560 +.. _#572: https://github.com/deanmalmgren/textract/pull/572 .. _#573: https://github.com/deanmalmgren/textract/pull/573 .. _#574: https://github.com/deanmalmgren/textract/pull/574 .. _#575: https://github.com/deanmalmgren/textract/pull/575 .. _#582: https://github.com/deanmalmgren/textract/pull/582 .. _#583: https://github.com/deanmalmgren/textract/pull/583 +.. _#585: https://github.com/deanmalmgren/textract/pull/585 .. _#588: https://github.com/deanmalmgren/textract/pull/588 -.. _#596: https://github.com/deanmalmgren/textract/pull/596 \ No newline at end of file +.. _#596: https://github.com/deanmalmgren/textract/pull/596 +.. _#600: https://github.com/deanmalmgren/textract/pull/600 \ No newline at end of file