From c46168d639e1be620b5b989b29dc651fdee600ea Mon Sep 17 00:00:00 2001 From: Abdul Moiz Muhammad Date: Sat, 19 Sep 2026 00:48:36 +0500 Subject: [PATCH 1/5] Fix gitignore pycache pattern, link LICENSE in README, test against Python 3.9-3.12 --- .github/workflows/ci.yml | 22 ++++++++++++++---- .gitignore | 10 ++++++-- README.md | 2 +- .../__pycache__/__init__.cpython-312.pyc | Bin 281 -> 0 bytes .../parsers/__pycache__/cnic.cpython-312.pyc | Bin 1246 -> 0 bytes .../__pycache__/intermediate.cpython-312.pyc | Bin 330 -> 0 bytes .../__pycache__/matric.cpython-312.pyc | Bin 324 -> 0 bytes 7 files changed, 26 insertions(+), 8 deletions(-) delete mode 100644 docling_pk/parsers/__pycache__/__init__.cpython-312.pyc delete mode 100644 docling_pk/parsers/__pycache__/cnic.cpython-312.pyc delete mode 100644 docling_pk/parsers/__pycache__/intermediate.cpython-312.pyc delete mode 100644 docling_pk/parsers/__pycache__/matric.cpython-312.pyc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4900b39..7e962ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,19 +9,31 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + fail-fast: false + steps: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ".[dev]" + pip install -r requirements.txt + + - name: Run tests with coverage + run: pytest -v --cov=docling_pk --cov-report=term-missing --cov-report=xml - - name: Run tests - run: pytest -v --cov=docling_pk --cov-report=term-missing \ No newline at end of file + - name: Upload coverage report + uses: actions/upload-artifact@v4 + with: + name: coverage-report-py${{ matrix.python-version }} + path: coverage.xml + if: always() \ No newline at end of file diff --git a/.gitignore b/.gitignore index b3a02f7..057506a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ *.docx -.__pycache__/ -.vscode/ \ No newline at end of file +__pycache__/ +.vscode/ +*.pyc +venv/ +*.egg-info/ +.pytest_cache/ +.coverage +coverage.xml diff --git a/README.md b/README.md index 0191c12..8a971e1 100644 --- a/README.md +++ b/README.md @@ -75,4 +75,4 @@ alone. ## License -TBD \ No newline at end of file +This project is licensed under the MIT License. See [LICENSE](LICENSE) for the full text. \ No newline at end of file diff --git a/docling_pk/parsers/__pycache__/__init__.cpython-312.pyc b/docling_pk/parsers/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 780ab0bc705b7c15eb446f23415e6de197d4ca88..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 281 zcmZuryH3ME5Zv`S1Ok!x1j<+f?gtPhB`uPhmE}8`gS)e}c7#&$6*N?IdilB8Y5z2#mLx#$L>({8odJqxW_dV9|Ma(4W- WKNR{>i1-GJS6IByj_;?uC*c;3Oh^#` diff --git a/docling_pk/parsers/__pycache__/cnic.cpython-312.pyc b/docling_pk/parsers/__pycache__/cnic.cpython-312.pyc deleted file mode 100644 index 81969ba410c53b1ed7ed3f2ba0f2c11311c86377..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1246 zcma)5OK1~Y7(Vk%nzrg>Kx=#!D`dnrq7_BNTyJl#s3liY5lvG|GUp^soJ_)*iH|yk zEUw^P8CSgv7vZj|b>UKXcBkBXF(HNMu^SN=f=HyUTzO8C@r8mv%>VfQ*E#=x=HLE) zAE0!>x|Lhv0eHy_m*lRpTA|83fBT%M2f9N+;0IXBMDqY?^>T*r)yKa3G+ zZoB&ziBw&r=U^0dll9XNc0ovx4N04^T1|Un{o{Oh8%F2v*?&0=Z<5Jvqd-PCo_15 z!-~V@JZ|1+33<1!$w4qDI6U6w@ZXL6w2)lS=X_UN>?-XlqTTD_j^GHGha$zLyRy5O zFZ`k-QorYGKlI+>CxF%yB>=NszGm~wq}%j1oNoa7{u}Ji{NbD0XWpJTX zBvPqZL>RN(XcA_lr`=fDeA1>_eVwXZtK;dM$@R1?i*0>7qYuA^W(s)}!X?v0ICQCi zFN9!0H?n4CxcEJFnxe%dwh*>LtnocR5(i{Rc$ps5VjmBH%ZvNCjEET4Eb`b*_- zQ{7otcUF@PHP}=~>gvd{y5}J*kH43}z>bB$W8ackJ3G0&A+h4$QXO9!e=t!yb8gum ze-EK0|+!_8f^vFXW3^1L1xeOMJ(VMgNFJ#*R)z z@LC24mW_!Bv)%jM26%wNcs*qwyod7zSTfKNyq%itU05;7S^~#$&q2>CptWSME^r;+ znEh+E76>)g?kV>@RW?=p^MRXz+UApwVo#@aj;W2sIT1A*QjIXRVX9e4n?GVlUC3on4O z@&YFvQzx9N-LRDJ``L~ye`VPraCGkD>4N*g%U;x1g6!*#`uj(#Y%3Ot% zip9+1iw`40-%(wP{WyDM$!{33tii%9ZJKg! z>xZ-UKbEyy7`ZEKHIp|OR>^L81=ds^Eud{45~^}q3voejI zm-WM0`ybA@+7{$2IaL*&N`2lg! Date: Sat, 19 Sep 2026 00:57:42 +0500 Subject: [PATCH 2/5] Add future annotations import for Python 3.9 compatibility --- docling_pk/extractor.py | 1 + docling_pk/ocr.py | 1 + docling_pk/parsers/_federal_board.py | 2 ++ docling_pk/parsers/cnic.py | 2 +- docling_pk/parsers/intermediate.py | 2 ++ docling_pk/parsers/matric.py | 2 ++ docling_pk/preprocessor.py | 1 + docling_pk/schema.py | 2 +- 8 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docling_pk/extractor.py b/docling_pk/extractor.py index 722c7f9..e87a954 100644 --- a/docling_pk/extractor.py +++ b/docling_pk/extractor.py @@ -8,6 +8,7 @@ from docling_pk.parsers import cnic, matric, intermediate from docling_pk.parsers._federal_board import detect_document_type from docling_pk.schema import DocumentResult, FieldResult +from __future__ import annotations SUPPORTED_TYPES = ("cnic", "matric", "intermediate") diff --git a/docling_pk/ocr.py b/docling_pk/ocr.py index 5f51215..ad5aba0 100644 --- a/docling_pk/ocr.py +++ b/docling_pk/ocr.py @@ -1,4 +1,5 @@ from typing import Iterable +from __future__ import annotations import numpy as np diff --git a/docling_pk/parsers/_federal_board.py b/docling_pk/parsers/_federal_board.py index 049e0fa..064bd4f 100644 --- a/docling_pk/parsers/_federal_board.py +++ b/docling_pk/parsers/_federal_board.py @@ -1,3 +1,5 @@ +from __future__ import annotations + def detect_document_type(raw_text: str) -> str: text = (raw_text or "").upper() if "INTERMEDIATE" in text: diff --git a/docling_pk/parsers/cnic.py b/docling_pk/parsers/cnic.py index a8282f5..feb2a54 100644 --- a/docling_pk/parsers/cnic.py +++ b/docling_pk/parsers/cnic.py @@ -1,7 +1,7 @@ import re from docling_pk.schema import FieldResult - +from __future__ import annotations CNIC_PATTERN = re.compile(r"\b\d{5}-\d{7}-\d\b") GENDER_PATTERN = re.compile(r"\b(?:gender\s*[:\-]?\s*)?(male|female|m|f)\b", re.IGNORECASE) diff --git a/docling_pk/parsers/intermediate.py b/docling_pk/parsers/intermediate.py index 4ee77f9..bdb9816 100644 --- a/docling_pk/parsers/intermediate.py +++ b/docling_pk/parsers/intermediate.py @@ -1,2 +1,4 @@ +from __future__ import annotations + def extract(raw_text: str) -> dict: return {} diff --git a/docling_pk/parsers/matric.py b/docling_pk/parsers/matric.py index 4ee77f9..bdb9816 100644 --- a/docling_pk/parsers/matric.py +++ b/docling_pk/parsers/matric.py @@ -1,2 +1,4 @@ +from __future__ import annotations + def extract(raw_text: str) -> dict: return {} diff --git a/docling_pk/preprocessor.py b/docling_pk/preprocessor.py index bd983ce..02f557f 100644 --- a/docling_pk/preprocessor.py +++ b/docling_pk/preprocessor.py @@ -1,4 +1,5 @@ from pathlib import Path +from __future__ import annotations import cv2 diff --git a/docling_pk/schema.py b/docling_pk/schema.py index 4ab4f01..e7088c7 100644 --- a/docling_pk/schema.py +++ b/docling_pk/schema.py @@ -6,7 +6,7 @@ is the one shape every other module builds against, so a change here affects the CLI, the tests, and every parser at once. """ - +from __future__ import annotations from dataclasses import dataclass, field from typing import Optional From 04d734f7a33f78055a4698e79938fdd9d679776b Mon Sep 17 00:00:00 2001 From: Abdul Moiz Muhammad Date: Sat, 26 Sep 2026 08:24:21 +0500 Subject: [PATCH 3/5] Restore real preprocessing, OCR, and parser logic overwritten by PR #9 --- docling_pk/ocr.py | 68 +++++++++-- docling_pk/parsers/__init__.py | 3 - docling_pk/parsers/_federal_board.py | 171 ++++++++++++++++++++++++++- docling_pk/parsers/cnic.py | 147 +++++++++++++++++++++-- docling_pk/parsers/intermediate.py | 20 +++- docling_pk/parsers/matric.py | 23 +++- docling_pk/preprocessor.py | 102 ++++++++++++++-- 7 files changed, 489 insertions(+), 45 deletions(-) diff --git a/docling_pk/ocr.py b/docling_pk/ocr.py index ad5aba0..4c551a4 100644 --- a/docling_pk/ocr.py +++ b/docling_pk/ocr.py @@ -1,17 +1,65 @@ -from typing import Iterable -from __future__ import annotations +""" +OCR wrapper for docling-pk. + +Wraps EasyOCR and returns plain text plus average confidence. The +reader is created once per process and reused. Languages are +configurable: CNIC needs English plus Urdu, certificates read better +English-only since the combined model introduces Arabic-script noise +into English words. + +Phone photos are sometimes shot in the wrong orientation. run_ocr +tries all four right-angle rotations and keeps whichever one EasyOCR +reads with the highest confidence times detection count. +""" import numpy as np +import easyocr + +_reader = None +_reader_languages = [None] + + +def _get_reader(languages: tuple = ("en", "ur")) -> easyocr.Reader: + global _reader + if _reader is None or _reader_languages[0] != languages: + _reader = easyocr.Reader(list(languages), gpu=True) + _reader_languages[0] = languages + return _reader + + +def _ocr_once(reader: easyocr.Reader, image: np.ndarray) -> tuple[str, float, int]: + detections = reader.readtext(image) + if not detections: + return "", 0.0, 0 + + lines = [text for (_, text, _) in detections] + scores = [score for (_, _, score) in detections] + raw_text = "\n".join(lines) + avg_confidence = sum(scores) / len(scores) + return raw_text, avg_confidence, len(detections) + +def run_ocr(image: np.ndarray, languages: tuple = ("en", "ur")) -> tuple[str, float]: + """ + Runs OCR on a preprocessed image, correcting 90-degree rotations. -def run_ocr(image: np.ndarray, languages: Iterable[str] = ("en",)) -> tuple[str, float]: - import easyocr + Args: + image: single-channel numpy array from preprocessor.preprocess(). + languages: EasyOCR language list. Use ("en",) for English-only + documents like certificates. - reader = easyocr.Reader(list(languages), gpu=False) - results = reader.readtext(image) + Returns: + A tuple of (raw_text, average_confidence) from whichever of the + four right-angle rotations scored best. + """ + reader = _get_reader(languages) + best_text, best_confidence, best_score = "", 0.0, -1.0 - text_parts = [item[1] for item in results] - confidences = [float(item[2]) for item in results] - avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0 + for k in range(4): + rotated = np.rot90(image, k=k) + text, confidence, count = _ocr_once(reader, rotated) + score = confidence * count + if score > best_score: + best_text, best_confidence, best_score = text, confidence, score - return "\n".join(text_parts), avg_confidence + return best_text, best_confidence diff --git a/docling_pk/parsers/__init__.py b/docling_pk/parsers/__init__.py index bcc5cba..e69de29 100644 --- a/docling_pk/parsers/__init__.py +++ b/docling_pk/parsers/__init__.py @@ -1,3 +0,0 @@ -from . import cnic, intermediate, matric - -__all__ = ["cnic", "matric", "intermediate"] diff --git a/docling_pk/parsers/_federal_board.py b/docling_pk/parsers/_federal_board.py index 064bd4f..afb5651 100644 --- a/docling_pk/parsers/_federal_board.py +++ b/docling_pk/parsers/_federal_board.py @@ -1,7 +1,172 @@ -from __future__ import annotations +""" +Shared field extraction for Federal Board of Intermediate and Secondary +Education (Islamabad) certificates. + +Digit matching restricted to ASCII 0-9 explicitly, not \\d, since \\d and +int() both silently accept Unicode digits (Arabic-Indic included), +which showed up in real testing as a silently wrong number instead of +a correctly failed match. + +Label patterns are kept loose since testing found labels misread in +ways that keep breaking stricter patterns ("Certiiicate", "Cerlificate", +"erlific3te"). Letter-for-digit misreads (O/o/l/I for 0/0/1/1) are +corrected inside number tokens, confirmed necessary on a real roll +number read as "1217O6" instead of "121706". The name/father-name +anchor tolerates common misspellings of "daughter" found across real +samples. The TOTAL row search stops at an all-caps subject-name line +(a hard boundary), confirmed necessary on a real sample where the true +total-possible value drifted far from the label and the search +otherwise crossed into a subject row and wrongly grabbed that +subject's own marks instead. +""" + +import re +from docling_pk.schema import FieldResult + +SERIAL_LABEL = re.compile(r"ser\w*\s*n[oa]", re.IGNORECASE) +CERTIFICATE_LABEL = re.compile(r"cer\w*\s*n[oa]", re.IGNORECASE) +ROLL_LABEL = re.compile(r"roll\s*n[oa]", re.IGNORECASE) +REGISTRATION_LABEL = re.compile(r"reg\w*\s*n[oa]", re.IGNORECASE) +GROUP_KEYWORDS = re.compile(r"\b(SCIENCE(?:\s+GENERAL)?|ARTS|COMMERCE|HUMANITIES)\b", re.IGNORECASE) +SESSION_YEAR = re.compile(r"annual\W*([0-9]{4})", re.IGNORECASE) +GRADE = re.compile(r"grade\s+([A-Z])\b", re.IGNORECASE) +DAUGHTER_OF_ANCHOR = re.compile(r"d[ao]ugh\w*\s*[o0]f", re.IGNORECASE) +DOB = re.compile(r"date of birth is\s+([0-9./-]+)", re.IGNORECASE) +TOTAL_LABEL = re.compile(r"\btotal\b", re.IGNORECASE) +NUMBER_RUN = re.compile(r"[0-9][0-9OolI./\s]{2,}[0-9]|[0-9]{3,}") +HARD_BOUNDARY = re.compile(r"^[A-Z\s]{4,}$") + +SKIP_LINE = re.compile(r"certif|cerif|^that$|board|education|islamabad|secondary|intermediate", re.IGNORECASE) + + +def _clean_number(raw: str) -> str: + corrected = raw.translate(str.maketrans("OolI", "0011")) + return re.sub(r"[^0-9/]", "", corrected) + + +def _find_number_near_label(lines: list[str], label_pattern: re.Pattern, window: int = 2) -> FieldResult: + for i, line in enumerate(lines): + if not label_pattern.search(line): + continue + + same_line = label_pattern.sub("", line) + match = NUMBER_RUN.search(same_line) + if match: + return FieldResult(value=_clean_number(match.group()), confidence=0.75) + + offsets = [] + for d in range(1, window + 1): + offsets.append(d) + offsets.append(-d) + for offset in offsets: + j = i + offset + if 0 <= j < len(lines): + match = NUMBER_RUN.search(lines[j]) + if match: + return FieldResult(value=_clean_number(match.group()), confidence=0.65) + return FieldResult(value=None, confidence=0.0) + + return FieldResult(value=None, confidence=0.0) + + +def _find_name_pair(lines: list[str]) -> tuple[FieldResult, FieldResult]: + anchor_idx = None + for i, line in enumerate(lines): + if DAUGHTER_OF_ANCHOR.search(line): + anchor_idx = i + break + + if anchor_idx is None: + return FieldResult(value=None, confidence=0.0), FieldResult(value=None, confidence=0.0) + + name = FieldResult(value=None, confidence=0.0) + for offset in (1, 2, 3): + j = anchor_idx - offset + if j < 0: + break + candidate = lines[j].strip(" '.-") + if len(candidate) >= 5 and not SKIP_LINE.search(candidate): + name = FieldResult(value=candidate, confidence=0.6) + break + + father_name = FieldResult(value=None, confidence=0.0) + for offset in (1, 2): + j = anchor_idx + offset + if j >= len(lines): + break + candidate = lines[j].strip(" '.-") + if len(candidate) >= 5 and not SKIP_LINE.search(candidate): + father_name = FieldResult(value=candidate, confidence=0.6) + break + + return name, father_name + + +def _find_total(lines: list[str]) -> tuple[FieldResult, FieldResult]: + for i, line in enumerate(lines): + if not TOTAL_LABEL.search(line): + continue + + numbers = [] + misses = 0 + for offset in range(1, 6): + j = i - offset + if j < 0: + break + candidate = lines[j].strip() + if HARD_BOUNDARY.match(candidate): + break + match = NUMBER_RUN.search(candidate) + if match: + cleaned = _clean_number(match.group()) + if cleaned: + numbers.append(int(cleaned)) + misses = 0 + else: + misses += 1 + if misses >= 2: + break + if len(numbers) == 2: + break + + if len(numbers) == 2: + possible, obtained = max(numbers), min(numbers) + return ( + FieldResult(value=str(possible), confidence=0.7), + FieldResult(value=str(obtained), confidence=0.7), + ) + return FieldResult(value=None, confidence=0.0), FieldResult(value=None, confidence=0.0) + + return FieldResult(value=None, confidence=0.0), FieldResult(value=None, confidence=0.0) + + +def extract_common_fields(raw_text: str) -> dict[str, FieldResult]: + lines = raw_text.splitlines() + + fields = { + "serial_number": _find_number_near_label(lines, SERIAL_LABEL), + "certificate_number": _find_number_near_label(lines, CERTIFICATE_LABEL), + "roll_number": _find_number_near_label(lines, ROLL_LABEL), + "registration_number": _find_number_near_label(lines, REGISTRATION_LABEL), + } + + group_match = GROUP_KEYWORDS.search(raw_text) + fields["group"] = FieldResult(value=group_match.group(1).upper(), confidence=0.75) if group_match else FieldResult(value=None, confidence=0.0) + + year_match = SESSION_YEAR.search(raw_text) + fields["session_year"] = FieldResult(value=year_match.group(1), confidence=0.8) if year_match else FieldResult(value=None, confidence=0.0) + + grade_match = GRADE.search(raw_text) + fields["grade"] = FieldResult(value=grade_match.group(1), confidence=0.6) if grade_match else FieldResult(value=None, confidence=0.0) + + fields["name"], fields["father_name"] = _find_name_pair(lines) + fields["total_marks_possible"], fields["total_marks_obtained"] = _find_total(lines) + fields["institute"] = FieldResult(value=None, confidence=0.0) + + return fields + def detect_document_type(raw_text: str) -> str: - text = (raw_text or "").upper() - if "INTERMEDIATE" in text: + if re.search(r"higher\s+secondary\s+school\s+certificate", raw_text, re.IGNORECASE): return "intermediate" return "matric" diff --git a/docling_pk/parsers/cnic.py b/docling_pk/parsers/cnic.py index feb2a54..7d59513 100644 --- a/docling_pk/parsers/cnic.py +++ b/docling_pk/parsers/cnic.py @@ -1,20 +1,147 @@ -import re +""" +CNIC field extraction. + +Labels are matched loosely since real OCR misreads them in small, +specific ways ("Gender" as "GenUer", "of" as "o{"/"o/", "Father Name" +as "Fathgr Name"/"Father Namg"). Dates and cnic_number use format-based +matching instead of label matching, since testing showed date labels +and values are read as two separate groups, not adjacent lines. + +Digit matching is restricted to ASCII 0-9 explicitly, not the regex \\d +shorthand. Python's \\d and int() both silently accept Unicode digits +(Arabic-Indic included), so a garbled OCR read could otherwise parse +as a different, wrong number with full confidence instead of correctly +failing to match. + +Gender's search window was widened after testing with the adaptive +threshold step removed: with cleaner OCR text, the M/F value can +appear further from its label than a 1-line window would catch. +""" +import re +from typing import Optional from docling_pk.schema import FieldResult -from __future__ import annotations -CNIC_PATTERN = re.compile(r"\b\d{5}-\d{7}-\d\b") -GENDER_PATTERN = re.compile(r"\b(?:gender\s*[:\-]?\s*)?(male|female|m|f)\b", re.IGNORECASE) +CNIC_PATTERN = re.compile(r"\b([0-9]{5}-[0-9]{7}-[0-9])\b") +DATE_PATTERN = re.compile(r"\b([0-9]{2})[.,]([0-9]{2})[.,]([0-9]{4})\b") + +FIELD_NAMES = [ + "name", "father_name", "cnic_number", + "date_of_birth", "date_of_issue", "date_of_expiry", + "gender", "address", +] + +FATHER_NAME_LABEL = re.compile(r"fath\w*", re.IGNORECASE) +NAME_LABEL = re.compile(r"\bname\b", re.IGNORECASE) +GENDER_LABEL = re.compile(r"gen.er", re.IGNORECASE) +ADDRESS_LABEL = re.compile(r"address", re.IGNORECASE) + + +def extract_cnic_number(raw_text: str) -> FieldResult: + """ + Extracts the CNIC number (format XXXXX-XXXXXXX-X). OCR sometimes + reads the dash as an underscore, colon, or backtick; all are + normalized before matching. + """ + normalized = raw_text.replace("_", "-").replace(":", "-").replace("`", "-") + match = CNIC_PATTERN.search(normalized) + if match: + return FieldResult(value=match.group(1), confidence=0.92) + return FieldResult(value=None, confidence=0.0) + + +def _value_after_label(lines: list[str], label_pattern: re.Pattern, exclude: re.Pattern = None) -> Optional[tuple[str, int]]: + for i, line in enumerate(lines): + if exclude and exclude.search(line): + continue + if label_pattern.search(line): + return line, i + return None + + +def _text_value(lines: list[str], label_pattern: re.Pattern, exclude: re.Pattern = None) -> FieldResult: + found = _value_after_label(lines, label_pattern, exclude) + if found is None: + return FieldResult(value=None, confidence=0.0) + + line, idx = found + same_line = label_pattern.sub("", line).strip(" :-") + if same_line: + return FieldResult(value=same_line, confidence=0.75) + + if idx + 1 < len(lines) and lines[idx + 1].strip(): + return FieldResult(value=lines[idx + 1].strip(), confidence=0.65) + + return FieldResult(value=None, confidence=0.0) + + +def _all_dates(raw_text: str) -> list[str]: + matches = DATE_PATTERN.findall(raw_text) + return [f"{d}.{m}.{y}" for (d, m, y) in matches] + + +def _dates_by_position(raw_text: str) -> tuple[FieldResult, FieldResult, FieldResult]: + """ + Assigns dates by order of appearance, not proximity to a label. + NADRA prints birth, issue, expiry in that fixed order, and testing + shows EasyOCR reads the value block in that order even when + separated from the labels. + """ + dates = _all_dates(raw_text) + results = [] + for i in range(3): + if i < len(dates): + results.append(FieldResult(value=dates[i], confidence=0.8)) + else: + results.append(FieldResult(value=None, confidence=0.0)) + return tuple(results) + + +def extract_gender(raw_text: str) -> FieldResult: + """ + Extracts gender as 'M' or 'F'. Searches up to 4 lines after the + label, not just the next line. + """ + lines = raw_text.splitlines() + found = _value_after_label(lines, GENDER_LABEL) + if found is None: + return FieldResult(value=None, confidence=0.0) + + line, idx = found + remainder = GENDER_LABEL.sub("", line).strip(" :-") + match = re.search(r"\b([MF])\b", remainder.upper()) + if match: + return FieldResult(value=match.group(1), confidence=0.8) + + for offset in range(1, 5): + j = idx + offset + if j >= len(lines): + break + match = re.search(r"\b([MF])\b", lines[j].strip().upper()) + if match: + return FieldResult(value=match.group(1), confidence=0.7) + + return FieldResult(value=None, confidence=0.0) def extract(raw_text: str) -> dict[str, FieldResult]: - cnic_match = CNIC_PATTERN.search(raw_text or "") - gender_match = GENDER_PATTERN.search(raw_text or "") + """ + Extracts all CNIC fields from raw OCR text. Missing fields return + FieldResult(None, 0.0) rather than raising. + """ + lines = raw_text.splitlines() - cnic_value = cnic_match.group(0) if cnic_match else None - gender_value = gender_match.group(1).upper() if gender_match else None + father_name = _text_value(lines, FATHER_NAME_LABEL) + name = _text_value(lines, NAME_LABEL, exclude=FATHER_NAME_LABEL) + dob, issue, expiry = _dates_by_position(raw_text) return { - "cnic_number": FieldResult(value=cnic_value, confidence=0.95 if cnic_value else 0.0), - "gender": FieldResult(value=gender_value, confidence=0.9 if gender_value else 0.0), + "name": name, + "father_name": father_name, + "cnic_number": extract_cnic_number(raw_text), + "date_of_birth": dob, + "date_of_issue": issue, + "date_of_expiry": expiry, + "gender": extract_gender(raw_text), + "address": _text_value(lines, ADDRESS_LABEL), } diff --git a/docling_pk/parsers/intermediate.py b/docling_pk/parsers/intermediate.py index bdb9816..cb12fc7 100644 --- a/docling_pk/parsers/intermediate.py +++ b/docling_pk/parsers/intermediate.py @@ -1,4 +1,18 @@ -from __future__ import annotations +""" +Intermediate (Higher Secondary School Certificate) extraction, Federal +Board Islamabad format. No date_of_birth field, this board does not +print it on the Intermediate certificate. +""" -def extract(raw_text: str) -> dict: - return {} +from docling_pk.schema import FieldResult +from docling_pk.parsers._federal_board import extract_common_fields + +FIELD_NAMES = [ + "serial_number", "certificate_number", "roll_number", "registration_number", + "group", "session_year", "grade", "name", "father_name", "institute", + "total_marks_possible", "total_marks_obtained", +] + + +def extract(raw_text: str) -> dict[str, FieldResult]: + return extract_common_fields(raw_text) diff --git a/docling_pk/parsers/matric.py b/docling_pk/parsers/matric.py index bdb9816..9f6565c 100644 --- a/docling_pk/parsers/matric.py +++ b/docling_pk/parsers/matric.py @@ -1,4 +1,21 @@ -from __future__ import annotations +""" +Matric (Secondary School Certificate) extraction, Federal Board Islamabad +format. Adds date_of_birth on top of the shared fields, since this board +prints DOB on the Matric certificate but not on the Intermediate one. +""" -def extract(raw_text: str) -> dict: - return {} +from docling_pk.schema import FieldResult +from docling_pk.parsers._federal_board import extract_common_fields, DOB + +FIELD_NAMES = [ + "serial_number", "certificate_number", "roll_number", "registration_number", + "group", "session_year", "grade", "name", "father_name", "institute", + "date_of_birth", "total_marks_possible", "total_marks_obtained", +] + + +def extract(raw_text: str) -> dict[str, FieldResult]: + fields = extract_common_fields(raw_text) + match = DOB.search(raw_text) + fields["date_of_birth"] = FieldResult(value=match.group(1), confidence=0.8) if match else FieldResult(value=None, confidence=0.0) + return fields diff --git a/docling_pk/preprocessor.py b/docling_pk/preprocessor.py index 02f557f..095b226 100644 --- a/docling_pk/preprocessor.py +++ b/docling_pk/preprocessor.py @@ -1,25 +1,101 @@ -from pathlib import Path -from __future__ import annotations +""" +Image preprocessing for docling-pk. + +Prepares a document photo for OCR. Order: grayscale, then blur check, +then deskew, then denoise. Adaptive thresholding was tested directly +(A/B test comparing thresholded output against plain grayscale on real +CNIC and certificate samples) and removed: it consistently lowered +EasyOCR's own confidence on every sample tested. EasyOCR's detector is +trained on natural photos, not pre-binarized ones, so feeding it our +own black-and-white conversion worked against it rather than for it. +""" import cv2 +import numpy as np +from pathlib import Path + +BLUR_VARIANCE_THRESHOLD = 35.0 class TooBlurryError(ValueError): - """Raised when an image is too blurry for reliable extraction.""" + """Raised when an image is too blurry to reliably OCR.""" + + +def preprocess(image_path: str) -> np.ndarray: + """ + Prepares a document image for OCR. + + Steps: + 1. Load and validate + 2. Grayscale + 3. Blur check + 4. Deskew + 5. Denoise + + Args: + image_path: path to the input image file. + Returns: + A single-channel numpy array ready for OCR. -def preprocess(image_path: str): + Raises: + FileNotFoundError: if image_path does not exist. + ValueError: if the file cannot be read as an image. + TooBlurryError: if the image is too blurry to reliably OCR. + """ path = Path(image_path) if not path.exists(): - raise FileNotFoundError(f"Image path does not exist: {image_path}") + raise FileNotFoundError(f"Image not found: {image_path}") + + img = cv2.imread(str(path)) + if img is None: + raise ValueError(f"Could not read image: {image_path}. Check the file is a valid PNG/JPG.") + + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + _check_blur(gray) + + deskewed = _deskew(gray) + denoised = cv2.fastNlMeansDenoising(deskewed, h=10) + return denoised + + +def _check_blur(gray: np.ndarray) -> None: + """ + Measures sharpness using the variance of the Laplacian. Threshold of + 35 picked from 4 real samples: two failing blurry images scored 12.9 + and 27.1, two working images scored 48.2 and 54.9. + """ + variance = cv2.Laplacian(gray, cv2.CV_64F).var() + if variance < BLUR_VARIANCE_THRESHOLD: + raise TooBlurryError( + f"Image too blurry to process (sharpness score {variance:.1f}, " + f"minimum {BLUR_VARIANCE_THRESHOLD}). Please retake the photo." + ) + + +def _deskew(gray: np.ndarray) -> np.ndarray: + """ + Estimates and corrects rotation using the minimum-area bounding box + of dark pixels. Known limitation, confirmed by direct testing: this + fails against cluttered or textured backgrounds, since the angle is + estimated from every dark pixel in the frame, not the document + specifically. v1 assumes a plain background. + """ + inverted = cv2.bitwise_not(gray) + _, binary = cv2.threshold(inverted, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) + coords = np.column_stack(np.where(binary > 0)) - image = cv2.imread(str(path)) - if image is None: - raise ValueError(f"Invalid image file: {image_path}") + if coords.shape[0] < 50: + return gray - gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - sharpness = float(cv2.Laplacian(gray, cv2.CV_64F).var()) - if sharpness < 25.0: - raise TooBlurryError("Image is too blurry to process reliably") + angle = cv2.minAreaRect(coords)[-1] + if angle < -45: + angle = -(90 + angle) + else: + angle = -angle - return image + (h, w) = gray.shape + center = (w // 2, h // 2) + matrix = cv2.getRotationMatrix2D(center, angle, 1.0) + rotated = cv2.warpAffine(gray, matrix, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE) + return rotated From 83194383a87f6cbcd785155605f2c3f7f5c5152e Mon Sep 17 00:00:00 2001 From: Abdul Moiz Muhammad Date: Sat, 26 Sep 2026 08:26:06 +0500 Subject: [PATCH 4/5] Remove leftover placeholder test --- tests/test_placeholder.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 tests/test_placeholder.py diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py deleted file mode 100644 index e11fa75..0000000 --- a/tests/test_placeholder.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Placeholder test so pytest has something to collect. pytest exits with -code 5 when zero tests exist at all, which is what happens with no test -suite merged yet. This gets replaced once tests/test_cnic.py (the -Integration Engineer's Week 2 deliverable) lands as its own PR. -""" - - -def test_placeholder(): - assert True \ No newline at end of file From 7e135009847d1ea61706cb0393c892087426eef8 Mon Sep 17 00:00:00 2001 From: Abdul Moiz Muhammad Date: Sat, 26 Sep 2026 23:20:52 +0500 Subject: [PATCH 5/5] Fix CLI test to handle click's version-inconsistent stderr capture --- tests/test_cli.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index d0289a8..2f2b045 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,6 +15,21 @@ COMMANDS = ["cnic", "matric", "intermediate", "degree"] +def _combined_output(result) -> str: + """ + Returns stdout plus stderr if available. click's CliRunner does not + consistently capture stderr separately across versions: older + versions merge stdout+stderr into one stream (result.output holds + both, result.stderr raises ValueError), newer versions capture them + separately. Checking both this way works either way, instead of + assuming one specific click version's behavior. + """ + combined = result.output + if result.stderr_bytes is not None: + combined += result.stderr + return combined + + @pytest.mark.parametrize("command", COMMANDS) def test_command_registered(command): """Each of the four document commands exists and shows help.""" @@ -48,4 +63,4 @@ def test_invalid_path_exits_with_error_code(command): """A path that does not exist exits with a distinct error code, not the placeholder code.""" result = runner.invoke(app, [command, "this/path/does/not/exist.jpg"]) assert result.exit_code == 2 - assert "does not exist" in result.stderr \ No newline at end of file + assert "does not exist" in _combined_output(result) \ No newline at end of file