diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4900b39..7e962ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,19 +9,31 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + fail-fast: false + steps: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ".[dev]" + pip install -r requirements.txt + + - name: Run tests with coverage + run: pytest -v --cov=docling_pk --cov-report=term-missing --cov-report=xml - - name: Run tests - run: pytest -v --cov=docling_pk --cov-report=term-missing \ No newline at end of file + - name: Upload coverage report + uses: actions/upload-artifact@v4 + with: + name: coverage-report-py${{ matrix.python-version }} + path: coverage.xml + if: always() \ No newline at end of file diff --git a/.gitignore b/.gitignore index b3a02f7..057506a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ *.docx -.__pycache__/ -.vscode/ \ No newline at end of file +__pycache__/ +.vscode/ +*.pyc +venv/ +*.egg-info/ +.pytest_cache/ +.coverage +coverage.xml diff --git a/README.md b/README.md index 0191c12..8a971e1 100644 --- a/README.md +++ b/README.md @@ -75,4 +75,4 @@ alone. ## License -TBD \ No newline at end of file +This project is licensed under the MIT License. See [LICENSE](LICENSE) for the full text. \ No newline at end of file diff --git a/docling_pk/extractor.py b/docling_pk/extractor.py index 722c7f9..e87a954 100644 --- a/docling_pk/extractor.py +++ b/docling_pk/extractor.py @@ -8,6 +8,7 @@ from docling_pk.parsers import cnic, matric, intermediate from docling_pk.parsers._federal_board import detect_document_type from docling_pk.schema import DocumentResult, FieldResult +from __future__ import annotations SUPPORTED_TYPES = ("cnic", "matric", "intermediate") diff --git a/docling_pk/ocr.py b/docling_pk/ocr.py index 5f51215..4c551a4 100644 --- a/docling_pk/ocr.py +++ b/docling_pk/ocr.py @@ -1,16 +1,65 @@ -from typing import Iterable +""" +OCR wrapper for docling-pk. + +Wraps EasyOCR and returns plain text plus average confidence. The +reader is created once per process and reused. Languages are +configurable: CNIC needs English plus Urdu, certificates read better +English-only since the combined model introduces Arabic-script noise +into English words. + +Phone photos are sometimes shot in the wrong orientation. run_ocr +tries all four right-angle rotations and keeps whichever one EasyOCR +reads with the highest confidence times detection count. +""" import numpy as np +import easyocr + +_reader = None +_reader_languages = [None] + + +def _get_reader(languages: tuple = ("en", "ur")) -> easyocr.Reader: + global _reader + if _reader is None or _reader_languages[0] != languages: + _reader = easyocr.Reader(list(languages), gpu=True) + _reader_languages[0] = languages + return _reader + + +def _ocr_once(reader: easyocr.Reader, image: np.ndarray) -> tuple[str, float, int]: + detections = reader.readtext(image) + if not detections: + return "", 0.0, 0 + + lines = [text for (_, text, _) in detections] + scores = [score for (_, _, score) in detections] + raw_text = "\n".join(lines) + avg_confidence = sum(scores) / len(scores) + return raw_text, avg_confidence, len(detections) + +def run_ocr(image: np.ndarray, languages: tuple = ("en", "ur")) -> tuple[str, float]: + """ + Runs OCR on a preprocessed image, correcting 90-degree rotations. -def run_ocr(image: np.ndarray, languages: Iterable[str] = ("en",)) -> tuple[str, float]: - import easyocr + Args: + image: single-channel numpy array from preprocessor.preprocess(). + languages: EasyOCR language list. Use ("en",) for English-only + documents like certificates. - reader = easyocr.Reader(list(languages), gpu=False) - results = reader.readtext(image) + Returns: + A tuple of (raw_text, average_confidence) from whichever of the + four right-angle rotations scored best. + """ + reader = _get_reader(languages) + best_text, best_confidence, best_score = "", 0.0, -1.0 - text_parts = [item[1] for item in results] - confidences = [float(item[2]) for item in results] - avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0 + for k in range(4): + rotated = np.rot90(image, k=k) + text, confidence, count = _ocr_once(reader, rotated) + score = confidence * count + if score > best_score: + best_text, best_confidence, best_score = text, confidence, score - return "\n".join(text_parts), avg_confidence + return best_text, best_confidence diff --git a/docling_pk/parsers/__init__.py b/docling_pk/parsers/__init__.py index bcc5cba..e69de29 100644 --- a/docling_pk/parsers/__init__.py +++ b/docling_pk/parsers/__init__.py @@ -1,3 +0,0 @@ -from . import cnic, intermediate, matric - -__all__ = ["cnic", "matric", "intermediate"] diff --git a/docling_pk/parsers/__pycache__/__init__.cpython-312.pyc b/docling_pk/parsers/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 780ab0b..0000000 Binary files a/docling_pk/parsers/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/docling_pk/parsers/__pycache__/cnic.cpython-312.pyc b/docling_pk/parsers/__pycache__/cnic.cpython-312.pyc deleted file mode 100644 index 81969ba..0000000 Binary files a/docling_pk/parsers/__pycache__/cnic.cpython-312.pyc and /dev/null differ diff --git a/docling_pk/parsers/__pycache__/intermediate.cpython-312.pyc b/docling_pk/parsers/__pycache__/intermediate.cpython-312.pyc deleted file mode 100644 index 12e2324..0000000 Binary files a/docling_pk/parsers/__pycache__/intermediate.cpython-312.pyc and /dev/null differ diff --git a/docling_pk/parsers/__pycache__/matric.cpython-312.pyc b/docling_pk/parsers/__pycache__/matric.cpython-312.pyc deleted file mode 100644 index 2c66430..0000000 Binary files a/docling_pk/parsers/__pycache__/matric.cpython-312.pyc and /dev/null differ diff --git a/docling_pk/parsers/_federal_board.py b/docling_pk/parsers/_federal_board.py index 049e0fa..afb5651 100644 --- a/docling_pk/parsers/_federal_board.py +++ b/docling_pk/parsers/_federal_board.py @@ -1,5 +1,172 @@ +""" +Shared field extraction for Federal Board of Intermediate and Secondary +Education (Islamabad) certificates. + +Digit matching restricted to ASCII 0-9 explicitly, not \\d, since \\d and +int() both silently accept Unicode digits (Arabic-Indic included), +which showed up in real testing as a silently wrong number instead of +a correctly failed match. + +Label patterns are kept loose since testing found labels misread in +ways that keep breaking stricter patterns ("Certiiicate", "Cerlificate", +"erlific3te"). Letter-for-digit misreads (O/o/l/I for 0/0/1/1) are +corrected inside number tokens, confirmed necessary on a real roll +number read as "1217O6" instead of "121706". The name/father-name +anchor tolerates common misspellings of "daughter" found across real +samples. The TOTAL row search stops at an all-caps subject-name line +(a hard boundary), confirmed necessary on a real sample where the true +total-possible value drifted far from the label and the search +otherwise crossed into a subject row and wrongly grabbed that +subject's own marks instead. +""" + +import re +from docling_pk.schema import FieldResult + +SERIAL_LABEL = re.compile(r"ser\w*\s*n[oa]", re.IGNORECASE) +CERTIFICATE_LABEL = re.compile(r"cer\w*\s*n[oa]", re.IGNORECASE) +ROLL_LABEL = re.compile(r"roll\s*n[oa]", re.IGNORECASE) +REGISTRATION_LABEL = re.compile(r"reg\w*\s*n[oa]", re.IGNORECASE) +GROUP_KEYWORDS = re.compile(r"\b(SCIENCE(?:\s+GENERAL)?|ARTS|COMMERCE|HUMANITIES)\b", re.IGNORECASE) +SESSION_YEAR = re.compile(r"annual\W*([0-9]{4})", re.IGNORECASE) +GRADE = re.compile(r"grade\s+([A-Z])\b", re.IGNORECASE) +DAUGHTER_OF_ANCHOR = re.compile(r"d[ao]ugh\w*\s*[o0]f", re.IGNORECASE) +DOB = re.compile(r"date of birth is\s+([0-9./-]+)", re.IGNORECASE) +TOTAL_LABEL = re.compile(r"\btotal\b", re.IGNORECASE) +NUMBER_RUN = re.compile(r"[0-9][0-9OolI./\s]{2,}[0-9]|[0-9]{3,}") +HARD_BOUNDARY = re.compile(r"^[A-Z\s]{4,}$") + +SKIP_LINE = re.compile(r"certif|cerif|^that$|board|education|islamabad|secondary|intermediate", re.IGNORECASE) + + +def _clean_number(raw: str) -> str: + corrected = raw.translate(str.maketrans("OolI", "0011")) + return re.sub(r"[^0-9/]", "", corrected) + + +def _find_number_near_label(lines: list[str], label_pattern: re.Pattern, window: int = 2) -> FieldResult: + for i, line in enumerate(lines): + if not label_pattern.search(line): + continue + + same_line = label_pattern.sub("", line) + match = NUMBER_RUN.search(same_line) + if match: + return FieldResult(value=_clean_number(match.group()), confidence=0.75) + + offsets = [] + for d in range(1, window + 1): + offsets.append(d) + offsets.append(-d) + for offset in offsets: + j = i + offset + if 0 <= j < len(lines): + match = NUMBER_RUN.search(lines[j]) + if match: + return FieldResult(value=_clean_number(match.group()), confidence=0.65) + return FieldResult(value=None, confidence=0.0) + + return FieldResult(value=None, confidence=0.0) + + +def _find_name_pair(lines: list[str]) -> tuple[FieldResult, FieldResult]: + anchor_idx = None + for i, line in enumerate(lines): + if DAUGHTER_OF_ANCHOR.search(line): + anchor_idx = i + break + + if anchor_idx is None: + return FieldResult(value=None, confidence=0.0), FieldResult(value=None, confidence=0.0) + + name = FieldResult(value=None, confidence=0.0) + for offset in (1, 2, 3): + j = anchor_idx - offset + if j < 0: + break + candidate = lines[j].strip(" '.-") + if len(candidate) >= 5 and not SKIP_LINE.search(candidate): + name = FieldResult(value=candidate, confidence=0.6) + break + + father_name = FieldResult(value=None, confidence=0.0) + for offset in (1, 2): + j = anchor_idx + offset + if j >= len(lines): + break + candidate = lines[j].strip(" '.-") + if len(candidate) >= 5 and not SKIP_LINE.search(candidate): + father_name = FieldResult(value=candidate, confidence=0.6) + break + + return name, father_name + + +def _find_total(lines: list[str]) -> tuple[FieldResult, FieldResult]: + for i, line in enumerate(lines): + if not TOTAL_LABEL.search(line): + continue + + numbers = [] + misses = 0 + for offset in range(1, 6): + j = i - offset + if j < 0: + break + candidate = lines[j].strip() + if HARD_BOUNDARY.match(candidate): + break + match = NUMBER_RUN.search(candidate) + if match: + cleaned = _clean_number(match.group()) + if cleaned: + numbers.append(int(cleaned)) + misses = 0 + else: + misses += 1 + if misses >= 2: + break + if len(numbers) == 2: + break + + if len(numbers) == 2: + possible, obtained = max(numbers), min(numbers) + return ( + FieldResult(value=str(possible), confidence=0.7), + FieldResult(value=str(obtained), confidence=0.7), + ) + return FieldResult(value=None, confidence=0.0), FieldResult(value=None, confidence=0.0) + + return FieldResult(value=None, confidence=0.0), FieldResult(value=None, confidence=0.0) + + +def extract_common_fields(raw_text: str) -> dict[str, FieldResult]: + lines = raw_text.splitlines() + + fields = { + "serial_number": _find_number_near_label(lines, SERIAL_LABEL), + "certificate_number": _find_number_near_label(lines, CERTIFICATE_LABEL), + "roll_number": _find_number_near_label(lines, ROLL_LABEL), + "registration_number": _find_number_near_label(lines, REGISTRATION_LABEL), + } + + group_match = GROUP_KEYWORDS.search(raw_text) + fields["group"] = FieldResult(value=group_match.group(1).upper(), confidence=0.75) if group_match else FieldResult(value=None, confidence=0.0) + + year_match = SESSION_YEAR.search(raw_text) + fields["session_year"] = FieldResult(value=year_match.group(1), confidence=0.8) if year_match else FieldResult(value=None, confidence=0.0) + + grade_match = GRADE.search(raw_text) + fields["grade"] = FieldResult(value=grade_match.group(1), confidence=0.6) if grade_match else FieldResult(value=None, confidence=0.0) + + fields["name"], fields["father_name"] = _find_name_pair(lines) + fields["total_marks_possible"], fields["total_marks_obtained"] = _find_total(lines) + fields["institute"] = FieldResult(value=None, confidence=0.0) + + return fields + + def detect_document_type(raw_text: str) -> str: - text = (raw_text or "").upper() - if "INTERMEDIATE" in text: + if re.search(r"higher\s+secondary\s+school\s+certificate", raw_text, re.IGNORECASE): return "intermediate" return "matric" diff --git a/docling_pk/parsers/cnic.py b/docling_pk/parsers/cnic.py index a8282f5..7d59513 100644 --- a/docling_pk/parsers/cnic.py +++ b/docling_pk/parsers/cnic.py @@ -1,20 +1,147 @@ -import re +""" +CNIC field extraction. + +Labels are matched loosely since real OCR misreads them in small, +specific ways ("Gender" as "GenUer", "of" as "o{"/"o/", "Father Name" +as "Fathgr Name"/"Father Namg"). Dates and cnic_number use format-based +matching instead of label matching, since testing showed date labels +and values are read as two separate groups, not adjacent lines. + +Digit matching is restricted to ASCII 0-9 explicitly, not the regex \\d +shorthand. Python's \\d and int() both silently accept Unicode digits +(Arabic-Indic included), so a garbled OCR read could otherwise parse +as a different, wrong number with full confidence instead of correctly +failing to match. +Gender's search window was widened after testing with the adaptive +threshold step removed: with cleaner OCR text, the M/F value can +appear further from its label than a 1-line window would catch. +""" + +import re +from typing import Optional from docling_pk.schema import FieldResult +CNIC_PATTERN = re.compile(r"\b([0-9]{5}-[0-9]{7}-[0-9])\b") +DATE_PATTERN = re.compile(r"\b([0-9]{2})[.,]([0-9]{2})[.,]([0-9]{4})\b") + +FIELD_NAMES = [ + "name", "father_name", "cnic_number", + "date_of_birth", "date_of_issue", "date_of_expiry", + "gender", "address", +] + +FATHER_NAME_LABEL = re.compile(r"fath\w*", re.IGNORECASE) +NAME_LABEL = re.compile(r"\bname\b", re.IGNORECASE) +GENDER_LABEL = re.compile(r"gen.er", re.IGNORECASE) +ADDRESS_LABEL = re.compile(r"address", re.IGNORECASE) + + +def extract_cnic_number(raw_text: str) -> FieldResult: + """ + Extracts the CNIC number (format XXXXX-XXXXXXX-X). OCR sometimes + reads the dash as an underscore, colon, or backtick; all are + normalized before matching. + """ + normalized = raw_text.replace("_", "-").replace(":", "-").replace("`", "-") + match = CNIC_PATTERN.search(normalized) + if match: + return FieldResult(value=match.group(1), confidence=0.92) + return FieldResult(value=None, confidence=0.0) + + +def _value_after_label(lines: list[str], label_pattern: re.Pattern, exclude: re.Pattern = None) -> Optional[tuple[str, int]]: + for i, line in enumerate(lines): + if exclude and exclude.search(line): + continue + if label_pattern.search(line): + return line, i + return None + + +def _text_value(lines: list[str], label_pattern: re.Pattern, exclude: re.Pattern = None) -> FieldResult: + found = _value_after_label(lines, label_pattern, exclude) + if found is None: + return FieldResult(value=None, confidence=0.0) + + line, idx = found + same_line = label_pattern.sub("", line).strip(" :-") + if same_line: + return FieldResult(value=same_line, confidence=0.75) + + if idx + 1 < len(lines) and lines[idx + 1].strip(): + return FieldResult(value=lines[idx + 1].strip(), confidence=0.65) + + return FieldResult(value=None, confidence=0.0) + + +def _all_dates(raw_text: str) -> list[str]: + matches = DATE_PATTERN.findall(raw_text) + return [f"{d}.{m}.{y}" for (d, m, y) in matches] + + +def _dates_by_position(raw_text: str) -> tuple[FieldResult, FieldResult, FieldResult]: + """ + Assigns dates by order of appearance, not proximity to a label. + NADRA prints birth, issue, expiry in that fixed order, and testing + shows EasyOCR reads the value block in that order even when + separated from the labels. + """ + dates = _all_dates(raw_text) + results = [] + for i in range(3): + if i < len(dates): + results.append(FieldResult(value=dates[i], confidence=0.8)) + else: + results.append(FieldResult(value=None, confidence=0.0)) + return tuple(results) + + +def extract_gender(raw_text: str) -> FieldResult: + """ + Extracts gender as 'M' or 'F'. Searches up to 4 lines after the + label, not just the next line. + """ + lines = raw_text.splitlines() + found = _value_after_label(lines, GENDER_LABEL) + if found is None: + return FieldResult(value=None, confidence=0.0) + + line, idx = found + remainder = GENDER_LABEL.sub("", line).strip(" :-") + match = re.search(r"\b([MF])\b", remainder.upper()) + if match: + return FieldResult(value=match.group(1), confidence=0.8) + + for offset in range(1, 5): + j = idx + offset + if j >= len(lines): + break + match = re.search(r"\b([MF])\b", lines[j].strip().upper()) + if match: + return FieldResult(value=match.group(1), confidence=0.7) -CNIC_PATTERN = re.compile(r"\b\d{5}-\d{7}-\d\b") -GENDER_PATTERN = re.compile(r"\b(?:gender\s*[:\-]?\s*)?(male|female|m|f)\b", re.IGNORECASE) + return FieldResult(value=None, confidence=0.0) def extract(raw_text: str) -> dict[str, FieldResult]: - cnic_match = CNIC_PATTERN.search(raw_text or "") - gender_match = GENDER_PATTERN.search(raw_text or "") + """ + Extracts all CNIC fields from raw OCR text. Missing fields return + FieldResult(None, 0.0) rather than raising. + """ + lines = raw_text.splitlines() - cnic_value = cnic_match.group(0) if cnic_match else None - gender_value = gender_match.group(1).upper() if gender_match else None + father_name = _text_value(lines, FATHER_NAME_LABEL) + name = _text_value(lines, NAME_LABEL, exclude=FATHER_NAME_LABEL) + dob, issue, expiry = _dates_by_position(raw_text) return { - "cnic_number": FieldResult(value=cnic_value, confidence=0.95 if cnic_value else 0.0), - "gender": FieldResult(value=gender_value, confidence=0.9 if gender_value else 0.0), + "name": name, + "father_name": father_name, + "cnic_number": extract_cnic_number(raw_text), + "date_of_birth": dob, + "date_of_issue": issue, + "date_of_expiry": expiry, + "gender": extract_gender(raw_text), + "address": _text_value(lines, ADDRESS_LABEL), } diff --git a/docling_pk/parsers/intermediate.py b/docling_pk/parsers/intermediate.py index 4ee77f9..cb12fc7 100644 --- a/docling_pk/parsers/intermediate.py +++ b/docling_pk/parsers/intermediate.py @@ -1,2 +1,18 @@ -def extract(raw_text: str) -> dict: - return {} +""" +Intermediate (Higher Secondary School Certificate) extraction, Federal +Board Islamabad format. No date_of_birth field, this board does not +print it on the Intermediate certificate. +""" + +from docling_pk.schema import FieldResult +from docling_pk.parsers._federal_board import extract_common_fields + +FIELD_NAMES = [ + "serial_number", "certificate_number", "roll_number", "registration_number", + "group", "session_year", "grade", "name", "father_name", "institute", + "total_marks_possible", "total_marks_obtained", +] + + +def extract(raw_text: str) -> dict[str, FieldResult]: + return extract_common_fields(raw_text) diff --git a/docling_pk/parsers/matric.py b/docling_pk/parsers/matric.py index 4ee77f9..9f6565c 100644 --- a/docling_pk/parsers/matric.py +++ b/docling_pk/parsers/matric.py @@ -1,2 +1,21 @@ -def extract(raw_text: str) -> dict: - return {} +""" +Matric (Secondary School Certificate) extraction, Federal Board Islamabad +format. Adds date_of_birth on top of the shared fields, since this board +prints DOB on the Matric certificate but not on the Intermediate one. +""" + +from docling_pk.schema import FieldResult +from docling_pk.parsers._federal_board import extract_common_fields, DOB + +FIELD_NAMES = [ + "serial_number", "certificate_number", "roll_number", "registration_number", + "group", "session_year", "grade", "name", "father_name", "institute", + "date_of_birth", "total_marks_possible", "total_marks_obtained", +] + + +def extract(raw_text: str) -> dict[str, FieldResult]: + fields = extract_common_fields(raw_text) + match = DOB.search(raw_text) + fields["date_of_birth"] = FieldResult(value=match.group(1), confidence=0.8) if match else FieldResult(value=None, confidence=0.0) + return fields diff --git a/docling_pk/preprocessor.py b/docling_pk/preprocessor.py index bd983ce..095b226 100644 --- a/docling_pk/preprocessor.py +++ b/docling_pk/preprocessor.py @@ -1,24 +1,101 @@ -from pathlib import Path +""" +Image preprocessing for docling-pk. + +Prepares a document photo for OCR. Order: grayscale, then blur check, +then deskew, then denoise. Adaptive thresholding was tested directly +(A/B test comparing thresholded output against plain grayscale on real +CNIC and certificate samples) and removed: it consistently lowered +EasyOCR's own confidence on every sample tested. EasyOCR's detector is +trained on natural photos, not pre-binarized ones, so feeding it our +own black-and-white conversion worked against it rather than for it. +""" import cv2 +import numpy as np +from pathlib import Path + +BLUR_VARIANCE_THRESHOLD = 35.0 class TooBlurryError(ValueError): - """Raised when an image is too blurry for reliable extraction.""" + """Raised when an image is too blurry to reliably OCR.""" + + +def preprocess(image_path: str) -> np.ndarray: + """ + Prepares a document image for OCR. + + Steps: + 1. Load and validate + 2. Grayscale + 3. Blur check + 4. Deskew + 5. Denoise + + Args: + image_path: path to the input image file. + Returns: + A single-channel numpy array ready for OCR. -def preprocess(image_path: str): + Raises: + FileNotFoundError: if image_path does not exist. + ValueError: if the file cannot be read as an image. + TooBlurryError: if the image is too blurry to reliably OCR. + """ path = Path(image_path) if not path.exists(): - raise FileNotFoundError(f"Image path does not exist: {image_path}") + raise FileNotFoundError(f"Image not found: {image_path}") + + img = cv2.imread(str(path)) + if img is None: + raise ValueError(f"Could not read image: {image_path}. Check the file is a valid PNG/JPG.") + + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + _check_blur(gray) + + deskewed = _deskew(gray) + denoised = cv2.fastNlMeansDenoising(deskewed, h=10) + return denoised + + +def _check_blur(gray: np.ndarray) -> None: + """ + Measures sharpness using the variance of the Laplacian. Threshold of + 35 picked from 4 real samples: two failing blurry images scored 12.9 + and 27.1, two working images scored 48.2 and 54.9. + """ + variance = cv2.Laplacian(gray, cv2.CV_64F).var() + if variance < BLUR_VARIANCE_THRESHOLD: + raise TooBlurryError( + f"Image too blurry to process (sharpness score {variance:.1f}, " + f"minimum {BLUR_VARIANCE_THRESHOLD}). Please retake the photo." + ) + + +def _deskew(gray: np.ndarray) -> np.ndarray: + """ + Estimates and corrects rotation using the minimum-area bounding box + of dark pixels. Known limitation, confirmed by direct testing: this + fails against cluttered or textured backgrounds, since the angle is + estimated from every dark pixel in the frame, not the document + specifically. v1 assumes a plain background. + """ + inverted = cv2.bitwise_not(gray) + _, binary = cv2.threshold(inverted, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) + coords = np.column_stack(np.where(binary > 0)) - image = cv2.imread(str(path)) - if image is None: - raise ValueError(f"Invalid image file: {image_path}") + if coords.shape[0] < 50: + return gray - gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - sharpness = float(cv2.Laplacian(gray, cv2.CV_64F).var()) - if sharpness < 25.0: - raise TooBlurryError("Image is too blurry to process reliably") + angle = cv2.minAreaRect(coords)[-1] + if angle < -45: + angle = -(90 + angle) + else: + angle = -angle - return image + (h, w) = gray.shape + center = (w // 2, h // 2) + matrix = cv2.getRotationMatrix2D(center, angle, 1.0) + rotated = cv2.warpAffine(gray, matrix, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE) + return rotated diff --git a/docling_pk/schema.py b/docling_pk/schema.py index 4ab4f01..e7088c7 100644 --- a/docling_pk/schema.py +++ b/docling_pk/schema.py @@ -6,7 +6,7 @@ is the one shape every other module builds against, so a change here affects the CLI, the tests, and every parser at once. """ - +from __future__ import annotations from dataclasses import dataclass, field from typing import Optional diff --git a/tests/test_cli.py b/tests/test_cli.py index d0289a8..2f2b045 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,6 +15,21 @@ COMMANDS = ["cnic", "matric", "intermediate", "degree"] +def _combined_output(result) -> str: + """ + Returns stdout plus stderr if available. click's CliRunner does not + consistently capture stderr separately across versions: older + versions merge stdout+stderr into one stream (result.output holds + both, result.stderr raises ValueError), newer versions capture them + separately. Checking both this way works either way, instead of + assuming one specific click version's behavior. + """ + combined = result.output + if result.stderr_bytes is not None: + combined += result.stderr + return combined + + @pytest.mark.parametrize("command", COMMANDS) def test_command_registered(command): """Each of the four document commands exists and shows help.""" @@ -48,4 +63,4 @@ def test_invalid_path_exits_with_error_code(command): """A path that does not exist exits with a distinct error code, not the placeholder code.""" result = runner.invoke(app, [command, "this/path/does/not/exist.jpg"]) assert result.exit_code == 2 - assert "does not exist" in result.stderr \ No newline at end of file + assert "does not exist" in _combined_output(result) \ No newline at end of file diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py deleted file mode 100644 index e11fa75..0000000 --- a/tests/test_placeholder.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Placeholder test so pytest has something to collect. pytest exits with -code 5 when zero tests exist at all, which is what happens with no test -suite merged yet. This gets replaced once tests/test_cnic.py (the -Integration Engineer's Week 2 deliverable) lands as its own PR. -""" - - -def test_placeholder(): - assert True \ No newline at end of file