Feat/v1 overhaul

.gitattributes

@@ -0,0 +1,5 @@
+* text=auto eol=lf
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.pdf binary

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1,24 @@
 ## What does this PR do?
-[One sentence: what does this PR add, fix, or change?]
-
-## Which module(s) does it touch?
-[List the files changed and why each was changed]
-
-## How did you test it?
-[What tests cover this? Run: pytest <test_file> and paste the result summary]
-
-## Edge cases handled:
-[What inputs could break this? How did you handle them?]
-
-## Checklist:
-- [ ] Tests added or updated
-- [ ] Docstrings on all new public functions
-- [ ] No unused imports
-- [ ] README updated if public API changed
-- [ ] Lead Engineer has reviewed and approved before submitting to director
+<!-- One or two sentences. -->
+
+## Modules touched
+<!-- Files changed and why. -->
+
+## How was it tested?
+<!-- pytest summary; coverage. -->
+
+## Benchmark impact
+<!-- Required if extraction logic changed: paste the OVERALL / per-type lines from
+     `python benchmarks/run_benchmark.py tests/fixtures/synthetic/labels.json`,
+     before and after. Real-sample numbers: aggregates only. -->
+
+## Edge cases handled
+<!-- What inputs could break this and how they behave now. -->
+
+## Checklist
+- [ ] `ruff check`, `ruff format --check`, `mypy src` pass
+- [ ] Tests added or updated; `pytest` passes
+- [ ] No field can come back `ok` with a guessed value (fallbacks are `low_confidence` with a reason)
+- [ ] Docstrings on new public functions (parameters, returns, example)
+- [ ] README / CHANGELOG updated if behaviour or the public API changed
+- [ ] **No real documents or personal data** in code, fixtures, logs or this description

.github/workflows/ci.yml

@@ -4,36 +4,128 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  test:
+  lint:
+    name: Lint and type-check
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: pip install -e ".[dev]"
+      - run: ruff check src tests benchmarks
+      - run: ruff format --check src tests benchmarks
+      - run: mypy src
+
+  test:
+    name: Tests (Python ${{ matrix.python-version }}, ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
+        os: [ubuntu-latest]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
-      fail-fast: false
-
+        include:
+          - os: windows-latest
+            python-version: "3.11"
+          - os: macos-latest
+            python-version: "3.11"
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-
-      - name: Install dependencies
+          cache: pip
+      - name: System libraries for OpenCV (Linux)
+        if: runner.os == 'Linux'
+        timeout-minutes: 5
+        # Usually preinstalled on GitHub runners: only touch apt when missing, so a
+        # slow package mirror can never stall the job.
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          if ldconfig -p | grep -q "libGL.so.1" && ldconfig -p | grep -q "libgthread-2.0.so.0"; then
+            echo "OpenCV system libraries already present"
+          else
+            sudo apt-get update -o Acquire::Retries=3 && sudo apt-get install -y --no-install-recommends libgl1 libglib2.0-0
+          fi
+      - run: python -m pip install --upgrade pip
+      - run: pip install -e ".[dev,pdf]"
+      - name: Unit tests with coverage gate (no OCR models)
+        run: pytest -m "not ocr" --cov=docling_pk --cov-report=term-missing --cov-report=xml --cov-fail-under=90
+      - uses: actions/upload-artifact@v4
+        if: always() && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
+        with:
+          name: coverage-xml
+          path: coverage.xml
 
-      - name: Run tests with coverage
-        run: pytest -v --cov=docling_pk --cov-report=term-missing --cov-report=xml
+  integration:
+    name: End-to-end OCR tests (RapidOCR on synthetic fixtures)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: System libraries for OpenCV (Linux)
+        if: runner.os == 'Linux'
+        timeout-minutes: 5
+        # Usually preinstalled on GitHub runners: only touch apt when missing, so a
+        # slow package mirror can never stall the job.
+        run: |
+          if ldconfig -p | grep -q "libGL.so.1" && ldconfig -p | grep -q "libgthread-2.0.so.0"; then
+            echo "OpenCV system libraries already present"
+          else
+            sudo apt-get update -o Acquire::Retries=3 && sudo apt-get install -y --no-install-recommends libgl1 libglib2.0-0
+          fi
+      - run: python -m pip install --upgrade pip
+      - run: pip install -e ".[dev,pdf]"
+      - run: pytest -m ocr -v
+      - name: Synthetic benchmark
+        run: python benchmarks/run_benchmark.py tests/fixtures/synthetic/labels.json --backend rapidocr --out benchmark_synthetic.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-synthetic
+          path: benchmark_synthetic.json
 
-      - name: Upload coverage report
-        uses: actions/upload-artifact@v4
+  package:
+    name: Build and verify distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          name: coverage-report-py${{ matrix.python-version }}
-          path: coverage.xml
-        if: always()
+          python-version: "3.11"
+      - name: System libraries for OpenCV (Linux)
+        timeout-minutes: 5
+        # Usually preinstalled on GitHub runners: only touch apt when missing, so a
+        # slow package mirror can never stall the job.
+        run: |
+          if ldconfig -p | grep -q "libGL.so.1" && ldconfig -p | grep -q "libgthread-2.0.so.0"; then
+            echo "OpenCV system libraries already present"
+          else
+            sudo apt-get update -o Acquire::Retries=3 && sudo apt-get install -y --no-install-recommends libgl1 libglib2.0-0
+          fi
+      - run: python -m pip install --upgrade pip build twine
+      - run: python -m build
+      - run: twine check --strict dist/*
+      - name: Install the wheel in a clean environment and smoke-test it
+        run: |
+          python -m venv /tmp/clean
+          /tmp/clean/bin/pip install dist/*.whl
+          cd /tmp
+          /tmp/clean/bin/python -c "import docling_pk; print(docling_pk.__version__)"
+          /tmp/clean/bin/docling-pk --version
+          /tmp/clean/bin/docling-pk extract "$GITHUB_WORKSPACE/tests/fixtures/synthetic/00_cnic_front_clean.jpg" --type cnic --no-verify --no-urdu
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/

.github/workflows/release.yml

@@ -0,0 +1,44 @@
+name: Release to PyPI
+
+# Publishes when a version tag (v1.0.0, v1.1.0, ...) is pushed.
+# Uses PyPI Trusted Publishing: no API token is stored in GitHub. One-time
+# setup on pypi.org: Account -> Publishing -> add a pending publisher with
+# owner "Inference-LAB", repository "docling-pk", workflow "release.yml",
+# environment "pypi".
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: python -m pip install --upgrade pip build twine
+      - run: python -m build
+      - run: twine check --strict dist/*
+      - name: Tag matches package version
+        run: |
+          VERSION=$(python -c "import re,pathlib;print(re.search(r'__version__ = \"(.+?)\"', pathlib.Path('src/docling_pk/__init__.py').read_text()).group(1))")
+          test "v$VERSION" = "${GITHUB_REF_NAME}" || { echo "Tag ${GITHUB_REF_NAME} != package version v$VERSION"; exit 1; }
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -1,9 +1,28 @@
-*.docx
+# Python
 __pycache__/
-.vscode/
-*.pyc
-venv/
+*.py[cod]
 *.egg-info/
+build/
+dist/
+.eggs/
+
+# Environments
+venv/
+.venv/
+env/
+
+# Tooling
 .pytest_cache/
+.ruff_cache/
+.mypy_cache/
 .coverage
 coverage.xml
+htmlcov/
+.vscode/
+.idea/
+
+# Documents and data that must never be committed: real people's documents
+*.docx
+private/
+private-eval/
+benchmarks/results/*.full.json

CHANGELOG.md

@@ -0,0 +1,51 @@
+# Changelog
+
+All notable changes to this project are documented here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
+[Semantic Versioning](https://semver.org/).
+
+## [1.0.0] - 2026-10
+
+First release on PyPI. Field accuracy on 16 real document photos went from
+34.9% (v0.1) to 94.4%, with 99.2% of fields marked `ok` correct.
+
+### Added
+- `extract()` public API accepting paths, bytes, NumPy arrays, PIL images,
+  PDFs and lists (CNIC front + back merged into one result).
+- `document_type="auto"` detection; aliases `ssc`, `hssc`, `fsc`, `inter`, `transcript`.
+- Degree and transcript extraction.
+- Per-field `status` and `reason`; `FieldResult.as_date()`; `DocumentResult.to_json()`
+  and dict-style access; marks table in `result.tables["subjects"]`.
+- RapidOCR backend (new default) and pluggable `OCRBackend` interface.
+- Cross-engine verification of CNIC numbers, dates, marks and IDs (with the `urdu` extra).
+- CNIC: label-geometry parsing, chronological date assignment, gender vs CNIC
+  parity check, back-side support, experimental Urdu address with normalization
+  and lexicon correction.
+- Certificates: column-aligned marks table validated against the TOTAL row,
+  subject sums and "marks in words"; board detection for FBISE and other boards.
+- Orientation (0/90/180/270) and skew correction that work on cluttered backgrounds.
+- Image-quality warnings (blur, exposure, glare, resolution).
+- CLI: `extract`, `batch`, `info`, `--version`, table output.
+- Synthetic document generator, 23 committed fixtures, benchmark runner and report.
+- Test suite (253 tests, 92% coverage), CI on Python 3.9–3.12 / Linux, Windows,
+  macOS, clean-install check, PyPI trusted-publishing release workflow.
+
+### Changed
+- EasyOCR moved to the optional `urdu` extra; the core install no longer needs PyTorch.
+- Blurry images produce warnings and partial results instead of being rejected.
+- Package moved to a `src/` layout; the CLI entry point is `docling_pk.cli:app`.
+
+### Fixed
+- `docling_pk.extractor` could not be imported (`from __future__` after other imports).
+- `from docling_pk import extract` did not exist.
+- The installed `docling-pk` command pointed to a module that was not packaged.
+- Orientation selection preferred Urdu-script noise on sideways reads, so the
+  cleanest CNIC photo extracted 0 of 8 fields.
+- Deskew was a no-op (or wrong) on OpenCV ≥ 4.5.
+- "Father Name" label minus "Father" was returned as the father's name ("Name").
+- Images with non-ASCII paths failed to load on Windows.
+
+## [0.1.0] - 2026-09
+
+Fellowship prototype: EasyOCR + regex CNIC parser, Federal Board certificate
+parser, CLI skeleton returning placeholder output.

CONTRIBUTING.md

@@ -0,0 +1,52 @@
+# Contributing to docling-pk
+
+## Setup
+
+```bash
+git clone https://github.com/Inference-LAB/docling-pk.git
+cd docling-pk
+python -m venv .venv && source .venv/bin/activate   # Windows: .venv\Scripts\activate
+pip install -e ".[dev,pdf]"          # add ,urdu to work on the Urdu address or verifier
+```
+
+## Before opening a PR
+
+```bash
+ruff check src tests benchmarks
+ruff format src tests benchmarks
+mypy src
+pytest -m "not ocr"                  # fast, no OCR models
+pytest                               # full suite incl. end-to-end OCR (downloads ~15 MB of models once)
+python benchmarks/run_benchmark.py tests/fixtures/synthetic/labels.json
+```
+
+CI runs all of these, enforces 90% coverage, and fails if any synthetic
+fixture returns a field with status `ok` and a wrong value. Paste the
+benchmark summary into the PR description whenever extraction logic changes.
+
+## Never commit real documents
+
+Real CNICs and certificates contain personal data. Do not add them to the
+repository, to issues, or to PR descriptions, even blurred. To evaluate on
+real samples, keep the images and a labels file outside the repo and run
+`benchmarks/run_benchmark.py /path/to/private/labels.json`; share only the
+aggregate numbers. For tests, add synthetic cases to
+`benchmarks/synthetic/generate.py` and regenerate the fixtures.
+
+## Adding a board, university or document type
+
+1. Collect a few real samples (privately) and label them.
+2. Run the benchmark to see where the current parser fails.
+3. Prefer **label vocabulary and validation** over coordinates: add labels to
+   the parser's label lists, formats to `validation/`, and checks that can turn
+   a wrong value into `low_confidence`.
+4. Add constructed-layout unit tests (see `tests/conftest.py`) and, if
+   possible, a synthetic fixture.
+5. Values found by fallbacks must be `low_confidence` with a `reason`.
+
+## Design rules
+
+- A field is never filled with a guess. Return `None` with a reason instead.
+- Status `ok` means it passed every check that applies to it.
+- Problems with the call raise; problems with the document go in `warnings`.
+- Keep the core install free of PyTorch; heavy models go in extras.