From 1fae7b9a1aeac9323c080cb996d8caa24f669394 Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Mon, 10 Aug 2026 15:54:40 +0530 Subject: [PATCH 1/7] cross-document validation Module --- lending-poc/.gitignore | 1 + lending-poc/alembic/env.py | 1 + ...f16_add_cases_documents_golden_records_.py | 106 +++++++ lending-poc/app/api/cases.py | 37 +++ lending-poc/app/config.py | 1 + lending-poc/app/main.py | 2 + lending-poc/app/matching/__init__.py | 0 lending-poc/app/matching/embeddings.py | 60 ++++ lending-poc/app/matching/exact.py | 100 ++++++ lending-poc/app/matching/fuzzy.py | 103 ++++++ lending-poc/app/models/__init__.py | 15 + lending-poc/app/models/case.py | 42 +++ lending-poc/app/models/document.py | 25 ++ lending-poc/app/models/golden_record.py | 32 ++ lending-poc/app/models/pipeline_result.py | 45 +++ lending-poc/app/models/types.py | 26 ++ lending-poc/app/models/validation_result.py | 29 ++ lending-poc/app/schemas/case.py | 38 +++ lending-poc/app/services/__init__.py | 0 .../app/services/business_validation.py | 233 ++++++++++++++ lending-poc/app/services/case_parsing.py | 110 +++++++ lending-poc/app/services/decision_engine.py | 46 +++ lending-poc/app/services/dto.py | 162 ++++++++++ lending-poc/app/services/golden_record.py | 82 +++++ .../app/services/identity_validation.py | 168 ++++++++++ lending-poc/app/services/persistence.py | 184 +++++++++++ lending-poc/app/services/pipeline.py | 56 ++++ lending-poc/app/services/scoring.py | 34 ++ lending-poc/app/services/validation_config.py | 40 +++ lending-poc/docker-compose.yml | 4 +- lending-poc/docs/Workflow.md | 294 ++++++++++++++++++ lending-poc/pyproject.toml | 4 + lending-poc/scripts/edge_case_scenarios.py | 149 +++++++++ lending-poc/scripts/run_demo.py | 71 +++++ lending-poc/scripts/sample_case.json | 136 ++++++++ 35 files changed, 2434 insertions(+), 2 deletions(-) create mode 100644 lending-poc/alembic/versions/12522c432f16_add_cases_documents_golden_records_.py create mode 100644 lending-poc/app/api/cases.py create mode 100644 lending-poc/app/matching/__init__.py create mode 100644 lending-poc/app/matching/embeddings.py create mode 100644 lending-poc/app/matching/exact.py create mode 100644 lending-poc/app/matching/fuzzy.py create mode 100644 lending-poc/app/models/case.py create mode 100644 lending-poc/app/models/document.py create mode 100644 lending-poc/app/models/golden_record.py create mode 100644 lending-poc/app/models/pipeline_result.py create mode 100644 lending-poc/app/models/types.py create mode 100644 lending-poc/app/models/validation_result.py create mode 100644 lending-poc/app/schemas/case.py create mode 100644 lending-poc/app/services/__init__.py create mode 100644 lending-poc/app/services/business_validation.py create mode 100644 lending-poc/app/services/case_parsing.py create mode 100644 lending-poc/app/services/decision_engine.py create mode 100644 lending-poc/app/services/dto.py create mode 100644 lending-poc/app/services/golden_record.py create mode 100644 lending-poc/app/services/identity_validation.py create mode 100644 lending-poc/app/services/persistence.py create mode 100644 lending-poc/app/services/pipeline.py create mode 100644 lending-poc/app/services/scoring.py create mode 100644 lending-poc/app/services/validation_config.py create mode 100644 lending-poc/docs/Workflow.md create mode 100644 lending-poc/scripts/edge_case_scenarios.py create mode 100644 lending-poc/scripts/run_demo.py create mode 100644 lending-poc/scripts/sample_case.json diff --git a/lending-poc/.gitignore b/lending-poc/.gitignore index 9aa5e0c..a9f203b 100644 --- a/lending-poc/.gitignore +++ b/lending-poc/.gitignore @@ -9,3 +9,4 @@ build/ .mypy_cache/ .pytest_cache/ .ruff_cache/ +venv/ \ No newline at end of file diff --git a/lending-poc/alembic/env.py b/lending-poc/alembic/env.py index 4a460b2..0e582ff 100644 --- a/lending-poc/alembic/env.py +++ b/lending-poc/alembic/env.py @@ -6,6 +6,7 @@ from app.config import settings from app.database import Base +import app.models # noqa: F401 (registers models on Base.metadata for autogenerate) config = context.config diff --git a/lending-poc/alembic/versions/12522c432f16_add_cases_documents_golden_records_.py b/lending-poc/alembic/versions/12522c432f16_add_cases_documents_golden_records_.py new file mode 100644 index 0000000..99a99ed --- /dev/null +++ b/lending-poc/alembic/versions/12522c432f16_add_cases_documents_golden_records_.py @@ -0,0 +1,106 @@ +"""add cases, documents, golden_records, validation_results, pipeline_results + +Revision ID: 12522c432f16 +Revises: +Create Date: 2026-08-10 11:51:33.283278 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +import pgvector.sqlalchemy +import app.models.types + +# revision identifiers, used by Alembic. +revision: str = '12522c432f16' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute('CREATE EXTENSION IF NOT EXISTS vector') + op.create_table('cases', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('applicant_ref', sa.String(), nullable=False), + sa.Column('status', sa.Enum('RECEIVED', 'RUNNING', 'PASS', 'FAIL', 'NEEDS_REVIEW', name='case_status'), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_cases_applicant_ref'), 'cases', ['applicant_ref'], unique=True) + op.create_table('documents', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('case_id', sa.UUID(), nullable=False), + sa.Column('doc_type', sa.Enum('AADHAAR', 'PAN', 'ADDRESS_PROOF', 'SALARY_SLIP', 'BANK_STATEMENT', name='doc_type'), nullable=False), + sa.Column('extracted_fields', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('source_file_ref', sa.String(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_documents_case_id'), 'documents', ['case_id'], unique=False) + op.create_table('golden_records', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('case_id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('address', sa.String(), nullable=True), + sa.Column('address_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=384), nullable=True), + sa.Column('aadhaar_number', app.models.types.EncryptedString(), nullable=True), + sa.Column('pan_number', app.models.types.EncryptedString(), nullable=True), + sa.Column('date_of_birth', sa.Date(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('case_id') + ) + op.create_table('pipeline_results', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('case_id', sa.UUID(), nullable=False), + sa.Column('overall_score', sa.Float(), nullable=False), + sa.Column('decision', sa.Enum('PASS', 'FAIL', 'NEEDS_REVIEW', name='decision'), nullable=False), + sa.Column('reasons', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('reviewer', sa.String(), nullable=True), + sa.Column('review_status', sa.Enum('PENDING', 'APPROVED', 'REJECTED', name='review_status'), nullable=True), + sa.Column('reviewer_remarks', sa.Text(), nullable=True), + sa.Column('reviewed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_pipeline_results_case_id'), 'pipeline_results', ['case_id'], unique=False) + op.create_table('validation_results', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('case_id', sa.UUID(), nullable=False), + sa.Column('document_id', sa.UUID(), nullable=True), + sa.Column('check_type', sa.Enum('NAME', 'ADDRESS', 'AADHAAR', 'PAN', 'DOB', 'EMPLOYER', 'SALARY_DATE', 'SALARY_CREDIT_COUNT', 'MANDATORY_PRESENCE', name='check_type'), nullable=False), + sa.Column('passed', sa.Boolean(), nullable=False), + sa.Column('score', sa.Float(), nullable=False), + sa.Column('evidence', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['document_id'], ['documents.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_validation_results_case_id'), 'validation_results', ['case_id'], unique=False) + op.create_index(op.f('ix_validation_results_document_id'), 'validation_results', ['document_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_validation_results_document_id'), table_name='validation_results') + op.drop_index(op.f('ix_validation_results_case_id'), table_name='validation_results') + op.drop_table('validation_results') + op.drop_index(op.f('ix_pipeline_results_case_id'), table_name='pipeline_results') + op.drop_table('pipeline_results') + op.drop_table('golden_records') + op.drop_index(op.f('ix_documents_case_id'), table_name='documents') + op.drop_table('documents') + op.drop_index(op.f('ix_cases_applicant_ref'), table_name='cases') + op.drop_table('cases') + # ### end Alembic commands ### diff --git a/lending-poc/app/api/cases.py b/lending-poc/app/api/cases.py new file mode 100644 index 0000000..3f96228 --- /dev/null +++ b/lending-poc/app/api/cases.py @@ -0,0 +1,37 @@ +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.database import get_db +from app.schemas.case import CaseCreateRequest, CaseCreateResponse, ValidationResultOut +from app.services.case_parsing import parse_case +from app.services.persistence import save_pipeline_result +from app.services.pipeline import run_pipeline + +router = APIRouter(tags=["cases"]) + + +@router.post("/cases", response_model=CaseCreateResponse) +async def create_case( + request: CaseCreateRequest, db: AsyncSession = Depends(get_db) +) -> CaseCreateResponse: + case_input = parse_case(request.model_dump()) + pipeline_result = run_pipeline(case_input) + case = await save_pipeline_result(db, case_input, pipeline_result) + + return CaseCreateResponse( + case_id=str(case.id), + applicant_ref=case_input.applicant_ref, + decision=pipeline_result.decision_result.decision.value, + overall_score=pipeline_result.decision_result.overall_score, + reasons=pipeline_result.decision_result.reasons, + validation_results=[ + ValidationResultOut( + check_type=r.check_type.value, + passed=r.passed, + score=r.score, + document_id=r.document_id, + evidence=r.evidence, + ) + for r in pipeline_result.validation_results + ], + ) diff --git a/lending-poc/app/config.py b/lending-poc/app/config.py index d8f7bdc..a0e69b7 100644 --- a/lending-poc/app/config.py +++ b/lending-poc/app/config.py @@ -11,6 +11,7 @@ class Settings(BaseSettings): DEBUG: bool = False DATABASE_URL: str = "postgresql+asyncpg://postgres:postgres@localhost:5432/lending_poc" LOG_LEVEL: str = "INFO" + ENCRYPTION_KEY: str = "" settings = Settings() diff --git a/lending-poc/app/main.py b/lending-poc/app/main.py index 8481002..dc9c4d0 100644 --- a/lending-poc/app/main.py +++ b/lending-poc/app/main.py @@ -4,6 +4,7 @@ from fastapi import FastAPI from sqlalchemy import text +from app.api.cases import router as cases_router from app.api.health import router as health_router from app.config import logger, settings from app.database import async_session, engine @@ -34,3 +35,4 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: ) app.include_router(health_router) +app.include_router(cases_router) diff --git a/lending-poc/app/matching/__init__.py b/lending-poc/app/matching/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lending-poc/app/matching/embeddings.py b/lending-poc/app/matching/embeddings.py new file mode 100644 index 0000000..329261f --- /dev/null +++ b/lending-poc/app/matching/embeddings.py @@ -0,0 +1,60 @@ +"""Address similarity via embeddings. + +Embeddings come from BAAI/bge-small-en-v1.5 (sentence-transformers), +running locally on CPU — no API key or network call per request. The +model is loaded once per process (module-level singleton) since load time +is the expensive part; encoding individual addresses is fast. +""" + +import math +import re +from functools import lru_cache + +EMBEDDING_MODEL_NAME = "BAAI/bge-small-en-v1.5" +EMBEDDING_DIMENSIONS = 384 + +# bge models are trained to prepend this instruction for retrieval queries. +_QUERY_PREFIX = "represent this sentence for searching relevant passages: " + + +@lru_cache(maxsize=1) +def _get_model(): + from sentence_transformers import SentenceTransformer + + return SentenceTransformer(EMBEDDING_MODEL_NAME) + + +def _normalize_address(address: str) -> str: + text = address.lower() + text = re.sub(r"[^a-z0-9\s]", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def get_address_embedding(address: str) -> list[float]: + normalized = _normalize_address(address) + if not normalized: + return [0.0] * EMBEDDING_DIMENSIONS + + model = _get_model() + vector = model.encode(_QUERY_PREFIX + normalized, normalize_embeddings=True) + return vector.tolist() + + +def cosine_similarity(vec_a: list[float], vec_b: list[float]) -> float: + if not vec_a or not vec_b or len(vec_a) != len(vec_b): + return 0.0 + dot = sum(a * b for a, b in zip(vec_a, vec_b)) + norm_a = math.sqrt(sum(a * a for a in vec_a)) + norm_b = math.sqrt(sum(b * b for b in vec_b)) + if norm_a == 0 or norm_b == 0: + return 0.0 + return dot / (norm_a * norm_b) + + +def address_similarity(address_a: str, address_b: str) -> float: + if not address_a or not address_b: + return 0.0 + vec_a = get_address_embedding(address_a) + vec_b = get_address_embedding(address_b) + similarity = cosine_similarity(vec_a, vec_b) + return max(0.0, similarity) diff --git a/lending-poc/app/matching/exact.py b/lending-poc/app/matching/exact.py new file mode 100644 index 0000000..4e5facb --- /dev/null +++ b/lending-poc/app/matching/exact.py @@ -0,0 +1,100 @@ +"""Exact-match checks for Aadhaar, PAN, and date of birth. + +Aadhaar numbers are often masked in extracted documents (e.g. +"XXXX XXXX 4321"). MatchResult is tri-state because a masked value can be +inconclusive rather than a clean match/mismatch. +""" + +from dataclasses import dataclass +from datetime import date +from enum import Enum + +MIN_OVERLAPPING_DIGITS = 4 + + +class MatchResult(str, Enum): + MATCH = "MATCH" + MISMATCH = "MISMATCH" + INCONCLUSIVE = "INCONCLUSIVE" + + +@dataclass +class ExactCheckOutcome: + result: MatchResult + reason: str | None = None + + +def _normalize(value: str) -> str: + return "".join(ch for ch in value.upper() if ch.isdigit() or ch == "X") + + +def _is_masked(value: str) -> bool: + return "X" in value + + +def aadhaar_match(golden: str | None, candidate: str | None) -> ExactCheckOutcome: + if not golden or not candidate: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "missing_value") + + g = _normalize(golden) + c = _normalize(candidate) + + if not _is_masked(g) and not _is_masked(c): + return ( + ExactCheckOutcome(MatchResult.MATCH) + if g == c + else ExactCheckOutcome(MatchResult.MISMATCH, "digits_differ") + ) + + if _is_masked(g) != _is_masked(c): + masked, unmasked = (g, c) if _is_masked(g) else (c, g) + trailing_digits = "".join(ch for ch in masked if ch != "X") + if not trailing_digits: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "no_unmasked_digits") + if len(unmasked) < len(trailing_digits): + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "unmasked_value_too_short") + suffix = unmasked[-len(trailing_digits):] + return ( + ExactCheckOutcome(MatchResult.MATCH) + if suffix == trailing_digits + else ExactCheckOutcome(MatchResult.MISMATCH, "suffix_digits_differ") + ) + + # Both masked: compare position-wise where both sides have a digit. + if len(g) != len(c): + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "masked_length_mismatch") + + overlapping = 0 + for gd, cd in zip(g, c): + if gd == "X" or cd == "X": + continue + overlapping += 1 + if gd != cd: + return ExactCheckOutcome(MatchResult.MISMATCH, "overlapping_digits_differ") + + if overlapping < MIN_OVERLAPPING_DIGITS: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "insufficient_unmasked_digits") + + return ExactCheckOutcome(MatchResult.MATCH) + + +def pan_match(golden: str | None, candidate: str | None) -> ExactCheckOutcome: + if not golden or not candidate: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "missing_value") + g = golden.strip().upper() + c = candidate.strip().upper() + return ( + ExactCheckOutcome(MatchResult.MATCH) + if g == c + else ExactCheckOutcome(MatchResult.MISMATCH, "pan_differs") + ) + + +def dob_match(golden: date | None, candidate: date | None) -> ExactCheckOutcome: + if golden is None or candidate is None: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "missing_value") + return ( + ExactCheckOutcome(MatchResult.MATCH) + if golden == candidate + else ExactCheckOutcome(MatchResult.MISMATCH, "dob_differs") + ) diff --git a/lending-poc/app/matching/fuzzy.py b/lending-poc/app/matching/fuzzy.py new file mode 100644 index 0000000..1264da1 --- /dev/null +++ b/lending-poc/app/matching/fuzzy.py @@ -0,0 +1,103 @@ +"""Name and employer string similarity. + +Base similarity comes from RapidFuzz's token_set_ratio (handles word +reordering and extra/missing tokens). On top of that, an initials-expansion +pass resolves abbreviations like "Ankita A.S" vs "Ankita Advitot Sunil", +which plain character-level fuzzy matching can't do on its own. +""" + +from rapidfuzz import fuzz + +MIN_OVERLAP_TOKEN = 1 + + +def _tokenize(name: str) -> list[str]: + return [tok.strip(".").lower() for tok in name.replace(".", ". ").split() if tok.strip(".")] + + +def _is_initial(token: str) -> bool: + return len(token) == 1 + + +def _initials_genuine_match(tokens_a: list[str], tokens_b: list[str]) -> bool: + """True if the side with more single-letter tokens is a genuine + initials-abbreviation of the other side's full-word tokens. + + Every initial on the shorter side must either (a) match the first + letter of a distinct, unused full word on the other side, or (b) have + no full word left to compare against at all (the other name simply + doesn't spell out that token, e.g. a middle/last name omitted + entirely). What's never tolerated is an initial whose letter + contradicts one of the available full words when a candidate exists + to compare against — that's what keeps a genuinely wrong initial + (e.g. "P" when the real name starts with "A") from being boosted. + """ + full_a = [t for t in tokens_a if not _is_initial(t)] + full_b = [t for t in tokens_b if not _is_initial(t)] + initials_a = [t for t in tokens_a if _is_initial(t)] + initials_b = [t for t in tokens_b if _is_initial(t)] + + # Pick the side that actually has initials to expand. + if initials_b and not initials_a: + initials, full = initials_b, full_a + elif initials_a and not initials_b: + initials, full = initials_a, full_b + else: + return False + + if not initials or not full: + return False + + # There must be at least one shared full token (anchor) between the + # two names, otherwise this is likely a different person entirely. + shared_full_words = set(full_a) & set(full_b) + if not shared_full_words: + return False + + # Words already spelled out identically on both sides (e.g. a shared + # first name) don't need to be re-derived from an initial — only the + # remaining full words are available to satisfy the initials. + remaining_full = list(full) + for word in shared_full_words: + if word in remaining_full: + remaining_full.remove(word) + + available_first_letters = [w[0] for w in remaining_full] + matched_any = False + for letter in initials: + if letter in available_first_letters: + available_first_letters.remove(letter) + matched_any = True + elif available_first_letters: + # A candidate full word exists but its first letter doesn't + # match this initial — a genuine contradiction, not boosted. + return False + # else: no remaining full word to check this initial against — + # tolerated as an omitted token, not a contradiction. + + return matched_any + + +def _base_token_set_ratio(a: str, b: str) -> float: + return float(fuzz.token_set_ratio(a.lower(), b.lower())) + + +def name_similarity(name_a: str, name_b: str) -> float: + if not name_a or not name_b: + return 0.0 + + base = _base_token_set_ratio(name_a, name_b) + + tokens_a = _tokenize(name_a) + tokens_b = _tokenize(name_b) + + if _initials_genuine_match(tokens_a, tokens_b): + return 100.0 + + return base + + +def employer_similarity(employer_a: str, employer_b: str) -> float: + if not employer_a or not employer_b: + return 0.0 + return _base_token_set_ratio(employer_a, employer_b) diff --git a/lending-poc/app/models/__init__.py b/lending-poc/app/models/__init__.py index e69de29..dacd302 100644 --- a/lending-poc/app/models/__init__.py +++ b/lending-poc/app/models/__init__.py @@ -0,0 +1,15 @@ +from app.models.case import Case, CaseStatus +from app.models.document import Document +from app.models.golden_record import GoldenRecord +from app.models.pipeline_result import PipelineResult, ReviewStatus +from app.models.validation_result import ValidationResult + +__all__ = [ + "Case", + "CaseStatus", + "Document", + "GoldenRecord", + "PipelineResult", + "ReviewStatus", + "ValidationResult", +] diff --git a/lending-poc/app/models/case.py b/lending-poc/app/models/case.py new file mode 100644 index 0000000..609c1b6 --- /dev/null +++ b/lending-poc/app/models/case.py @@ -0,0 +1,42 @@ +import uuid +from datetime import datetime +from enum import Enum + +from sqlalchemy import DateTime, Enum as SAEnum, String, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.database import Base + + +class CaseStatus(str, Enum): + RECEIVED = "RECEIVED" + RUNNING = "RUNNING" + PASS = "PASS" + FAIL = "FAIL" + NEEDS_REVIEW = "NEEDS_REVIEW" + + +class Case(Base): + __tablename__ = "cases" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + applicant_ref: Mapped[str] = mapped_column(String, unique=True, index=True, nullable=False) + status: Mapped[CaseStatus] = mapped_column( + SAEnum(CaseStatus, name="case_status"), nullable=False, default=CaseStatus.RECEIVED + ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + + documents: Mapped[list["Document"]] = relationship(back_populates="case", cascade="all, delete-orphan") + golden_record: Mapped["GoldenRecord | None"] = relationship( + back_populates="case", cascade="all, delete-orphan", uselist=False + ) + validation_results: Mapped[list["ValidationResult"]] = relationship( + back_populates="case", cascade="all, delete-orphan" + ) + pipeline_results: Mapped[list["PipelineResult"]] = relationship( + back_populates="case", cascade="all, delete-orphan" + ) diff --git a/lending-poc/app/models/document.py b/lending-poc/app/models/document.py new file mode 100644 index 0000000..0c52c71 --- /dev/null +++ b/lending-poc/app/models/document.py @@ -0,0 +1,25 @@ +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, Enum as SAEnum, ForeignKey, String, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.database import Base +from app.services.dto import DocType + + +class Document(Base): + __tablename__ = "documents" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + case_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True + ) + doc_type: Mapped[DocType] = mapped_column(SAEnum(DocType, name="doc_type"), nullable=False) + extracted_fields: Mapped[dict] = mapped_column(JSONB, nullable=False) + source_file_ref: Mapped[str] = mapped_column(String, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + case: Mapped["Case"] = relationship(back_populates="documents") + validation_results: Mapped[list["ValidationResult"]] = relationship(back_populates="document") diff --git a/lending-poc/app/models/golden_record.py b/lending-poc/app/models/golden_record.py new file mode 100644 index 0000000..d7f4058 --- /dev/null +++ b/lending-poc/app/models/golden_record.py @@ -0,0 +1,32 @@ +import uuid +from datetime import date, datetime + +from pgvector.sqlalchemy import Vector +from sqlalchemy import Date, DateTime, ForeignKey, String, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.database import Base +from app.matching.embeddings import EMBEDDING_DIMENSIONS +from app.models.types import EncryptedString + + +class GoldenRecord(Base): + __tablename__ = "golden_records" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + case_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), unique=True, nullable=False + ) + name: Mapped[str | None] = mapped_column(String, nullable=True) + address: Mapped[str | None] = mapped_column(String, nullable=True) + address_embedding: Mapped[list[float] | None] = mapped_column(Vector(EMBEDDING_DIMENSIONS), nullable=True) + aadhaar_number: Mapped[str | None] = mapped_column(EncryptedString, nullable=True) + pan_number: Mapped[str | None] = mapped_column(EncryptedString, nullable=True) + date_of_birth: Mapped[date | None] = mapped_column(Date, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + + case: Mapped["Case"] = relationship(back_populates="golden_record") diff --git a/lending-poc/app/models/pipeline_result.py b/lending-poc/app/models/pipeline_result.py new file mode 100644 index 0000000..9c722e7 --- /dev/null +++ b/lending-poc/app/models/pipeline_result.py @@ -0,0 +1,45 @@ +"""DB model for a pipeline run. + +Named `PipelineResult` to match the ERD/table name. This collides with the +in-memory `app.services.dto.PipelineResult` dataclass — import one or both +qualified (`from app.models import pipeline_result as pipeline_result_model`) +in any module that needs both. +""" + +import uuid +from datetime import datetime +from enum import Enum + +from sqlalchemy import DateTime, Enum as SAEnum, Float, ForeignKey, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.database import Base +from app.services.dto import Decision + + +class ReviewStatus(str, Enum): + PENDING = "PENDING" + APPROVED = "APPROVED" + REJECTED = "REJECTED" + + +class PipelineResult(Base): + __tablename__ = "pipeline_results" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + case_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True + ) + overall_score: Mapped[float] = mapped_column(Float, nullable=False) + decision: Mapped[Decision] = mapped_column(SAEnum(Decision, name="decision"), nullable=False) + reasons: Mapped[list | None] = mapped_column(JSONB, nullable=True) + reviewer: Mapped[str | None] = mapped_column(String, nullable=True) + review_status: Mapped[ReviewStatus | None] = mapped_column( + SAEnum(ReviewStatus, name="review_status"), nullable=True + ) + reviewer_remarks: Mapped[str | None] = mapped_column(Text, nullable=True) + reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + case: Mapped["Case"] = relationship(back_populates="pipeline_results") diff --git a/lending-poc/app/models/types.py b/lending-poc/app/models/types.py new file mode 100644 index 0000000..7dbb1ab --- /dev/null +++ b/lending-poc/app/models/types.py @@ -0,0 +1,26 @@ +from cryptography.fernet import Fernet +from sqlalchemy import String +from sqlalchemy.types import TypeDecorator + +from app.config import settings + + +class EncryptedString(TypeDecorator): + """Stores strings encrypted at rest (Fernet/AES) via ENCRYPTION_KEY. + + Transparent to callers: reads/writes plain str in Python, ciphertext + in the DB column. + """ + + impl = String + cache_ok = True + + def process_bind_param(self, value: str | None, dialect) -> str | None: + if value is None: + return None + return Fernet(settings.ENCRYPTION_KEY).encrypt(value.encode("utf-8")).decode("utf-8") + + def process_result_value(self, value: str | None, dialect) -> str | None: + if value is None: + return None + return Fernet(settings.ENCRYPTION_KEY).decrypt(value.encode("utf-8")).decode("utf-8") diff --git a/lending-poc/app/models/validation_result.py b/lending-poc/app/models/validation_result.py new file mode 100644 index 0000000..9abda31 --- /dev/null +++ b/lending-poc/app/models/validation_result.py @@ -0,0 +1,29 @@ +import uuid +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, Enum as SAEnum, Float, ForeignKey, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.database import Base +from app.services.dto import CheckType + + +class ValidationResult(Base): + __tablename__ = "validation_results" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + case_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True + ) + document_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("documents.id", ondelete="CASCADE"), nullable=True, index=True + ) + check_type: Mapped[CheckType] = mapped_column(SAEnum(CheckType, name="check_type"), nullable=False) + passed: Mapped[bool] = mapped_column(Boolean, nullable=False) + score: Mapped[float] = mapped_column(Float, nullable=False) + evidence: Mapped[dict | None] = mapped_column(JSONB, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + case: Mapped["Case"] = relationship(back_populates="validation_results") + document: Mapped["Document | None"] = relationship(back_populates="validation_results") diff --git a/lending-poc/app/schemas/case.py b/lending-poc/app/schemas/case.py new file mode 100644 index 0000000..2b3d03e --- /dev/null +++ b/lending-poc/app/schemas/case.py @@ -0,0 +1,38 @@ +"""Request/response schemas for POST /cases. + +Mirrors the JSON shape documented in docs/Workflow.md and used by +scripts/sample_case.json — one applicant_ref plus a flat list of documents. +""" + +from typing import Any + +from pydantic import BaseModel + + +class DocumentIn(BaseModel): + doc_type: str + extracted_fields: dict[str, Any] | None = None + source_file_ref: str | None = None + salary_slips: list[dict[str, Any]] | None = None # only present when doc_type == SALARY_SLIP + + +class CaseCreateRequest(BaseModel): + applicant_ref: str + documents: list[DocumentIn] + + +class ValidationResultOut(BaseModel): + check_type: str + passed: bool + score: float + document_id: str | None = None + evidence: dict[str, Any] | None = None + + +class CaseCreateResponse(BaseModel): + case_id: str + applicant_ref: str + decision: str + overall_score: float + reasons: list[str] + validation_results: list[ValidationResultOut] diff --git a/lending-poc/app/services/__init__.py b/lending-poc/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lending-poc/app/services/business_validation.py b/lending-poc/app/services/business_validation.py new file mode 100644 index 0000000..be47f14 --- /dev/null +++ b/lending-poc/app/services/business_validation.py @@ -0,0 +1,233 @@ +"""Checks employer + salary consistency between salary slips and the bank +statement. No specific payroll day is assumed anywhere: each slip is +matched against bank transactions inside a broad, month-level window. +""" + +from calendar import monthrange +from datetime import date, timedelta + +from app.matching.fuzzy import employer_similarity +from app.services import validation_config as cfg +from app.services.dto import ( + BankStatementDoc, + BankTransaction, + CaseInput, + CheckType, + SalarySlipDoc, + ValidationResult, +) + + +def _add_months(d: date, months: int) -> date: + month_index = d.month - 1 + months + year = d.year + month_index // 12 + month = month_index % 12 + 1 + return date(year, month, 1) + + +def _month_window(salary_month: date) -> tuple[date, date]: + window_start = date(salary_month.year, salary_month.month, 1) - timedelta( + days=cfg.SALARY_CREDIT_BUFFER_DAYS + ) + window_end_month = _add_months(salary_month, cfg.SALARY_CREDIT_EXTRA_MONTHS) + last_day = monthrange(window_end_month.year, window_end_month.month)[1] + window_end = date(window_end_month.year, window_end_month.month, last_day) + return window_start, window_end + + +def _within_amount_tolerance(amount: float, expected: float) -> bool: + if not expected: + return False + pct_diff = abs(amount - expected) / expected * 100.0 + return pct_diff <= cfg.SALARY_AMOUNT_TOLERANCE_PCT + + +def _candidate_transactions( + window: tuple[date, date], transactions: list[BankTransaction], expected_amount: float | None +) -> list[BankTransaction]: + """Transactions inside the month-window AND within salary-amount + tolerance of what this specific slip declared. The tolerance gate is + on amount alone -- a large reimbursement/bonus/advance with a + coincidentally close amount is excluded here, before narration + similarity ever gets a vote, so it can never mask a genuinely missing + salary credit. + """ + start, end = window + return [ + txn + for txn in transactions + if txn.amount is not None + and txn.txn_date is not None + and txn.amount > 0 + and start <= txn.txn_date <= end + and (expected_amount is None or _within_amount_tolerance(txn.amount, expected_amount)) + ] + + +def _amount_closeness(amount: float, expected: float) -> float: + if not expected: + return 0.0 + closeness = 100.0 * (1 - abs(amount - expected) / expected) + return max(0.0, min(100.0, closeness)) + + +def _score_transaction(txn: BankTransaction, employer_name: str, expected_amount: float) -> float: + employer_score = employer_similarity(employer_name, txn.narration) if employer_name else 0.0 + amount_score = _amount_closeness(txn.amount, expected_amount) + return ( + cfg.TXN_SELECTION_EMPLOYER_WEIGHT * employer_score + + cfg.TXN_SELECTION_AMOUNT_WEIGHT * amount_score + ) + + +def _select_best_transaction( + candidates: list[BankTransaction], employer_name: str | None, expected_amount: float | None +) -> tuple[BankTransaction, float] | None: + if not candidates or expected_amount is None: + return None + scored = [(txn, _score_transaction(txn, employer_name, expected_amount)) for txn in candidates] + best_txn, best_score = max(scored, key=lambda pair: pair[1]) + if best_score < cfg.TXN_SELECTION_MIN_SCORE: + return None + return best_txn, best_score + + +def _validate_salary_slip( + slip: SalarySlipDoc, bank_statement: BankStatementDoc, used_transaction_ids: set[int] +) -> ValidationResult: + """Matches one slip against the bank statement. + + `used_transaction_ids` (by `id(txn)`) tracks credits already claimed by + an earlier slip in this same case, since overlapping month-windows mean + the same credit could otherwise be double-counted as evidence for two + different declared months of income. + """ + if slip.salary_month is None: + return ValidationResult( + check_type=CheckType.SALARY_DATE, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="missing_salary_month", + ) + + window = _month_window(slip.salary_month) + candidates = [ + txn + for txn in _candidate_transactions(window, bank_statement.transactions, slip.net_salary) + if id(txn) not in used_transaction_ids + ] + selection = _select_best_transaction(candidates, slip.employer_name, slip.net_salary) + + if selection is None: + return ValidationResult( + check_type=CheckType.SALARY_DATE, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="no_matching_credit_in_window", + evidence={"window": window}, + ) + + txn, score = selection + used_transaction_ids.add(id(txn)) + return ValidationResult( + check_type=CheckType.SALARY_DATE, + passed=True, + score=score, + document_id=slip.doc_id, + evidence={"matched_transaction": txn}, + ) + + +def _employer_match_for_slip( + slip: SalarySlipDoc, slip_result: ValidationResult +) -> ValidationResult: + """Verifies one slip's declared employer against its OWN matched bank + transaction's narration only — never against another month's slip. + + Employer consistency is judged month-by-month, on purpose: an + applicant switching jobs mid-history is normal and legitimate, so May's + employer claim is never compared to June's. Each month stands on its + own evidence. + """ + if not slip.employer_name: + return ValidationResult( + check_type=CheckType.EMPLOYER, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="missing_employer_name", + ) + + if not slip_result.passed or not slip_result.evidence: + return ValidationResult( + check_type=CheckType.EMPLOYER, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="no_matching_credit_to_verify_employer_against", + ) + + matched_txn = slip_result.evidence.get("matched_transaction") + score = employer_similarity(slip.employer_name, matched_txn.narration) + passed = score >= cfg.EMPLOYER_MATCH_THRESHOLD + return ValidationResult( + check_type=CheckType.EMPLOYER, + passed=passed, + score=score, + document_id=slip.doc_id, + failure_reason=None if passed else "employer_narration_mismatch", + ) + + +def _salary_credit_count( + salary_slips: list[SalarySlipDoc], + bank_statement: BankStatementDoc, + slip_results: list[ValidationResult], +) -> ValidationResult: + total_slips = len(salary_slips) + matched_slips = sum(1 for r in slip_results if r.passed) + confidence_score = (matched_slips / total_slips * 100.0) if total_slips else 0.0 + + dates = [txn.txn_date for txn in bank_statement.transactions if txn.txn_date is not None] + stmt_duration = {"start": min(dates), "end": max(dates)} if dates else None + + return ValidationResult( + check_type=CheckType.SALARY_CREDIT_COUNT, + passed=(matched_slips == total_slips), + score=confidence_score, + evidence={ + "stmt_duration": stmt_duration, + "no_of_matches": matched_slips, + "total_slips": total_slips, + "confidence_score": confidence_score, + }, + ) + + +def run_business_validation(case: CaseInput) -> list[ValidationResult]: + results: list[ValidationResult] = [] + + if not case.salary_slips or not case.bank_statement: + return results + + used_transaction_ids: set[int] = set() + ordered_slips = sorted( + case.salary_slips, key=lambda slip: slip.salary_month or date.max + ) + slip_results_by_doc_id = { + slip.doc_id: _validate_salary_slip(slip, case.bank_statement, used_transaction_ids) + for slip in ordered_slips + } + # Preserve the original slip order in the output, independent of the + # chronological order used to resolve which slip claims which credit. + slip_results = [slip_results_by_doc_id[slip.doc_id] for slip in case.salary_slips] + results.extend(slip_results) + + for slip in case.salary_slips: + results.append(_employer_match_for_slip(slip, slip_results_by_doc_id[slip.doc_id])) + + results.append(_salary_credit_count(case.salary_slips, case.bank_statement, slip_results)) + + return results diff --git a/lending-poc/app/services/case_parsing.py b/lending-poc/app/services/case_parsing.py new file mode 100644 index 0000000..dd8db9d --- /dev/null +++ b/lending-poc/app/services/case_parsing.py @@ -0,0 +1,110 @@ +"""Parses the raw request JSON shape (see docs/Workflow.md) into CaseInput. + +Shared by the POST /cases endpoint and scripts/run_demo.py so both use +identical parsing rules. +""" + +from datetime import date, datetime + +from app.services.dto import ( + AadhaarDoc, + AddressProofDoc, + BankStatementDoc, + BankTransaction, + CaseInput, + PanDoc, + SalarySlipDoc, +) + + +def _get(fields: dict, key: str) -> str | None: + """Null-safe field lookup: a missing key, JSON null, and an empty/ + whitespace-only string are all treated as no value.""" + value = fields.get(key) + if value is None: + return None + if isinstance(value, str) and not value.strip(): + return None + return value + + +def _parse_date(value: str | None) -> date | None: + if value is None: + return None + return datetime.strptime(value, "%Y-%m-%d").date() + + +def _parse_month(value: str | None) -> date | None: + if value is None: + return None + return datetime.strptime(value, "%Y-%m").date().replace(day=1) + + +def _parse_float(value) -> float | None: + if value is None or (isinstance(value, str) and not value.strip()): + return None + return float(value) + + +def parse_case(payload: dict) -> CaseInput: + case = CaseInput(applicant_ref=payload["applicant_ref"]) + + for doc in payload["documents"]: + doc_type = doc["doc_type"] + + if doc_type == "AADHAAR": + fields = doc["extracted_fields"] + case.aadhaar = AadhaarDoc( + name=_get(fields, "name"), + address=_get(fields, "address"), + aadhaar_number=_get(fields, "aadhaar_number"), + date_of_birth=_parse_date(_get(fields, "date_of_birth")), + source_file_ref=doc["source_file_ref"], + ) + + elif doc_type == "PAN": + fields = doc["extracted_fields"] + case.pan = PanDoc( + name=_get(fields, "name"), + pan_number=_get(fields, "pan_number"), + source_file_ref=doc["source_file_ref"], + ) + + elif doc_type == "ADDRESS_PROOF": + fields = doc["extracted_fields"] + case.address_proof = AddressProofDoc( + address=_get(fields, "address"), + source_file_ref=doc["source_file_ref"], + ) + + elif doc_type == "SALARY_SLIP": + for i, slip in enumerate(doc["salary_slips"]): + fields = slip["extracted_fields"] + case.salary_slips.append( + SalarySlipDoc( + employer_name=_get(fields, "employer_name"), + net_salary=_parse_float(_get(fields, "net_salary")), + salary_month=_parse_month(_get(fields, "salary_month")), + source_file_ref=slip["source_file_ref"], + doc_id=f"SALARY_SLIP-{i}", + name=_get(fields, "name"), + ) + ) + + elif doc_type == "BANK_STATEMENT": + fields = doc["extracted_fields"] + transactions = [ + BankTransaction( + narration=_get(txn, "narration"), + amount=_parse_float(_get(txn, "amount")), + txn_date=_parse_date(_get(txn, "date")), + ) + for txn in fields.get("transactions", []) + ] + case.bank_statement = BankStatementDoc( + transactions=transactions, + source_file_ref=doc["source_file_ref"], + name=_get(fields, "name"), + ) + + return case diff --git a/lending-poc/app/services/decision_engine.py b/lending-poc/app/services/decision_engine.py new file mode 100644 index 0000000..be8abd5 --- /dev/null +++ b/lending-poc/app/services/decision_engine.py @@ -0,0 +1,46 @@ +"""Final PASS / FAIL / NEEDS_REVIEW logic.""" + +from app.services import validation_config as cfg +from app.services.dto import CheckType, Decision, DecisionResult, ScoreResult, ValidationResult + +MANDATORY_CHECK_TYPES = {CheckType.NAME, CheckType.AADHAAR, CheckType.PAN, CheckType.DOB} + + +def make_decision( + score: ScoreResult, validation_results: list[ValidationResult] +) -> DecisionResult: + reasons: list[str] = [] + + mandatory_failures = [ + r + for r in validation_results + if r.check_type in MANDATORY_CHECK_TYPES + and not r.passed + and r.failure_reason == "missing_in_golden_record" + ] + if mandatory_failures: + reasons = [f"MANDATORY_FIELD_MISSING:{r.check_type.value}" for r in mandatory_failures] + return DecisionResult( + decision=Decision.FAIL, reasons=reasons, overall_score=score.overall_score + ) + + if score.overall_score >= cfg.DECISION_PASS_THRESHOLD: + return DecisionResult(decision=Decision.PASS, reasons=["score_meets_pass_threshold"], overall_score=score.overall_score) + + if score.overall_score < cfg.DECISION_FAIL_THRESHOLD: + return DecisionResult( + decision=Decision.FAIL, + reasons=["score_below_fail_threshold"], + overall_score=score.overall_score, + ) + + failing_checks = [ + f"{r.check_type.value}:{r.failure_reason or 'below_threshold'}" + for r in validation_results + if not r.passed + ] + return DecisionResult( + decision=Decision.NEEDS_REVIEW, + reasons=failing_checks or ["score_in_review_band"], + overall_score=score.overall_score, + ) diff --git a/lending-poc/app/services/dto.py b/lending-poc/app/services/dto.py new file mode 100644 index 0000000..acc1d48 --- /dev/null +++ b/lending-poc/app/services/dto.py @@ -0,0 +1,162 @@ +"""Plain in-memory data structures for the validation pipeline. + +No database/ORM involved yet — these dataclasses are what the extraction +pipeline's JSON gets parsed into, and what every service function passes +around. When persistence is added later, these become the shape that gets +mapped to/from the DB models, but the validation logic itself does not +change. +""" + +from dataclasses import dataclass, field +from datetime import date +from enum import Enum + + +class DocType(str, Enum): + AADHAAR = "AADHAAR" + PAN = "PAN" + ADDRESS_PROOF = "ADDRESS_PROOF" + SALARY_SLIP = "SALARY_SLIP" + BANK_STATEMENT = "BANK_STATEMENT" + + +class CheckType(str, Enum): + NAME = "NAME" + ADDRESS = "ADDRESS" + AADHAAR = "AADHAAR" + PAN = "PAN" + DOB = "DOB" + EMPLOYER = "EMPLOYER" + SALARY_DATE = "SALARY_DATE" + SALARY_CREDIT_COUNT = "SALARY_CREDIT_COUNT" + MANDATORY_PRESENCE = "MANDATORY_PRESENCE" + + +class Decision(str, Enum): + PASS = "PASS" + FAIL = "FAIL" + NEEDS_REVIEW = "NEEDS_REVIEW" + + +@dataclass +class AadhaarDoc: + source_file_ref: str + name: str | None = None + address: str | None = None + aadhaar_number: str | None = None + date_of_birth: date | None = None + doc_id: str = "AADHAAR" + + +@dataclass +class PanDoc: + source_file_ref: str + name: str | None = None + pan_number: str | None = None + doc_id: str = "PAN" + + +@dataclass +class AddressProofDoc: + source_file_ref: str + address: str | None = None + doc_id: str = "ADDRESS_PROOF" + + +@dataclass +class SalarySlipDoc: + source_file_ref: str + doc_id: str + employer_name: str | None = None + net_salary: float | None = None + salary_month: date | None = None # first-of-month + name: str | None = None + + +@dataclass +class BankTransaction: + narration: str | None = None + amount: float | None = None + txn_date: date | None = None + + +@dataclass +class BankStatementDoc: + transactions: list[BankTransaction] + source_file_ref: str + doc_id: str = "BANK_STATEMENT" + name: str | None = None + + +@dataclass +class CaseInput: + applicant_ref: str + aadhaar: AadhaarDoc | None = None + pan: PanDoc | None = None + address_proof: AddressProofDoc | None = None + salary_slips: list[SalarySlipDoc] = field(default_factory=list) + bank_statement: BankStatementDoc | None = None + + def present_doc_types(self) -> set[str]: + present = set() + if self.aadhaar: + present.add(DocType.AADHAAR.value) + if self.pan: + present.add(DocType.PAN.value) + if self.address_proof: + present.add(DocType.ADDRESS_PROOF.value) + if self.salary_slips: + present.add(DocType.SALARY_SLIP.value) + if self.bank_statement: + present.add(DocType.BANK_STATEMENT.value) + return present + + +@dataclass +class GoldenRecord: + name: str | None = None + name_source: str | None = None + first_name: str | None = None + middle_name: str | None = None + last_name: str | None = None + address: str | None = None + address_source: str | None = None + address_embedding: list[float] = field(default_factory=list) + date_of_birth: date | None = None + dob_source: str | None = None + aadhaar_number: str | None = None + aadhaar_source: str | None = None + pan_number: str | None = None + pan_source: str | None = None + + +@dataclass +class ValidationResult: + check_type: CheckType + passed: bool + score: float + document_id: str | None = None + failure_reason: str | None = None + evidence: dict | None = None # in-memory only; becomes real columns/FKs once persisted + + +@dataclass +class ScoreResult: + overall_score: float + component_scores: dict[str, float] + + +@dataclass +class DecisionResult: + decision: Decision + reasons: list[str] + overall_score: float + + +@dataclass +class PipelineResult: + golden_record: GoldenRecord | None + validation_results: list[ValidationResult] + score_result: ScoreResult | None + decision_result: DecisionResult + audit_log: list[str] diff --git a/lending-poc/app/services/golden_record.py b/lending-poc/app/services/golden_record.py new file mode 100644 index 0000000..c676e81 --- /dev/null +++ b/lending-poc/app/services/golden_record.py @@ -0,0 +1,82 @@ +"""Builds the one trusted identity profile (Golden Record) per applicant. + +Field precedence: AADHAAR is the primary source for address/DOB, with +ADDRESS_PROOF as a fallback for address. For name specifically: when both +AADHAAR and PAN carry a name, and they're recognizably the same person +(per fuzzy.name_similarity), the fuller of the two (more name tokens) is +preferred as the golden name — e.g. "Ankita Sunil Advitot" over "Ankita +Advitot" — since the fuller version carries strictly more identity +information. If the two names don't look related, AADHAAR stays primary +and the mismatch is left for identity_validation's NAME check to flag, +rather than silently adopting an unrelated "fuller" name. +""" + +from app.matching.embeddings import get_address_embedding +from app.matching.fuzzy import name_similarity +from app.services import validation_config as cfg +from app.services.dto import CaseInput, GoldenRecord + +FULLER_NAME_RELATEDNESS_THRESHOLD = cfg.NAME_MATCH_THRESHOLD + + +def _split_name(full_name: str) -> tuple[str | None, str | None, str | None]: + tokens = full_name.split() + if not tokens: + return None, None, None + if len(tokens) == 1: + return tokens[0], None, None + if len(tokens) == 2: + return tokens[0], None, tokens[1] + return tokens[0], " ".join(tokens[1:-1]), tokens[-1] + + +def _choose_name(aadhaar_name: str | None, pan_name: str | None) -> tuple[str | None, str]: + """Returns (chosen_name, source) where source is "AADHAAR" or "PAN".""" + if aadhaar_name and pan_name: + related = name_similarity(aadhaar_name, pan_name) >= FULLER_NAME_RELATEDNESS_THRESHOLD + if related and len(pan_name.split()) > len(aadhaar_name.split()): + return pan_name, "PAN" + return aadhaar_name, "AADHAAR" + if aadhaar_name: + return aadhaar_name, "AADHAAR" + if pan_name: + return pan_name, "PAN" + return None, "AADHAAR" + + +def build_golden_record(case: CaseInput) -> GoldenRecord: + golden = GoldenRecord() + + aadhaar_name = case.aadhaar.name if case.aadhaar else None + pan_name = case.pan.name if case.pan else None + chosen_name, name_source_type = _choose_name(aadhaar_name, pan_name) + + if chosen_name is not None: + golden.name = chosen_name + golden.name_source = case.aadhaar.doc_id if name_source_type == "AADHAAR" else case.pan.doc_id + golden.first_name, golden.middle_name, golden.last_name = _split_name(chosen_name) + + if case.aadhaar and case.aadhaar.address is not None: + golden.address = case.aadhaar.address + golden.address_source = case.aadhaar.doc_id + + if case.aadhaar and case.aadhaar.date_of_birth is not None: + golden.date_of_birth = case.aadhaar.date_of_birth + golden.dob_source = case.aadhaar.doc_id + + if case.aadhaar and case.aadhaar.aadhaar_number is not None: + golden.aadhaar_number = case.aadhaar.aadhaar_number + golden.aadhaar_source = case.aadhaar.doc_id + + if golden.address is None and case.address_proof and case.address_proof.address is not None: + golden.address = case.address_proof.address + golden.address_source = case.address_proof.doc_id + + if case.pan and case.pan.pan_number is not None: + golden.pan_number = case.pan.pan_number + golden.pan_source = case.pan.doc_id + + if golden.address: + golden.address_embedding = get_address_embedding(golden.address) + + return golden diff --git a/lending-poc/app/services/identity_validation.py b/lending-poc/app/services/identity_validation.py new file mode 100644 index 0000000..8b544a9 --- /dev/null +++ b/lending-poc/app/services/identity_validation.py @@ -0,0 +1,168 @@ +"""Checks every document agrees with the Golden Record: name, address, +Aadhaar, PAN, DOB. Also enforces that mandatory identity fields exist on +the Golden Record at all, regardless of why they're missing. +""" + +from app.matching import exact, fuzzy +from app.matching.embeddings import address_similarity +from app.services import validation_config as cfg +from app.services.dto import CaseInput, CheckType, GoldenRecord, ValidationResult + +MANDATORY_GOLDEN_FIELDS = { + CheckType.NAME: "name", + CheckType.AADHAAR: "aadhaar_number", + CheckType.PAN: "pan_number", + CheckType.DOB: "date_of_birth", +} + + +def check_mandatory_presence(golden: GoldenRecord) -> list[ValidationResult]: + results = [] + for check_type, field_name in MANDATORY_GOLDEN_FIELDS.items(): + if getattr(golden, field_name) is None: + results.append( + ValidationResult( + check_type=check_type, + passed=False, + score=0.0, + failure_reason="missing_in_golden_record", + ) + ) + return results + + +def _exact_result_to_validation(check_type: CheckType, outcome, document_id: str) -> ValidationResult: + passed = outcome.result == exact.MatchResult.MATCH + score = 100.0 if passed else (50.0 if outcome.result == exact.MatchResult.INCONCLUSIVE else 0.0) + return ValidationResult( + check_type=check_type, + passed=passed, + score=score, + document_id=document_id, + failure_reason=None if passed else outcome.reason, + ) + + +def validate_document_against_golden( + document_id: str, + doc_name: str | None, + doc_address: str | None, + doc_aadhaar: str | None, + doc_pan: str | None, + doc_dob, + golden: GoldenRecord, +) -> list[ValidationResult]: + results = [] + + if doc_name is not None and golden.name is not None: + score = fuzzy.name_similarity(golden.name, doc_name) + passed = score >= cfg.NAME_MATCH_THRESHOLD + results.append( + ValidationResult( + check_type=CheckType.NAME, + passed=passed, + score=score, + document_id=document_id, + failure_reason=None if passed else "name_below_threshold", + ) + ) + + if doc_address is not None and golden.address is not None: + similarity = address_similarity(golden.address, doc_address) + score = similarity * 100.0 + passed = similarity >= cfg.ADDRESS_SIMILARITY_THRESHOLD + results.append( + ValidationResult( + check_type=CheckType.ADDRESS, + passed=passed, + score=score, + document_id=document_id, + failure_reason=None if passed else "address_below_threshold", + ) + ) + + if doc_aadhaar is not None: + outcome = exact.aadhaar_match(golden.aadhaar_number, doc_aadhaar) + results.append(_exact_result_to_validation(CheckType.AADHAAR, outcome, document_id)) + + if doc_pan is not None: + outcome = exact.pan_match(golden.pan_number, doc_pan) + results.append(_exact_result_to_validation(CheckType.PAN, outcome, document_id)) + + if doc_dob is not None: + outcome = exact.dob_match(golden.date_of_birth, doc_dob) + results.append(_exact_result_to_validation(CheckType.DOB, outcome, document_id)) + + return results + + +def run_identity_validation(case: CaseInput, golden: GoldenRecord) -> list[ValidationResult]: + results: list[ValidationResult] = [] + results.extend(check_mandatory_presence(golden)) + + if case.aadhaar: + results.extend( + validate_document_against_golden( + document_id=case.aadhaar.doc_id, + doc_name=case.aadhaar.name, + doc_address=case.aadhaar.address, + doc_aadhaar=case.aadhaar.aadhaar_number, + doc_pan=None, + doc_dob=case.aadhaar.date_of_birth, + golden=golden, + ) + ) + + if case.pan: + results.extend( + validate_document_against_golden( + document_id=case.pan.doc_id, + doc_name=case.pan.name, + doc_address=None, + doc_aadhaar=None, + doc_pan=case.pan.pan_number, + doc_dob=None, + golden=golden, + ) + ) + + if case.address_proof: + results.extend( + validate_document_against_golden( + document_id=case.address_proof.doc_id, + doc_name=None, + doc_address=case.address_proof.address, + doc_aadhaar=None, + doc_pan=None, + doc_dob=None, + golden=golden, + ) + ) + + for slip in case.salary_slips: + results.extend( + validate_document_against_golden( + document_id=slip.doc_id, + doc_name=slip.name, + doc_address=None, + doc_aadhaar=None, + doc_pan=None, + doc_dob=None, + golden=golden, + ) + ) + + if case.bank_statement: + results.extend( + validate_document_against_golden( + document_id=case.bank_statement.doc_id, + doc_name=case.bank_statement.name, + doc_address=None, + doc_aadhaar=None, + doc_pan=None, + doc_dob=None, + golden=golden, + ) + ) + + return results diff --git a/lending-poc/app/services/persistence.py b/lending-poc/app/services/persistence.py new file mode 100644 index 0000000..60be46d --- /dev/null +++ b/lending-poc/app/services/persistence.py @@ -0,0 +1,184 @@ +"""Persists one pipeline run (CaseInput + PipelineResult) to the database. + +The in-memory dataclasses in app.services.dto reference documents by their +string doc_id (e.g. "AADHAAR", "SALARY_SLIP-0"). This module inserts the +Document rows first and keeps a doc_id -> Document.id map so +ValidationResult.document_id (also a doc_id string) can be resolved to the +real foreign key. +""" + +import dataclasses +import datetime +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.case import Case, CaseStatus +from app.models.document import Document +from app.models.golden_record import GoldenRecord as GoldenRecordModel +from app.models.pipeline_result import PipelineResult as PipelineResultModel +from app.models.validation_result import ValidationResult as ValidationResultModel +from app.services.dto import CaseInput, Decision, DocType +from app.services.dto import PipelineResult as PipelineResultDTO + +_DECISION_TO_CASE_STATUS = { + Decision.PASS: CaseStatus.PASS, + Decision.FAIL: CaseStatus.FAIL, + Decision.NEEDS_REVIEW: CaseStatus.NEEDS_REVIEW, +} + + +def _json_safe(value): + """Recursively converts dataclasses/dates/tuples in evidence dicts into + plain JSON-serializable values, since business_validation/scoring build + evidence out of BankTransaction dataclasses and date objects for + in-memory use. + """ + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return {k: _json_safe(v) for k, v in dataclasses.asdict(value).items()} + if isinstance(value, (datetime.date, datetime.datetime)): + return value.isoformat() + if isinstance(value, dict): + return {k: _json_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(v) for v in value] + return value + + +def _document_rows(case: CaseInput) -> list[tuple[str, Document]]: + """Returns (doc_id, Document) pairs for every document present on the case.""" + rows: list[tuple[str, Document]] = [] + + if case.aadhaar: + rows.append(( + case.aadhaar.doc_id, + Document( + doc_type=DocType.AADHAAR, + source_file_ref=case.aadhaar.source_file_ref, + extracted_fields={ + "name": case.aadhaar.name, + "address": case.aadhaar.address, + "aadhaar_number": case.aadhaar.aadhaar_number, + "date_of_birth": case.aadhaar.date_of_birth.isoformat() if case.aadhaar.date_of_birth else None, + }, + ), + )) + + if case.pan: + rows.append(( + case.pan.doc_id, + Document( + doc_type=DocType.PAN, + source_file_ref=case.pan.source_file_ref, + extracted_fields={"name": case.pan.name, "pan_number": case.pan.pan_number}, + ), + )) + + if case.address_proof: + rows.append(( + case.address_proof.doc_id, + Document( + doc_type=DocType.ADDRESS_PROOF, + source_file_ref=case.address_proof.source_file_ref, + extracted_fields={"address": case.address_proof.address}, + ), + )) + + for slip in case.salary_slips: + rows.append(( + slip.doc_id, + Document( + doc_type=DocType.SALARY_SLIP, + source_file_ref=slip.source_file_ref, + extracted_fields={ + "employee_name": slip.name, + "employer": slip.employer_name, + "net_salary": slip.net_salary, + "salary_month": slip.salary_month.isoformat() if slip.salary_month else None, + }, + ), + )) + + if case.bank_statement: + rows.append(( + case.bank_statement.doc_id, + Document( + doc_type=DocType.BANK_STATEMENT, + source_file_ref=case.bank_statement.source_file_ref, + extracted_fields={ + "account_holder": case.bank_statement.name, + "transactions": [ + { + "narration": txn.narration, + "amount": txn.amount, + "date": txn.txn_date.isoformat() if txn.txn_date else None, + } + for txn in case.bank_statement.transactions + ], + }, + ), + )) + + return rows + + +async def save_pipeline_result( + db: AsyncSession, case_input: CaseInput, pipeline_result: PipelineResultDTO +) -> Case: + """Persists one pipeline run in a single transaction and returns the Case row. + + Caller owns the commit (via app.database.get_db, which commits on + success / rolls back on error). + """ + case = Case( + applicant_ref=case_input.applicant_ref, + status=_DECISION_TO_CASE_STATUS[pipeline_result.decision_result.decision], + ) + db.add(case) + await db.flush() # assigns case.id + + doc_id_to_pk: dict[str, uuid.UUID] = {} + for doc_id, document in _document_rows(case_input): + document.case_id = case.id + db.add(document) + await db.flush() # assigns document.id + doc_id_to_pk[doc_id] = document.id + + golden = pipeline_result.golden_record + if golden is not None: + db.add( + GoldenRecordModel( + case_id=case.id, + name=golden.name, + address=golden.address, + address_embedding=golden.address_embedding or None, + aadhaar_number=golden.aadhaar_number, + pan_number=golden.pan_number, + date_of_birth=golden.date_of_birth, + ) + ) + + for result in pipeline_result.validation_results: + db.add( + ValidationResultModel( + case_id=case.id, + document_id=doc_id_to_pk.get(result.document_id) if result.document_id else None, + check_type=result.check_type, + passed=result.passed, + score=result.score, + evidence=_json_safe(result.evidence) if result.evidence else None, + ) + ) + + score_result = pipeline_result.score_result + db.add( + PipelineResultModel( + case_id=case.id, + overall_score=score_result.overall_score if score_result else 0.0, + decision=pipeline_result.decision_result.decision, + reasons=pipeline_result.decision_result.reasons, + ) + ) + + await db.flush() + return case diff --git a/lending-poc/app/services/pipeline.py b/lending-poc/app/services/pipeline.py new file mode 100644 index 0000000..bdee270 --- /dev/null +++ b/lending-poc/app/services/pipeline.py @@ -0,0 +1,56 @@ +"""Orchestrates Golden Record -> Identity -> Business -> Scoring -> +Decision in one call, with a simple in-memory audit log. +""" + +from app.services import business_validation, decision_engine, golden_record, scoring +from app.services import validation_config as cfg +from app.services.dto import CaseInput, Decision, DecisionResult, PipelineResult +from app.services.identity_validation import run_identity_validation + + +def run_pipeline(case: CaseInput) -> PipelineResult: + audit_log: list[str] = [] + audit_log.append(f"INGEST: case_created applicant_ref={case.applicant_ref}") + + present = case.present_doc_types() + missing = set(cfg.REQUIRED_DOCUMENT_TYPES) - present + if missing: + reasons = [f"MISSING_DOCUMENT:{doc_type}" for doc_type in sorted(missing)] + audit_log.append(f"PIPELINE: precheck_failed missing={sorted(missing)}") + decision_result = DecisionResult(decision=Decision.FAIL, reasons=reasons, overall_score=0.0) + return PipelineResult( + golden_record=None, + validation_results=[], + score_result=None, + decision_result=decision_result, + audit_log=audit_log, + ) + + golden = golden_record.build_golden_record(case) + audit_log.append( + f"GOLDEN_RECORD: built name={golden.name!r} address={golden.address!r}" + ) + + identity_results = run_identity_validation(case, golden) + audit_log.append(f"IDENTITY_VALIDATION: {len(identity_results)} checks run") + + business_results = business_validation.run_business_validation(case) + audit_log.append(f"BUSINESS_VALIDATION: {len(business_results)} checks run") + + all_results = identity_results + business_results + + score_result = scoring.compute_score(all_results) + audit_log.append(f"SCORING: overall_score={score_result.overall_score:.2f}") + + decision_result = decision_engine.make_decision(score_result, all_results) + audit_log.append( + f"DECISION: {decision_result.decision.value} reasons={decision_result.reasons}" + ) + + return PipelineResult( + golden_record=golden, + validation_results=all_results, + score_result=score_result, + decision_result=decision_result, + audit_log=audit_log, + ) diff --git a/lending-poc/app/services/scoring.py b/lending-poc/app/services/scoring.py new file mode 100644 index 0000000..6f79533 --- /dev/null +++ b/lending-poc/app/services/scoring.py @@ -0,0 +1,34 @@ +"""Combines all validation checks into one weighted confidence number.""" + +from collections import defaultdict + +from app.services import validation_config as cfg +from app.services.dto import ScoreResult, ValidationResult + + +def compute_score(validation_results: list[ValidationResult]) -> ScoreResult: + scores_by_type: dict[str, list[float]] = defaultdict(list) + for result in validation_results: + scores_by_type[result.check_type.value].append(result.score) + + mean_by_type = { + check_type: sum(scores) / len(scores) for check_type, scores in scores_by_type.items() + } + + observed_weight_total = sum( + weight for check_type, weight in cfg.VALIDATION_WEIGHTS.items() if check_type in mean_by_type + ) + + if observed_weight_total == 0: + return ScoreResult(overall_score=0.0, component_scores={}) + + overall_score = ( + sum( + cfg.VALIDATION_WEIGHTS[check_type] * mean_by_type[check_type] + for check_type in mean_by_type + if check_type in cfg.VALIDATION_WEIGHTS + ) + / observed_weight_total + ) + + return ScoreResult(overall_score=overall_score, component_scores=mean_by_type) diff --git a/lending-poc/app/services/validation_config.py b/lending-poc/app/services/validation_config.py new file mode 100644 index 0000000..e306d5c --- /dev/null +++ b/lending-poc/app/services/validation_config.py @@ -0,0 +1,40 @@ +"""Tunable thresholds/weights for the validation pipeline. + +Plain constants for this standalone, no-DB run. When this integrates with +the rest of the app, these move into app/config.py (pydantic Settings, +loaded from .env) without changing any service logic that reads them. +""" + +REQUIRED_DOCUMENT_TYPES = ["AADHAAR", "PAN", "SALARY_SLIP", "BANK_STATEMENT"] + +NAME_MATCH_THRESHOLD = 85.0 +EMPLOYER_MATCH_THRESHOLD = 80.0 +ADDRESS_SIMILARITY_THRESHOLD = 0.55 # cosine, 0-1 (stub embeddings are coarser than real ones) + +SALARY_CREDIT_EXTRA_MONTHS = 1 +SALARY_CREDIT_BUFFER_DAYS = 5 +TXN_SELECTION_EMPLOYER_WEIGHT = 0.70 +TXN_SELECTION_AMOUNT_WEIGHT = 0.30 +TXN_SELECTION_MIN_SCORE = 60.0 + +# A transaction must land within this percentage of the slip's declared +# net_salary to be eligible as a salary-credit match AT ALL, independent +# of how well its narration scores. This is what stops a same-employer +# reimbursement/bonus/advance with a coincidentally close amount from +# masking a genuinely missing salary credit -- the gate is on the amount +# itself, not on narration keywords (which don't generalize across +# employers/languages/formats). +SALARY_AMOUNT_TOLERANCE_PCT = 3.0 + +VALIDATION_WEIGHTS = { + "NAME": 0.15, + "ADDRESS": 0.10, + "AADHAAR": 0.15, + "PAN": 0.15, + "DOB": 0.10, + "EMPLOYER": 0.10, + "SALARY_CREDIT_COUNT": 0.25, +} + +DECISION_PASS_THRESHOLD = 90.0 +DECISION_FAIL_THRESHOLD = 60.0 # below -> FAIL, between -> NEEDS_REVIEW diff --git a/lending-poc/docker-compose.yml b/lending-poc/docker-compose.yml index 9b576a3..52cfaa9 100644 --- a/lending-poc/docker-compose.yml +++ b/lending-poc/docker-compose.yml @@ -1,12 +1,12 @@ services: db: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: lending_poc ports: - - "5432:5432" + - "55432:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: diff --git a/lending-poc/docs/Workflow.md b/lending-poc/docs/Workflow.md new file mode 100644 index 0000000..43cb8e0 --- /dev/null +++ b/lending-poc/docs/Workflow.md @@ -0,0 +1,294 @@ +# Document Validation & Decision Engine — How It Works + +## In one sentence + +You give it a loan applicant's documents (already extracted into structured +fields by an OCR/extraction pipeline elsewhere) — Aadhaar, PAN, address +proof, salary slips, bank statement — and it gives back one verdict: **PASS, +FAIL, or NEEDS_REVIEW**, plus a full breakdown of exactly which checks +passed, which failed, and why. + +It does not read PDFs or images itself. It assumes that's already done and +works only with the structured JSON that comes out of that step. + +--- + +## Input + +One JSON object per applicant ("case"), shaped like this: + +```json +{ + "applicant_ref": "APP-2026-00123", + "documents": [ + { + "doc_type": "AADHAAR", + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "address": "Flat 204, Green Heights, Baner, Pune, Maharashtra 411045", + "aadhaar_number": "XXXX XXXX 4321", + "date_of_birth": "1995-03-14" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/aadhaar_front.pdf" + }, + { + "doc_type": "PAN", + "extracted_fields": { + "name": "Sneha Lokhande", + "pan_number": "ABCDE1234F" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/pan_card.pdf" + }, + { + "doc_type": "ADDRESS_PROOF", + "extracted_fields": { + "address": "Apartment 204, Green Heights, Baner, Pune, MH 411045" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/address_proof.pdf" + }, + { + "doc_type": "SALARY_SLIP", + "salary_slips": [ + { + "extracted_fields": { + "employer_name": "ABC Technologies Pvt Ltd", + "net_salary": 75000, + "salary_month": "2026-05" + }, + "source_file_ref": "s3://kyc-docs/.../salary_slip_may.pdf" + } + ] + }, + { + "doc_type": "BANK_STATEMENT", + "extracted_fields": { + "transactions": [ + { "narration": "ABC TECHNOLOGIES SALARY MAY", "amount": 75000, "date": "2026-06-02" }, + { "narration": "SWIGGY ORDER PAYMENT", "amount": -450, "date": "2026-06-10" } + ] + }, + "source_file_ref": "s3://kyc-docs/.../bank_statement.pdf" + } + ] +} +``` + +A few things worth knowing about this shape: + +- **Any field can be missing or blank.** Real extraction output is messy — + an address might not be captured, a DOB might be unreadable. The engine + treats a missing key, an empty string, and an explicit `null` all the + same way: as "no value," never as a crash. +- **`SALARY_SLIP` is the one document type that can repeat** — an applicant + usually has several months of salary slips, each with its own + `net_salary`, `employer_name`, and `salary_month`. +- **`source_file_ref`** is just kept for traceability (which physical file + a fact came from) — it doesn't affect any decision. + +--- + +## Output + +One decision object, plus a full audit trail of every intermediate +check. In shape: + +``` +Decision: PASS | FAIL | NEEDS_REVIEW +Overall score: 0-100 +Reasons: [ "SALARY_DATE:no_matching_credit_in_window", ... ] + +Golden Record: the one trusted identity profile built from all documents + (name, address, DOB, Aadhaar, PAN — each tagged with which + document it came from) + +Validation results: one row per individual check, e.g. + [PASS] NAME score=100.00 doc=AADHAAR + [FAIL] SALARY_DATE score=0.00 doc=SALARY_SLIP-3 reason=no_matching_credit_in_window + +Score breakdown: the weighted average per check type + NAME 100, ADDRESS 88, AADHAAR 100, ... SALARY_CREDIT_COUNT 75 +``` + +Nothing here is a black box — every number in the final score traces back +to one specific, inspectable check on one specific document. + +--- + +## How it works — the seven stages + +``` + Documents in + │ + ▼ + ① Golden Record Build ONE trusted identity from all ID documents + │ + ▼ + ② Identity Validation Does every document agree with the Golden Record? + │ + ▼ + ③ Business Validation Do salary slips have matching bank credits? + │ + ▼ + ④ Scoring Combine every check into one weighted number + │ + ▼ + ⑤ Decision PASS / FAIL / NEEDS_REVIEW + │ + ▼ + ⑥ Review (if needed) A human resolves NEEDS_REVIEW cases + │ + ▼ + ⑦ Audit Every step is permanently logged +``` + +### ① Golden Record — "who is this person, really?" + +Aadhaar and PAN often disagree slightly on name (e.g. Aadhaar has a full +name, PAN has an abbreviated one). Rather than blindly trusting one +document, the engine builds a single reconciled identity: + +- **Name**: if Aadhaar and PAN both have a name and they're clearly the + same person, the **fuller** one wins (more words = more information) — + e.g. "Sneha Sunil Lokhande" beats "Sneha Lokhande". It's then split into + first / middle / last name. If the two names don't look related at all, + Aadhaar stays the trusted source and the mismatch gets flagged later. +- **Address, DOB, Aadhaar number**: taken from Aadhaar; address falls back + to the address-proof document if Aadhaar didn't have one. +- **PAN number**: taken from the PAN document. +- Every field remembers **which document it came from**, for traceability. + +**Mandatory rule**: if name, Aadhaar number, PAN number, or DOB is missing +from *every* document that could supply it, the case is an automatic FAIL — +these are non-negotiable identity anchors. + +### ② Identity Validation — "does every document agree?" + +Every document is compared against the Golden Record: + +| Check | How it's compared | +|---|---| +| **Name** | Fuzzy text match, tolerant of reordering ("Lokhande Sneha" = "Sneha Lokhande") and abbreviation ("Sneha S.L." = "Sneha Sunil Lokhande") | +| **Address** | AI embedding similarity — tolerant of different wording ("Flat 204" vs "Apartment 204") since it compares *meaning*, not exact text | +| **Aadhaar number** | Exact match, with a fallback for masked numbers (e.g. "XXXX XXXX 4321" matches a full number ending in 4321) | +| **PAN number** | Exact match | +| **DOB** | Exact match | + +Each of these runs on *every* document that carries that field — Aadhaar, +PAN, address proof, even the salary slips and bank statement if they +happen to carry a name too. + +### ③ Business Validation — "is the declared income real?" + +This is the more involved stage, because real payroll data is messy: every +company pays on a different day of the month, so the engine never assumes +a fixed payday. + +For each salary slip: + +1. **Build a window** around that slip's month — a few days before the + month starts, through the end of the *following* month. This is wide + enough to catch "pays on the 1st" and "pays on the 5th of next month" + equally, without any company-specific rules. +2. **Find candidate bank transactions** inside that window that are: + - credits (positive amount), and + - within **3% of the declared net salary** — this is a hard cutoff. A + transaction outside this range is never eligible, no matter how good + its description looks. (This exists specifically to stop a + same-employer reimbursement or bonus with a coincidentally close + amount from being mistaken for the real salary payment.) +3. **Score each remaining candidate** — 70% on how well the transaction's + description matches the employer name, 30% on how close the amount is — + and pick the best one. +4. **A transaction can only be used once.** If two slips' windows overlap + (common, since windows span two months), an earlier slip claims its + match first so the same bank credit can't "prove" two different months + of income. +5. If no transaction survives, that slip's `SALARY_DATE` check fails — + this is a real, visible finding, not silently skipped. + +**Employer check** — run separately, *per slip, per month* — verifies that +month's declared employer against *that month's own* matched bank credit +only. It deliberately never compares one month's employer to another's, +because switching jobs mid-history is normal, not suspicious. + +**SALARY_CREDIT_COUNT** — one case-level summary: "N salary slips were +declared, how many actually got a verified bank credit?" If an applicant +declares 4 months but only 3 have proof, this shows `3/4 = 75%`. Note that +this **does not by itself fail the case** — it only lowers the overall +score. Missing one month's proof out of several is common and shouldn't +be treated the same as missing proof entirely. + +### ④ Scoring — one number from many checks + +Every check above produces a score from 0–100. These are grouped by check +type, averaged, and combined into one overall score using fixed weights: + +| Check | Weight | +|---|---| +| SALARY_CREDIT_COUNT | 25% (highest — this is the core "is the income real" signal) | +| NAME | 15% | +| AADHAAR | 15% | +| PAN | 15% | +| ADDRESS | 10% | +| DOB | 10% | +| EMPLOYER | 10% | + +### ⑤ Decision — turning the score into a verdict + +1. If name, Aadhaar, PAN, or DOB is missing entirely → **FAIL**, regardless + of score. +2. If overall score ≥ 90 → **PASS**. +3. If overall score < 60 → **FAIL**. +4. Otherwise → **NEEDS_REVIEW** — the case isn't clean enough to auto-pass, + but isn't bad enough to auto-reject either. A human should look at it. + +### ⑥ Review + +Cases marked NEEDS_REVIEW wait in a queue for a human reviewer, who sees +exactly which checks failed and why, and makes the final call. + +### ⑦ Audit + +Every stage — ingest, golden record built, each validation run, the final +score, the decision — is logged permanently. Nothing happens invisibly; +you can always answer "why did this case get this verdict?" after the +fact. + +--- + +## Worked example + +A real (deliberately imperfect) case run through the engine: + +- Aadhaar: "Sneha Sunil Lokhande" — PAN: "Sneha Lokhande" (shorter) → + Golden Record keeps Aadhaar's fuller name. **PASS.** +- Address worded differently across documents → matched by meaning, not + exact text. **PASS.** +- 4 salary months declared: March (clean), April (employer switched from + ABC to Nimbus, both fully verified), May (a decoy reimbursement in the + same window as the real salary — correctly ignored), June (**no matching + bank credit exists at all**). +- Result: June's `SALARY_DATE` check **fails** honestly. + `SALARY_CREDIT_COUNT` drops to 75% (3 of 4 months proven). Two months' + `EMPLOYER` checks land just under threshold due to how the company names + compare textually. +- None of this is a single catastrophic failure — it's a mix of solid and + shaky signals. **Final verdict: NEEDS_REVIEW at 88.4/100** — sent to a + human rather than auto-approved or auto-rejected. + +This is the intended behavior: the engine doesn't try to force every case +into a clean PASS or FAIL. Genuinely ambiguous evidence should produce an +ambiguous verdict. + +--- + +## Try it yourself + +```bash +cd lending-poc +python3 scripts/run_demo.py +``` + +This runs `scripts/sample_case.json` (the example above, in full) through +every stage and prints the Golden Record, every individual check, the +score breakdown, and the final decision. diff --git a/lending-poc/pyproject.toml b/lending-poc/pyproject.toml index e5522fa..babdcb8 100644 --- a/lending-poc/pyproject.toml +++ b/lending-poc/pyproject.toml @@ -11,6 +11,10 @@ dependencies = [ "alembic>=1.14", "pydantic-settings>=2.0", "python-dotenv>=1.0", + "pgvector>=0.3", + "cryptography>=43.0", + "sentence-transformers>=3.0", + "rapidfuzz>=3.0", ] [project.optional-dependencies] diff --git a/lending-poc/scripts/edge_case_scenarios.py b/lending-poc/scripts/edge_case_scenarios.py new file mode 100644 index 0000000..a02f0dc --- /dev/null +++ b/lending-poc/scripts/edge_case_scenarios.py @@ -0,0 +1,149 @@ +"""Runs a battery of edge-case scenarios through the validation pipeline, +starting from the base sample_case.json and mutating it per scenario, to +show exactly what today's code does and does not handle correctly. + +Usage: + python scripts/edge_case_scenarios.py +""" + +import copy +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from scripts.run_demo import parse_case # noqa: E402 +from app.services.pipeline import run_pipeline # noqa: E402 + + +def load_base() -> dict: + path = Path(__file__).resolve().parent / "sample_case.json" + return json.loads(path.read_text()) + + +def summarize(label: str, payload: dict) -> None: + case = parse_case(payload) + result = run_pipeline(case) + + print("=" * 78) + print(label) + print("=" * 78) + print(f" decision = {result.decision_result.decision.value} " + f"score = {result.decision_result.overall_score:.2f}") + print(f" reasons = {result.decision_result.reasons}") + + for r in result.validation_results: + if r.check_type.value in ("SALARY_DATE", "EMPLOYER", "SALARY_CREDIT_COUNT"): + status = "PASS" if r.passed else "FAIL" + doc = f" doc={r.document_id}" if r.document_id else "" + print(f" [{status}] {r.check_type.value:<20} score={r.score:6.2f}{doc}") + if r.evidence and "matched_transaction" in r.evidence: + txn = r.evidence["matched_transaction"] + print(f" -> matched: {txn.narration!r} amount={txn.amount} date={txn.txn_date}") + elif r.evidence and r.check_type.value == "SALARY_CREDIT_COUNT": + print(f" evidence: {r.evidence}") + print() + + +def scenario_reimbursement_ambiguity() -> None: + """A reimbursement credit sits inside the same month-window as the real + salary credit, from the same employer. Does the 70/30 scoring correctly + prefer the salary-sized credit over the reimbursement? + """ + payload = load_base() + # Base data already has this: May slip window contains both the + # "SALARY MAY" (75000) and "REIMBURSEMENT" (4500) transactions. + summarize("1. Reimbursement in same window as real salary credit (baseline)", payload) + + # Harder version: reimbursement amount is much closer to net_salary, + # to see if amount-closeness alone could fool the selection. + payload2 = load_base() + for doc in payload2["documents"]: + if doc["doc_type"] == "BANK_STATEMENT": + for txn in doc["extracted_fields"]["transactions"]: + if "REIMBURSEMENT" in txn["narration"]: + txn["amount"] = 74000 # suspiciously close to net_salary (75000) + summarize( + "1b. Reimbursement amount very close to net_salary (adversarial)", payload2 + ) + + +def scenario_partial_slip_match() -> None: + """3 salary slips declared, bank statement only backs 2 of them. + Per spec (README 'Open decisions'), this must NOT hard-fail the case — + only lower SALARY_CREDIT_COUNT's score. Confirm that's actually true. + """ + payload = load_base() + for doc in payload["documents"]: + if doc["doc_type"] == "SALARY_SLIP": + doc["salary_slips"].append( + { + "extracted_fields": { + "employer_name": "ABC Technologies Pvt Ltd", + "net_salary": 75000, + "salary_month": "2026-07", + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_july.pdf", + } + ) + # No corresponding July bank credit exists in the base data. + summarize("2. 3 slips declared, only 2 have matching bank credits", payload) + + +def scenario_salary_amount_changed() -> None: + """Net salary changes month-to-month (raise or deduction). Each slip's + amount-closeness should be judged against its OWN declared net_salary, + not a fixed/previous value. + """ + payload = load_base() + for doc in payload["documents"]: + if doc["doc_type"] == "SALARY_SLIP": + for slip in doc["salary_slips"]: + if slip["extracted_fields"]["salary_month"] == "2026-06": + slip["extracted_fields"]["net_salary"] = 82000 # a raise + if doc["doc_type"] == "BANK_STATEMENT": + for txn in doc["extracted_fields"]["transactions"]: + if "JUN" in txn["narration"]: + txn["amount"] = 82000 # bank credit reflects the raise + summarize("3. Salary raised in June (75000 -> 82000), bank credit matches", payload) + + # Now the adversarial version: slip says raise, but bank credit still + # shows the OLD amount -- should this still confidently match? + payload2 = load_base() + for doc in payload2["documents"]: + if doc["doc_type"] == "SALARY_SLIP": + for slip in doc["salary_slips"]: + if slip["extracted_fields"]["salary_month"] == "2026-06": + slip["extracted_fields"]["net_salary"] = 82000 + # bank statement untouched: still shows 75000 for June + summarize( + "3b. Slip claims raise to 82000 but bank credit still shows 75000", payload2 + ) + + +def scenario_employer_switch() -> None: + """Applicant switched jobs: earlier slips are Employer A, most recent + slip(s) are Employer B, with matching bank credits from each. Does + EMPLOYER consistency correctly distinguish "job switch" from "identity + fraud", or does it just uniformly fail? + """ + payload = load_base() + for doc in payload["documents"]: + if doc["doc_type"] == "SALARY_SLIP": + # May slip: old employer. June slip: new employer (switched). + for slip in doc["salary_slips"]: + if slip["extracted_fields"]["salary_month"] == "2026-06": + slip["extracted_fields"]["employer_name"] = "NextGen Solutions Pvt Ltd" + if doc["doc_type"] == "BANK_STATEMENT": + for txn in doc["extracted_fields"]["transactions"]: + if "JUN" in txn["narration"]: + txn["narration"] = "NEXTGEN SOLUTIONS SALARY JUN" + summarize("4. Employer switched between May (ABC) and June (NextGen)", payload) + + +if __name__ == "__main__": + scenario_reimbursement_ambiguity() + scenario_partial_slip_match() + scenario_salary_amount_changed() + scenario_employer_switch() diff --git a/lending-poc/scripts/run_demo.py b/lending-poc/scripts/run_demo.py new file mode 100644 index 0000000..ec7500e --- /dev/null +++ b/lending-poc/scripts/run_demo.py @@ -0,0 +1,71 @@ +"""Runs one dummy case through the validation pipeline end-to-end, with no +database involved. Parses scripts/sample_case.json (the README's exact +sample payload) into in-memory DTOs, runs the pipeline, and prints every +intermediate result plus the final decision. + +Usage: + python scripts/run_demo.py +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from app.services.case_parsing import parse_case # noqa: E402 +from app.services.pipeline import run_pipeline # noqa: E402 + + +def main() -> None: + sample_path = Path(__file__).resolve().parent / "sample_case.json" + payload = json.loads(sample_path.read_text()) + case = parse_case(payload) + + result = run_pipeline(case) + + print("=" * 70) + print(f"Applicant: {case.applicant_ref}") + print("=" * 70) + + print("\n--- Audit log ---") + for line in result.audit_log: + print(f" {line}") + + if result.golden_record: + g = result.golden_record + print("\n--- Golden Record ---") + print(f" name = {g.name!r} (source: {g.name_source})") + print(f" first_name = {g.first_name!r}") + print(f" middle_name = {g.middle_name!r}") + print(f" last_name = {g.last_name!r}") + print(f" address = {g.address!r} (source: {g.address_source})") + print(f" date_of_birth = {g.date_of_birth} (source: {g.dob_source})") + print(f" aadhaar_number = {g.aadhaar_number!r} (source: {g.aadhaar_source})") + print(f" pan_number = {g.pan_number!r} (source: {g.pan_source})") + + print("\n--- Validation results ---") + for r in result.validation_results: + status = "PASS" if r.passed else "FAIL" + doc = f" doc={r.document_id}" if r.document_id else "" + reason = f" reason={r.failure_reason}" if r.failure_reason else "" + print(f" [{status}] {r.check_type.value:<22} score={r.score:6.2f}{doc}{reason}") + if r.evidence: + print(f" evidence={r.evidence}") + + if result.score_result: + print("\n--- Score ---") + print(f" overall_score = {result.score_result.overall_score:.2f}") + print(" component_scores:") + for check_type, score in result.score_result.component_scores.items(): + print(f" {check_type:<22} {score:.2f}") + + print("\n--- Decision ---") + print(f" decision = {result.decision_result.decision.value}") + print(f" overall_score = {result.decision_result.overall_score:.2f}") + print(f" reasons = {result.decision_result.reasons}") + print("=" * 70) + + +if __name__ == "__main__": + main() diff --git a/lending-poc/scripts/sample_case.json b/lending-poc/scripts/sample_case.json new file mode 100644 index 0000000..7ee78f3 --- /dev/null +++ b/lending-poc/scripts/sample_case.json @@ -0,0 +1,136 @@ +{ + "_test_design_notes": { + "purpose": "Realistic mixed-signal case for one applicant, NOT a clean happy-path. Exercises pass, fail, and boundary conditions across identity and business validation in a single run. These notes describe VERIFIED behavior (actually run through the pipeline), not assumptions -- update them if the matching logic changes.", + "planted_conditions": { + "AADHAAR_name": "Sneha Sunil Lokhande (full 3-token name, becomes golden name source) -> PASS", + "PAN_name": "Sneha Lokhande (2 tokens, shorter than Aadhaar -> Aadhaar stays golden per fuller-name rule) -> PASS", + "ADDRESS_PROOF": "worded differently (Apartment vs Flat, MH vs Maharashtra) -> PASS via embedding similarity (~77, above the 55 threshold), not exact match", + "BANK_STATEMENT_name": "Lokhande S. S. (surname-first + initials) -> PASS, tests initials-expansion with reordering", + "MARCH_slip": "clean month, ABC Technologies, exact amount -> SALARY_DATE PASS (86.0), EMPLOYER PASS (80.0, right at the threshold -- 4-word company name vs narration missing 'Pvt Ltd' keeps token_set_ratio near the boundary, not a clean 100)", + "APRIL_slip": "legitimate employer switch mid-history (ABC Technologies -> Nimbus Retail Pvt Ltd), bank narration reflects the new employer correctly -> SALARY_DATE PASS (83.5), but EMPLOYER actually FAILS (76.47, just under the 80 threshold): 'Nimbus Retail' is shorter than 'ABC Technologies', so the missing 'Pvt Ltd' tokens cost proportionally more of the token_set_ratio score. Intentionally left as-is rather than tuned to pass -- demonstrates the matcher is sensitive to company-name length, a real calibration edge worth knowing about, not a hidden bug.", + "MAY_slip": "reimbursement decoy in the same window as the real salary credit (4500 vs expected 78000) -> the real salary credit is correctly selected over the reimbursement (SALARY_DATE PASS, 83.5); EMPLOYER also fails here for the same short-company-name reason as April (76.47)", + "JUNE_slip": "genuinely missing bank credit -- no transaction anywhere near the window matches -- SALARY_DATE FAILS for this slip specifically (0.0, reason=no_matching_credit_in_window), EMPLOYER also fails as an unavoidable consequence (nothing to verify employer against). Contributes to a lowered SALARY_CREDIT_COUNT (3/4 = 75%) but does NOT hard-fail the whole case, per spec -- SALARY_CREDIT_COUNT only feeds the weighted score.", + "noise_transactions": "a rent debit, two small unrelated debits, and an unrelated company's credit (QUANTUM CONSULTING REFERRAL BONUS) sitting outside every slip's window -- all correctly ignored/never selected as a match", + "amount_tolerance_gate": "SALARY_AMOUNT_TOLERANCE_PCT (3%) in validation_config.py hard-excludes any transaction whose amount is not within 3% of the slip's declared net_salary from even being a match candidate, regardless of narration/employer score. This is why the May reimbursement (4500 vs expected 78000, ~94% off) can never be mistaken for the real salary credit -- and why a decoy transaction closer in amount (tested separately in edge_case_scenarios.py) is also correctly excluded rather than relying on narration keywords like 'REIMBURSEMENT', which don't generalize." + }, + "expected_overall_decision": "NEEDS_REVIEW at ~88.4 overall score -- driven by the June missing credit plus two borderline EMPLOYER checks, not a single dominant failure. Re-verify this note against actual output after any change to matching/scoring/decision code: run `python scripts/run_demo.py` and diff the printed Decision block against this note." + }, + "applicant_ref": "APP-2026-00134", + "documents": [ + { + "doc_type": "AADHAAR", + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "address": "Flat 204, Green Heights, Baner, Pune, Maharashtra 411045", + "aadhaar_number": "XXXX XXXX 4321", + "date_of_birth": "1995-03-14" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/aadhaar_front.pdf" + }, + { + "doc_type": "PAN", + "extracted_fields": { + "name": "Sneha Lokhande", + "pan_number": "ABCDE1234F" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/pan_card.pdf" + }, + { + "doc_type": "ADDRESS_PROOF", + "extracted_fields": { + "address": "Apartment 204, Green Heights, Baner, Pune, MH 411045" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/address_proof.pdf" + }, + { + "doc_type": "SALARY_SLIP", + "salary_slips": [ + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "ABC Technologies Pvt Ltd", + "net_salary": 75000, + "salary_month": "2026-03" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_march.pdf" + }, + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "Nimbus Retail Pvt Ltd", + "net_salary": 79000, + "salary_month": "2026-04" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_april.pdf" + }, + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "Nimbus Retail Pvt Ltd", + "net_salary": 78000, + "salary_month": "2026-05" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_may.pdf" + }, + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "Nimbus Retail Pvt Ltd", + "net_salary": 78000, + "salary_month": "2026-06" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_june.pdf" + } + ] + }, + { + "doc_type": "BANK_STATEMENT", + "extracted_fields": { + "name": "Lokhande S. S.", + "transactions": [ + { + "narration": "ABC TECHNOLOGIES SALARY MAR", + "amount": 75000, + "date": "2026-04-01" + }, + { + "narration": "HOUSE RENT EMI DEBIT", + "amount": -18000, + "date": "2026-04-03" + }, + { + "narration": "NIMBUS RETAIL SALARY APR", + "amount": 79000, + "date": "2026-05-02" + }, + { + "narration": "ZEPTO MART PAYMENT", + "amount": -620, + "date": "2026-05-08" + }, + { + "narration": "NIMBUS RETAIL SALARY MAY", + "amount": 78000, + "date": "2026-06-02" + }, + { + "narration": "NIMBUS RETAIL REIMBURSEMENT TRAVEL", + "amount": 4500, + "date": "2026-06-05" + }, + { + "narration": "SWIGGY ORDER PAYMENT", + "amount": -450, + "date": "2026-06-10" + }, + { + "narration": "QUANTUM CONSULTING REFERRAL BONUS", + "amount": 15000, + "date": "2026-07-04" + } + ] + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/bank_statement_mar_to_jul.pdf" + } + ] +} From 0e61f28b911fa921fed2dc9551f1739df5b64711 Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Thu, 13 Aug 2026 12:06:12 +0530 Subject: [PATCH 2/7] fix: github comments resolved --- lending-poc/app/api/cases.py | 3 ++- lending-poc/app/models/types.py | 19 +++++++++++++-- .../app/services/business_validation.py | 9 +++++++ lending-poc/app/services/persistence.py | 24 +++---------------- lending-poc/app/utils/json_safe.py | 19 +++++++++++++++ 5 files changed, 50 insertions(+), 24 deletions(-) create mode 100644 lending-poc/app/utils/json_safe.py diff --git a/lending-poc/app/api/cases.py b/lending-poc/app/api/cases.py index 3f96228..1572f99 100644 --- a/lending-poc/app/api/cases.py +++ b/lending-poc/app/api/cases.py @@ -6,6 +6,7 @@ from app.services.case_parsing import parse_case from app.services.persistence import save_pipeline_result from app.services.pipeline import run_pipeline +from app.utils.json_safe import json_safe router = APIRouter(tags=["cases"]) @@ -30,7 +31,7 @@ async def create_case( passed=r.passed, score=r.score, document_id=r.document_id, - evidence=r.evidence, + evidence=json_safe(r.evidence) if r.evidence else None, ) for r in pipeline_result.validation_results ], diff --git a/lending-poc/app/models/types.py b/lending-poc/app/models/types.py index 7dbb1ab..eb7c77e 100644 --- a/lending-poc/app/models/types.py +++ b/lending-poc/app/models/types.py @@ -4,6 +4,21 @@ from app.config import settings +_fernet: Fernet | None = None + + +def _get_fernet() -> Fernet: + global _fernet + if _fernet is None: + if not settings.ENCRYPTION_KEY: + raise ValueError( + "ENCRYPTION_KEY is not set. Configure a valid Fernet key " + "(see cryptography.fernet.Fernet.generate_key()) before " + "reading/writing encrypted columns." + ) + _fernet = Fernet(settings.ENCRYPTION_KEY) + return _fernet + class EncryptedString(TypeDecorator): """Stores strings encrypted at rest (Fernet/AES) via ENCRYPTION_KEY. @@ -18,9 +33,9 @@ class EncryptedString(TypeDecorator): def process_bind_param(self, value: str | None, dialect) -> str | None: if value is None: return None - return Fernet(settings.ENCRYPTION_KEY).encrypt(value.encode("utf-8")).decode("utf-8") + return _get_fernet().encrypt(value.encode("utf-8")).decode("utf-8") def process_result_value(self, value: str | None, dialect) -> str | None: if value is None: return None - return Fernet(settings.ENCRYPTION_KEY).decrypt(value.encode("utf-8")).decode("utf-8") + return _get_fernet().decrypt(value.encode("utf-8")).decode("utf-8") diff --git a/lending-poc/app/services/business_validation.py b/lending-poc/app/services/business_validation.py index be47f14..ed58b07 100644 --- a/lending-poc/app/services/business_validation.py +++ b/lending-poc/app/services/business_validation.py @@ -111,6 +111,15 @@ def _validate_salary_slip( failure_reason="missing_salary_month", ) + if slip.net_salary is None: + return ValidationResult( + check_type=CheckType.SALARY_DATE, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="missing_net_salary", + ) + window = _month_window(slip.salary_month) candidates = [ txn diff --git a/lending-poc/app/services/persistence.py b/lending-poc/app/services/persistence.py index 60be46d..477326b 100644 --- a/lending-poc/app/services/persistence.py +++ b/lending-poc/app/services/persistence.py @@ -7,8 +7,6 @@ real foreign key. """ -import dataclasses -import datetime import uuid from sqlalchemy.ext.asyncio import AsyncSession @@ -20,6 +18,7 @@ from app.models.validation_result import ValidationResult as ValidationResultModel from app.services.dto import CaseInput, Decision, DocType from app.services.dto import PipelineResult as PipelineResultDTO +from app.utils.json_safe import json_safe _DECISION_TO_CASE_STATUS = { Decision.PASS: CaseStatus.PASS, @@ -28,23 +27,6 @@ } -def _json_safe(value): - """Recursively converts dataclasses/dates/tuples in evidence dicts into - plain JSON-serializable values, since business_validation/scoring build - evidence out of BankTransaction dataclasses and date objects for - in-memory use. - """ - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return {k: _json_safe(v) for k, v in dataclasses.asdict(value).items()} - if isinstance(value, (datetime.date, datetime.datetime)): - return value.isoformat() - if isinstance(value, dict): - return {k: _json_safe(v) for k, v in value.items()} - if isinstance(value, (list, tuple)): - return [_json_safe(v) for v in value] - return value - - def _document_rows(case: CaseInput) -> list[tuple[str, Document]]: """Returns (doc_id, Document) pairs for every document present on the case.""" rows: list[tuple[str, Document]] = [] @@ -106,7 +88,7 @@ def _document_rows(case: CaseInput) -> list[tuple[str, Document]]: doc_type=DocType.BANK_STATEMENT, source_file_ref=case.bank_statement.source_file_ref, extracted_fields={ - "account_holder": case.bank_statement.name, + "name": case.bank_statement.name, "transactions": [ { "narration": txn.narration, @@ -166,7 +148,7 @@ async def save_pipeline_result( check_type=result.check_type, passed=result.passed, score=result.score, - evidence=_json_safe(result.evidence) if result.evidence else None, + evidence=json_safe(result.evidence) if result.evidence else None, ) ) diff --git a/lending-poc/app/utils/json_safe.py b/lending-poc/app/utils/json_safe.py new file mode 100644 index 0000000..06825a5 --- /dev/null +++ b/lending-poc/app/utils/json_safe.py @@ -0,0 +1,19 @@ +"""Recursively converts dataclasses/dates/tuples into plain +JSON-serializable values, since validation/scoring build evidence out of +dataclasses (e.g. BankTransaction) and date objects for in-memory use. +""" + +import dataclasses +import datetime + + +def json_safe(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return {k: json_safe(v) for k, v in dataclasses.asdict(value).items()} + if isinstance(value, (datetime.date, datetime.datetime)): + return value.isoformat() + if isinstance(value, dict): + return {k: json_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [json_safe(v) for v in value] + return value From c86035c8b84f3c504c97ad0b2855865bcf3ea8e7 Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Thu, 13 Aug 2026 12:34:14 +0530 Subject: [PATCH 3/7] fix: removed duplicate DB setup code --- ...f16_add_cases_documents_golden_records_.py | 106 ------------------ lending-poc/app/models/__init__.py | 15 --- lending-poc/app/models/case.py | 42 ------- lending-poc/app/models/document.py | 25 ----- lending-poc/app/models/golden_record.py | 32 ------ lending-poc/app/models/pipeline_result.py | 45 -------- lending-poc/app/models/types.py | 41 ------- lending-poc/app/models/validation_result.py | 29 ----- lending-poc/app/services/persistence.py | 10 +- 9 files changed, 5 insertions(+), 340 deletions(-) delete mode 100644 lending-poc/alembic/versions/12522c432f16_add_cases_documents_golden_records_.py delete mode 100644 lending-poc/app/models/__init__.py delete mode 100644 lending-poc/app/models/case.py delete mode 100644 lending-poc/app/models/document.py delete mode 100644 lending-poc/app/models/golden_record.py delete mode 100644 lending-poc/app/models/pipeline_result.py delete mode 100644 lending-poc/app/models/types.py delete mode 100644 lending-poc/app/models/validation_result.py diff --git a/lending-poc/alembic/versions/12522c432f16_add_cases_documents_golden_records_.py b/lending-poc/alembic/versions/12522c432f16_add_cases_documents_golden_records_.py deleted file mode 100644 index 99a99ed..0000000 --- a/lending-poc/alembic/versions/12522c432f16_add_cases_documents_golden_records_.py +++ /dev/null @@ -1,106 +0,0 @@ -"""add cases, documents, golden_records, validation_results, pipeline_results - -Revision ID: 12522c432f16 -Revises: -Create Date: 2026-08-10 11:51:33.283278 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql -import pgvector.sqlalchemy -import app.models.types - -# revision identifiers, used by Alembic. -revision: str = '12522c432f16' -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.execute('CREATE EXTENSION IF NOT EXISTS vector') - op.create_table('cases', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('applicant_ref', sa.String(), nullable=False), - sa.Column('status', sa.Enum('RECEIVED', 'RUNNING', 'PASS', 'FAIL', 'NEEDS_REVIEW', name='case_status'), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_cases_applicant_ref'), 'cases', ['applicant_ref'], unique=True) - op.create_table('documents', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('case_id', sa.UUID(), nullable=False), - sa.Column('doc_type', sa.Enum('AADHAAR', 'PAN', 'ADDRESS_PROOF', 'SALARY_SLIP', 'BANK_STATEMENT', name='doc_type'), nullable=False), - sa.Column('extracted_fields', postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column('source_file_ref', sa.String(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_documents_case_id'), 'documents', ['case_id'], unique=False) - op.create_table('golden_records', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('case_id', sa.UUID(), nullable=False), - sa.Column('name', sa.String(), nullable=True), - sa.Column('address', sa.String(), nullable=True), - sa.Column('address_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=384), nullable=True), - sa.Column('aadhaar_number', app.models.types.EncryptedString(), nullable=True), - sa.Column('pan_number', app.models.types.EncryptedString(), nullable=True), - sa.Column('date_of_birth', sa.Date(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('case_id') - ) - op.create_table('pipeline_results', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('case_id', sa.UUID(), nullable=False), - sa.Column('overall_score', sa.Float(), nullable=False), - sa.Column('decision', sa.Enum('PASS', 'FAIL', 'NEEDS_REVIEW', name='decision'), nullable=False), - sa.Column('reasons', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('reviewer', sa.String(), nullable=True), - sa.Column('review_status', sa.Enum('PENDING', 'APPROVED', 'REJECTED', name='review_status'), nullable=True), - sa.Column('reviewer_remarks', sa.Text(), nullable=True), - sa.Column('reviewed_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_pipeline_results_case_id'), 'pipeline_results', ['case_id'], unique=False) - op.create_table('validation_results', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('case_id', sa.UUID(), nullable=False), - sa.Column('document_id', sa.UUID(), nullable=True), - sa.Column('check_type', sa.Enum('NAME', 'ADDRESS', 'AADHAAR', 'PAN', 'DOB', 'EMPLOYER', 'SALARY_DATE', 'SALARY_CREDIT_COUNT', 'MANDATORY_PRESENCE', name='check_type'), nullable=False), - sa.Column('passed', sa.Boolean(), nullable=False), - sa.Column('score', sa.Float(), nullable=False), - sa.Column('evidence', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), - sa.ForeignKeyConstraint(['document_id'], ['documents.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_validation_results_case_id'), 'validation_results', ['case_id'], unique=False) - op.create_index(op.f('ix_validation_results_document_id'), 'validation_results', ['document_id'], unique=False) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_validation_results_document_id'), table_name='validation_results') - op.drop_index(op.f('ix_validation_results_case_id'), table_name='validation_results') - op.drop_table('validation_results') - op.drop_index(op.f('ix_pipeline_results_case_id'), table_name='pipeline_results') - op.drop_table('pipeline_results') - op.drop_table('golden_records') - op.drop_index(op.f('ix_documents_case_id'), table_name='documents') - op.drop_table('documents') - op.drop_index(op.f('ix_cases_applicant_ref'), table_name='cases') - op.drop_table('cases') - # ### end Alembic commands ### diff --git a/lending-poc/app/models/__init__.py b/lending-poc/app/models/__init__.py deleted file mode 100644 index dacd302..0000000 --- a/lending-poc/app/models/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.models.case import Case, CaseStatus -from app.models.document import Document -from app.models.golden_record import GoldenRecord -from app.models.pipeline_result import PipelineResult, ReviewStatus -from app.models.validation_result import ValidationResult - -__all__ = [ - "Case", - "CaseStatus", - "Document", - "GoldenRecord", - "PipelineResult", - "ReviewStatus", - "ValidationResult", -] diff --git a/lending-poc/app/models/case.py b/lending-poc/app/models/case.py deleted file mode 100644 index 609c1b6..0000000 --- a/lending-poc/app/models/case.py +++ /dev/null @@ -1,42 +0,0 @@ -import uuid -from datetime import datetime -from enum import Enum - -from sqlalchemy import DateTime, Enum as SAEnum, String, func -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from app.database import Base - - -class CaseStatus(str, Enum): - RECEIVED = "RECEIVED" - RUNNING = "RUNNING" - PASS = "PASS" - FAIL = "FAIL" - NEEDS_REVIEW = "NEEDS_REVIEW" - - -class Case(Base): - __tablename__ = "cases" - - id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - applicant_ref: Mapped[str] = mapped_column(String, unique=True, index=True, nullable=False) - status: Mapped[CaseStatus] = mapped_column( - SAEnum(CaseStatus, name="case_status"), nullable=False, default=CaseStatus.RECEIVED - ) - created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() - ) - - documents: Mapped[list["Document"]] = relationship(back_populates="case", cascade="all, delete-orphan") - golden_record: Mapped["GoldenRecord | None"] = relationship( - back_populates="case", cascade="all, delete-orphan", uselist=False - ) - validation_results: Mapped[list["ValidationResult"]] = relationship( - back_populates="case", cascade="all, delete-orphan" - ) - pipeline_results: Mapped[list["PipelineResult"]] = relationship( - back_populates="case", cascade="all, delete-orphan" - ) diff --git a/lending-poc/app/models/document.py b/lending-poc/app/models/document.py deleted file mode 100644 index 0c52c71..0000000 --- a/lending-poc/app/models/document.py +++ /dev/null @@ -1,25 +0,0 @@ -import uuid -from datetime import datetime - -from sqlalchemy import DateTime, Enum as SAEnum, ForeignKey, String, func -from sqlalchemy.dialects.postgresql import JSONB, UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from app.database import Base -from app.services.dto import DocType - - -class Document(Base): - __tablename__ = "documents" - - id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - case_id: Mapped[uuid.UUID] = mapped_column( - UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True - ) - doc_type: Mapped[DocType] = mapped_column(SAEnum(DocType, name="doc_type"), nullable=False) - extracted_fields: Mapped[dict] = mapped_column(JSONB, nullable=False) - source_file_ref: Mapped[str] = mapped_column(String, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) - - case: Mapped["Case"] = relationship(back_populates="documents") - validation_results: Mapped[list["ValidationResult"]] = relationship(back_populates="document") diff --git a/lending-poc/app/models/golden_record.py b/lending-poc/app/models/golden_record.py deleted file mode 100644 index d7f4058..0000000 --- a/lending-poc/app/models/golden_record.py +++ /dev/null @@ -1,32 +0,0 @@ -import uuid -from datetime import date, datetime - -from pgvector.sqlalchemy import Vector -from sqlalchemy import Date, DateTime, ForeignKey, String, func -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from app.database import Base -from app.matching.embeddings import EMBEDDING_DIMENSIONS -from app.models.types import EncryptedString - - -class GoldenRecord(Base): - __tablename__ = "golden_records" - - id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - case_id: Mapped[uuid.UUID] = mapped_column( - UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), unique=True, nullable=False - ) - name: Mapped[str | None] = mapped_column(String, nullable=True) - address: Mapped[str | None] = mapped_column(String, nullable=True) - address_embedding: Mapped[list[float] | None] = mapped_column(Vector(EMBEDDING_DIMENSIONS), nullable=True) - aadhaar_number: Mapped[str | None] = mapped_column(EncryptedString, nullable=True) - pan_number: Mapped[str | None] = mapped_column(EncryptedString, nullable=True) - date_of_birth: Mapped[date | None] = mapped_column(Date, nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() - ) - - case: Mapped["Case"] = relationship(back_populates="golden_record") diff --git a/lending-poc/app/models/pipeline_result.py b/lending-poc/app/models/pipeline_result.py deleted file mode 100644 index 9c722e7..0000000 --- a/lending-poc/app/models/pipeline_result.py +++ /dev/null @@ -1,45 +0,0 @@ -"""DB model for a pipeline run. - -Named `PipelineResult` to match the ERD/table name. This collides with the -in-memory `app.services.dto.PipelineResult` dataclass — import one or both -qualified (`from app.models import pipeline_result as pipeline_result_model`) -in any module that needs both. -""" - -import uuid -from datetime import datetime -from enum import Enum - -from sqlalchemy import DateTime, Enum as SAEnum, Float, ForeignKey, String, Text, func -from sqlalchemy.dialects.postgresql import JSONB, UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from app.database import Base -from app.services.dto import Decision - - -class ReviewStatus(str, Enum): - PENDING = "PENDING" - APPROVED = "APPROVED" - REJECTED = "REJECTED" - - -class PipelineResult(Base): - __tablename__ = "pipeline_results" - - id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - case_id: Mapped[uuid.UUID] = mapped_column( - UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True - ) - overall_score: Mapped[float] = mapped_column(Float, nullable=False) - decision: Mapped[Decision] = mapped_column(SAEnum(Decision, name="decision"), nullable=False) - reasons: Mapped[list | None] = mapped_column(JSONB, nullable=True) - reviewer: Mapped[str | None] = mapped_column(String, nullable=True) - review_status: Mapped[ReviewStatus | None] = mapped_column( - SAEnum(ReviewStatus, name="review_status"), nullable=True - ) - reviewer_remarks: Mapped[str | None] = mapped_column(Text, nullable=True) - reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) - - case: Mapped["Case"] = relationship(back_populates="pipeline_results") diff --git a/lending-poc/app/models/types.py b/lending-poc/app/models/types.py deleted file mode 100644 index eb7c77e..0000000 --- a/lending-poc/app/models/types.py +++ /dev/null @@ -1,41 +0,0 @@ -from cryptography.fernet import Fernet -from sqlalchemy import String -from sqlalchemy.types import TypeDecorator - -from app.config import settings - -_fernet: Fernet | None = None - - -def _get_fernet() -> Fernet: - global _fernet - if _fernet is None: - if not settings.ENCRYPTION_KEY: - raise ValueError( - "ENCRYPTION_KEY is not set. Configure a valid Fernet key " - "(see cryptography.fernet.Fernet.generate_key()) before " - "reading/writing encrypted columns." - ) - _fernet = Fernet(settings.ENCRYPTION_KEY) - return _fernet - - -class EncryptedString(TypeDecorator): - """Stores strings encrypted at rest (Fernet/AES) via ENCRYPTION_KEY. - - Transparent to callers: reads/writes plain str in Python, ciphertext - in the DB column. - """ - - impl = String - cache_ok = True - - def process_bind_param(self, value: str | None, dialect) -> str | None: - if value is None: - return None - return _get_fernet().encrypt(value.encode("utf-8")).decode("utf-8") - - def process_result_value(self, value: str | None, dialect) -> str | None: - if value is None: - return None - return _get_fernet().decrypt(value.encode("utf-8")).decode("utf-8") diff --git a/lending-poc/app/models/validation_result.py b/lending-poc/app/models/validation_result.py deleted file mode 100644 index 9abda31..0000000 --- a/lending-poc/app/models/validation_result.py +++ /dev/null @@ -1,29 +0,0 @@ -import uuid -from datetime import datetime - -from sqlalchemy import Boolean, DateTime, Enum as SAEnum, Float, ForeignKey, func -from sqlalchemy.dialects.postgresql import JSONB, UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from app.database import Base -from app.services.dto import CheckType - - -class ValidationResult(Base): - __tablename__ = "validation_results" - - id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - case_id: Mapped[uuid.UUID] = mapped_column( - UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True - ) - document_id: Mapped[uuid.UUID | None] = mapped_column( - UUID(as_uuid=True), ForeignKey("documents.id", ondelete="CASCADE"), nullable=True, index=True - ) - check_type: Mapped[CheckType] = mapped_column(SAEnum(CheckType, name="check_type"), nullable=False) - passed: Mapped[bool] = mapped_column(Boolean, nullable=False) - score: Mapped[float] = mapped_column(Float, nullable=False) - evidence: Mapped[dict | None] = mapped_column(JSONB, nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) - - case: Mapped["Case"] = relationship(back_populates="validation_results") - document: Mapped["Document | None"] = relationship(back_populates="validation_results") diff --git a/lending-poc/app/services/persistence.py b/lending-poc/app/services/persistence.py index 477326b..14c412e 100644 --- a/lending-poc/app/services/persistence.py +++ b/lending-poc/app/services/persistence.py @@ -11,14 +11,14 @@ from sqlalchemy.ext.asyncio import AsyncSession -from app.models.case import Case, CaseStatus -from app.models.document import Document -from app.models.golden_record import GoldenRecord as GoldenRecordModel -from app.models.pipeline_result import PipelineResult as PipelineResultModel -from app.models.validation_result import ValidationResult as ValidationResultModel from app.services.dto import CaseInput, Decision, DocType from app.services.dto import PipelineResult as PipelineResultDTO from app.utils.json_safe import json_safe +from db.models.case import Case, CaseStatus +from db.models.document import Document +from db.models.golden_record import GoldenRecord as GoldenRecordModel +from db.models.pipeline_result import PipelineResult as PipelineResultModel +from db.models.validation_result import ValidationResult as ValidationResultModel _DECISION_TO_CASE_STATUS = { Decision.PASS: CaseStatus.PASS, From 50ab5e1855735e989236f884443c3378362d8626 Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Thu, 13 Aug 2026 15:09:14 +0530 Subject: [PATCH 4/7] fix: solved github comments --- lending-poc/app/api/cases.py | 7 +- lending-poc/app/config.py | 2 +- lending-poc/app/schemas/case.py | 9 +- lending-poc/app/services/case_parsing.py | 18 +- lending-poc/app/services/dto.py | 10 +- lending-poc/app/services/persistence.py | 4 +- lending-poc/db/config.py | 2 +- ..._make_document_source_file_ref_nullable.py | 25 ++ lending-poc/db/models/document.py | 2 +- lending-poc/docs/Workflow.md | 294 ------------------ lending-poc/scripts/edge_case_scenarios.py | 149 --------- lending-poc/scripts/run_demo.py | 71 ----- 12 files changed, 57 insertions(+), 536 deletions(-) create mode 100644 lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py delete mode 100644 lending-poc/docs/Workflow.md delete mode 100644 lending-poc/scripts/edge_case_scenarios.py delete mode 100644 lending-poc/scripts/run_demo.py diff --git a/lending-poc/app/api/cases.py b/lending-poc/app/api/cases.py index 1572f99..4d32522 100644 --- a/lending-poc/app/api/cases.py +++ b/lending-poc/app/api/cases.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from sqlalchemy.ext.asyncio import AsyncSession from app.database import get_db @@ -15,7 +15,10 @@ async def create_case( request: CaseCreateRequest, db: AsyncSession = Depends(get_db) ) -> CaseCreateResponse: - case_input = parse_case(request.model_dump()) + try: + case_input = parse_case(request.model_dump()) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc pipeline_result = run_pipeline(case_input) case = await save_pipeline_result(db, case_input, pipeline_result) diff --git a/lending-poc/app/config.py b/lending-poc/app/config.py index 777bbb0..90c1779 100644 --- a/lending-poc/app/config.py +++ b/lending-poc/app/config.py @@ -11,7 +11,7 @@ class Settings(BaseSettings): DEBUG: bool = False DATABASE_URL: str LOG_LEVEL: str = "INFO" - ENCRYPTION_KEY: str = "" + ENCRYPTION_KEY: str settings = Settings() diff --git a/lending-poc/app/schemas/case.py b/lending-poc/app/schemas/case.py index 2b3d03e..c56e86c 100644 --- a/lending-poc/app/schemas/case.py +++ b/lending-poc/app/schemas/case.py @@ -9,11 +9,16 @@ from pydantic import BaseModel +class SalarySlipIn(BaseModel): + extracted_fields: dict[str, Any] + source_file_ref: str | None = None + + class DocumentIn(BaseModel): doc_type: str - extracted_fields: dict[str, Any] | None = None + extracted_fields: dict[str, Any] source_file_ref: str | None = None - salary_slips: list[dict[str, Any]] | None = None # only present when doc_type == SALARY_SLIP + salary_slips: list[SalarySlipIn] | None = None # only present when doc_type == SALARY_SLIP class CaseCreateRequest(BaseModel): diff --git a/lending-poc/app/services/case_parsing.py b/lending-poc/app/services/case_parsing.py index dd8db9d..b8a0c33 100644 --- a/lending-poc/app/services/case_parsing.py +++ b/lending-poc/app/services/case_parsing.py @@ -1,7 +1,6 @@ """Parses the raw request JSON shape (see docs/Workflow.md) into CaseInput. -Shared by the POST /cases endpoint and scripts/run_demo.py so both use -identical parsing rules. +Used by the POST /cases endpoint. """ from datetime import date, datetime @@ -59,7 +58,7 @@ def parse_case(payload: dict) -> CaseInput: address=_get(fields, "address"), aadhaar_number=_get(fields, "aadhaar_number"), date_of_birth=_parse_date(_get(fields, "date_of_birth")), - source_file_ref=doc["source_file_ref"], + source_file_ref=doc.get("source_file_ref"), ) elif doc_type == "PAN": @@ -67,25 +66,28 @@ def parse_case(payload: dict) -> CaseInput: case.pan = PanDoc( name=_get(fields, "name"), pan_number=_get(fields, "pan_number"), - source_file_ref=doc["source_file_ref"], + source_file_ref=doc.get("source_file_ref"), ) elif doc_type == "ADDRESS_PROOF": fields = doc["extracted_fields"] case.address_proof = AddressProofDoc( address=_get(fields, "address"), - source_file_ref=doc["source_file_ref"], + source_file_ref=doc.get("source_file_ref"), ) elif doc_type == "SALARY_SLIP": - for i, slip in enumerate(doc["salary_slips"]): + slips = doc.get("salary_slips") or [] + if not slips: + raise ValueError("SALARY_SLIP document must include at least one entry in salary_slips") + for i, slip in enumerate(slips): fields = slip["extracted_fields"] case.salary_slips.append( SalarySlipDoc( employer_name=_get(fields, "employer_name"), net_salary=_parse_float(_get(fields, "net_salary")), salary_month=_parse_month(_get(fields, "salary_month")), - source_file_ref=slip["source_file_ref"], + source_file_ref=slip.get("source_file_ref"), doc_id=f"SALARY_SLIP-{i}", name=_get(fields, "name"), ) @@ -103,7 +105,7 @@ def parse_case(payload: dict) -> CaseInput: ] case.bank_statement = BankStatementDoc( transactions=transactions, - source_file_ref=doc["source_file_ref"], + source_file_ref=doc.get("source_file_ref"), name=_get(fields, "name"), ) diff --git a/lending-poc/app/services/dto.py b/lending-poc/app/services/dto.py index acc1d48..cfd8374 100644 --- a/lending-poc/app/services/dto.py +++ b/lending-poc/app/services/dto.py @@ -40,37 +40,37 @@ class Decision(str, Enum): @dataclass class AadhaarDoc: - source_file_ref: str name: str | None = None address: str | None = None aadhaar_number: str | None = None date_of_birth: date | None = None + source_file_ref: str | None = None doc_id: str = "AADHAAR" @dataclass class PanDoc: - source_file_ref: str name: str | None = None pan_number: str | None = None + source_file_ref: str | None = None doc_id: str = "PAN" @dataclass class AddressProofDoc: - source_file_ref: str address: str | None = None + source_file_ref: str | None = None doc_id: str = "ADDRESS_PROOF" @dataclass class SalarySlipDoc: - source_file_ref: str doc_id: str employer_name: str | None = None net_salary: float | None = None salary_month: date | None = None # first-of-month name: str | None = None + source_file_ref: str | None = None @dataclass @@ -83,9 +83,9 @@ class BankTransaction: @dataclass class BankStatementDoc: transactions: list[BankTransaction] - source_file_ref: str doc_id: str = "BANK_STATEMENT" name: str | None = None + source_file_ref: str | None = None @dataclass diff --git a/lending-poc/app/services/persistence.py b/lending-poc/app/services/persistence.py index 14c412e..4f5890a 100644 --- a/lending-poc/app/services/persistence.py +++ b/lending-poc/app/services/persistence.py @@ -73,8 +73,8 @@ def _document_rows(case: CaseInput) -> list[tuple[str, Document]]: doc_type=DocType.SALARY_SLIP, source_file_ref=slip.source_file_ref, extracted_fields={ - "employee_name": slip.name, - "employer": slip.employer_name, + "name": slip.name, + "employer_name": slip.employer_name, "net_salary": slip.net_salary, "salary_month": slip.salary_month.isoformat() if slip.salary_month else None, }, diff --git a/lending-poc/db/config.py b/lending-poc/db/config.py index 7109786..20e4dba 100644 --- a/lending-poc/db/config.py +++ b/lending-poc/db/config.py @@ -6,7 +6,7 @@ class Settings(BaseSettings): DEBUG: bool = False DATABASE_URL: str - ENCRYPTION_KEY: str = "" + ENCRYPTION_KEY: str settings = Settings() diff --git a/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py b/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py new file mode 100644 index 0000000..3189c29 --- /dev/null +++ b/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py @@ -0,0 +1,25 @@ +"""make document source_file_ref nullable + +Revision ID: 0006_doc_source_ref_nullable +Revises: 0005_add_validation_results +Create Date: 2026-08-13 00:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "0006_doc_source_ref_nullable" +down_revision: Union[str, None] = "0005_add_validation_results" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("documents", "source_file_ref", existing_type=sa.String(), nullable=True) + + +def downgrade() -> None: + op.alter_column("documents", "source_file_ref", existing_type=sa.String(), nullable=False) diff --git a/lending-poc/db/models/document.py b/lending-poc/db/models/document.py index 2a4069e..de7f35b 100644 --- a/lending-poc/db/models/document.py +++ b/lending-poc/db/models/document.py @@ -18,7 +18,7 @@ class Document(Base): ) doc_type: Mapped[DocType] = mapped_column(SAEnum(DocType, name="doc_type"), nullable=False) extracted_fields: Mapped[dict] = mapped_column(JSONB, nullable=False) - source_file_ref: Mapped[str] = mapped_column(String, nullable=False) + source_file_ref: Mapped[str | None] = mapped_column(String, nullable=True) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) case: Mapped["Case"] = relationship(back_populates="documents") diff --git a/lending-poc/docs/Workflow.md b/lending-poc/docs/Workflow.md deleted file mode 100644 index 43cb8e0..0000000 --- a/lending-poc/docs/Workflow.md +++ /dev/null @@ -1,294 +0,0 @@ -# Document Validation & Decision Engine — How It Works - -## In one sentence - -You give it a loan applicant's documents (already extracted into structured -fields by an OCR/extraction pipeline elsewhere) — Aadhaar, PAN, address -proof, salary slips, bank statement — and it gives back one verdict: **PASS, -FAIL, or NEEDS_REVIEW**, plus a full breakdown of exactly which checks -passed, which failed, and why. - -It does not read PDFs or images itself. It assumes that's already done and -works only with the structured JSON that comes out of that step. - ---- - -## Input - -One JSON object per applicant ("case"), shaped like this: - -```json -{ - "applicant_ref": "APP-2026-00123", - "documents": [ - { - "doc_type": "AADHAAR", - "extracted_fields": { - "name": "Sneha Sunil Lokhande", - "address": "Flat 204, Green Heights, Baner, Pune, Maharashtra 411045", - "aadhaar_number": "XXXX XXXX 4321", - "date_of_birth": "1995-03-14" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00123/aadhaar_front.pdf" - }, - { - "doc_type": "PAN", - "extracted_fields": { - "name": "Sneha Lokhande", - "pan_number": "ABCDE1234F" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00123/pan_card.pdf" - }, - { - "doc_type": "ADDRESS_PROOF", - "extracted_fields": { - "address": "Apartment 204, Green Heights, Baner, Pune, MH 411045" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00123/address_proof.pdf" - }, - { - "doc_type": "SALARY_SLIP", - "salary_slips": [ - { - "extracted_fields": { - "employer_name": "ABC Technologies Pvt Ltd", - "net_salary": 75000, - "salary_month": "2026-05" - }, - "source_file_ref": "s3://kyc-docs/.../salary_slip_may.pdf" - } - ] - }, - { - "doc_type": "BANK_STATEMENT", - "extracted_fields": { - "transactions": [ - { "narration": "ABC TECHNOLOGIES SALARY MAY", "amount": 75000, "date": "2026-06-02" }, - { "narration": "SWIGGY ORDER PAYMENT", "amount": -450, "date": "2026-06-10" } - ] - }, - "source_file_ref": "s3://kyc-docs/.../bank_statement.pdf" - } - ] -} -``` - -A few things worth knowing about this shape: - -- **Any field can be missing or blank.** Real extraction output is messy — - an address might not be captured, a DOB might be unreadable. The engine - treats a missing key, an empty string, and an explicit `null` all the - same way: as "no value," never as a crash. -- **`SALARY_SLIP` is the one document type that can repeat** — an applicant - usually has several months of salary slips, each with its own - `net_salary`, `employer_name`, and `salary_month`. -- **`source_file_ref`** is just kept for traceability (which physical file - a fact came from) — it doesn't affect any decision. - ---- - -## Output - -One decision object, plus a full audit trail of every intermediate -check. In shape: - -``` -Decision: PASS | FAIL | NEEDS_REVIEW -Overall score: 0-100 -Reasons: [ "SALARY_DATE:no_matching_credit_in_window", ... ] - -Golden Record: the one trusted identity profile built from all documents - (name, address, DOB, Aadhaar, PAN — each tagged with which - document it came from) - -Validation results: one row per individual check, e.g. - [PASS] NAME score=100.00 doc=AADHAAR - [FAIL] SALARY_DATE score=0.00 doc=SALARY_SLIP-3 reason=no_matching_credit_in_window - -Score breakdown: the weighted average per check type - NAME 100, ADDRESS 88, AADHAAR 100, ... SALARY_CREDIT_COUNT 75 -``` - -Nothing here is a black box — every number in the final score traces back -to one specific, inspectable check on one specific document. - ---- - -## How it works — the seven stages - -``` - Documents in - │ - ▼ - ① Golden Record Build ONE trusted identity from all ID documents - │ - ▼ - ② Identity Validation Does every document agree with the Golden Record? - │ - ▼ - ③ Business Validation Do salary slips have matching bank credits? - │ - ▼ - ④ Scoring Combine every check into one weighted number - │ - ▼ - ⑤ Decision PASS / FAIL / NEEDS_REVIEW - │ - ▼ - ⑥ Review (if needed) A human resolves NEEDS_REVIEW cases - │ - ▼ - ⑦ Audit Every step is permanently logged -``` - -### ① Golden Record — "who is this person, really?" - -Aadhaar and PAN often disagree slightly on name (e.g. Aadhaar has a full -name, PAN has an abbreviated one). Rather than blindly trusting one -document, the engine builds a single reconciled identity: - -- **Name**: if Aadhaar and PAN both have a name and they're clearly the - same person, the **fuller** one wins (more words = more information) — - e.g. "Sneha Sunil Lokhande" beats "Sneha Lokhande". It's then split into - first / middle / last name. If the two names don't look related at all, - Aadhaar stays the trusted source and the mismatch gets flagged later. -- **Address, DOB, Aadhaar number**: taken from Aadhaar; address falls back - to the address-proof document if Aadhaar didn't have one. -- **PAN number**: taken from the PAN document. -- Every field remembers **which document it came from**, for traceability. - -**Mandatory rule**: if name, Aadhaar number, PAN number, or DOB is missing -from *every* document that could supply it, the case is an automatic FAIL — -these are non-negotiable identity anchors. - -### ② Identity Validation — "does every document agree?" - -Every document is compared against the Golden Record: - -| Check | How it's compared | -|---|---| -| **Name** | Fuzzy text match, tolerant of reordering ("Lokhande Sneha" = "Sneha Lokhande") and abbreviation ("Sneha S.L." = "Sneha Sunil Lokhande") | -| **Address** | AI embedding similarity — tolerant of different wording ("Flat 204" vs "Apartment 204") since it compares *meaning*, not exact text | -| **Aadhaar number** | Exact match, with a fallback for masked numbers (e.g. "XXXX XXXX 4321" matches a full number ending in 4321) | -| **PAN number** | Exact match | -| **DOB** | Exact match | - -Each of these runs on *every* document that carries that field — Aadhaar, -PAN, address proof, even the salary slips and bank statement if they -happen to carry a name too. - -### ③ Business Validation — "is the declared income real?" - -This is the more involved stage, because real payroll data is messy: every -company pays on a different day of the month, so the engine never assumes -a fixed payday. - -For each salary slip: - -1. **Build a window** around that slip's month — a few days before the - month starts, through the end of the *following* month. This is wide - enough to catch "pays on the 1st" and "pays on the 5th of next month" - equally, without any company-specific rules. -2. **Find candidate bank transactions** inside that window that are: - - credits (positive amount), and - - within **3% of the declared net salary** — this is a hard cutoff. A - transaction outside this range is never eligible, no matter how good - its description looks. (This exists specifically to stop a - same-employer reimbursement or bonus with a coincidentally close - amount from being mistaken for the real salary payment.) -3. **Score each remaining candidate** — 70% on how well the transaction's - description matches the employer name, 30% on how close the amount is — - and pick the best one. -4. **A transaction can only be used once.** If two slips' windows overlap - (common, since windows span two months), an earlier slip claims its - match first so the same bank credit can't "prove" two different months - of income. -5. If no transaction survives, that slip's `SALARY_DATE` check fails — - this is a real, visible finding, not silently skipped. - -**Employer check** — run separately, *per slip, per month* — verifies that -month's declared employer against *that month's own* matched bank credit -only. It deliberately never compares one month's employer to another's, -because switching jobs mid-history is normal, not suspicious. - -**SALARY_CREDIT_COUNT** — one case-level summary: "N salary slips were -declared, how many actually got a verified bank credit?" If an applicant -declares 4 months but only 3 have proof, this shows `3/4 = 75%`. Note that -this **does not by itself fail the case** — it only lowers the overall -score. Missing one month's proof out of several is common and shouldn't -be treated the same as missing proof entirely. - -### ④ Scoring — one number from many checks - -Every check above produces a score from 0–100. These are grouped by check -type, averaged, and combined into one overall score using fixed weights: - -| Check | Weight | -|---|---| -| SALARY_CREDIT_COUNT | 25% (highest — this is the core "is the income real" signal) | -| NAME | 15% | -| AADHAAR | 15% | -| PAN | 15% | -| ADDRESS | 10% | -| DOB | 10% | -| EMPLOYER | 10% | - -### ⑤ Decision — turning the score into a verdict - -1. If name, Aadhaar, PAN, or DOB is missing entirely → **FAIL**, regardless - of score. -2. If overall score ≥ 90 → **PASS**. -3. If overall score < 60 → **FAIL**. -4. Otherwise → **NEEDS_REVIEW** — the case isn't clean enough to auto-pass, - but isn't bad enough to auto-reject either. A human should look at it. - -### ⑥ Review - -Cases marked NEEDS_REVIEW wait in a queue for a human reviewer, who sees -exactly which checks failed and why, and makes the final call. - -### ⑦ Audit - -Every stage — ingest, golden record built, each validation run, the final -score, the decision — is logged permanently. Nothing happens invisibly; -you can always answer "why did this case get this verdict?" after the -fact. - ---- - -## Worked example - -A real (deliberately imperfect) case run through the engine: - -- Aadhaar: "Sneha Sunil Lokhande" — PAN: "Sneha Lokhande" (shorter) → - Golden Record keeps Aadhaar's fuller name. **PASS.** -- Address worded differently across documents → matched by meaning, not - exact text. **PASS.** -- 4 salary months declared: March (clean), April (employer switched from - ABC to Nimbus, both fully verified), May (a decoy reimbursement in the - same window as the real salary — correctly ignored), June (**no matching - bank credit exists at all**). -- Result: June's `SALARY_DATE` check **fails** honestly. - `SALARY_CREDIT_COUNT` drops to 75% (3 of 4 months proven). Two months' - `EMPLOYER` checks land just under threshold due to how the company names - compare textually. -- None of this is a single catastrophic failure — it's a mix of solid and - shaky signals. **Final verdict: NEEDS_REVIEW at 88.4/100** — sent to a - human rather than auto-approved or auto-rejected. - -This is the intended behavior: the engine doesn't try to force every case -into a clean PASS or FAIL. Genuinely ambiguous evidence should produce an -ambiguous verdict. - ---- - -## Try it yourself - -```bash -cd lending-poc -python3 scripts/run_demo.py -``` - -This runs `scripts/sample_case.json` (the example above, in full) through -every stage and prints the Golden Record, every individual check, the -score breakdown, and the final decision. diff --git a/lending-poc/scripts/edge_case_scenarios.py b/lending-poc/scripts/edge_case_scenarios.py deleted file mode 100644 index a02f0dc..0000000 --- a/lending-poc/scripts/edge_case_scenarios.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Runs a battery of edge-case scenarios through the validation pipeline, -starting from the base sample_case.json and mutating it per scenario, to -show exactly what today's code does and does not handle correctly. - -Usage: - python scripts/edge_case_scenarios.py -""" - -import copy -import json -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from scripts.run_demo import parse_case # noqa: E402 -from app.services.pipeline import run_pipeline # noqa: E402 - - -def load_base() -> dict: - path = Path(__file__).resolve().parent / "sample_case.json" - return json.loads(path.read_text()) - - -def summarize(label: str, payload: dict) -> None: - case = parse_case(payload) - result = run_pipeline(case) - - print("=" * 78) - print(label) - print("=" * 78) - print(f" decision = {result.decision_result.decision.value} " - f"score = {result.decision_result.overall_score:.2f}") - print(f" reasons = {result.decision_result.reasons}") - - for r in result.validation_results: - if r.check_type.value in ("SALARY_DATE", "EMPLOYER", "SALARY_CREDIT_COUNT"): - status = "PASS" if r.passed else "FAIL" - doc = f" doc={r.document_id}" if r.document_id else "" - print(f" [{status}] {r.check_type.value:<20} score={r.score:6.2f}{doc}") - if r.evidence and "matched_transaction" in r.evidence: - txn = r.evidence["matched_transaction"] - print(f" -> matched: {txn.narration!r} amount={txn.amount} date={txn.txn_date}") - elif r.evidence and r.check_type.value == "SALARY_CREDIT_COUNT": - print(f" evidence: {r.evidence}") - print() - - -def scenario_reimbursement_ambiguity() -> None: - """A reimbursement credit sits inside the same month-window as the real - salary credit, from the same employer. Does the 70/30 scoring correctly - prefer the salary-sized credit over the reimbursement? - """ - payload = load_base() - # Base data already has this: May slip window contains both the - # "SALARY MAY" (75000) and "REIMBURSEMENT" (4500) transactions. - summarize("1. Reimbursement in same window as real salary credit (baseline)", payload) - - # Harder version: reimbursement amount is much closer to net_salary, - # to see if amount-closeness alone could fool the selection. - payload2 = load_base() - for doc in payload2["documents"]: - if doc["doc_type"] == "BANK_STATEMENT": - for txn in doc["extracted_fields"]["transactions"]: - if "REIMBURSEMENT" in txn["narration"]: - txn["amount"] = 74000 # suspiciously close to net_salary (75000) - summarize( - "1b. Reimbursement amount very close to net_salary (adversarial)", payload2 - ) - - -def scenario_partial_slip_match() -> None: - """3 salary slips declared, bank statement only backs 2 of them. - Per spec (README 'Open decisions'), this must NOT hard-fail the case — - only lower SALARY_CREDIT_COUNT's score. Confirm that's actually true. - """ - payload = load_base() - for doc in payload["documents"]: - if doc["doc_type"] == "SALARY_SLIP": - doc["salary_slips"].append( - { - "extracted_fields": { - "employer_name": "ABC Technologies Pvt Ltd", - "net_salary": 75000, - "salary_month": "2026-07", - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_july.pdf", - } - ) - # No corresponding July bank credit exists in the base data. - summarize("2. 3 slips declared, only 2 have matching bank credits", payload) - - -def scenario_salary_amount_changed() -> None: - """Net salary changes month-to-month (raise or deduction). Each slip's - amount-closeness should be judged against its OWN declared net_salary, - not a fixed/previous value. - """ - payload = load_base() - for doc in payload["documents"]: - if doc["doc_type"] == "SALARY_SLIP": - for slip in doc["salary_slips"]: - if slip["extracted_fields"]["salary_month"] == "2026-06": - slip["extracted_fields"]["net_salary"] = 82000 # a raise - if doc["doc_type"] == "BANK_STATEMENT": - for txn in doc["extracted_fields"]["transactions"]: - if "JUN" in txn["narration"]: - txn["amount"] = 82000 # bank credit reflects the raise - summarize("3. Salary raised in June (75000 -> 82000), bank credit matches", payload) - - # Now the adversarial version: slip says raise, but bank credit still - # shows the OLD amount -- should this still confidently match? - payload2 = load_base() - for doc in payload2["documents"]: - if doc["doc_type"] == "SALARY_SLIP": - for slip in doc["salary_slips"]: - if slip["extracted_fields"]["salary_month"] == "2026-06": - slip["extracted_fields"]["net_salary"] = 82000 - # bank statement untouched: still shows 75000 for June - summarize( - "3b. Slip claims raise to 82000 but bank credit still shows 75000", payload2 - ) - - -def scenario_employer_switch() -> None: - """Applicant switched jobs: earlier slips are Employer A, most recent - slip(s) are Employer B, with matching bank credits from each. Does - EMPLOYER consistency correctly distinguish "job switch" from "identity - fraud", or does it just uniformly fail? - """ - payload = load_base() - for doc in payload["documents"]: - if doc["doc_type"] == "SALARY_SLIP": - # May slip: old employer. June slip: new employer (switched). - for slip in doc["salary_slips"]: - if slip["extracted_fields"]["salary_month"] == "2026-06": - slip["extracted_fields"]["employer_name"] = "NextGen Solutions Pvt Ltd" - if doc["doc_type"] == "BANK_STATEMENT": - for txn in doc["extracted_fields"]["transactions"]: - if "JUN" in txn["narration"]: - txn["narration"] = "NEXTGEN SOLUTIONS SALARY JUN" - summarize("4. Employer switched between May (ABC) and June (NextGen)", payload) - - -if __name__ == "__main__": - scenario_reimbursement_ambiguity() - scenario_partial_slip_match() - scenario_salary_amount_changed() - scenario_employer_switch() diff --git a/lending-poc/scripts/run_demo.py b/lending-poc/scripts/run_demo.py deleted file mode 100644 index ec7500e..0000000 --- a/lending-poc/scripts/run_demo.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Runs one dummy case through the validation pipeline end-to-end, with no -database involved. Parses scripts/sample_case.json (the README's exact -sample payload) into in-memory DTOs, runs the pipeline, and prints every -intermediate result plus the final decision. - -Usage: - python scripts/run_demo.py -""" - -import json -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from app.services.case_parsing import parse_case # noqa: E402 -from app.services.pipeline import run_pipeline # noqa: E402 - - -def main() -> None: - sample_path = Path(__file__).resolve().parent / "sample_case.json" - payload = json.loads(sample_path.read_text()) - case = parse_case(payload) - - result = run_pipeline(case) - - print("=" * 70) - print(f"Applicant: {case.applicant_ref}") - print("=" * 70) - - print("\n--- Audit log ---") - for line in result.audit_log: - print(f" {line}") - - if result.golden_record: - g = result.golden_record - print("\n--- Golden Record ---") - print(f" name = {g.name!r} (source: {g.name_source})") - print(f" first_name = {g.first_name!r}") - print(f" middle_name = {g.middle_name!r}") - print(f" last_name = {g.last_name!r}") - print(f" address = {g.address!r} (source: {g.address_source})") - print(f" date_of_birth = {g.date_of_birth} (source: {g.dob_source})") - print(f" aadhaar_number = {g.aadhaar_number!r} (source: {g.aadhaar_source})") - print(f" pan_number = {g.pan_number!r} (source: {g.pan_source})") - - print("\n--- Validation results ---") - for r in result.validation_results: - status = "PASS" if r.passed else "FAIL" - doc = f" doc={r.document_id}" if r.document_id else "" - reason = f" reason={r.failure_reason}" if r.failure_reason else "" - print(f" [{status}] {r.check_type.value:<22} score={r.score:6.2f}{doc}{reason}") - if r.evidence: - print(f" evidence={r.evidence}") - - if result.score_result: - print("\n--- Score ---") - print(f" overall_score = {result.score_result.overall_score:.2f}") - print(" component_scores:") - for check_type, score in result.score_result.component_scores.items(): - print(f" {check_type:<22} {score:.2f}") - - print("\n--- Decision ---") - print(f" decision = {result.decision_result.decision.value}") - print(f" overall_score = {result.decision_result.overall_score:.2f}") - print(f" reasons = {result.decision_result.reasons}") - print("=" * 70) - - -if __name__ == "__main__": - main() From 2505a6122d0ac8752f31d1744b365b019ae1723f Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Thu, 13 Aug 2026 17:09:34 +0530 Subject: [PATCH 5/7] docs: added 2 docs for API and implementation doc --- lending-poc/docs/cases_api.md | 124 ++++++++++++++++++++++++++++ lending-poc/docs/features.md | 149 ++++++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 lending-poc/docs/cases_api.md create mode 100644 lending-poc/docs/features.md diff --git a/lending-poc/docs/cases_api.md b/lending-poc/docs/cases_api.md new file mode 100644 index 0000000..9bca958 --- /dev/null +++ b/lending-poc/docs/cases_api.md @@ -0,0 +1,124 @@ +# `/cases` API + +## What this PR does + +Adds the `POST /cases` endpoint, the first user-facing entry point into the lending validation pipeline. It accepts an applicant's KYC and income documents (Aadhaar, PAN, address proof, salary slips, bank statement) as JSON, runs them through identity and business validation, computes an overall confidence score, and returns a PASS / FAIL / NEEDS_REVIEW decision. The full case, its documents, golden record, and validation results are persisted to the database in one transaction. + +## Endpoint + +`POST /cases` + +### Request + +```json +{ + "applicant_ref": "APP-2026-00134", + "documents": [ + { + "doc_type": "AADHAAR", + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "address": "Flat 204, Green Heights, Baner, Pune, Maharashtra 411045", + "aadhaar_number": "XXXX XXXX 4321", + "date_of_birth": "1995-03-14" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/aadhaar_front.pdf" + }, + { + "doc_type": "PAN", + "extracted_fields": { + "name": "Sneha Lokhande", + "pan_number": "ABCDE1234F" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/pan_card.pdf" + }, + { + "doc_type": "ADDRESS_PROOF", + "extracted_fields": { + "address": "Apartment 204, Green Heights, Baner, Pune, MH 411045" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/address_proof.pdf" + }, + { + "doc_type": "SALARY_SLIP", + "salary_slips": [ + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "ABC Technologies Pvt Ltd", + "net_salary": 75000, + "salary_month": "2026-03" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_march.pdf" + } + ] + }, + { + "doc_type": "BANK_STATEMENT", + "extracted_fields": { + "name": "Lokhande S. S.", + "transactions": [ + { "narration": "ABC TECHNOLOGIES SALARY MAR", "amount": 75000, "date": "2026-04-01" }, + { "narration": "HOUSE RENT EMI DEBIT", "amount": -18000, "date": "2026-04-03" } + ] + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/bank_statement_mar_to_jul.pdf" + } + ] +} +``` + +Notes: +- `applicant_ref` and `documents` are required. +- Every document needs `doc_type` and `extracted_fields`; `source_file_ref` is optional. +- `SALARY_SLIP` is the only `doc_type` that carries a `salary_slips` array instead of a flat `extracted_fields` — a case can include multiple salary slips (one per month). +- Required document types for a case to proceed: `AADHAAR`, `PAN`, `SALARY_SLIP`, `BANK_STATEMENT`. `ADDRESS_PROOF` is optional (used as an address fallback). + +### Response `200 OK` + +```json +{ + "case_id": "6e2f6c2a-6a3a-4c1d-9a4b-6f2b6c2a6a3a", + "applicant_ref": "APP-2026-00134", + "decision": "NEEDS_REVIEW", + "overall_score": 88.4, + "reasons": [ + "SALARY_DATE:no_matching_credit_in_window", + "EMPLOYER:employer_narration_mismatch" + ], + "validation_results": [ + { + "check_type": "NAME", + "passed": true, + "score": 94.5, + "document_id": "PAN", + "evidence": null + }, + { + "check_type": "SALARY_DATE", + "passed": false, + "score": 0.0, + "document_id": "SALARY_SLIP-3", + "evidence": { "window": ["2026-05-27", "2026-07-31"] } + } + ] +} +``` + +| Field | Type | Description | +|---|---|---| +| `case_id` | string (UUID) | Primary key of the persisted `Case` row. | +| `applicant_ref` | string | Echoed back from the request. | +| `decision` | string | One of `PASS`, `FAIL`, `NEEDS_REVIEW`. | +| `overall_score` | float | Weighted score (0–100) across all validation checks. | +| `reasons` | string[] | Why this decision was reached (e.g. failing checks, or which mandatory field is missing). | +| `validation_results` | array | One entry per check run, with `check_type`, `passed`, `score`, the `document_id` it applies to, and any supporting `evidence`. | + +### Error responses + +| Status | When | +|---|---| +| `400 Bad Request` | The request body fails semantic parsing in `parse_case` (e.g. malformed/missing required fields inside `extracted_fields`). | +| `422 Unprocessable Entity` | The request body fails schema validation (wrong types, missing `applicant_ref`/`documents`). | + +If any of `AADHAAR`, `PAN`, `SALARY_SLIP`, `BANK_STATEMENT` is missing from `documents`, the pipeline still returns `200 OK` with `decision: "FAIL"` and reasons like `MISSING_DOCUMENT:PAN` — this is a business decision, not an HTTP error. diff --git a/lending-poc/docs/features.md b/lending-poc/docs/features.md new file mode 100644 index 0000000..4b5be5f --- /dev/null +++ b/lending-poc/docs/features.md @@ -0,0 +1,149 @@ +# Lending POC — Feature Guide + +This document explains, in detail, what the validation pipeline behind `POST /cases` does and how each piece works. See [cases_api.md](cases_api.md) for the request/response contract. + +## 1. Pipeline overview + +`app/services/pipeline.py` orchestrates one end-to-end run in this order: + +``` +INGEST -> required-document precheck -> GOLDEN RECORD -> IDENTITY VALIDATION + -> BUSINESS VALIDATION -> SCORING -> DECISION +``` + +Every step appends a line to an in-memory `audit_log`, giving a readable trail of what happened for a given case (currently returned internally on `PipelineResult`, not yet exposed on the API response). + +### 1.1 Required-document precheck + +Before anything else runs, the pipeline checks that all of `AADHAAR`, `PAN`, `SALARY_SLIP`, `BANK_STATEMENT` are present (`validation_config.REQUIRED_DOCUMENT_TYPES`). If any are missing, the pipeline short-circuits with `decision=FAIL`, `overall_score=0.0`, and reasons like `MISSING_DOCUMENT:PAN` — no golden record or checks are computed. + +## 2. Golden Record (`app/services/golden_record.py`) + +The Golden Record is the single trusted identity profile for the applicant, built by merging the KYC documents: + +- **Address & DOB**: sourced from `AADHAAR`. If Aadhaar has no address, `ADDRESS_PROOF` is used as a fallback. +- **Aadhaar number**: from `AADHAAR`. +- **PAN number**: from `PAN`. +- **Name**: the more interesting case. + - If only one of Aadhaar/PAN has a name, that one is used. + - If both have a name, and they're recognizably the same person (`fuzzy.name_similarity` >= `NAME_MATCH_THRESHOLD`, 85), the **fuller** name (more tokens) wins — e.g. "Sneha Sunil Lokhande" over "Sneha Lokhande" — because it carries strictly more identity information. + - If the two names *aren't* recognizably related, Aadhaar stays authoritative and the mismatch is left for the NAME identity check to flag, rather than silently trusting an unrelated "fuller" name. +- The chosen name is split into `first_name` / `middle_name` / `last_name`. +- If an address was resolved, an address embedding is computed (`app.matching.embeddings.get_address_embedding`) and stored for later similarity checks. + +Each golden field also records its `*_source` (which document it came from), useful for traceability. + +## 3. Identity Validation (`app/services/identity_validation.py`) + +Two parts: + +### 3.1 Mandatory presence check + +Regardless of *why* a field is missing, the Golden Record must have `name`, `aadhaar_number`, `pan_number`, and `date_of_birth`. Any missing field produces a failed `ValidationResult` with `failure_reason="missing_in_golden_record"` — and, critically, this is one of the few failures that can force an overall `FAIL` decision outright (see §6). + +### 3.2 Per-document cross-checks against the Golden Record + +Every document that carries an identity field is compared against the Golden Record: + +| Document | Fields checked | +|---|---| +| AADHAAR | name, address, aadhaar_number, DOB | +| PAN | name, pan_number | +| ADDRESS_PROOF | address | +| Each SALARY_SLIP | name | +| BANK_STATEMENT | name | + +Matching strategies (`app/matching/`): +- **NAME** — fuzzy string similarity (`fuzzy.name_similarity`), handles reordering (surname-first), initials, and minor spelling differences. Passes at >= 85. +- **ADDRESS** — embedding cosine similarity (`embeddings.address_similarity`), tolerant of differently-worded but equivalent addresses (e.g. "Apartment" vs "Flat", "MH" vs "Maharashtra"). Passes at >= 0.55 similarity (scored as similarity × 100). +- **AADHAAR / PAN / DOB** — exact matching (`app.matching.exact`). Result is `MATCH` (score 100), `NO_MATCH` (score 0), or `INCONCLUSIVE` (score 50, e.g. one side missing/unparseable). + +## 4. Business Validation (`app/services/business_validation.py`) + +Verifies that declared income (salary slips) is corroborated by actual bank activity. Only runs if both salary slips and a bank statement are present. + +### 4.1 Salary-credit matching (`SALARY_DATE` check) + +For each salary slip: +1. A **month-level search window** is built around the slip's declared `salary_month`: starts `SALARY_CREDIT_BUFFER_DAYS` (5) days before the month begins, and extends `SALARY_CREDIT_EXTRA_MONTHS` (1) month past it. No specific payroll day is assumed. +2. Within that window, candidate transactions must be **credits** (`amount > 0`) and within `SALARY_AMOUNT_TOLERANCE_PCT` (3%) of the slip's declared `net_salary`. This amount gate is applied *before* narration scoring — it's what prevents a same-employer reimbursement/bonus with a coincidentally close amount from masking a genuinely missing salary credit. +3. Among eligible candidates, each is scored as a weighted blend of employer-name similarity to the transaction narration (`TXN_SELECTION_EMPLOYER_WEIGHT`, 0.70) and amount closeness (`TXN_SELECTION_AMOUNT_WEIGHT`, 0.30). The highest-scoring candidate is selected, provided its score clears `TXN_SELECTION_MIN_SCORE` (60). +4. Once a transaction is claimed by a slip, it's excluded from consideration for other slips in the same case (prevents one bank credit being counted as evidence for two different months, which can happen since windows overlap). +5. If no eligible transaction is found, the check fails with `failure_reason="no_matching_credit_in_window"`. + +Slips are resolved in chronological order (earliest `salary_month` first) so earlier months get first claim on ambiguous transactions, but results are returned in the original request order. + +### 4.2 Employer consistency (`EMPLOYER` check) + +Each slip's declared `employer_name` is compared — via fuzzy similarity — only against the narration of **that same slip's own matched transaction** (never against another month's slip or employer). This is intentional: a legitimate employer switch mid-history (e.g. a job change) should not penalize either month. Passes at similarity >= `EMPLOYER_MATCH_THRESHOLD` (80). If the slip had no matched transaction to begin with, this check automatically fails with `failure_reason="no_matching_credit_to_verify_employer_against"`. + +### 4.3 Salary credit count (`SALARY_CREDIT_COUNT` check) + +An aggregate check: `matched_slips / total_slips × 100`. It passes only if *every* slip matched a transaction, but a partial match (e.g. 3 of 4 months) doesn't hard-fail the case — it only lowers this component's score, which feeds into the weighted overall score. Evidence includes the bank statement's observed date range and match counts. + +## 5. Scoring (`app/services/scoring.py`) + +Given every `ValidationResult` produced above: +1. Scores are grouped by `check_type` and averaged (e.g. if 4 salary slips each produced a `SALARY_DATE` score, they're averaged into one `SALARY_DATE` component score). +2. Each component is weighted per `VALIDATION_WEIGHTS`: + + | Check | Weight | + |---|---| + | NAME | 0.15 | + | ADDRESS | 0.10 | + | AADHAAR | 0.15 | + | PAN | 0.15 | + | DOB | 0.10 | + | EMPLOYER | 0.10 | + | SALARY_CREDIT_COUNT | 0.25 | + +3. The overall score is the weighted average, **renormalized over only the check types actually observed** in this case (so a case missing an optional check type doesn't get unfairly diluted by a zero for a check that never ran). Note `SALARY_DATE` itself isn't in the weight table — it gates whether a credit was found at all, but the weighted score is driven by `EMPLOYER` and `SALARY_CREDIT_COUNT`. + +## 6. Decision Engine (`app/services/decision_engine.py`) + +Final decision logic, in priority order: + +1. **Hard FAIL** — if any mandatory identity field (`NAME`, `AADHAAR`, `PAN`, `DOB`) is missing from the Golden Record entirely (`failure_reason="missing_in_golden_record"`), the case fails immediately regardless of score. Reasons: `MANDATORY_FIELD_MISSING:`. +2. **PASS** — if `overall_score >= DECISION_PASS_THRESHOLD` (90). +3. **FAIL** — if `overall_score < DECISION_FAIL_THRESHOLD` (60). +4. **NEEDS_REVIEW** — anything in between (60–90). Reasons list every individual failing check as `:`. + +## 7. Persistence (`app/services/persistence.py`) + +A successful pipeline run is persisted in a single DB transaction: +- One `Case` row (`applicant_ref`, `status` derived from the decision: PASS/FAIL/NEEDS_REVIEW). +- One `Document` row per submitted document (including one per salary slip), storing `extracted_fields` as JSON. +- One `GoldenRecord` row (name, address + embedding, Aadhaar/PAN numbers, DOB). +- One `ValidationResult` row per check performed, linked back to the specific document it was evaluated against where applicable. +- One `PipelineResult` row with the overall score, decision, and reasons. + +Document primary keys are resolved via an in-memory `doc_id -> Document.id` map so validation results (which reference documents by string `doc_id` like `"SALARY_SLIP-2"`) can be foreign-keyed correctly. + +## 8. Supported document types + +| `doc_type` | Purpose | +|---|---| +| `AADHAAR` | Primary identity source (name, address, DOB, Aadhaar number) | +| `PAN` | Secondary identity source (name, PAN number) | +| `ADDRESS_PROOF` | Address fallback if Aadhaar has none | +| `SALARY_SLIP` | Declared income; multiple allowed per case (one per month) | +| `BANK_STATEMENT` | Source of truth for actual salary credits | + +## 9. Configuration reference (`app/services/validation_config.py`) + +All thresholds/weights are centralized here as plain constants (intended to move into `app/config.py` / environment-driven settings as the app matures, without touching service logic): + +| Constant | Value | Meaning | +|---|---|---| +| `NAME_MATCH_THRESHOLD` | 85.0 | Min fuzzy score for NAME to pass | +| `EMPLOYER_MATCH_THRESHOLD` | 80.0 | Min fuzzy score for EMPLOYER to pass | +| `ADDRESS_SIMILARITY_THRESHOLD` | 0.55 | Min cosine similarity for ADDRESS to pass | +| `SALARY_CREDIT_EXTRA_MONTHS` | 1 | Months the salary-credit search window extends past the declared month | +| `SALARY_CREDIT_BUFFER_DAYS` | 5 | Days the window starts before the declared month | +| `TXN_SELECTION_EMPLOYER_WEIGHT` | 0.70 | Weight of narration similarity in transaction selection | +| `TXN_SELECTION_AMOUNT_WEIGHT` | 0.30 | Weight of amount closeness in transaction selection | +| `TXN_SELECTION_MIN_SCORE` | 60.0 | Min blended score for a transaction to be selected | +| `SALARY_AMOUNT_TOLERANCE_PCT` | 3.0 | Max % difference between transaction amount and declared net salary to be eligible at all | +| `DECISION_PASS_THRESHOLD` | 90.0 | Min overall score for PASS | +| `DECISION_FAIL_THRESHOLD` | 60.0 | Below this, FAIL; between this and PASS threshold, NEEDS_REVIEW | +| `REQUIRED_DOCUMENT_TYPES` | AADHAAR, PAN, SALARY_SLIP, BANK_STATEMENT | Documents that must be present for the pipeline to proceed | From e97f1859ebc26de1ea3a764ce7d223e7a0b96516 Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Fri, 14 Aug 2026 14:53:48 +0530 Subject: [PATCH 6/7] fix: case.py added specific attribute and fixed attributeError --- lending-poc/app/schemas/case.py | 88 ++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 7 deletions(-) diff --git a/lending-poc/app/schemas/case.py b/lending-poc/app/schemas/case.py index c56e86c..55d33db 100644 --- a/lending-poc/app/schemas/case.py +++ b/lending-poc/app/schemas/case.py @@ -2,23 +2,97 @@ Mirrors the JSON shape documented in docs/Workflow.md and used by scripts/sample_case.json — one applicant_ref plus a flat list of documents. + +`DocumentIn` is a discriminated union keyed on doc_type: each document kind +gets its own extracted-fields shape, and only SALARY_SLIP carries +salary_slips (and no top-level extracted_fields, matching the documented +sample). """ -from typing import Any +from typing import Annotated, Any, Literal, Union + +from pydantic import BaseModel, Field + + +class BankTransactionIn(BaseModel): + narration: str | None = None + amount: float | None = None + date: str | None = None + + +class AadhaarFieldsIn(BaseModel): + name: str | None = None + address: str | None = None + aadhaar_number: str | None = None + date_of_birth: str | None = None + -from pydantic import BaseModel +class PanFieldsIn(BaseModel): + name: str | None = None + pan_number: str | None = None + + +class AddressProofFieldsIn(BaseModel): + address: str | None = None + + +class SalarySlipFieldsIn(BaseModel): + name: str | None = None + employer_name: str | None = None + net_salary: float | str | None = None + salary_month: str | None = None + + +class BankStatementFieldsIn(BaseModel): + name: str | None = None + transactions: list[BankTransactionIn] = Field(default_factory=list) class SalarySlipIn(BaseModel): - extracted_fields: dict[str, Any] + extracted_fields: SalarySlipFieldsIn source_file_ref: str | None = None -class DocumentIn(BaseModel): - doc_type: str - extracted_fields: dict[str, Any] +class AadhaarDocumentIn(BaseModel): + doc_type: Literal["AADHAAR"] + extracted_fields: AadhaarFieldsIn source_file_ref: str | None = None - salary_slips: list[SalarySlipIn] | None = None # only present when doc_type == SALARY_SLIP + + +class PanDocumentIn(BaseModel): + doc_type: Literal["PAN"] + extracted_fields: PanFieldsIn + source_file_ref: str | None = None + + +class AddressProofDocumentIn(BaseModel): + doc_type: Literal["ADDRESS_PROOF"] + extracted_fields: AddressProofFieldsIn + source_file_ref: str | None = None + + +class SalarySlipDocumentIn(BaseModel): + doc_type: Literal["SALARY_SLIP"] + salary_slips: list[SalarySlipIn] + source_file_ref: str | None = None + + +class BankStatementDocumentIn(BaseModel): + doc_type: Literal["BANK_STATEMENT"] + extracted_fields: BankStatementFieldsIn + source_file_ref: str | None = None + + +DocumentIn = Annotated[ + Union[ + AadhaarDocumentIn, + PanDocumentIn, + AddressProofDocumentIn, + SalarySlipDocumentIn, + BankStatementDocumentIn, + ], + Field(discriminator="doc_type"), +] class CaseCreateRequest(BaseModel): From a96bd3c8ff8640e735c022105a258802975bf3e5 Mon Sep 17 00:00:00 2001 From: Vishwajeetsingh Desurkar Date: Fri, 14 Aug 2026 15:38:36 +0530 Subject: [PATCH 7/7] fix: enforce MIN_OVERLAPPING_DIGITS for masked Aadhaar and redact PAN in JSONB - aadhaar_match: return INCONCLUSIVE when a masked value exposes fewer than MIN_OVERLAPPING_DIGITS visible digits, preventing a single-digit suffix from producing a definitive MATCH - persistence: mask all but the last 4 chars of PAN before writing to the documents.extracted_fields JSONB column; full value is encrypted in GoldenRecord --- lending-poc/app/matching/exact.py | 2 ++ lending-poc/app/services/persistence.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lending-poc/app/matching/exact.py b/lending-poc/app/matching/exact.py index 4e5facb..190b3e7 100644 --- a/lending-poc/app/matching/exact.py +++ b/lending-poc/app/matching/exact.py @@ -51,6 +51,8 @@ def aadhaar_match(golden: str | None, candidate: str | None) -> ExactCheckOutcom trailing_digits = "".join(ch for ch in masked if ch != "X") if not trailing_digits: return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "no_unmasked_digits") + if len(trailing_digits) < MIN_OVERLAPPING_DIGITS: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "insufficient_unmasked_digits") if len(unmasked) < len(trailing_digits): return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "unmasked_value_too_short") suffix = unmasked[-len(trailing_digits):] diff --git a/lending-poc/app/services/persistence.py b/lending-poc/app/services/persistence.py index 4f5890a..6e41534 100644 --- a/lending-poc/app/services/persistence.py +++ b/lending-poc/app/services/persistence.py @@ -27,6 +27,18 @@ } +def _mask_pan(pan: str | None) -> str | None: + """Mask all but the last 4 characters of a PAN for JSONB storage. + + The full value is encrypted in the GoldenRecord; the document payload + only needs enough for audit trail without exposing the raw PAN. + """ + if not pan: + return pan + visible = min(4, len(pan)) + return "X" * (len(pan) - visible) + pan[-visible:] + + def _document_rows(case: CaseInput) -> list[tuple[str, Document]]: """Returns (doc_id, Document) pairs for every document present on the case.""" rows: list[tuple[str, Document]] = [] @@ -52,7 +64,7 @@ def _document_rows(case: CaseInput) -> list[tuple[str, Document]]: Document( doc_type=DocType.PAN, source_file_ref=case.pan.source_file_ref, - extracted_fields={"name": case.pan.name, "pan_number": case.pan.pan_number}, + extracted_fields={"name": case.pan.name, "pan_number": _mask_pan(case.pan.pan_number)}, ), ))