diff --git a/lending-poc/app/api/cases.py b/lending-poc/app/api/cases.py new file mode 100644 index 0000000..4d32522 --- /dev/null +++ b/lending-poc/app/api/cases.py @@ -0,0 +1,41 @@ +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.ext.asyncio import AsyncSession + +from app.database import get_db +from app.schemas.case import CaseCreateRequest, CaseCreateResponse, ValidationResultOut +from app.services.case_parsing import parse_case +from app.services.persistence import save_pipeline_result +from app.services.pipeline import run_pipeline +from app.utils.json_safe import json_safe + +router = APIRouter(tags=["cases"]) + + +@router.post("/cases", response_model=CaseCreateResponse) +async def create_case( + request: CaseCreateRequest, db: AsyncSession = Depends(get_db) +) -> CaseCreateResponse: + try: + case_input = parse_case(request.model_dump()) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + pipeline_result = run_pipeline(case_input) + case = await save_pipeline_result(db, case_input, pipeline_result) + + return CaseCreateResponse( + case_id=str(case.id), + applicant_ref=case_input.applicant_ref, + decision=pipeline_result.decision_result.decision.value, + overall_score=pipeline_result.decision_result.overall_score, + reasons=pipeline_result.decision_result.reasons, + validation_results=[ + ValidationResultOut( + check_type=r.check_type.value, + passed=r.passed, + score=r.score, + document_id=r.document_id, + evidence=json_safe(r.evidence) if r.evidence else None, + ) + for r in pipeline_result.validation_results + ], + ) diff --git a/lending-poc/app/config.py b/lending-poc/app/config.py index 0daa2c9..90c1779 100644 --- a/lending-poc/app/config.py +++ b/lending-poc/app/config.py @@ -11,6 +11,7 @@ class Settings(BaseSettings): DEBUG: bool = False DATABASE_URL: str LOG_LEVEL: str = "INFO" + ENCRYPTION_KEY: str settings = Settings() diff --git a/lending-poc/app/main.py b/lending-poc/app/main.py index 8481002..dc9c4d0 100644 --- a/lending-poc/app/main.py +++ b/lending-poc/app/main.py @@ -4,6 +4,7 @@ from fastapi import FastAPI from sqlalchemy import text +from app.api.cases import router as cases_router from app.api.health import router as health_router from app.config import logger, settings from app.database import async_session, engine @@ -34,3 +35,4 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: ) app.include_router(health_router) +app.include_router(cases_router) diff --git a/lending-poc/app/models/__init__.py b/lending-poc/app/matching/__init__.py similarity index 100% rename from lending-poc/app/models/__init__.py rename to lending-poc/app/matching/__init__.py diff --git a/lending-poc/app/matching/embeddings.py b/lending-poc/app/matching/embeddings.py new file mode 100644 index 0000000..329261f --- /dev/null +++ b/lending-poc/app/matching/embeddings.py @@ -0,0 +1,60 @@ +"""Address similarity via embeddings. + +Embeddings come from BAAI/bge-small-en-v1.5 (sentence-transformers), +running locally on CPU — no API key or network call per request. The +model is loaded once per process (module-level singleton) since load time +is the expensive part; encoding individual addresses is fast. +""" + +import math +import re +from functools import lru_cache + +EMBEDDING_MODEL_NAME = "BAAI/bge-small-en-v1.5" +EMBEDDING_DIMENSIONS = 384 + +# bge models are trained to prepend this instruction for retrieval queries. +_QUERY_PREFIX = "represent this sentence for searching relevant passages: " + + +@lru_cache(maxsize=1) +def _get_model(): + from sentence_transformers import SentenceTransformer + + return SentenceTransformer(EMBEDDING_MODEL_NAME) + + +def _normalize_address(address: str) -> str: + text = address.lower() + text = re.sub(r"[^a-z0-9\s]", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def get_address_embedding(address: str) -> list[float]: + normalized = _normalize_address(address) + if not normalized: + return [0.0] * EMBEDDING_DIMENSIONS + + model = _get_model() + vector = model.encode(_QUERY_PREFIX + normalized, normalize_embeddings=True) + return vector.tolist() + + +def cosine_similarity(vec_a: list[float], vec_b: list[float]) -> float: + if not vec_a or not vec_b or len(vec_a) != len(vec_b): + return 0.0 + dot = sum(a * b for a, b in zip(vec_a, vec_b)) + norm_a = math.sqrt(sum(a * a for a in vec_a)) + norm_b = math.sqrt(sum(b * b for b in vec_b)) + if norm_a == 0 or norm_b == 0: + return 0.0 + return dot / (norm_a * norm_b) + + +def address_similarity(address_a: str, address_b: str) -> float: + if not address_a or not address_b: + return 0.0 + vec_a = get_address_embedding(address_a) + vec_b = get_address_embedding(address_b) + similarity = cosine_similarity(vec_a, vec_b) + return max(0.0, similarity) diff --git a/lending-poc/app/matching/exact.py b/lending-poc/app/matching/exact.py new file mode 100644 index 0000000..190b3e7 --- /dev/null +++ b/lending-poc/app/matching/exact.py @@ -0,0 +1,102 @@ +"""Exact-match checks for Aadhaar, PAN, and date of birth. + +Aadhaar numbers are often masked in extracted documents (e.g. +"XXXX XXXX 4321"). MatchResult is tri-state because a masked value can be +inconclusive rather than a clean match/mismatch. +""" + +from dataclasses import dataclass +from datetime import date +from enum import Enum + +MIN_OVERLAPPING_DIGITS = 4 + + +class MatchResult(str, Enum): + MATCH = "MATCH" + MISMATCH = "MISMATCH" + INCONCLUSIVE = "INCONCLUSIVE" + + +@dataclass +class ExactCheckOutcome: + result: MatchResult + reason: str | None = None + + +def _normalize(value: str) -> str: + return "".join(ch for ch in value.upper() if ch.isdigit() or ch == "X") + + +def _is_masked(value: str) -> bool: + return "X" in value + + +def aadhaar_match(golden: str | None, candidate: str | None) -> ExactCheckOutcome: + if not golden or not candidate: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "missing_value") + + g = _normalize(golden) + c = _normalize(candidate) + + if not _is_masked(g) and not _is_masked(c): + return ( + ExactCheckOutcome(MatchResult.MATCH) + if g == c + else ExactCheckOutcome(MatchResult.MISMATCH, "digits_differ") + ) + + if _is_masked(g) != _is_masked(c): + masked, unmasked = (g, c) if _is_masked(g) else (c, g) + trailing_digits = "".join(ch for ch in masked if ch != "X") + if not trailing_digits: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "no_unmasked_digits") + if len(trailing_digits) < MIN_OVERLAPPING_DIGITS: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "insufficient_unmasked_digits") + if len(unmasked) < len(trailing_digits): + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "unmasked_value_too_short") + suffix = unmasked[-len(trailing_digits):] + return ( + ExactCheckOutcome(MatchResult.MATCH) + if suffix == trailing_digits + else ExactCheckOutcome(MatchResult.MISMATCH, "suffix_digits_differ") + ) + + # Both masked: compare position-wise where both sides have a digit. + if len(g) != len(c): + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "masked_length_mismatch") + + overlapping = 0 + for gd, cd in zip(g, c): + if gd == "X" or cd == "X": + continue + overlapping += 1 + if gd != cd: + return ExactCheckOutcome(MatchResult.MISMATCH, "overlapping_digits_differ") + + if overlapping < MIN_OVERLAPPING_DIGITS: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "insufficient_unmasked_digits") + + return ExactCheckOutcome(MatchResult.MATCH) + + +def pan_match(golden: str | None, candidate: str | None) -> ExactCheckOutcome: + if not golden or not candidate: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "missing_value") + g = golden.strip().upper() + c = candidate.strip().upper() + return ( + ExactCheckOutcome(MatchResult.MATCH) + if g == c + else ExactCheckOutcome(MatchResult.MISMATCH, "pan_differs") + ) + + +def dob_match(golden: date | None, candidate: date | None) -> ExactCheckOutcome: + if golden is None or candidate is None: + return ExactCheckOutcome(MatchResult.INCONCLUSIVE, "missing_value") + return ( + ExactCheckOutcome(MatchResult.MATCH) + if golden == candidate + else ExactCheckOutcome(MatchResult.MISMATCH, "dob_differs") + ) diff --git a/lending-poc/app/matching/fuzzy.py b/lending-poc/app/matching/fuzzy.py new file mode 100644 index 0000000..1264da1 --- /dev/null +++ b/lending-poc/app/matching/fuzzy.py @@ -0,0 +1,103 @@ +"""Name and employer string similarity. + +Base similarity comes from RapidFuzz's token_set_ratio (handles word +reordering and extra/missing tokens). On top of that, an initials-expansion +pass resolves abbreviations like "Ankita A.S" vs "Ankita Advitot Sunil", +which plain character-level fuzzy matching can't do on its own. +""" + +from rapidfuzz import fuzz + +MIN_OVERLAP_TOKEN = 1 + + +def _tokenize(name: str) -> list[str]: + return [tok.strip(".").lower() for tok in name.replace(".", ". ").split() if tok.strip(".")] + + +def _is_initial(token: str) -> bool: + return len(token) == 1 + + +def _initials_genuine_match(tokens_a: list[str], tokens_b: list[str]) -> bool: + """True if the side with more single-letter tokens is a genuine + initials-abbreviation of the other side's full-word tokens. + + Every initial on the shorter side must either (a) match the first + letter of a distinct, unused full word on the other side, or (b) have + no full word left to compare against at all (the other name simply + doesn't spell out that token, e.g. a middle/last name omitted + entirely). What's never tolerated is an initial whose letter + contradicts one of the available full words when a candidate exists + to compare against — that's what keeps a genuinely wrong initial + (e.g. "P" when the real name starts with "A") from being boosted. + """ + full_a = [t for t in tokens_a if not _is_initial(t)] + full_b = [t for t in tokens_b if not _is_initial(t)] + initials_a = [t for t in tokens_a if _is_initial(t)] + initials_b = [t for t in tokens_b if _is_initial(t)] + + # Pick the side that actually has initials to expand. + if initials_b and not initials_a: + initials, full = initials_b, full_a + elif initials_a and not initials_b: + initials, full = initials_a, full_b + else: + return False + + if not initials or not full: + return False + + # There must be at least one shared full token (anchor) between the + # two names, otherwise this is likely a different person entirely. + shared_full_words = set(full_a) & set(full_b) + if not shared_full_words: + return False + + # Words already spelled out identically on both sides (e.g. a shared + # first name) don't need to be re-derived from an initial — only the + # remaining full words are available to satisfy the initials. + remaining_full = list(full) + for word in shared_full_words: + if word in remaining_full: + remaining_full.remove(word) + + available_first_letters = [w[0] for w in remaining_full] + matched_any = False + for letter in initials: + if letter in available_first_letters: + available_first_letters.remove(letter) + matched_any = True + elif available_first_letters: + # A candidate full word exists but its first letter doesn't + # match this initial — a genuine contradiction, not boosted. + return False + # else: no remaining full word to check this initial against — + # tolerated as an omitted token, not a contradiction. + + return matched_any + + +def _base_token_set_ratio(a: str, b: str) -> float: + return float(fuzz.token_set_ratio(a.lower(), b.lower())) + + +def name_similarity(name_a: str, name_b: str) -> float: + if not name_a or not name_b: + return 0.0 + + base = _base_token_set_ratio(name_a, name_b) + + tokens_a = _tokenize(name_a) + tokens_b = _tokenize(name_b) + + if _initials_genuine_match(tokens_a, tokens_b): + return 100.0 + + return base + + +def employer_similarity(employer_a: str, employer_b: str) -> float: + if not employer_a or not employer_b: + return 0.0 + return _base_token_set_ratio(employer_a, employer_b) diff --git a/lending-poc/app/schemas/case.py b/lending-poc/app/schemas/case.py new file mode 100644 index 0000000..55d33db --- /dev/null +++ b/lending-poc/app/schemas/case.py @@ -0,0 +1,117 @@ +"""Request/response schemas for POST /cases. + +Mirrors the JSON shape documented in docs/Workflow.md and used by +scripts/sample_case.json — one applicant_ref plus a flat list of documents. + +`DocumentIn` is a discriminated union keyed on doc_type: each document kind +gets its own extracted-fields shape, and only SALARY_SLIP carries +salary_slips (and no top-level extracted_fields, matching the documented +sample). +""" + +from typing import Annotated, Any, Literal, Union + +from pydantic import BaseModel, Field + + +class BankTransactionIn(BaseModel): + narration: str | None = None + amount: float | None = None + date: str | None = None + + +class AadhaarFieldsIn(BaseModel): + name: str | None = None + address: str | None = None + aadhaar_number: str | None = None + date_of_birth: str | None = None + + +class PanFieldsIn(BaseModel): + name: str | None = None + pan_number: str | None = None + + +class AddressProofFieldsIn(BaseModel): + address: str | None = None + + +class SalarySlipFieldsIn(BaseModel): + name: str | None = None + employer_name: str | None = None + net_salary: float | str | None = None + salary_month: str | None = None + + +class BankStatementFieldsIn(BaseModel): + name: str | None = None + transactions: list[BankTransactionIn] = Field(default_factory=list) + + +class SalarySlipIn(BaseModel): + extracted_fields: SalarySlipFieldsIn + source_file_ref: str | None = None + + +class AadhaarDocumentIn(BaseModel): + doc_type: Literal["AADHAAR"] + extracted_fields: AadhaarFieldsIn + source_file_ref: str | None = None + + +class PanDocumentIn(BaseModel): + doc_type: Literal["PAN"] + extracted_fields: PanFieldsIn + source_file_ref: str | None = None + + +class AddressProofDocumentIn(BaseModel): + doc_type: Literal["ADDRESS_PROOF"] + extracted_fields: AddressProofFieldsIn + source_file_ref: str | None = None + + +class SalarySlipDocumentIn(BaseModel): + doc_type: Literal["SALARY_SLIP"] + salary_slips: list[SalarySlipIn] + source_file_ref: str | None = None + + +class BankStatementDocumentIn(BaseModel): + doc_type: Literal["BANK_STATEMENT"] + extracted_fields: BankStatementFieldsIn + source_file_ref: str | None = None + + +DocumentIn = Annotated[ + Union[ + AadhaarDocumentIn, + PanDocumentIn, + AddressProofDocumentIn, + SalarySlipDocumentIn, + BankStatementDocumentIn, + ], + Field(discriminator="doc_type"), +] + + +class CaseCreateRequest(BaseModel): + applicant_ref: str + documents: list[DocumentIn] + + +class ValidationResultOut(BaseModel): + check_type: str + passed: bool + score: float + document_id: str | None = None + evidence: dict[str, Any] | None = None + + +class CaseCreateResponse(BaseModel): + case_id: str + applicant_ref: str + decision: str + overall_score: float + reasons: list[str] + validation_results: list[ValidationResultOut] diff --git a/lending-poc/app/services/__init__.py b/lending-poc/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lending-poc/app/services/business_validation.py b/lending-poc/app/services/business_validation.py new file mode 100644 index 0000000..ed58b07 --- /dev/null +++ b/lending-poc/app/services/business_validation.py @@ -0,0 +1,242 @@ +"""Checks employer + salary consistency between salary slips and the bank +statement. No specific payroll day is assumed anywhere: each slip is +matched against bank transactions inside a broad, month-level window. +""" + +from calendar import monthrange +from datetime import date, timedelta + +from app.matching.fuzzy import employer_similarity +from app.services import validation_config as cfg +from app.services.dto import ( + BankStatementDoc, + BankTransaction, + CaseInput, + CheckType, + SalarySlipDoc, + ValidationResult, +) + + +def _add_months(d: date, months: int) -> date: + month_index = d.month - 1 + months + year = d.year + month_index // 12 + month = month_index % 12 + 1 + return date(year, month, 1) + + +def _month_window(salary_month: date) -> tuple[date, date]: + window_start = date(salary_month.year, salary_month.month, 1) - timedelta( + days=cfg.SALARY_CREDIT_BUFFER_DAYS + ) + window_end_month = _add_months(salary_month, cfg.SALARY_CREDIT_EXTRA_MONTHS) + last_day = monthrange(window_end_month.year, window_end_month.month)[1] + window_end = date(window_end_month.year, window_end_month.month, last_day) + return window_start, window_end + + +def _within_amount_tolerance(amount: float, expected: float) -> bool: + if not expected: + return False + pct_diff = abs(amount - expected) / expected * 100.0 + return pct_diff <= cfg.SALARY_AMOUNT_TOLERANCE_PCT + + +def _candidate_transactions( + window: tuple[date, date], transactions: list[BankTransaction], expected_amount: float | None +) -> list[BankTransaction]: + """Transactions inside the month-window AND within salary-amount + tolerance of what this specific slip declared. The tolerance gate is + on amount alone -- a large reimbursement/bonus/advance with a + coincidentally close amount is excluded here, before narration + similarity ever gets a vote, so it can never mask a genuinely missing + salary credit. + """ + start, end = window + return [ + txn + for txn in transactions + if txn.amount is not None + and txn.txn_date is not None + and txn.amount > 0 + and start <= txn.txn_date <= end + and (expected_amount is None or _within_amount_tolerance(txn.amount, expected_amount)) + ] + + +def _amount_closeness(amount: float, expected: float) -> float: + if not expected: + return 0.0 + closeness = 100.0 * (1 - abs(amount - expected) / expected) + return max(0.0, min(100.0, closeness)) + + +def _score_transaction(txn: BankTransaction, employer_name: str, expected_amount: float) -> float: + employer_score = employer_similarity(employer_name, txn.narration) if employer_name else 0.0 + amount_score = _amount_closeness(txn.amount, expected_amount) + return ( + cfg.TXN_SELECTION_EMPLOYER_WEIGHT * employer_score + + cfg.TXN_SELECTION_AMOUNT_WEIGHT * amount_score + ) + + +def _select_best_transaction( + candidates: list[BankTransaction], employer_name: str | None, expected_amount: float | None +) -> tuple[BankTransaction, float] | None: + if not candidates or expected_amount is None: + return None + scored = [(txn, _score_transaction(txn, employer_name, expected_amount)) for txn in candidates] + best_txn, best_score = max(scored, key=lambda pair: pair[1]) + if best_score < cfg.TXN_SELECTION_MIN_SCORE: + return None + return best_txn, best_score + + +def _validate_salary_slip( + slip: SalarySlipDoc, bank_statement: BankStatementDoc, used_transaction_ids: set[int] +) -> ValidationResult: + """Matches one slip against the bank statement. + + `used_transaction_ids` (by `id(txn)`) tracks credits already claimed by + an earlier slip in this same case, since overlapping month-windows mean + the same credit could otherwise be double-counted as evidence for two + different declared months of income. + """ + if slip.salary_month is None: + return ValidationResult( + check_type=CheckType.SALARY_DATE, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="missing_salary_month", + ) + + if slip.net_salary is None: + return ValidationResult( + check_type=CheckType.SALARY_DATE, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="missing_net_salary", + ) + + window = _month_window(slip.salary_month) + candidates = [ + txn + for txn in _candidate_transactions(window, bank_statement.transactions, slip.net_salary) + if id(txn) not in used_transaction_ids + ] + selection = _select_best_transaction(candidates, slip.employer_name, slip.net_salary) + + if selection is None: + return ValidationResult( + check_type=CheckType.SALARY_DATE, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="no_matching_credit_in_window", + evidence={"window": window}, + ) + + txn, score = selection + used_transaction_ids.add(id(txn)) + return ValidationResult( + check_type=CheckType.SALARY_DATE, + passed=True, + score=score, + document_id=slip.doc_id, + evidence={"matched_transaction": txn}, + ) + + +def _employer_match_for_slip( + slip: SalarySlipDoc, slip_result: ValidationResult +) -> ValidationResult: + """Verifies one slip's declared employer against its OWN matched bank + transaction's narration only — never against another month's slip. + + Employer consistency is judged month-by-month, on purpose: an + applicant switching jobs mid-history is normal and legitimate, so May's + employer claim is never compared to June's. Each month stands on its + own evidence. + """ + if not slip.employer_name: + return ValidationResult( + check_type=CheckType.EMPLOYER, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="missing_employer_name", + ) + + if not slip_result.passed or not slip_result.evidence: + return ValidationResult( + check_type=CheckType.EMPLOYER, + passed=False, + score=0.0, + document_id=slip.doc_id, + failure_reason="no_matching_credit_to_verify_employer_against", + ) + + matched_txn = slip_result.evidence.get("matched_transaction") + score = employer_similarity(slip.employer_name, matched_txn.narration) + passed = score >= cfg.EMPLOYER_MATCH_THRESHOLD + return ValidationResult( + check_type=CheckType.EMPLOYER, + passed=passed, + score=score, + document_id=slip.doc_id, + failure_reason=None if passed else "employer_narration_mismatch", + ) + + +def _salary_credit_count( + salary_slips: list[SalarySlipDoc], + bank_statement: BankStatementDoc, + slip_results: list[ValidationResult], +) -> ValidationResult: + total_slips = len(salary_slips) + matched_slips = sum(1 for r in slip_results if r.passed) + confidence_score = (matched_slips / total_slips * 100.0) if total_slips else 0.0 + + dates = [txn.txn_date for txn in bank_statement.transactions if txn.txn_date is not None] + stmt_duration = {"start": min(dates), "end": max(dates)} if dates else None + + return ValidationResult( + check_type=CheckType.SALARY_CREDIT_COUNT, + passed=(matched_slips == total_slips), + score=confidence_score, + evidence={ + "stmt_duration": stmt_duration, + "no_of_matches": matched_slips, + "total_slips": total_slips, + "confidence_score": confidence_score, + }, + ) + + +def run_business_validation(case: CaseInput) -> list[ValidationResult]: + results: list[ValidationResult] = [] + + if not case.salary_slips or not case.bank_statement: + return results + + used_transaction_ids: set[int] = set() + ordered_slips = sorted( + case.salary_slips, key=lambda slip: slip.salary_month or date.max + ) + slip_results_by_doc_id = { + slip.doc_id: _validate_salary_slip(slip, case.bank_statement, used_transaction_ids) + for slip in ordered_slips + } + # Preserve the original slip order in the output, independent of the + # chronological order used to resolve which slip claims which credit. + slip_results = [slip_results_by_doc_id[slip.doc_id] for slip in case.salary_slips] + results.extend(slip_results) + + for slip in case.salary_slips: + results.append(_employer_match_for_slip(slip, slip_results_by_doc_id[slip.doc_id])) + + results.append(_salary_credit_count(case.salary_slips, case.bank_statement, slip_results)) + + return results diff --git a/lending-poc/app/services/case_parsing.py b/lending-poc/app/services/case_parsing.py new file mode 100644 index 0000000..b8a0c33 --- /dev/null +++ b/lending-poc/app/services/case_parsing.py @@ -0,0 +1,112 @@ +"""Parses the raw request JSON shape (see docs/Workflow.md) into CaseInput. + +Used by the POST /cases endpoint. +""" + +from datetime import date, datetime + +from app.services.dto import ( + AadhaarDoc, + AddressProofDoc, + BankStatementDoc, + BankTransaction, + CaseInput, + PanDoc, + SalarySlipDoc, +) + + +def _get(fields: dict, key: str) -> str | None: + """Null-safe field lookup: a missing key, JSON null, and an empty/ + whitespace-only string are all treated as no value.""" + value = fields.get(key) + if value is None: + return None + if isinstance(value, str) and not value.strip(): + return None + return value + + +def _parse_date(value: str | None) -> date | None: + if value is None: + return None + return datetime.strptime(value, "%Y-%m-%d").date() + + +def _parse_month(value: str | None) -> date | None: + if value is None: + return None + return datetime.strptime(value, "%Y-%m").date().replace(day=1) + + +def _parse_float(value) -> float | None: + if value is None or (isinstance(value, str) and not value.strip()): + return None + return float(value) + + +def parse_case(payload: dict) -> CaseInput: + case = CaseInput(applicant_ref=payload["applicant_ref"]) + + for doc in payload["documents"]: + doc_type = doc["doc_type"] + + if doc_type == "AADHAAR": + fields = doc["extracted_fields"] + case.aadhaar = AadhaarDoc( + name=_get(fields, "name"), + address=_get(fields, "address"), + aadhaar_number=_get(fields, "aadhaar_number"), + date_of_birth=_parse_date(_get(fields, "date_of_birth")), + source_file_ref=doc.get("source_file_ref"), + ) + + elif doc_type == "PAN": + fields = doc["extracted_fields"] + case.pan = PanDoc( + name=_get(fields, "name"), + pan_number=_get(fields, "pan_number"), + source_file_ref=doc.get("source_file_ref"), + ) + + elif doc_type == "ADDRESS_PROOF": + fields = doc["extracted_fields"] + case.address_proof = AddressProofDoc( + address=_get(fields, "address"), + source_file_ref=doc.get("source_file_ref"), + ) + + elif doc_type == "SALARY_SLIP": + slips = doc.get("salary_slips") or [] + if not slips: + raise ValueError("SALARY_SLIP document must include at least one entry in salary_slips") + for i, slip in enumerate(slips): + fields = slip["extracted_fields"] + case.salary_slips.append( + SalarySlipDoc( + employer_name=_get(fields, "employer_name"), + net_salary=_parse_float(_get(fields, "net_salary")), + salary_month=_parse_month(_get(fields, "salary_month")), + source_file_ref=slip.get("source_file_ref"), + doc_id=f"SALARY_SLIP-{i}", + name=_get(fields, "name"), + ) + ) + + elif doc_type == "BANK_STATEMENT": + fields = doc["extracted_fields"] + transactions = [ + BankTransaction( + narration=_get(txn, "narration"), + amount=_parse_float(_get(txn, "amount")), + txn_date=_parse_date(_get(txn, "date")), + ) + for txn in fields.get("transactions", []) + ] + case.bank_statement = BankStatementDoc( + transactions=transactions, + source_file_ref=doc.get("source_file_ref"), + name=_get(fields, "name"), + ) + + return case diff --git a/lending-poc/app/services/decision_engine.py b/lending-poc/app/services/decision_engine.py new file mode 100644 index 0000000..be8abd5 --- /dev/null +++ b/lending-poc/app/services/decision_engine.py @@ -0,0 +1,46 @@ +"""Final PASS / FAIL / NEEDS_REVIEW logic.""" + +from app.services import validation_config as cfg +from app.services.dto import CheckType, Decision, DecisionResult, ScoreResult, ValidationResult + +MANDATORY_CHECK_TYPES = {CheckType.NAME, CheckType.AADHAAR, CheckType.PAN, CheckType.DOB} + + +def make_decision( + score: ScoreResult, validation_results: list[ValidationResult] +) -> DecisionResult: + reasons: list[str] = [] + + mandatory_failures = [ + r + for r in validation_results + if r.check_type in MANDATORY_CHECK_TYPES + and not r.passed + and r.failure_reason == "missing_in_golden_record" + ] + if mandatory_failures: + reasons = [f"MANDATORY_FIELD_MISSING:{r.check_type.value}" for r in mandatory_failures] + return DecisionResult( + decision=Decision.FAIL, reasons=reasons, overall_score=score.overall_score + ) + + if score.overall_score >= cfg.DECISION_PASS_THRESHOLD: + return DecisionResult(decision=Decision.PASS, reasons=["score_meets_pass_threshold"], overall_score=score.overall_score) + + if score.overall_score < cfg.DECISION_FAIL_THRESHOLD: + return DecisionResult( + decision=Decision.FAIL, + reasons=["score_below_fail_threshold"], + overall_score=score.overall_score, + ) + + failing_checks = [ + f"{r.check_type.value}:{r.failure_reason or 'below_threshold'}" + for r in validation_results + if not r.passed + ] + return DecisionResult( + decision=Decision.NEEDS_REVIEW, + reasons=failing_checks or ["score_in_review_band"], + overall_score=score.overall_score, + ) diff --git a/lending-poc/app/services/dto.py b/lending-poc/app/services/dto.py new file mode 100644 index 0000000..cfd8374 --- /dev/null +++ b/lending-poc/app/services/dto.py @@ -0,0 +1,162 @@ +"""Plain in-memory data structures for the validation pipeline. + +No database/ORM involved yet — these dataclasses are what the extraction +pipeline's JSON gets parsed into, and what every service function passes +around. When persistence is added later, these become the shape that gets +mapped to/from the DB models, but the validation logic itself does not +change. +""" + +from dataclasses import dataclass, field +from datetime import date +from enum import Enum + + +class DocType(str, Enum): + AADHAAR = "AADHAAR" + PAN = "PAN" + ADDRESS_PROOF = "ADDRESS_PROOF" + SALARY_SLIP = "SALARY_SLIP" + BANK_STATEMENT = "BANK_STATEMENT" + + +class CheckType(str, Enum): + NAME = "NAME" + ADDRESS = "ADDRESS" + AADHAAR = "AADHAAR" + PAN = "PAN" + DOB = "DOB" + EMPLOYER = "EMPLOYER" + SALARY_DATE = "SALARY_DATE" + SALARY_CREDIT_COUNT = "SALARY_CREDIT_COUNT" + MANDATORY_PRESENCE = "MANDATORY_PRESENCE" + + +class Decision(str, Enum): + PASS = "PASS" + FAIL = "FAIL" + NEEDS_REVIEW = "NEEDS_REVIEW" + + +@dataclass +class AadhaarDoc: + name: str | None = None + address: str | None = None + aadhaar_number: str | None = None + date_of_birth: date | None = None + source_file_ref: str | None = None + doc_id: str = "AADHAAR" + + +@dataclass +class PanDoc: + name: str | None = None + pan_number: str | None = None + source_file_ref: str | None = None + doc_id: str = "PAN" + + +@dataclass +class AddressProofDoc: + address: str | None = None + source_file_ref: str | None = None + doc_id: str = "ADDRESS_PROOF" + + +@dataclass +class SalarySlipDoc: + doc_id: str + employer_name: str | None = None + net_salary: float | None = None + salary_month: date | None = None # first-of-month + name: str | None = None + source_file_ref: str | None = None + + +@dataclass +class BankTransaction: + narration: str | None = None + amount: float | None = None + txn_date: date | None = None + + +@dataclass +class BankStatementDoc: + transactions: list[BankTransaction] + doc_id: str = "BANK_STATEMENT" + name: str | None = None + source_file_ref: str | None = None + + +@dataclass +class CaseInput: + applicant_ref: str + aadhaar: AadhaarDoc | None = None + pan: PanDoc | None = None + address_proof: AddressProofDoc | None = None + salary_slips: list[SalarySlipDoc] = field(default_factory=list) + bank_statement: BankStatementDoc | None = None + + def present_doc_types(self) -> set[str]: + present = set() + if self.aadhaar: + present.add(DocType.AADHAAR.value) + if self.pan: + present.add(DocType.PAN.value) + if self.address_proof: + present.add(DocType.ADDRESS_PROOF.value) + if self.salary_slips: + present.add(DocType.SALARY_SLIP.value) + if self.bank_statement: + present.add(DocType.BANK_STATEMENT.value) + return present + + +@dataclass +class GoldenRecord: + name: str | None = None + name_source: str | None = None + first_name: str | None = None + middle_name: str | None = None + last_name: str | None = None + address: str | None = None + address_source: str | None = None + address_embedding: list[float] = field(default_factory=list) + date_of_birth: date | None = None + dob_source: str | None = None + aadhaar_number: str | None = None + aadhaar_source: str | None = None + pan_number: str | None = None + pan_source: str | None = None + + +@dataclass +class ValidationResult: + check_type: CheckType + passed: bool + score: float + document_id: str | None = None + failure_reason: str | None = None + evidence: dict | None = None # in-memory only; becomes real columns/FKs once persisted + + +@dataclass +class ScoreResult: + overall_score: float + component_scores: dict[str, float] + + +@dataclass +class DecisionResult: + decision: Decision + reasons: list[str] + overall_score: float + + +@dataclass +class PipelineResult: + golden_record: GoldenRecord | None + validation_results: list[ValidationResult] + score_result: ScoreResult | None + decision_result: DecisionResult + audit_log: list[str] diff --git a/lending-poc/app/services/golden_record.py b/lending-poc/app/services/golden_record.py new file mode 100644 index 0000000..c676e81 --- /dev/null +++ b/lending-poc/app/services/golden_record.py @@ -0,0 +1,82 @@ +"""Builds the one trusted identity profile (Golden Record) per applicant. + +Field precedence: AADHAAR is the primary source for address/DOB, with +ADDRESS_PROOF as a fallback for address. For name specifically: when both +AADHAAR and PAN carry a name, and they're recognizably the same person +(per fuzzy.name_similarity), the fuller of the two (more name tokens) is +preferred as the golden name — e.g. "Ankita Sunil Advitot" over "Ankita +Advitot" — since the fuller version carries strictly more identity +information. If the two names don't look related, AADHAAR stays primary +and the mismatch is left for identity_validation's NAME check to flag, +rather than silently adopting an unrelated "fuller" name. +""" + +from app.matching.embeddings import get_address_embedding +from app.matching.fuzzy import name_similarity +from app.services import validation_config as cfg +from app.services.dto import CaseInput, GoldenRecord + +FULLER_NAME_RELATEDNESS_THRESHOLD = cfg.NAME_MATCH_THRESHOLD + + +def _split_name(full_name: str) -> tuple[str | None, str | None, str | None]: + tokens = full_name.split() + if not tokens: + return None, None, None + if len(tokens) == 1: + return tokens[0], None, None + if len(tokens) == 2: + return tokens[0], None, tokens[1] + return tokens[0], " ".join(tokens[1:-1]), tokens[-1] + + +def _choose_name(aadhaar_name: str | None, pan_name: str | None) -> tuple[str | None, str]: + """Returns (chosen_name, source) where source is "AADHAAR" or "PAN".""" + if aadhaar_name and pan_name: + related = name_similarity(aadhaar_name, pan_name) >= FULLER_NAME_RELATEDNESS_THRESHOLD + if related and len(pan_name.split()) > len(aadhaar_name.split()): + return pan_name, "PAN" + return aadhaar_name, "AADHAAR" + if aadhaar_name: + return aadhaar_name, "AADHAAR" + if pan_name: + return pan_name, "PAN" + return None, "AADHAAR" + + +def build_golden_record(case: CaseInput) -> GoldenRecord: + golden = GoldenRecord() + + aadhaar_name = case.aadhaar.name if case.aadhaar else None + pan_name = case.pan.name if case.pan else None + chosen_name, name_source_type = _choose_name(aadhaar_name, pan_name) + + if chosen_name is not None: + golden.name = chosen_name + golden.name_source = case.aadhaar.doc_id if name_source_type == "AADHAAR" else case.pan.doc_id + golden.first_name, golden.middle_name, golden.last_name = _split_name(chosen_name) + + if case.aadhaar and case.aadhaar.address is not None: + golden.address = case.aadhaar.address + golden.address_source = case.aadhaar.doc_id + + if case.aadhaar and case.aadhaar.date_of_birth is not None: + golden.date_of_birth = case.aadhaar.date_of_birth + golden.dob_source = case.aadhaar.doc_id + + if case.aadhaar and case.aadhaar.aadhaar_number is not None: + golden.aadhaar_number = case.aadhaar.aadhaar_number + golden.aadhaar_source = case.aadhaar.doc_id + + if golden.address is None and case.address_proof and case.address_proof.address is not None: + golden.address = case.address_proof.address + golden.address_source = case.address_proof.doc_id + + if case.pan and case.pan.pan_number is not None: + golden.pan_number = case.pan.pan_number + golden.pan_source = case.pan.doc_id + + if golden.address: + golden.address_embedding = get_address_embedding(golden.address) + + return golden diff --git a/lending-poc/app/services/identity_validation.py b/lending-poc/app/services/identity_validation.py new file mode 100644 index 0000000..8b544a9 --- /dev/null +++ b/lending-poc/app/services/identity_validation.py @@ -0,0 +1,168 @@ +"""Checks every document agrees with the Golden Record: name, address, +Aadhaar, PAN, DOB. Also enforces that mandatory identity fields exist on +the Golden Record at all, regardless of why they're missing. +""" + +from app.matching import exact, fuzzy +from app.matching.embeddings import address_similarity +from app.services import validation_config as cfg +from app.services.dto import CaseInput, CheckType, GoldenRecord, ValidationResult + +MANDATORY_GOLDEN_FIELDS = { + CheckType.NAME: "name", + CheckType.AADHAAR: "aadhaar_number", + CheckType.PAN: "pan_number", + CheckType.DOB: "date_of_birth", +} + + +def check_mandatory_presence(golden: GoldenRecord) -> list[ValidationResult]: + results = [] + for check_type, field_name in MANDATORY_GOLDEN_FIELDS.items(): + if getattr(golden, field_name) is None: + results.append( + ValidationResult( + check_type=check_type, + passed=False, + score=0.0, + failure_reason="missing_in_golden_record", + ) + ) + return results + + +def _exact_result_to_validation(check_type: CheckType, outcome, document_id: str) -> ValidationResult: + passed = outcome.result == exact.MatchResult.MATCH + score = 100.0 if passed else (50.0 if outcome.result == exact.MatchResult.INCONCLUSIVE else 0.0) + return ValidationResult( + check_type=check_type, + passed=passed, + score=score, + document_id=document_id, + failure_reason=None if passed else outcome.reason, + ) + + +def validate_document_against_golden( + document_id: str, + doc_name: str | None, + doc_address: str | None, + doc_aadhaar: str | None, + doc_pan: str | None, + doc_dob, + golden: GoldenRecord, +) -> list[ValidationResult]: + results = [] + + if doc_name is not None and golden.name is not None: + score = fuzzy.name_similarity(golden.name, doc_name) + passed = score >= cfg.NAME_MATCH_THRESHOLD + results.append( + ValidationResult( + check_type=CheckType.NAME, + passed=passed, + score=score, + document_id=document_id, + failure_reason=None if passed else "name_below_threshold", + ) + ) + + if doc_address is not None and golden.address is not None: + similarity = address_similarity(golden.address, doc_address) + score = similarity * 100.0 + passed = similarity >= cfg.ADDRESS_SIMILARITY_THRESHOLD + results.append( + ValidationResult( + check_type=CheckType.ADDRESS, + passed=passed, + score=score, + document_id=document_id, + failure_reason=None if passed else "address_below_threshold", + ) + ) + + if doc_aadhaar is not None: + outcome = exact.aadhaar_match(golden.aadhaar_number, doc_aadhaar) + results.append(_exact_result_to_validation(CheckType.AADHAAR, outcome, document_id)) + + if doc_pan is not None: + outcome = exact.pan_match(golden.pan_number, doc_pan) + results.append(_exact_result_to_validation(CheckType.PAN, outcome, document_id)) + + if doc_dob is not None: + outcome = exact.dob_match(golden.date_of_birth, doc_dob) + results.append(_exact_result_to_validation(CheckType.DOB, outcome, document_id)) + + return results + + +def run_identity_validation(case: CaseInput, golden: GoldenRecord) -> list[ValidationResult]: + results: list[ValidationResult] = [] + results.extend(check_mandatory_presence(golden)) + + if case.aadhaar: + results.extend( + validate_document_against_golden( + document_id=case.aadhaar.doc_id, + doc_name=case.aadhaar.name, + doc_address=case.aadhaar.address, + doc_aadhaar=case.aadhaar.aadhaar_number, + doc_pan=None, + doc_dob=case.aadhaar.date_of_birth, + golden=golden, + ) + ) + + if case.pan: + results.extend( + validate_document_against_golden( + document_id=case.pan.doc_id, + doc_name=case.pan.name, + doc_address=None, + doc_aadhaar=None, + doc_pan=case.pan.pan_number, + doc_dob=None, + golden=golden, + ) + ) + + if case.address_proof: + results.extend( + validate_document_against_golden( + document_id=case.address_proof.doc_id, + doc_name=None, + doc_address=case.address_proof.address, + doc_aadhaar=None, + doc_pan=None, + doc_dob=None, + golden=golden, + ) + ) + + for slip in case.salary_slips: + results.extend( + validate_document_against_golden( + document_id=slip.doc_id, + doc_name=slip.name, + doc_address=None, + doc_aadhaar=None, + doc_pan=None, + doc_dob=None, + golden=golden, + ) + ) + + if case.bank_statement: + results.extend( + validate_document_against_golden( + document_id=case.bank_statement.doc_id, + doc_name=case.bank_statement.name, + doc_address=None, + doc_aadhaar=None, + doc_pan=None, + doc_dob=None, + golden=golden, + ) + ) + + return results diff --git a/lending-poc/app/services/persistence.py b/lending-poc/app/services/persistence.py new file mode 100644 index 0000000..6e41534 --- /dev/null +++ b/lending-poc/app/services/persistence.py @@ -0,0 +1,178 @@ +"""Persists one pipeline run (CaseInput + PipelineResult) to the database. + +The in-memory dataclasses in app.services.dto reference documents by their +string doc_id (e.g. "AADHAAR", "SALARY_SLIP-0"). This module inserts the +Document rows first and keeps a doc_id -> Document.id map so +ValidationResult.document_id (also a doc_id string) can be resolved to the +real foreign key. +""" + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.services.dto import CaseInput, Decision, DocType +from app.services.dto import PipelineResult as PipelineResultDTO +from app.utils.json_safe import json_safe +from db.models.case import Case, CaseStatus +from db.models.document import Document +from db.models.golden_record import GoldenRecord as GoldenRecordModel +from db.models.pipeline_result import PipelineResult as PipelineResultModel +from db.models.validation_result import ValidationResult as ValidationResultModel + +_DECISION_TO_CASE_STATUS = { + Decision.PASS: CaseStatus.PASS, + Decision.FAIL: CaseStatus.FAIL, + Decision.NEEDS_REVIEW: CaseStatus.NEEDS_REVIEW, +} + + +def _mask_pan(pan: str | None) -> str | None: + """Mask all but the last 4 characters of a PAN for JSONB storage. + + The full value is encrypted in the GoldenRecord; the document payload + only needs enough for audit trail without exposing the raw PAN. + """ + if not pan: + return pan + visible = min(4, len(pan)) + return "X" * (len(pan) - visible) + pan[-visible:] + + +def _document_rows(case: CaseInput) -> list[tuple[str, Document]]: + """Returns (doc_id, Document) pairs for every document present on the case.""" + rows: list[tuple[str, Document]] = [] + + if case.aadhaar: + rows.append(( + case.aadhaar.doc_id, + Document( + doc_type=DocType.AADHAAR, + source_file_ref=case.aadhaar.source_file_ref, + extracted_fields={ + "name": case.aadhaar.name, + "address": case.aadhaar.address, + "aadhaar_number": case.aadhaar.aadhaar_number, + "date_of_birth": case.aadhaar.date_of_birth.isoformat() if case.aadhaar.date_of_birth else None, + }, + ), + )) + + if case.pan: + rows.append(( + case.pan.doc_id, + Document( + doc_type=DocType.PAN, + source_file_ref=case.pan.source_file_ref, + extracted_fields={"name": case.pan.name, "pan_number": _mask_pan(case.pan.pan_number)}, + ), + )) + + if case.address_proof: + rows.append(( + case.address_proof.doc_id, + Document( + doc_type=DocType.ADDRESS_PROOF, + source_file_ref=case.address_proof.source_file_ref, + extracted_fields={"address": case.address_proof.address}, + ), + )) + + for slip in case.salary_slips: + rows.append(( + slip.doc_id, + Document( + doc_type=DocType.SALARY_SLIP, + source_file_ref=slip.source_file_ref, + extracted_fields={ + "name": slip.name, + "employer_name": slip.employer_name, + "net_salary": slip.net_salary, + "salary_month": slip.salary_month.isoformat() if slip.salary_month else None, + }, + ), + )) + + if case.bank_statement: + rows.append(( + case.bank_statement.doc_id, + Document( + doc_type=DocType.BANK_STATEMENT, + source_file_ref=case.bank_statement.source_file_ref, + extracted_fields={ + "name": case.bank_statement.name, + "transactions": [ + { + "narration": txn.narration, + "amount": txn.amount, + "date": txn.txn_date.isoformat() if txn.txn_date else None, + } + for txn in case.bank_statement.transactions + ], + }, + ), + )) + + return rows + + +async def save_pipeline_result( + db: AsyncSession, case_input: CaseInput, pipeline_result: PipelineResultDTO +) -> Case: + """Persists one pipeline run in a single transaction and returns the Case row. + + Caller owns the commit (via app.database.get_db, which commits on + success / rolls back on error). + """ + case = Case( + applicant_ref=case_input.applicant_ref, + status=_DECISION_TO_CASE_STATUS[pipeline_result.decision_result.decision], + ) + db.add(case) + await db.flush() # assigns case.id + + doc_id_to_pk: dict[str, uuid.UUID] = {} + for doc_id, document in _document_rows(case_input): + document.case_id = case.id + db.add(document) + await db.flush() # assigns document.id + doc_id_to_pk[doc_id] = document.id + + golden = pipeline_result.golden_record + if golden is not None: + db.add( + GoldenRecordModel( + case_id=case.id, + name=golden.name, + address=golden.address, + address_embedding=golden.address_embedding or None, + aadhaar_number=golden.aadhaar_number, + pan_number=golden.pan_number, + date_of_birth=golden.date_of_birth, + ) + ) + + for result in pipeline_result.validation_results: + db.add( + ValidationResultModel( + case_id=case.id, + document_id=doc_id_to_pk.get(result.document_id) if result.document_id else None, + check_type=result.check_type, + passed=result.passed, + score=result.score, + evidence=json_safe(result.evidence) if result.evidence else None, + ) + ) + + score_result = pipeline_result.score_result + db.add( + PipelineResultModel( + case_id=case.id, + overall_score=score_result.overall_score if score_result else 0.0, + decision=pipeline_result.decision_result.decision, + reasons=pipeline_result.decision_result.reasons, + ) + ) + + await db.flush() + return case diff --git a/lending-poc/app/services/pipeline.py b/lending-poc/app/services/pipeline.py new file mode 100644 index 0000000..bdee270 --- /dev/null +++ b/lending-poc/app/services/pipeline.py @@ -0,0 +1,56 @@ +"""Orchestrates Golden Record -> Identity -> Business -> Scoring -> +Decision in one call, with a simple in-memory audit log. +""" + +from app.services import business_validation, decision_engine, golden_record, scoring +from app.services import validation_config as cfg +from app.services.dto import CaseInput, Decision, DecisionResult, PipelineResult +from app.services.identity_validation import run_identity_validation + + +def run_pipeline(case: CaseInput) -> PipelineResult: + audit_log: list[str] = [] + audit_log.append(f"INGEST: case_created applicant_ref={case.applicant_ref}") + + present = case.present_doc_types() + missing = set(cfg.REQUIRED_DOCUMENT_TYPES) - present + if missing: + reasons = [f"MISSING_DOCUMENT:{doc_type}" for doc_type in sorted(missing)] + audit_log.append(f"PIPELINE: precheck_failed missing={sorted(missing)}") + decision_result = DecisionResult(decision=Decision.FAIL, reasons=reasons, overall_score=0.0) + return PipelineResult( + golden_record=None, + validation_results=[], + score_result=None, + decision_result=decision_result, + audit_log=audit_log, + ) + + golden = golden_record.build_golden_record(case) + audit_log.append( + f"GOLDEN_RECORD: built name={golden.name!r} address={golden.address!r}" + ) + + identity_results = run_identity_validation(case, golden) + audit_log.append(f"IDENTITY_VALIDATION: {len(identity_results)} checks run") + + business_results = business_validation.run_business_validation(case) + audit_log.append(f"BUSINESS_VALIDATION: {len(business_results)} checks run") + + all_results = identity_results + business_results + + score_result = scoring.compute_score(all_results) + audit_log.append(f"SCORING: overall_score={score_result.overall_score:.2f}") + + decision_result = decision_engine.make_decision(score_result, all_results) + audit_log.append( + f"DECISION: {decision_result.decision.value} reasons={decision_result.reasons}" + ) + + return PipelineResult( + golden_record=golden, + validation_results=all_results, + score_result=score_result, + decision_result=decision_result, + audit_log=audit_log, + ) diff --git a/lending-poc/app/services/scoring.py b/lending-poc/app/services/scoring.py new file mode 100644 index 0000000..6f79533 --- /dev/null +++ b/lending-poc/app/services/scoring.py @@ -0,0 +1,34 @@ +"""Combines all validation checks into one weighted confidence number.""" + +from collections import defaultdict + +from app.services import validation_config as cfg +from app.services.dto import ScoreResult, ValidationResult + + +def compute_score(validation_results: list[ValidationResult]) -> ScoreResult: + scores_by_type: dict[str, list[float]] = defaultdict(list) + for result in validation_results: + scores_by_type[result.check_type.value].append(result.score) + + mean_by_type = { + check_type: sum(scores) / len(scores) for check_type, scores in scores_by_type.items() + } + + observed_weight_total = sum( + weight for check_type, weight in cfg.VALIDATION_WEIGHTS.items() if check_type in mean_by_type + ) + + if observed_weight_total == 0: + return ScoreResult(overall_score=0.0, component_scores={}) + + overall_score = ( + sum( + cfg.VALIDATION_WEIGHTS[check_type] * mean_by_type[check_type] + for check_type in mean_by_type + if check_type in cfg.VALIDATION_WEIGHTS + ) + / observed_weight_total + ) + + return ScoreResult(overall_score=overall_score, component_scores=mean_by_type) diff --git a/lending-poc/app/services/validation_config.py b/lending-poc/app/services/validation_config.py new file mode 100644 index 0000000..e306d5c --- /dev/null +++ b/lending-poc/app/services/validation_config.py @@ -0,0 +1,40 @@ +"""Tunable thresholds/weights for the validation pipeline. + +Plain constants for this standalone, no-DB run. When this integrates with +the rest of the app, these move into app/config.py (pydantic Settings, +loaded from .env) without changing any service logic that reads them. +""" + +REQUIRED_DOCUMENT_TYPES = ["AADHAAR", "PAN", "SALARY_SLIP", "BANK_STATEMENT"] + +NAME_MATCH_THRESHOLD = 85.0 +EMPLOYER_MATCH_THRESHOLD = 80.0 +ADDRESS_SIMILARITY_THRESHOLD = 0.55 # cosine, 0-1 (stub embeddings are coarser than real ones) + +SALARY_CREDIT_EXTRA_MONTHS = 1 +SALARY_CREDIT_BUFFER_DAYS = 5 +TXN_SELECTION_EMPLOYER_WEIGHT = 0.70 +TXN_SELECTION_AMOUNT_WEIGHT = 0.30 +TXN_SELECTION_MIN_SCORE = 60.0 + +# A transaction must land within this percentage of the slip's declared +# net_salary to be eligible as a salary-credit match AT ALL, independent +# of how well its narration scores. This is what stops a same-employer +# reimbursement/bonus/advance with a coincidentally close amount from +# masking a genuinely missing salary credit -- the gate is on the amount +# itself, not on narration keywords (which don't generalize across +# employers/languages/formats). +SALARY_AMOUNT_TOLERANCE_PCT = 3.0 + +VALIDATION_WEIGHTS = { + "NAME": 0.15, + "ADDRESS": 0.10, + "AADHAAR": 0.15, + "PAN": 0.15, + "DOB": 0.10, + "EMPLOYER": 0.10, + "SALARY_CREDIT_COUNT": 0.25, +} + +DECISION_PASS_THRESHOLD = 90.0 +DECISION_FAIL_THRESHOLD = 60.0 # below -> FAIL, between -> NEEDS_REVIEW diff --git a/lending-poc/app/utils/json_safe.py b/lending-poc/app/utils/json_safe.py new file mode 100644 index 0000000..06825a5 --- /dev/null +++ b/lending-poc/app/utils/json_safe.py @@ -0,0 +1,19 @@ +"""Recursively converts dataclasses/dates/tuples into plain +JSON-serializable values, since validation/scoring build evidence out of +dataclasses (e.g. BankTransaction) and date objects for in-memory use. +""" + +import dataclasses +import datetime + + +def json_safe(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return {k: json_safe(v) for k, v in dataclasses.asdict(value).items()} + if isinstance(value, (datetime.date, datetime.datetime)): + return value.isoformat() + if isinstance(value, dict): + return {k: json_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [json_safe(v) for v in value] + return value diff --git a/lending-poc/db/config.py b/lending-poc/db/config.py index 7109786..20e4dba 100644 --- a/lending-poc/db/config.py +++ b/lending-poc/db/config.py @@ -6,7 +6,7 @@ class Settings(BaseSettings): DEBUG: bool = False DATABASE_URL: str - ENCRYPTION_KEY: str = "" + ENCRYPTION_KEY: str settings = Settings() diff --git a/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py b/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py new file mode 100644 index 0000000..3189c29 --- /dev/null +++ b/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py @@ -0,0 +1,25 @@ +"""make document source_file_ref nullable + +Revision ID: 0006_doc_source_ref_nullable +Revises: 0005_add_validation_results +Create Date: 2026-08-13 00:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "0006_doc_source_ref_nullable" +down_revision: Union[str, None] = "0005_add_validation_results" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("documents", "source_file_ref", existing_type=sa.String(), nullable=True) + + +def downgrade() -> None: + op.alter_column("documents", "source_file_ref", existing_type=sa.String(), nullable=False) diff --git a/lending-poc/db/models/document.py b/lending-poc/db/models/document.py index 2a4069e..de7f35b 100644 --- a/lending-poc/db/models/document.py +++ b/lending-poc/db/models/document.py @@ -18,7 +18,7 @@ class Document(Base): ) doc_type: Mapped[DocType] = mapped_column(SAEnum(DocType, name="doc_type"), nullable=False) extracted_fields: Mapped[dict] = mapped_column(JSONB, nullable=False) - source_file_ref: Mapped[str] = mapped_column(String, nullable=False) + source_file_ref: Mapped[str | None] = mapped_column(String, nullable=True) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) case: Mapped["Case"] = relationship(back_populates="documents") diff --git a/lending-poc/docs/cases_api.md b/lending-poc/docs/cases_api.md new file mode 100644 index 0000000..9bca958 --- /dev/null +++ b/lending-poc/docs/cases_api.md @@ -0,0 +1,124 @@ +# `/cases` API + +## What this PR does + +Adds the `POST /cases` endpoint, the first user-facing entry point into the lending validation pipeline. It accepts an applicant's KYC and income documents (Aadhaar, PAN, address proof, salary slips, bank statement) as JSON, runs them through identity and business validation, computes an overall confidence score, and returns a PASS / FAIL / NEEDS_REVIEW decision. The full case, its documents, golden record, and validation results are persisted to the database in one transaction. + +## Endpoint + +`POST /cases` + +### Request + +```json +{ + "applicant_ref": "APP-2026-00134", + "documents": [ + { + "doc_type": "AADHAAR", + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "address": "Flat 204, Green Heights, Baner, Pune, Maharashtra 411045", + "aadhaar_number": "XXXX XXXX 4321", + "date_of_birth": "1995-03-14" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/aadhaar_front.pdf" + }, + { + "doc_type": "PAN", + "extracted_fields": { + "name": "Sneha Lokhande", + "pan_number": "ABCDE1234F" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/pan_card.pdf" + }, + { + "doc_type": "ADDRESS_PROOF", + "extracted_fields": { + "address": "Apartment 204, Green Heights, Baner, Pune, MH 411045" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/address_proof.pdf" + }, + { + "doc_type": "SALARY_SLIP", + "salary_slips": [ + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "ABC Technologies Pvt Ltd", + "net_salary": 75000, + "salary_month": "2026-03" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_march.pdf" + } + ] + }, + { + "doc_type": "BANK_STATEMENT", + "extracted_fields": { + "name": "Lokhande S. S.", + "transactions": [ + { "narration": "ABC TECHNOLOGIES SALARY MAR", "amount": 75000, "date": "2026-04-01" }, + { "narration": "HOUSE RENT EMI DEBIT", "amount": -18000, "date": "2026-04-03" } + ] + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/bank_statement_mar_to_jul.pdf" + } + ] +} +``` + +Notes: +- `applicant_ref` and `documents` are required. +- Every document needs `doc_type` and `extracted_fields`; `source_file_ref` is optional. +- `SALARY_SLIP` is the only `doc_type` that carries a `salary_slips` array instead of a flat `extracted_fields` — a case can include multiple salary slips (one per month). +- Required document types for a case to proceed: `AADHAAR`, `PAN`, `SALARY_SLIP`, `BANK_STATEMENT`. `ADDRESS_PROOF` is optional (used as an address fallback). + +### Response `200 OK` + +```json +{ + "case_id": "6e2f6c2a-6a3a-4c1d-9a4b-6f2b6c2a6a3a", + "applicant_ref": "APP-2026-00134", + "decision": "NEEDS_REVIEW", + "overall_score": 88.4, + "reasons": [ + "SALARY_DATE:no_matching_credit_in_window", + "EMPLOYER:employer_narration_mismatch" + ], + "validation_results": [ + { + "check_type": "NAME", + "passed": true, + "score": 94.5, + "document_id": "PAN", + "evidence": null + }, + { + "check_type": "SALARY_DATE", + "passed": false, + "score": 0.0, + "document_id": "SALARY_SLIP-3", + "evidence": { "window": ["2026-05-27", "2026-07-31"] } + } + ] +} +``` + +| Field | Type | Description | +|---|---|---| +| `case_id` | string (UUID) | Primary key of the persisted `Case` row. | +| `applicant_ref` | string | Echoed back from the request. | +| `decision` | string | One of `PASS`, `FAIL`, `NEEDS_REVIEW`. | +| `overall_score` | float | Weighted score (0–100) across all validation checks. | +| `reasons` | string[] | Why this decision was reached (e.g. failing checks, or which mandatory field is missing). | +| `validation_results` | array | One entry per check run, with `check_type`, `passed`, `score`, the `document_id` it applies to, and any supporting `evidence`. | + +### Error responses + +| Status | When | +|---|---| +| `400 Bad Request` | The request body fails semantic parsing in `parse_case` (e.g. malformed/missing required fields inside `extracted_fields`). | +| `422 Unprocessable Entity` | The request body fails schema validation (wrong types, missing `applicant_ref`/`documents`). | + +If any of `AADHAAR`, `PAN`, `SALARY_SLIP`, `BANK_STATEMENT` is missing from `documents`, the pipeline still returns `200 OK` with `decision: "FAIL"` and reasons like `MISSING_DOCUMENT:PAN` — this is a business decision, not an HTTP error. diff --git a/lending-poc/docs/features.md b/lending-poc/docs/features.md new file mode 100644 index 0000000..4b5be5f --- /dev/null +++ b/lending-poc/docs/features.md @@ -0,0 +1,149 @@ +# Lending POC — Feature Guide + +This document explains, in detail, what the validation pipeline behind `POST /cases` does and how each piece works. See [cases_api.md](cases_api.md) for the request/response contract. + +## 1. Pipeline overview + +`app/services/pipeline.py` orchestrates one end-to-end run in this order: + +``` +INGEST -> required-document precheck -> GOLDEN RECORD -> IDENTITY VALIDATION + -> BUSINESS VALIDATION -> SCORING -> DECISION +``` + +Every step appends a line to an in-memory `audit_log`, giving a readable trail of what happened for a given case (currently returned internally on `PipelineResult`, not yet exposed on the API response). + +### 1.1 Required-document precheck + +Before anything else runs, the pipeline checks that all of `AADHAAR`, `PAN`, `SALARY_SLIP`, `BANK_STATEMENT` are present (`validation_config.REQUIRED_DOCUMENT_TYPES`). If any are missing, the pipeline short-circuits with `decision=FAIL`, `overall_score=0.0`, and reasons like `MISSING_DOCUMENT:PAN` — no golden record or checks are computed. + +## 2. Golden Record (`app/services/golden_record.py`) + +The Golden Record is the single trusted identity profile for the applicant, built by merging the KYC documents: + +- **Address & DOB**: sourced from `AADHAAR`. If Aadhaar has no address, `ADDRESS_PROOF` is used as a fallback. +- **Aadhaar number**: from `AADHAAR`. +- **PAN number**: from `PAN`. +- **Name**: the more interesting case. + - If only one of Aadhaar/PAN has a name, that one is used. + - If both have a name, and they're recognizably the same person (`fuzzy.name_similarity` >= `NAME_MATCH_THRESHOLD`, 85), the **fuller** name (more tokens) wins — e.g. "Sneha Sunil Lokhande" over "Sneha Lokhande" — because it carries strictly more identity information. + - If the two names *aren't* recognizably related, Aadhaar stays authoritative and the mismatch is left for the NAME identity check to flag, rather than silently trusting an unrelated "fuller" name. +- The chosen name is split into `first_name` / `middle_name` / `last_name`. +- If an address was resolved, an address embedding is computed (`app.matching.embeddings.get_address_embedding`) and stored for later similarity checks. + +Each golden field also records its `*_source` (which document it came from), useful for traceability. + +## 3. Identity Validation (`app/services/identity_validation.py`) + +Two parts: + +### 3.1 Mandatory presence check + +Regardless of *why* a field is missing, the Golden Record must have `name`, `aadhaar_number`, `pan_number`, and `date_of_birth`. Any missing field produces a failed `ValidationResult` with `failure_reason="missing_in_golden_record"` — and, critically, this is one of the few failures that can force an overall `FAIL` decision outright (see §6). + +### 3.2 Per-document cross-checks against the Golden Record + +Every document that carries an identity field is compared against the Golden Record: + +| Document | Fields checked | +|---|---| +| AADHAAR | name, address, aadhaar_number, DOB | +| PAN | name, pan_number | +| ADDRESS_PROOF | address | +| Each SALARY_SLIP | name | +| BANK_STATEMENT | name | + +Matching strategies (`app/matching/`): +- **NAME** — fuzzy string similarity (`fuzzy.name_similarity`), handles reordering (surname-first), initials, and minor spelling differences. Passes at >= 85. +- **ADDRESS** — embedding cosine similarity (`embeddings.address_similarity`), tolerant of differently-worded but equivalent addresses (e.g. "Apartment" vs "Flat", "MH" vs "Maharashtra"). Passes at >= 0.55 similarity (scored as similarity × 100). +- **AADHAAR / PAN / DOB** — exact matching (`app.matching.exact`). Result is `MATCH` (score 100), `NO_MATCH` (score 0), or `INCONCLUSIVE` (score 50, e.g. one side missing/unparseable). + +## 4. Business Validation (`app/services/business_validation.py`) + +Verifies that declared income (salary slips) is corroborated by actual bank activity. Only runs if both salary slips and a bank statement are present. + +### 4.1 Salary-credit matching (`SALARY_DATE` check) + +For each salary slip: +1. A **month-level search window** is built around the slip's declared `salary_month`: starts `SALARY_CREDIT_BUFFER_DAYS` (5) days before the month begins, and extends `SALARY_CREDIT_EXTRA_MONTHS` (1) month past it. No specific payroll day is assumed. +2. Within that window, candidate transactions must be **credits** (`amount > 0`) and within `SALARY_AMOUNT_TOLERANCE_PCT` (3%) of the slip's declared `net_salary`. This amount gate is applied *before* narration scoring — it's what prevents a same-employer reimbursement/bonus with a coincidentally close amount from masking a genuinely missing salary credit. +3. Among eligible candidates, each is scored as a weighted blend of employer-name similarity to the transaction narration (`TXN_SELECTION_EMPLOYER_WEIGHT`, 0.70) and amount closeness (`TXN_SELECTION_AMOUNT_WEIGHT`, 0.30). The highest-scoring candidate is selected, provided its score clears `TXN_SELECTION_MIN_SCORE` (60). +4. Once a transaction is claimed by a slip, it's excluded from consideration for other slips in the same case (prevents one bank credit being counted as evidence for two different months, which can happen since windows overlap). +5. If no eligible transaction is found, the check fails with `failure_reason="no_matching_credit_in_window"`. + +Slips are resolved in chronological order (earliest `salary_month` first) so earlier months get first claim on ambiguous transactions, but results are returned in the original request order. + +### 4.2 Employer consistency (`EMPLOYER` check) + +Each slip's declared `employer_name` is compared — via fuzzy similarity — only against the narration of **that same slip's own matched transaction** (never against another month's slip or employer). This is intentional: a legitimate employer switch mid-history (e.g. a job change) should not penalize either month. Passes at similarity >= `EMPLOYER_MATCH_THRESHOLD` (80). If the slip had no matched transaction to begin with, this check automatically fails with `failure_reason="no_matching_credit_to_verify_employer_against"`. + +### 4.3 Salary credit count (`SALARY_CREDIT_COUNT` check) + +An aggregate check: `matched_slips / total_slips × 100`. It passes only if *every* slip matched a transaction, but a partial match (e.g. 3 of 4 months) doesn't hard-fail the case — it only lowers this component's score, which feeds into the weighted overall score. Evidence includes the bank statement's observed date range and match counts. + +## 5. Scoring (`app/services/scoring.py`) + +Given every `ValidationResult` produced above: +1. Scores are grouped by `check_type` and averaged (e.g. if 4 salary slips each produced a `SALARY_DATE` score, they're averaged into one `SALARY_DATE` component score). +2. Each component is weighted per `VALIDATION_WEIGHTS`: + + | Check | Weight | + |---|---| + | NAME | 0.15 | + | ADDRESS | 0.10 | + | AADHAAR | 0.15 | + | PAN | 0.15 | + | DOB | 0.10 | + | EMPLOYER | 0.10 | + | SALARY_CREDIT_COUNT | 0.25 | + +3. The overall score is the weighted average, **renormalized over only the check types actually observed** in this case (so a case missing an optional check type doesn't get unfairly diluted by a zero for a check that never ran). Note `SALARY_DATE` itself isn't in the weight table — it gates whether a credit was found at all, but the weighted score is driven by `EMPLOYER` and `SALARY_CREDIT_COUNT`. + +## 6. Decision Engine (`app/services/decision_engine.py`) + +Final decision logic, in priority order: + +1. **Hard FAIL** — if any mandatory identity field (`NAME`, `AADHAAR`, `PAN`, `DOB`) is missing from the Golden Record entirely (`failure_reason="missing_in_golden_record"`), the case fails immediately regardless of score. Reasons: `MANDATORY_FIELD_MISSING:`. +2. **PASS** — if `overall_score >= DECISION_PASS_THRESHOLD` (90). +3. **FAIL** — if `overall_score < DECISION_FAIL_THRESHOLD` (60). +4. **NEEDS_REVIEW** — anything in between (60–90). Reasons list every individual failing check as `:`. + +## 7. Persistence (`app/services/persistence.py`) + +A successful pipeline run is persisted in a single DB transaction: +- One `Case` row (`applicant_ref`, `status` derived from the decision: PASS/FAIL/NEEDS_REVIEW). +- One `Document` row per submitted document (including one per salary slip), storing `extracted_fields` as JSON. +- One `GoldenRecord` row (name, address + embedding, Aadhaar/PAN numbers, DOB). +- One `ValidationResult` row per check performed, linked back to the specific document it was evaluated against where applicable. +- One `PipelineResult` row with the overall score, decision, and reasons. + +Document primary keys are resolved via an in-memory `doc_id -> Document.id` map so validation results (which reference documents by string `doc_id` like `"SALARY_SLIP-2"`) can be foreign-keyed correctly. + +## 8. Supported document types + +| `doc_type` | Purpose | +|---|---| +| `AADHAAR` | Primary identity source (name, address, DOB, Aadhaar number) | +| `PAN` | Secondary identity source (name, PAN number) | +| `ADDRESS_PROOF` | Address fallback if Aadhaar has none | +| `SALARY_SLIP` | Declared income; multiple allowed per case (one per month) | +| `BANK_STATEMENT` | Source of truth for actual salary credits | + +## 9. Configuration reference (`app/services/validation_config.py`) + +All thresholds/weights are centralized here as plain constants (intended to move into `app/config.py` / environment-driven settings as the app matures, without touching service logic): + +| Constant | Value | Meaning | +|---|---|---| +| `NAME_MATCH_THRESHOLD` | 85.0 | Min fuzzy score for NAME to pass | +| `EMPLOYER_MATCH_THRESHOLD` | 80.0 | Min fuzzy score for EMPLOYER to pass | +| `ADDRESS_SIMILARITY_THRESHOLD` | 0.55 | Min cosine similarity for ADDRESS to pass | +| `SALARY_CREDIT_EXTRA_MONTHS` | 1 | Months the salary-credit search window extends past the declared month | +| `SALARY_CREDIT_BUFFER_DAYS` | 5 | Days the window starts before the declared month | +| `TXN_SELECTION_EMPLOYER_WEIGHT` | 0.70 | Weight of narration similarity in transaction selection | +| `TXN_SELECTION_AMOUNT_WEIGHT` | 0.30 | Weight of amount closeness in transaction selection | +| `TXN_SELECTION_MIN_SCORE` | 60.0 | Min blended score for a transaction to be selected | +| `SALARY_AMOUNT_TOLERANCE_PCT` | 3.0 | Max % difference between transaction amount and declared net salary to be eligible at all | +| `DECISION_PASS_THRESHOLD` | 90.0 | Min overall score for PASS | +| `DECISION_FAIL_THRESHOLD` | 60.0 | Below this, FAIL; between this and PASS threshold, NEEDS_REVIEW | +| `REQUIRED_DOCUMENT_TYPES` | AADHAAR, PAN, SALARY_SLIP, BANK_STATEMENT | Documents that must be present for the pipeline to proceed | diff --git a/lending-poc/pyproject.toml b/lending-poc/pyproject.toml index 268ce22..babdcb8 100644 --- a/lending-poc/pyproject.toml +++ b/lending-poc/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "python-dotenv>=1.0", "pgvector>=0.3", "cryptography>=43.0", + "sentence-transformers>=3.0", + "rapidfuzz>=3.0", ] [project.optional-dependencies] diff --git a/lending-poc/scripts/sample_case.json b/lending-poc/scripts/sample_case.json new file mode 100644 index 0000000..7ee78f3 --- /dev/null +++ b/lending-poc/scripts/sample_case.json @@ -0,0 +1,136 @@ +{ + "_test_design_notes": { + "purpose": "Realistic mixed-signal case for one applicant, NOT a clean happy-path. Exercises pass, fail, and boundary conditions across identity and business validation in a single run. These notes describe VERIFIED behavior (actually run through the pipeline), not assumptions -- update them if the matching logic changes.", + "planted_conditions": { + "AADHAAR_name": "Sneha Sunil Lokhande (full 3-token name, becomes golden name source) -> PASS", + "PAN_name": "Sneha Lokhande (2 tokens, shorter than Aadhaar -> Aadhaar stays golden per fuller-name rule) -> PASS", + "ADDRESS_PROOF": "worded differently (Apartment vs Flat, MH vs Maharashtra) -> PASS via embedding similarity (~77, above the 55 threshold), not exact match", + "BANK_STATEMENT_name": "Lokhande S. S. (surname-first + initials) -> PASS, tests initials-expansion with reordering", + "MARCH_slip": "clean month, ABC Technologies, exact amount -> SALARY_DATE PASS (86.0), EMPLOYER PASS (80.0, right at the threshold -- 4-word company name vs narration missing 'Pvt Ltd' keeps token_set_ratio near the boundary, not a clean 100)", + "APRIL_slip": "legitimate employer switch mid-history (ABC Technologies -> Nimbus Retail Pvt Ltd), bank narration reflects the new employer correctly -> SALARY_DATE PASS (83.5), but EMPLOYER actually FAILS (76.47, just under the 80 threshold): 'Nimbus Retail' is shorter than 'ABC Technologies', so the missing 'Pvt Ltd' tokens cost proportionally more of the token_set_ratio score. Intentionally left as-is rather than tuned to pass -- demonstrates the matcher is sensitive to company-name length, a real calibration edge worth knowing about, not a hidden bug.", + "MAY_slip": "reimbursement decoy in the same window as the real salary credit (4500 vs expected 78000) -> the real salary credit is correctly selected over the reimbursement (SALARY_DATE PASS, 83.5); EMPLOYER also fails here for the same short-company-name reason as April (76.47)", + "JUNE_slip": "genuinely missing bank credit -- no transaction anywhere near the window matches -- SALARY_DATE FAILS for this slip specifically (0.0, reason=no_matching_credit_in_window), EMPLOYER also fails as an unavoidable consequence (nothing to verify employer against). Contributes to a lowered SALARY_CREDIT_COUNT (3/4 = 75%) but does NOT hard-fail the whole case, per spec -- SALARY_CREDIT_COUNT only feeds the weighted score.", + "noise_transactions": "a rent debit, two small unrelated debits, and an unrelated company's credit (QUANTUM CONSULTING REFERRAL BONUS) sitting outside every slip's window -- all correctly ignored/never selected as a match", + "amount_tolerance_gate": "SALARY_AMOUNT_TOLERANCE_PCT (3%) in validation_config.py hard-excludes any transaction whose amount is not within 3% of the slip's declared net_salary from even being a match candidate, regardless of narration/employer score. This is why the May reimbursement (4500 vs expected 78000, ~94% off) can never be mistaken for the real salary credit -- and why a decoy transaction closer in amount (tested separately in edge_case_scenarios.py) is also correctly excluded rather than relying on narration keywords like 'REIMBURSEMENT', which don't generalize." + }, + "expected_overall_decision": "NEEDS_REVIEW at ~88.4 overall score -- driven by the June missing credit plus two borderline EMPLOYER checks, not a single dominant failure. Re-verify this note against actual output after any change to matching/scoring/decision code: run `python scripts/run_demo.py` and diff the printed Decision block against this note." + }, + "applicant_ref": "APP-2026-00134", + "documents": [ + { + "doc_type": "AADHAAR", + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "address": "Flat 204, Green Heights, Baner, Pune, Maharashtra 411045", + "aadhaar_number": "XXXX XXXX 4321", + "date_of_birth": "1995-03-14" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/aadhaar_front.pdf" + }, + { + "doc_type": "PAN", + "extracted_fields": { + "name": "Sneha Lokhande", + "pan_number": "ABCDE1234F" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/pan_card.pdf" + }, + { + "doc_type": "ADDRESS_PROOF", + "extracted_fields": { + "address": "Apartment 204, Green Heights, Baner, Pune, MH 411045" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/address_proof.pdf" + }, + { + "doc_type": "SALARY_SLIP", + "salary_slips": [ + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "ABC Technologies Pvt Ltd", + "net_salary": 75000, + "salary_month": "2026-03" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_march.pdf" + }, + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "Nimbus Retail Pvt Ltd", + "net_salary": 79000, + "salary_month": "2026-04" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_april.pdf" + }, + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "Nimbus Retail Pvt Ltd", + "net_salary": 78000, + "salary_month": "2026-05" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_may.pdf" + }, + { + "extracted_fields": { + "name": "Sneha Sunil Lokhande", + "employer_name": "Nimbus Retail Pvt Ltd", + "net_salary": 78000, + "salary_month": "2026-06" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/salary_slip_june.pdf" + } + ] + }, + { + "doc_type": "BANK_STATEMENT", + "extracted_fields": { + "name": "Lokhande S. S.", + "transactions": [ + { + "narration": "ABC TECHNOLOGIES SALARY MAR", + "amount": 75000, + "date": "2026-04-01" + }, + { + "narration": "HOUSE RENT EMI DEBIT", + "amount": -18000, + "date": "2026-04-03" + }, + { + "narration": "NIMBUS RETAIL SALARY APR", + "amount": 79000, + "date": "2026-05-02" + }, + { + "narration": "ZEPTO MART PAYMENT", + "amount": -620, + "date": "2026-05-08" + }, + { + "narration": "NIMBUS RETAIL SALARY MAY", + "amount": 78000, + "date": "2026-06-02" + }, + { + "narration": "NIMBUS RETAIL REIMBURSEMENT TRAVEL", + "amount": 4500, + "date": "2026-06-05" + }, + { + "narration": "SWIGGY ORDER PAYMENT", + "amount": -450, + "date": "2026-06-10" + }, + { + "narration": "QUANTUM CONSULTING REFERRAL BONUS", + "amount": 15000, + "date": "2026-07-04" + } + ] + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00123/bank_statement_mar_to_jul.pdf" + } + ] +}