From 808272b7e2f666d8f0edea1fd0cdc7aff5197bbf Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Mon, 10 Aug 2026 21:35:52 +0530 Subject: [PATCH 1/4] feat: DB migrations and DB setup --- lending-poc/.gitignore | 1 + .../versions/.gitkeep => db/__init__.py} | 0 lending-poc/{ => db}/alembic.ini | 4 +- lending-poc/db/config.py | 12 ++ lending-poc/db/database.py | 26 +++++ lending-poc/{alembic => db/migrations}/env.py | 5 +- .../{alembic => db/migrations}/script.py.mako | 0 ...f16_add_cases_documents_golden_records_.py | 106 ++++++++++++++++++ lending-poc/db/models/__init__.py | 19 ++++ lending-poc/db/models/case.py | 42 +++++++ lending-poc/db/models/document.py | 25 +++++ lending-poc/db/models/enums.py | 33 ++++++ lending-poc/db/models/golden_record.py | 36 ++++++ lending-poc/db/models/pipeline_result.py | 45 ++++++++ lending-poc/db/models/types.py | 26 +++++ lending-poc/db/models/validation_result.py | 29 +++++ lending-poc/docker-compose.yml | 4 +- lending-poc/pyproject.toml | 2 + 18 files changed, 409 insertions(+), 6 deletions(-) rename lending-poc/{alembic/versions/.gitkeep => db/__init__.py} (100%) rename lending-poc/{ => db}/alembic.ini (88%) create mode 100644 lending-poc/db/config.py create mode 100644 lending-poc/db/database.py rename lending-poc/{alembic => db/migrations}/env.py (89%) rename lending-poc/{alembic => db/migrations}/script.py.mako (100%) create mode 100644 lending-poc/db/migrations/versions/12522c432f16_add_cases_documents_golden_records_.py create mode 100644 lending-poc/db/models/__init__.py create mode 100644 lending-poc/db/models/case.py create mode 100644 lending-poc/db/models/document.py create mode 100644 lending-poc/db/models/enums.py create mode 100644 lending-poc/db/models/golden_record.py create mode 100644 lending-poc/db/models/pipeline_result.py create mode 100644 lending-poc/db/models/types.py create mode 100644 lending-poc/db/models/validation_result.py diff --git a/lending-poc/.gitignore b/lending-poc/.gitignore index 9aa5e0c..a9f203b 100644 --- a/lending-poc/.gitignore +++ b/lending-poc/.gitignore @@ -9,3 +9,4 @@ build/ .mypy_cache/ .pytest_cache/ .ruff_cache/ +venv/ \ No newline at end of file diff --git a/lending-poc/alembic/versions/.gitkeep b/lending-poc/db/__init__.py similarity index 100% rename from lending-poc/alembic/versions/.gitkeep rename to lending-poc/db/__init__.py diff --git a/lending-poc/alembic.ini b/lending-poc/db/alembic.ini similarity index 88% rename from lending-poc/alembic.ini rename to lending-poc/db/alembic.ini index 40234c6..87d079e 100644 --- a/lending-poc/alembic.ini +++ b/lending-poc/db/alembic.ini @@ -1,6 +1,6 @@ [alembic] -script_location = alembic -prepend_sys_path = . +script_location = %(here)s/migrations +prepend_sys_path = %(here)s/.. sqlalchemy.url = driver://user:pass@localhost/dbname [loggers] diff --git a/lending-poc/db/config.py b/lending-poc/db/config.py new file mode 100644 index 0000000..77e2805 --- /dev/null +++ b/lending-poc/db/config.py @@ -0,0 +1,12 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + DEBUG: bool = False + DATABASE_URL: str = "postgresql+asyncpg://postgres:postgres@localhost:5432/lending_poc" + ENCRYPTION_KEY: str = "" + + +settings = Settings() diff --git a/lending-poc/db/database.py b/lending-poc/db/database.py new file mode 100644 index 0000000..b6e3e58 --- /dev/null +++ b/lending-poc/db/database.py @@ -0,0 +1,26 @@ +from collections.abc import AsyncGenerator + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.orm import DeclarativeBase + +from db.config import settings + +engine = create_async_engine(settings.DATABASE_URL, echo=settings.DEBUG) + +async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + +class Base(DeclarativeBase): + pass + + +async def get_db() -> AsyncGenerator[AsyncSession, None]: + session = async_session() + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + finally: + await session.close() diff --git a/lending-poc/alembic/env.py b/lending-poc/db/migrations/env.py similarity index 89% rename from lending-poc/alembic/env.py rename to lending-poc/db/migrations/env.py index 4a460b2..c9071f9 100644 --- a/lending-poc/alembic/env.py +++ b/lending-poc/db/migrations/env.py @@ -4,8 +4,9 @@ from alembic import context from sqlalchemy.ext.asyncio import create_async_engine -from app.config import settings -from app.database import Base +from db.config import settings +from db.database import Base +import db.models # noqa: F401 (registers models on Base.metadata for autogenerate) config = context.config diff --git a/lending-poc/alembic/script.py.mako b/lending-poc/db/migrations/script.py.mako similarity index 100% rename from lending-poc/alembic/script.py.mako rename to lending-poc/db/migrations/script.py.mako diff --git a/lending-poc/db/migrations/versions/12522c432f16_add_cases_documents_golden_records_.py b/lending-poc/db/migrations/versions/12522c432f16_add_cases_documents_golden_records_.py new file mode 100644 index 0000000..4b1ed75 --- /dev/null +++ b/lending-poc/db/migrations/versions/12522c432f16_add_cases_documents_golden_records_.py @@ -0,0 +1,106 @@ +"""add cases, documents, golden_records, validation_results, pipeline_results + +Revision ID: 12522c432f16 +Revises: +Create Date: 2026-08-10 11:51:33.283278 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +import pgvector.sqlalchemy +import db.models.types + +# revision identifiers, used by Alembic. +revision: str = '12522c432f16' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute('CREATE EXTENSION IF NOT EXISTS vector') + op.create_table('cases', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('applicant_ref', sa.String(), nullable=False), + sa.Column('status', sa.Enum('RECEIVED', 'RUNNING', 'PASS', 'FAIL', 'NEEDS_REVIEW', name='case_status'), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_cases_applicant_ref'), 'cases', ['applicant_ref'], unique=True) + op.create_table('documents', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('case_id', sa.UUID(), nullable=False), + sa.Column('doc_type', sa.Enum('AADHAAR', 'PAN', 'ADDRESS_PROOF', 'SALARY_SLIP', 'BANK_STATEMENT', name='doc_type'), nullable=False), + sa.Column('extracted_fields', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('source_file_ref', sa.String(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_documents_case_id'), 'documents', ['case_id'], unique=False) + op.create_table('golden_records', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('case_id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('address', sa.String(), nullable=True), + sa.Column('address_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=384), nullable=True), + sa.Column('aadhaar_number', db.models.types.EncryptedString(), nullable=True), + sa.Column('pan_number', db.models.types.EncryptedString(), nullable=True), + sa.Column('date_of_birth', sa.Date(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('case_id') + ) + op.create_table('pipeline_results', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('case_id', sa.UUID(), nullable=False), + sa.Column('overall_score', sa.Float(), nullable=False), + sa.Column('decision', sa.Enum('PASS', 'FAIL', 'NEEDS_REVIEW', name='decision'), nullable=False), + sa.Column('reasons', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('reviewer', sa.String(), nullable=True), + sa.Column('review_status', sa.Enum('PENDING', 'APPROVED', 'REJECTED', name='review_status'), nullable=True), + sa.Column('reviewer_remarks', sa.Text(), nullable=True), + sa.Column('reviewed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_pipeline_results_case_id'), 'pipeline_results', ['case_id'], unique=False) + op.create_table('validation_results', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('case_id', sa.UUID(), nullable=False), + sa.Column('document_id', sa.UUID(), nullable=True), + sa.Column('check_type', sa.Enum('NAME', 'ADDRESS', 'AADHAAR', 'PAN', 'DOB', 'EMPLOYER', 'SALARY_DATE', 'SALARY_CREDIT_COUNT', 'MANDATORY_PRESENCE', name='check_type'), nullable=False), + sa.Column('passed', sa.Boolean(), nullable=False), + sa.Column('score', sa.Float(), nullable=False), + sa.Column('evidence', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['document_id'], ['documents.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_validation_results_case_id'), 'validation_results', ['case_id'], unique=False) + op.create_index(op.f('ix_validation_results_document_id'), 'validation_results', ['document_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_validation_results_document_id'), table_name='validation_results') + op.drop_index(op.f('ix_validation_results_case_id'), table_name='validation_results') + op.drop_table('validation_results') + op.drop_index(op.f('ix_pipeline_results_case_id'), table_name='pipeline_results') + op.drop_table('pipeline_results') + op.drop_table('golden_records') + op.drop_index(op.f('ix_documents_case_id'), table_name='documents') + op.drop_table('documents') + op.drop_index(op.f('ix_cases_applicant_ref'), table_name='cases') + op.drop_table('cases') + # ### end Alembic commands ### diff --git a/lending-poc/db/models/__init__.py b/lending-poc/db/models/__init__.py new file mode 100644 index 0000000..4690ba3 --- /dev/null +++ b/lending-poc/db/models/__init__.py @@ -0,0 +1,19 @@ +from db.models.case import Case, CaseStatus +from db.models.document import Document +from db.models.enums import CheckType, Decision, DocType +from db.models.golden_record import GoldenRecord +from db.models.pipeline_result import PipelineResult, ReviewStatus +from db.models.validation_result import ValidationResult + +__all__ = [ + "Case", + "CaseStatus", + "CheckType", + "Decision", + "Document", + "DocType", + "GoldenRecord", + "PipelineResult", + "ReviewStatus", + "ValidationResult", +] diff --git a/lending-poc/db/models/case.py b/lending-poc/db/models/case.py new file mode 100644 index 0000000..8984198 --- /dev/null +++ b/lending-poc/db/models/case.py @@ -0,0 +1,42 @@ +import uuid +from datetime import datetime +from enum import Enum + +from sqlalchemy import DateTime, Enum as SAEnum, String, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from db.database import Base + + +class CaseStatus(str, Enum): + RECEIVED = "RECEIVED" + RUNNING = "RUNNING" + PASS = "PASS" + FAIL = "FAIL" + NEEDS_REVIEW = "NEEDS_REVIEW" + + +class Case(Base): + __tablename__ = "cases" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + applicant_ref: Mapped[str] = mapped_column(String, unique=True, index=True, nullable=False) + status: Mapped[CaseStatus] = mapped_column( + SAEnum(CaseStatus, name="case_status"), nullable=False, default=CaseStatus.RECEIVED + ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + + documents: Mapped[list["Document"]] = relationship(back_populates="case", cascade="all, delete-orphan") + golden_record: Mapped["GoldenRecord | None"] = relationship( + back_populates="case", cascade="all, delete-orphan", uselist=False + ) + validation_results: Mapped[list["ValidationResult"]] = relationship( + back_populates="case", cascade="all, delete-orphan" + ) + pipeline_results: Mapped[list["PipelineResult"]] = relationship( + back_populates="case", cascade="all, delete-orphan" + ) diff --git a/lending-poc/db/models/document.py b/lending-poc/db/models/document.py new file mode 100644 index 0000000..2a4069e --- /dev/null +++ b/lending-poc/db/models/document.py @@ -0,0 +1,25 @@ +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, Enum as SAEnum, ForeignKey, String, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from db.database import Base +from db.models.enums import DocType + + +class Document(Base): + __tablename__ = "documents" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + case_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True + ) + doc_type: Mapped[DocType] = mapped_column(SAEnum(DocType, name="doc_type"), nullable=False) + extracted_fields: Mapped[dict] = mapped_column(JSONB, nullable=False) + source_file_ref: Mapped[str] = mapped_column(String, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + case: Mapped["Case"] = relationship(back_populates="documents") + validation_results: Mapped[list["ValidationResult"]] = relationship(back_populates="document") diff --git a/lending-poc/db/models/enums.py b/lending-poc/db/models/enums.py new file mode 100644 index 0000000..c888a81 --- /dev/null +++ b/lending-poc/db/models/enums.py @@ -0,0 +1,33 @@ +"""Enums shared by DB models and the validation pipeline's in-memory DTOs. + +Owned by the DB layer since they back Postgres enum columns; the pipeline +layer (app.services.dto) imports these rather than redefining them. +""" + +from enum import Enum + + +class DocType(str, Enum): + AADHAAR = "AADHAAR" + PAN = "PAN" + ADDRESS_PROOF = "ADDRESS_PROOF" + SALARY_SLIP = "SALARY_SLIP" + BANK_STATEMENT = "BANK_STATEMENT" + + +class CheckType(str, Enum): + NAME = "NAME" + ADDRESS = "ADDRESS" + AADHAAR = "AADHAAR" + PAN = "PAN" + DOB = "DOB" + EMPLOYER = "EMPLOYER" + SALARY_DATE = "SALARY_DATE" + SALARY_CREDIT_COUNT = "SALARY_CREDIT_COUNT" + MANDATORY_PRESENCE = "MANDATORY_PRESENCE" + + +class Decision(str, Enum): + PASS = "PASS" + FAIL = "FAIL" + NEEDS_REVIEW = "NEEDS_REVIEW" diff --git a/lending-poc/db/models/golden_record.py b/lending-poc/db/models/golden_record.py new file mode 100644 index 0000000..ddbce3f --- /dev/null +++ b/lending-poc/db/models/golden_record.py @@ -0,0 +1,36 @@ +import uuid +from datetime import date, datetime + +from pgvector.sqlalchemy import Vector +from sqlalchemy import Date, DateTime, ForeignKey, String, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from db.database import Base +from db.models.types import EncryptedString + +# Dimensionality of the BAAI/bge-small-en-v1.5 embedding used for address +# matching (app.matching.embeddings) — duplicated here as a plain constant +# so the DB models don't depend on the matching/ML package. +EMBEDDING_DIMENSIONS = 384 + + +class GoldenRecord(Base): + __tablename__ = "golden_records" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + case_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), unique=True, nullable=False + ) + name: Mapped[str | None] = mapped_column(String, nullable=True) + address: Mapped[str | None] = mapped_column(String, nullable=True) + address_embedding: Mapped[list[float] | None] = mapped_column(Vector(EMBEDDING_DIMENSIONS), nullable=True) + aadhaar_number: Mapped[str | None] = mapped_column(EncryptedString, nullable=True) + pan_number: Mapped[str | None] = mapped_column(EncryptedString, nullable=True) + date_of_birth: Mapped[date | None] = mapped_column(Date, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + + case: Mapped["Case"] = relationship(back_populates="golden_record") diff --git a/lending-poc/db/models/pipeline_result.py b/lending-poc/db/models/pipeline_result.py new file mode 100644 index 0000000..6f18099 --- /dev/null +++ b/lending-poc/db/models/pipeline_result.py @@ -0,0 +1,45 @@ +"""DB model for a pipeline run. + +Named `PipelineResult` to match the ERD/table name. This collides with the +in-memory `app.services.dto.PipelineResult` dataclass — import one or both +qualified (`from app.models import pipeline_result as pipeline_result_model`) +in any module that needs both. +""" + +import uuid +from datetime import datetime +from enum import Enum + +from sqlalchemy import DateTime, Enum as SAEnum, Float, ForeignKey, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from db.database import Base +from db.models.enums import Decision + + +class ReviewStatus(str, Enum): + PENDING = "PENDING" + APPROVED = "APPROVED" + REJECTED = "REJECTED" + + +class PipelineResult(Base): + __tablename__ = "pipeline_results" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + case_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True + ) + overall_score: Mapped[float] = mapped_column(Float, nullable=False) + decision: Mapped[Decision] = mapped_column(SAEnum(Decision, name="decision"), nullable=False) + reasons: Mapped[list | None] = mapped_column(JSONB, nullable=True) + reviewer: Mapped[str | None] = mapped_column(String, nullable=True) + review_status: Mapped[ReviewStatus | None] = mapped_column( + SAEnum(ReviewStatus, name="review_status"), nullable=True + ) + reviewer_remarks: Mapped[str | None] = mapped_column(Text, nullable=True) + reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + case: Mapped["Case"] = relationship(back_populates="pipeline_results") diff --git a/lending-poc/db/models/types.py b/lending-poc/db/models/types.py new file mode 100644 index 0000000..e37ecb2 --- /dev/null +++ b/lending-poc/db/models/types.py @@ -0,0 +1,26 @@ +from cryptography.fernet import Fernet +from sqlalchemy import String +from sqlalchemy.types import TypeDecorator + +from db.config import settings + + +class EncryptedString(TypeDecorator): + """Stores strings encrypted at rest (Fernet/AES) via ENCRYPTION_KEY. + + Transparent to callers: reads/writes plain str in Python, ciphertext + in the DB column. + """ + + impl = String + cache_ok = True + + def process_bind_param(self, value: str | None, dialect) -> str | None: + if value is None: + return None + return Fernet(settings.ENCRYPTION_KEY).encrypt(value.encode("utf-8")).decode("utf-8") + + def process_result_value(self, value: str | None, dialect) -> str | None: + if value is None: + return None + return Fernet(settings.ENCRYPTION_KEY).decrypt(value.encode("utf-8")).decode("utf-8") diff --git a/lending-poc/db/models/validation_result.py b/lending-poc/db/models/validation_result.py new file mode 100644 index 0000000..ce09c9a --- /dev/null +++ b/lending-poc/db/models/validation_result.py @@ -0,0 +1,29 @@ +import uuid +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, Enum as SAEnum, Float, ForeignKey, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from db.database import Base +from db.models.enums import CheckType + + +class ValidationResult(Base): + __tablename__ = "validation_results" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + case_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("cases.id", ondelete="CASCADE"), nullable=False, index=True + ) + document_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("documents.id", ondelete="CASCADE"), nullable=True, index=True + ) + check_type: Mapped[CheckType] = mapped_column(SAEnum(CheckType, name="check_type"), nullable=False) + passed: Mapped[bool] = mapped_column(Boolean, nullable=False) + score: Mapped[float] = mapped_column(Float, nullable=False) + evidence: Mapped[dict | None] = mapped_column(JSONB, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + case: Mapped["Case"] = relationship(back_populates="validation_results") + document: Mapped["Document | None"] = relationship(back_populates="validation_results") diff --git a/lending-poc/docker-compose.yml b/lending-poc/docker-compose.yml index 9b576a3..52cfaa9 100644 --- a/lending-poc/docker-compose.yml +++ b/lending-poc/docker-compose.yml @@ -1,12 +1,12 @@ services: db: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: lending_poc ports: - - "5432:5432" + - "55432:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: diff --git a/lending-poc/pyproject.toml b/lending-poc/pyproject.toml index e5522fa..268ce22 100644 --- a/lending-poc/pyproject.toml +++ b/lending-poc/pyproject.toml @@ -11,6 +11,8 @@ dependencies = [ "alembic>=1.14", "pydantic-settings>=2.0", "python-dotenv>=1.0", + "pgvector>=0.3", + "cryptography>=43.0", ] [project.optional-dependencies] From 01cd970fc129e722445eaa85dc93719e90533ca8 Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Mon, 10 Aug 2026 21:45:17 +0530 Subject: [PATCH 2/4] fix: seperated the database migrations files --- .../db/migrations/versions/0001_add_cases.py | 41 +++++++ .../migrations/versions/0002_add_documents.py | 43 +++++++ .../versions/0003_add_golden_records.py | 49 ++++++++ .../versions/0004_add_pipeline_results.py | 46 ++++++++ .../versions/0005_add_validation_results.py | 61 ++++++++++ ...f16_add_cases_documents_golden_records_.py | 106 ------------------ 6 files changed, 240 insertions(+), 106 deletions(-) create mode 100644 lending-poc/db/migrations/versions/0001_add_cases.py create mode 100644 lending-poc/db/migrations/versions/0002_add_documents.py create mode 100644 lending-poc/db/migrations/versions/0003_add_golden_records.py create mode 100644 lending-poc/db/migrations/versions/0004_add_pipeline_results.py create mode 100644 lending-poc/db/migrations/versions/0005_add_validation_results.py delete mode 100644 lending-poc/db/migrations/versions/12522c432f16_add_cases_documents_golden_records_.py diff --git a/lending-poc/db/migrations/versions/0001_add_cases.py b/lending-poc/db/migrations/versions/0001_add_cases.py new file mode 100644 index 0000000..44a4576 --- /dev/null +++ b/lending-poc/db/migrations/versions/0001_add_cases.py @@ -0,0 +1,41 @@ +"""add cases + +Revision ID: 0001_add_cases +Revises: +Create Date: 2026-08-10 11:51:33.283278 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "0001_add_cases" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + op.create_table( + "cases", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("applicant_ref", sa.String(), nullable=False), + sa.Column( + "status", + sa.Enum("RECEIVED", "RUNNING", "PASS", "FAIL", "NEEDS_REVIEW", name="case_status"), + nullable=False, + ), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_cases_applicant_ref"), "cases", ["applicant_ref"], unique=True) + + +def downgrade() -> None: + op.drop_index(op.f("ix_cases_applicant_ref"), table_name="cases") + op.drop_table("cases") + op.execute("DROP TYPE IF EXISTS case_status") diff --git a/lending-poc/db/migrations/versions/0002_add_documents.py b/lending-poc/db/migrations/versions/0002_add_documents.py new file mode 100644 index 0000000..f9d624e --- /dev/null +++ b/lending-poc/db/migrations/versions/0002_add_documents.py @@ -0,0 +1,43 @@ +"""add documents + +Revision ID: 0002_add_documents +Revises: 0001_add_cases +Create Date: 2026-08-10 11:51:33.283278 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "0002_add_documents" +down_revision: Union[str, None] = "0001_add_cases" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "documents", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("case_id", sa.UUID(), nullable=False), + sa.Column( + "doc_type", + sa.Enum("AADHAAR", "PAN", "ADDRESS_PROOF", "SALARY_SLIP", "BANK_STATEMENT", name="doc_type"), + nullable=False, + ), + sa.Column("extracted_fields", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("source_file_ref", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["case_id"], ["cases.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_documents_case_id"), "documents", ["case_id"], unique=False) + + +def downgrade() -> None: + op.drop_index(op.f("ix_documents_case_id"), table_name="documents") + op.drop_table("documents") + op.execute("DROP TYPE IF EXISTS doc_type") diff --git a/lending-poc/db/migrations/versions/0003_add_golden_records.py b/lending-poc/db/migrations/versions/0003_add_golden_records.py new file mode 100644 index 0000000..20070cb --- /dev/null +++ b/lending-poc/db/migrations/versions/0003_add_golden_records.py @@ -0,0 +1,49 @@ +"""add golden_records + +Revision ID: 0003_add_golden_records +Revises: 0002_add_documents +Create Date: 2026-08-10 11:51:33.283278 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import pgvector.sqlalchemy + +import db.models.types + +# revision identifiers, used by Alembic. +revision: str = "0003_add_golden_records" +down_revision: Union[str, None] = "0002_add_documents" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "golden_records", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("case_id", sa.UUID(), nullable=False), + sa.Column("name", sa.String(), nullable=True), + sa.Column("address", sa.String(), nullable=True), + sa.Column("address_embedding", pgvector.sqlalchemy.vector.VECTOR(dim=384), nullable=True), + sa.Column("aadhaar_number", db.models.types.EncryptedString(), nullable=True), + sa.Column("pan_number", db.models.types.EncryptedString(), nullable=True), + sa.Column("date_of_birth", sa.Date(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + onupdate=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint(["case_id"], ["cases.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("case_id"), + ) + + +def downgrade() -> None: + op.drop_table("golden_records") diff --git a/lending-poc/db/migrations/versions/0004_add_pipeline_results.py b/lending-poc/db/migrations/versions/0004_add_pipeline_results.py new file mode 100644 index 0000000..63cd56d --- /dev/null +++ b/lending-poc/db/migrations/versions/0004_add_pipeline_results.py @@ -0,0 +1,46 @@ +"""add pipeline_results + +Revision ID: 0004_add_pipeline_results +Revises: 0003_add_golden_records +Create Date: 2026-08-10 11:51:33.283278 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "0004_add_pipeline_results" +down_revision: Union[str, None] = "0003_add_golden_records" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "pipeline_results", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("case_id", sa.UUID(), nullable=False), + sa.Column("overall_score", sa.Float(), nullable=False), + sa.Column("decision", sa.Enum("PASS", "FAIL", "NEEDS_REVIEW", name="decision"), nullable=False), + sa.Column("reasons", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("reviewer", sa.String(), nullable=True), + sa.Column( + "review_status", sa.Enum("PENDING", "APPROVED", "REJECTED", name="review_status"), nullable=True + ), + sa.Column("reviewer_remarks", sa.Text(), nullable=True), + sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["case_id"], ["cases.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_pipeline_results_case_id"), "pipeline_results", ["case_id"], unique=False) + + +def downgrade() -> None: + op.drop_index(op.f("ix_pipeline_results_case_id"), table_name="pipeline_results") + op.drop_table("pipeline_results") + op.execute("DROP TYPE IF EXISTS decision") + op.execute("DROP TYPE IF EXISTS review_status") diff --git a/lending-poc/db/migrations/versions/0005_add_validation_results.py b/lending-poc/db/migrations/versions/0005_add_validation_results.py new file mode 100644 index 0000000..58a6cab --- /dev/null +++ b/lending-poc/db/migrations/versions/0005_add_validation_results.py @@ -0,0 +1,61 @@ +"""add validation_results + +Revision ID: 0005_add_validation_results +Revises: 0004_add_pipeline_results +Create Date: 2026-08-10 11:51:33.283278 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "0005_add_validation_results" +down_revision: Union[str, None] = "0004_add_pipeline_results" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "validation_results", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("case_id", sa.UUID(), nullable=False), + sa.Column("document_id", sa.UUID(), nullable=True), + sa.Column( + "check_type", + sa.Enum( + "NAME", + "ADDRESS", + "AADHAAR", + "PAN", + "DOB", + "EMPLOYER", + "SALARY_DATE", + "SALARY_CREDIT_COUNT", + "MANDATORY_PRESENCE", + name="check_type", + ), + nullable=False, + ), + sa.Column("passed", sa.Boolean(), nullable=False), + sa.Column("score", sa.Float(), nullable=False), + sa.Column("evidence", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["case_id"], ["cases.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["document_id"], ["documents.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_validation_results_case_id"), "validation_results", ["case_id"], unique=False) + op.create_index( + op.f("ix_validation_results_document_id"), "validation_results", ["document_id"], unique=False + ) + + +def downgrade() -> None: + op.drop_index(op.f("ix_validation_results_document_id"), table_name="validation_results") + op.drop_index(op.f("ix_validation_results_case_id"), table_name="validation_results") + op.drop_table("validation_results") + op.execute("DROP TYPE IF EXISTS check_type") diff --git a/lending-poc/db/migrations/versions/12522c432f16_add_cases_documents_golden_records_.py b/lending-poc/db/migrations/versions/12522c432f16_add_cases_documents_golden_records_.py deleted file mode 100644 index 4b1ed75..0000000 --- a/lending-poc/db/migrations/versions/12522c432f16_add_cases_documents_golden_records_.py +++ /dev/null @@ -1,106 +0,0 @@ -"""add cases, documents, golden_records, validation_results, pipeline_results - -Revision ID: 12522c432f16 -Revises: -Create Date: 2026-08-10 11:51:33.283278 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql -import pgvector.sqlalchemy -import db.models.types - -# revision identifiers, used by Alembic. -revision: str = '12522c432f16' -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.execute('CREATE EXTENSION IF NOT EXISTS vector') - op.create_table('cases', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('applicant_ref', sa.String(), nullable=False), - sa.Column('status', sa.Enum('RECEIVED', 'RUNNING', 'PASS', 'FAIL', 'NEEDS_REVIEW', name='case_status'), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_cases_applicant_ref'), 'cases', ['applicant_ref'], unique=True) - op.create_table('documents', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('case_id', sa.UUID(), nullable=False), - sa.Column('doc_type', sa.Enum('AADHAAR', 'PAN', 'ADDRESS_PROOF', 'SALARY_SLIP', 'BANK_STATEMENT', name='doc_type'), nullable=False), - sa.Column('extracted_fields', postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column('source_file_ref', sa.String(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_documents_case_id'), 'documents', ['case_id'], unique=False) - op.create_table('golden_records', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('case_id', sa.UUID(), nullable=False), - sa.Column('name', sa.String(), nullable=True), - sa.Column('address', sa.String(), nullable=True), - sa.Column('address_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=384), nullable=True), - sa.Column('aadhaar_number', db.models.types.EncryptedString(), nullable=True), - sa.Column('pan_number', db.models.types.EncryptedString(), nullable=True), - sa.Column('date_of_birth', sa.Date(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('case_id') - ) - op.create_table('pipeline_results', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('case_id', sa.UUID(), nullable=False), - sa.Column('overall_score', sa.Float(), nullable=False), - sa.Column('decision', sa.Enum('PASS', 'FAIL', 'NEEDS_REVIEW', name='decision'), nullable=False), - sa.Column('reasons', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('reviewer', sa.String(), nullable=True), - sa.Column('review_status', sa.Enum('PENDING', 'APPROVED', 'REJECTED', name='review_status'), nullable=True), - sa.Column('reviewer_remarks', sa.Text(), nullable=True), - sa.Column('reviewed_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_pipeline_results_case_id'), 'pipeline_results', ['case_id'], unique=False) - op.create_table('validation_results', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('case_id', sa.UUID(), nullable=False), - sa.Column('document_id', sa.UUID(), nullable=True), - sa.Column('check_type', sa.Enum('NAME', 'ADDRESS', 'AADHAAR', 'PAN', 'DOB', 'EMPLOYER', 'SALARY_DATE', 'SALARY_CREDIT_COUNT', 'MANDATORY_PRESENCE', name='check_type'), nullable=False), - sa.Column('passed', sa.Boolean(), nullable=False), - sa.Column('score', sa.Float(), nullable=False), - sa.Column('evidence', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['case_id'], ['cases.id'], ondelete='CASCADE'), - sa.ForeignKeyConstraint(['document_id'], ['documents.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_validation_results_case_id'), 'validation_results', ['case_id'], unique=False) - op.create_index(op.f('ix_validation_results_document_id'), 'validation_results', ['document_id'], unique=False) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_validation_results_document_id'), table_name='validation_results') - op.drop_index(op.f('ix_validation_results_case_id'), table_name='validation_results') - op.drop_table('validation_results') - op.drop_index(op.f('ix_pipeline_results_case_id'), table_name='pipeline_results') - op.drop_table('pipeline_results') - op.drop_table('golden_records') - op.drop_index(op.f('ix_documents_case_id'), table_name='documents') - op.drop_table('documents') - op.drop_index(op.f('ix_cases_applicant_ref'), table_name='cases') - op.drop_table('cases') - # ### end Alembic commands ### From 531948230eb9fc23f7a230dd131e1e7e609b32da Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Tue, 11 Aug 2026 22:14:42 +0530 Subject: [PATCH 3/4] Fix PR review comments: Fernet key handling, misleading docstring, and hardcoded DB config --- lending-poc/.env.example | 14 ++- lending-poc/app/config.py | 2 +- lending-poc/app/utils/json_safe.py | 19 ++++ lending-poc/db/config.py | 2 +- lending-poc/db/models/pipeline_result.py | 8 +- lending-poc/db/models/types.py | 10 +- lending-poc/docker-compose.yml | 16 +-- lending-poc/scripts/sample_case_fail.json | 81 ++++++++++++++ .../scripts/sample_case_needs_review.json | 95 +++++++++++++++++ lending-poc/scripts/sample_case_pass.json | 100 ++++++++++++++++++ 10 files changed, 332 insertions(+), 15 deletions(-) create mode 100644 lending-poc/app/utils/json_safe.py create mode 100644 lending-poc/scripts/sample_case_fail.json create mode 100644 lending-poc/scripts/sample_case_needs_review.json create mode 100644 lending-poc/scripts/sample_case_pass.json diff --git a/lending-poc/.env.example b/lending-poc/.env.example index 0fafff7..91f109c 100644 --- a/lending-poc/.env.example +++ b/lending-poc/.env.example @@ -1,4 +1,16 @@ -DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/lending_poc +# Used when running the app/Alembic from the host (matches docker-compose's published port). +DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:55432/lending_poc +# Used by the `app` service inside docker-compose (talks to `db` over the Docker network). +DATABASE_URL_DOCKER=postgresql+asyncpg://postgres:postgres@db:5432/lending_poc + +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=lending_poc +POSTGRES_HOST_PORT=55432 + +# Generate with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +ENCRYPTION_KEY= + APP_NAME=lending-poc APP_VERSION=0.1.0 DEBUG=true diff --git a/lending-poc/app/config.py b/lending-poc/app/config.py index d8f7bdc..0daa2c9 100644 --- a/lending-poc/app/config.py +++ b/lending-poc/app/config.py @@ -9,7 +9,7 @@ class Settings(BaseSettings): APP_NAME: str = "lending-poc" APP_VERSION: str = "0.1.0" DEBUG: bool = False - DATABASE_URL: str = "postgresql+asyncpg://postgres:postgres@localhost:5432/lending_poc" + DATABASE_URL: str LOG_LEVEL: str = "INFO" diff --git a/lending-poc/app/utils/json_safe.py b/lending-poc/app/utils/json_safe.py new file mode 100644 index 0000000..06825a5 --- /dev/null +++ b/lending-poc/app/utils/json_safe.py @@ -0,0 +1,19 @@ +"""Recursively converts dataclasses/dates/tuples into plain +JSON-serializable values, since validation/scoring build evidence out of +dataclasses (e.g. BankTransaction) and date objects for in-memory use. +""" + +import dataclasses +import datetime + + +def json_safe(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return {k: json_safe(v) for k, v in dataclasses.asdict(value).items()} + if isinstance(value, (datetime.date, datetime.datetime)): + return value.isoformat() + if isinstance(value, dict): + return {k: json_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [json_safe(v) for v in value] + return value diff --git a/lending-poc/db/config.py b/lending-poc/db/config.py index 77e2805..7109786 100644 --- a/lending-poc/db/config.py +++ b/lending-poc/db/config.py @@ -5,7 +5,7 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") DEBUG: bool = False - DATABASE_URL: str = "postgresql+asyncpg://postgres:postgres@localhost:5432/lending_poc" + DATABASE_URL: str ENCRYPTION_KEY: str = "" diff --git a/lending-poc/db/models/pipeline_result.py b/lending-poc/db/models/pipeline_result.py index 6f18099..9f821f8 100644 --- a/lending-poc/db/models/pipeline_result.py +++ b/lending-poc/db/models/pipeline_result.py @@ -1,9 +1,9 @@ """DB model for a pipeline run. -Named `PipelineResult` to match the ERD/table name. This collides with the -in-memory `app.services.dto.PipelineResult` dataclass — import one or both -qualified (`from app.models import pipeline_result as pipeline_result_model`) -in any module that needs both. +Named `PipelineResult` to match the ERD/table name. If an in-memory DTO with +the same name is introduced elsewhere, import one or both qualified +(`from db.models import pipeline_result as pipeline_result_model`) in any +module that needs both. """ import uuid diff --git a/lending-poc/db/models/types.py b/lending-poc/db/models/types.py index e37ecb2..1aa98c3 100644 --- a/lending-poc/db/models/types.py +++ b/lending-poc/db/models/types.py @@ -5,6 +5,12 @@ from db.config import settings +def _get_fernet() -> Fernet: + if not settings.ENCRYPTION_KEY: + raise ValueError("ENCRYPTION_KEY is not set; cannot encrypt/decrypt EncryptedString columns.") + return Fernet(settings.ENCRYPTION_KEY.encode("utf-8")) + + class EncryptedString(TypeDecorator): """Stores strings encrypted at rest (Fernet/AES) via ENCRYPTION_KEY. @@ -18,9 +24,9 @@ class EncryptedString(TypeDecorator): def process_bind_param(self, value: str | None, dialect) -> str | None: if value is None: return None - return Fernet(settings.ENCRYPTION_KEY).encrypt(value.encode("utf-8")).decode("utf-8") + return _get_fernet().encrypt(value.encode("utf-8")).decode("utf-8") def process_result_value(self, value: str | None, dialect) -> str | None: if value is None: return None - return Fernet(settings.ENCRYPTION_KEY).decrypt(value.encode("utf-8")).decode("utf-8") + return _get_fernet().decrypt(value.encode("utf-8")).decode("utf-8") diff --git a/lending-poc/docker-compose.yml b/lending-poc/docker-compose.yml index 52cfaa9..38239e6 100644 --- a/lending-poc/docker-compose.yml +++ b/lending-poc/docker-compose.yml @@ -1,16 +1,18 @@ services: db: image: pgvector/pgvector:pg16 + env_file: + - .env environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: lending_poc + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-lending_poc} ports: - - "55432:5432" + - "${POSTGRES_HOST_PORT:-55432}:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres}"] interval: 5s timeout: 5s retries: 5 @@ -20,8 +22,10 @@ services: command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload ports: - "8000:8000" + env_file: + - .env environment: - DATABASE_URL: postgresql+asyncpg://postgres:postgres@db:5432/lending_poc + DATABASE_URL: ${DATABASE_URL_DOCKER:-postgresql+asyncpg://postgres:postgres@db:5432/lending_poc} volumes: - .:/app depends_on: diff --git a/lending-poc/scripts/sample_case_fail.json b/lending-poc/scripts/sample_case_fail.json new file mode 100644 index 0000000..c28d74c --- /dev/null +++ b/lending-poc/scripts/sample_case_fail.json @@ -0,0 +1,81 @@ +{ + "_test_design_notes": { + "purpose": "Clean fail-path case. Two independent failure triggers stacked: (1) Aadhaar date_of_birth is missing -> golden_record.date_of_birth stays None -> MANDATORY_FIELD_MISSING:DOB hard-fails the case in decision_engine regardless of score; (2) PAN name doesn't match the applicant (different person, NAME score 35.29 doc=PAN), Aadhaar address vs address proof is a different city (ADDRESS score 53.50, below the 0.55 threshold), and neither salary slip has a matching bank credit (both SALARY_DATE and EMPLOYER FAIL at 0.0, SALARY_CREDIT_COUNT 0.0) -- so even ignoring the mandatory-field short-circuit, the underlying score (50.73) is already under DECISION_FAIL_THRESHOLD (60.0).", + "expected_overall_decision": "FAIL at 50.73 overall score, reasons=['MANDATORY_FIELD_MISSING:DOB'] -- re-verify with: python scripts/run_demo.py scripts/sample_case_fail.json" + }, + "applicant_ref": "APP-2026-00201", + "documents": [ + { + "doc_type": "AADHAAR", + "extracted_fields": { + "name": "Priya Deshmukh", + "address": "House 8, Lakeview Colony, Nagpur, Maharashtra 440001", + "aadhaar_number": "9988 7766 5544", + "date_of_birth": null + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00201/aadhaar_front.pdf" + }, + { + "doc_type": "PAN", + "extracted_fields": { + "name": "Priya Deshmukh", + "pan_number": "ZZTOP1111Q" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00201/pan_card.pdf" + }, + { + "doc_type": "ADDRESS_PROOF", + "extracted_fields": { + "address": "House 10, Lakeview Colony,Iran , Iran" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00201/address_proof.pdf" + }, + { + "doc_type": "SALARY_SLIP", + "salary_slips": [ + { + "extracted_fields": { + "name": "Priya Deshmukh", + "employer_name": "Falcon Industries Pvt Ltd", + "net_salary": 60000, + "salary_month": "2026-03" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00201/salary_slip_march.pdf" + }, + { + "extracted_fields": { + "name": "Priya Deshmukh", + "employer_name": "Falcon Industries Pvt Ltd", + "net_salary": 60000, + "salary_month": "2026-04" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00201/salary_slip_april.pdf" + } + ] + }, + { + "doc_type": "BANK_STATEMENT", + "extracted_fields": { + "name": "Priya Deshmukh", + "transactions": [ + { + "narration": "Falcon Industries Pvt Ltd", + "amount": 60000, + "date": "2026-12-12" + }, + { + "narration": "ELECTRICITY BILL PAYMENT", + "amount": -2200, + "date": "2026-04-10" + }, + { + "narration": "UNKNOWN CREDIT NEFT", + "amount": 12000, + "date": "2026-04-15" + } + ] + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00201/bank_statement_mar_to_apr.pdf" + } + ] +} diff --git a/lending-poc/scripts/sample_case_needs_review.json b/lending-poc/scripts/sample_case_needs_review.json new file mode 100644 index 0000000..3cc4234 --- /dev/null +++ b/lending-poc/scripts/sample_case_needs_review.json @@ -0,0 +1,95 @@ +{ + "_test_design_notes": { + "purpose": "Borderline case: no mandatory field is missing (so the hard-fail short-circuit never triggers), but one check fails so the weighted score settles inside the NEEDS_REVIEW band (DECISION_FAIL_THRESHOLD 60.0 <= score < DECISION_PASS_THRESHOLD 90.0). Verified behavior (actually run through the pipeline): all identity checks (NAME, ADDRESS, AADHAAR, PAN, DOB) PASS -- ADDRESS_PROOF wording differs from Aadhaar (different locality: Kothrud vs Warje) but still clears the 0.55 embedding-similarity threshold at ~56.8, just above the line. The April slip's employer_name is shortened to 'Crestline' (missing 'Pvt Ltd') vs the bank narration 'CRESTLINE SALARY APR' -- this still scores EMPLOYER 100.0 via token_set_ratio (all of Crestline's tokens are a subset of the narration's), so shortening the name alone did NOT create a borderline EMPLOYER score as originally intended; left as-is since it demonstrates the matcher tolerates missing trailing suffix tokens when every present token matches. The actual driver of NEEDS_REVIEW is the May slip: no bank transaction anywhere near its window matches -> SALARY_DATE FAILS (0.0, reason=no_matching_credit_in_window), EMPLOYER FAILS as a consequence (nothing to verify against), and SALARY_CREDIT_COUNT drops to 2/3 (66.67).", + "expected_overall_decision": "NEEDS_REVIEW at ~86.17 overall score -- re-verify with: python scripts/run_demo.py scripts/sample_case_needs_review.json (or the inline pipeline invocation used to design this fixture)" + }, + "applicant_ref": "APP-2026-00202", + "documents": [ + { + "doc_type": "AADHAAR", + "extracted_fields": { + "name": "Anjali Ramesh Patil", + "address": "Flat 7, Silver Oak Society, Kothrud, Pune, Maharashtra 411038", + "aadhaar_number": "5566 7788 9900", + "date_of_birth": "1992-11-02" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00202/aadhaar_front.pdf" + }, + { + "doc_type": "PAN", + "extracted_fields": { + "name": "Anjali Ramesh Patil", + "pan_number": "ARPAT2025K" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00202/pan_card.pdf" + }, + { + "doc_type": "ADDRESS_PROOF", + "extracted_fields": { + "address": "Building B, Riverside Enclave, Warje, Pune, Maharashtra 411058" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00202/address_proof.pdf" + }, + { + "doc_type": "SALARY_SLIP", + "salary_slips": [ + { + "extracted_fields": { + "name": "Anjali Ramesh Patil", + "employer_name": "Crestline Systems Pvt Ltd", + "net_salary": 68000, + "salary_month": "2026-03" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00202/salary_slip_march.pdf" + }, + { + "extracted_fields": { + "name": "Anjali Ramesh Patil", + "employer_name": "Crestline Systems Pvt Ltd", + "net_salary": 68000, + "salary_month": "2026-04" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00202/salary_slip_april.pdf" + }, + { + "extracted_fields": { + "name": "Anjali Ramesh Patil", + "employer_name": "Crestline Systems Pvt Ltd", + "net_salary": 68000, + "salary_month": "2026-05" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00202/salary_slip_may.pdf" + } + ] + }, + { + "doc_type": "BANK_STATEMENT", + "extracted_fields": { + "name": "Anjali Ramesh Patil", + "transactions": [ + { + "narration": "Crestline Systems Pvt Ltd", + "amount": 68000, + "date": "2026-03-03" + }, + { + "narration": "Crestline Systems", + "amount": 68000, + "date": "2026-04-04" + }, + { + "narration": "Crestline Systems Pvt Ltd", + "amount": 68000, + "date": "2026-05-05" + }, + { + "narration": "ONLINE SHOPPING PAYMENT", + "amount": -3200, + "date": "2026-05-10" + } + ] + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00202/bank_statement_mar_to_may.pdf" + } + ] +} diff --git a/lending-poc/scripts/sample_case_pass.json b/lending-poc/scripts/sample_case_pass.json new file mode 100644 index 0000000..a9a4642 --- /dev/null +++ b/lending-poc/scripts/sample_case_pass.json @@ -0,0 +1,100 @@ +{ + "_test_design_notes": { + "purpose": "Clean happy-path case: every identity document agrees with the Golden Record, every salary slip has a matching bank credit with a clearly-passing employer name, no mandatory fields missing. Verified behavior: every single check (NAME, ADDRESS, AADHAAR, PAN, DOB, EMPLOYER, SALARY_DATE, SALARY_CREDIT_COUNT) scores exactly 100.0.", + "expected_overall_decision": "PASS at 100.0 overall score, reasons=['score_meets_pass_threshold'] -- re-verify with: python scripts/run_demo.py scripts/sample_case_pass.json" + }, + "applicant_ref": "APP-2026-00200", + "documents": [ + { + "doc_type": "AADHAAR", + "extracted_fields": { + "name": "Rohit Kumar Sharma", + "address": "Flat 12, Sunrise Apartments, Andheri East, Mumbai, Maharashtra 400069", + "aadhaar_number": "1234 5678 9123", + "date_of_birth": "1990-06-21" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00200/aadhaar_front.pdf" + }, + { + "doc_type": "PAN", + "extracted_fields": { + "name": "Rohit Kumar Sharma", + "pan_number": "PQRSX9876Z" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00200/pan_card.pdf" + }, + { + "doc_type": "ADDRESS_PROOF", + "extracted_fields": { + "address": "Flat 12, Sunrise Apartments, Andheri East, Mumbai, Maharashtra 400069" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00200/address_proof.pdf" + }, + { + "doc_type": "SALARY_SLIP", + "salary_slips": [ + { + "extracted_fields": { + "name": "Rohit Kumar Sharma", + "employer_name": "Orion Software Pvt Ltd", + "net_salary": 90000, + "salary_month": "2026-03" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00200/salary_slip_march.pdf" + }, + { + "extracted_fields": { + "name": "Rohit Kumar Sharma", + "employer_name": "Orion Software Pvt Ltd", + "net_salary": 90000, + "salary_month": "2026-04" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00200/salary_slip_april.pdf" + }, + { + "extracted_fields": { + "name": "Rohit Kumar Sharma", + "employer_name": "Orion Software Pvt Ltd", + "net_salary": 92000, + "salary_month": "2026-05" + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00200/salary_slip_may.pdf" + } + ] + }, + { + "doc_type": "BANK_STATEMENT", + "extracted_fields": { + "name": "Rohit Kumar Sharma", + "transactions": [ + { + "narration": "ORION SOFTWARE PVT LTD SALARY MAR", + "amount": 90000, + "date": "2026-04-01" + }, + { + "narration": "HOUSE RENT EMI DEBIT", + "amount": -20000, + "date": "2026-04-03" + }, + { + "narration": "ORION SOFTWARE PVT LTD SALARY APR", + "amount": 90000, + "date": "2026-05-01" + }, + { + "narration": "GROCERY STORE PAYMENT", + "amount": -1500, + "date": "2026-05-06" + }, + { + "narration": "ORION SOFTWARE PVT LTD SALARY MAY", + "amount": 92000, + "date": "2026-06-01" + } + ] + }, + "source_file_ref": "s3://kyc-docs/APP-2026-00200/bank_statement_mar_to_jun.pdf" + } + ] +} From 798e08f33d824fc9122dd511764aa1ed9f7a55c3 Mon Sep 17 00:00:00 2001 From: Ankita-Advitot Date: Wed, 12 Aug 2026 10:20:46 +0530 Subject: [PATCH 4/4] fix: deleted unecessary files committed to --- lending-poc/.env.example | 17 --- lending-poc/app/utils/json_safe.py | 19 ---- lending-poc/scripts/sample_case_fail.json | 81 -------------- .../scripts/sample_case_needs_review.json | 95 ----------------- lending-poc/scripts/sample_case_pass.json | 100 ------------------ 5 files changed, 312 deletions(-) delete mode 100644 lending-poc/.env.example delete mode 100644 lending-poc/app/utils/json_safe.py delete mode 100644 lending-poc/scripts/sample_case_fail.json delete mode 100644 lending-poc/scripts/sample_case_needs_review.json delete mode 100644 lending-poc/scripts/sample_case_pass.json diff --git a/lending-poc/.env.example b/lending-poc/.env.example deleted file mode 100644 index 91f109c..0000000 --- a/lending-poc/.env.example +++ /dev/null @@ -1,17 +0,0 @@ -# Used when running the app/Alembic from the host (matches docker-compose's published port). -DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:55432/lending_poc -# Used by the `app` service inside docker-compose (talks to `db` over the Docker network). -DATABASE_URL_DOCKER=postgresql+asyncpg://postgres:postgres@db:5432/lending_poc - -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DB=lending_poc -POSTGRES_HOST_PORT=55432 - -# Generate with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" -ENCRYPTION_KEY= - -APP_NAME=lending-poc -APP_VERSION=0.1.0 -DEBUG=true -LOG_LEVEL=INFO diff --git a/lending-poc/app/utils/json_safe.py b/lending-poc/app/utils/json_safe.py deleted file mode 100644 index 06825a5..0000000 --- a/lending-poc/app/utils/json_safe.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Recursively converts dataclasses/dates/tuples into plain -JSON-serializable values, since validation/scoring build evidence out of -dataclasses (e.g. BankTransaction) and date objects for in-memory use. -""" - -import dataclasses -import datetime - - -def json_safe(value): - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return {k: json_safe(v) for k, v in dataclasses.asdict(value).items()} - if isinstance(value, (datetime.date, datetime.datetime)): - return value.isoformat() - if isinstance(value, dict): - return {k: json_safe(v) for k, v in value.items()} - if isinstance(value, (list, tuple)): - return [json_safe(v) for v in value] - return value diff --git a/lending-poc/scripts/sample_case_fail.json b/lending-poc/scripts/sample_case_fail.json deleted file mode 100644 index c28d74c..0000000 --- a/lending-poc/scripts/sample_case_fail.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "_test_design_notes": { - "purpose": "Clean fail-path case. Two independent failure triggers stacked: (1) Aadhaar date_of_birth is missing -> golden_record.date_of_birth stays None -> MANDATORY_FIELD_MISSING:DOB hard-fails the case in decision_engine regardless of score; (2) PAN name doesn't match the applicant (different person, NAME score 35.29 doc=PAN), Aadhaar address vs address proof is a different city (ADDRESS score 53.50, below the 0.55 threshold), and neither salary slip has a matching bank credit (both SALARY_DATE and EMPLOYER FAIL at 0.0, SALARY_CREDIT_COUNT 0.0) -- so even ignoring the mandatory-field short-circuit, the underlying score (50.73) is already under DECISION_FAIL_THRESHOLD (60.0).", - "expected_overall_decision": "FAIL at 50.73 overall score, reasons=['MANDATORY_FIELD_MISSING:DOB'] -- re-verify with: python scripts/run_demo.py scripts/sample_case_fail.json" - }, - "applicant_ref": "APP-2026-00201", - "documents": [ - { - "doc_type": "AADHAAR", - "extracted_fields": { - "name": "Priya Deshmukh", - "address": "House 8, Lakeview Colony, Nagpur, Maharashtra 440001", - "aadhaar_number": "9988 7766 5544", - "date_of_birth": null - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00201/aadhaar_front.pdf" - }, - { - "doc_type": "PAN", - "extracted_fields": { - "name": "Priya Deshmukh", - "pan_number": "ZZTOP1111Q" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00201/pan_card.pdf" - }, - { - "doc_type": "ADDRESS_PROOF", - "extracted_fields": { - "address": "House 10, Lakeview Colony,Iran , Iran" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00201/address_proof.pdf" - }, - { - "doc_type": "SALARY_SLIP", - "salary_slips": [ - { - "extracted_fields": { - "name": "Priya Deshmukh", - "employer_name": "Falcon Industries Pvt Ltd", - "net_salary": 60000, - "salary_month": "2026-03" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00201/salary_slip_march.pdf" - }, - { - "extracted_fields": { - "name": "Priya Deshmukh", - "employer_name": "Falcon Industries Pvt Ltd", - "net_salary": 60000, - "salary_month": "2026-04" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00201/salary_slip_april.pdf" - } - ] - }, - { - "doc_type": "BANK_STATEMENT", - "extracted_fields": { - "name": "Priya Deshmukh", - "transactions": [ - { - "narration": "Falcon Industries Pvt Ltd", - "amount": 60000, - "date": "2026-12-12" - }, - { - "narration": "ELECTRICITY BILL PAYMENT", - "amount": -2200, - "date": "2026-04-10" - }, - { - "narration": "UNKNOWN CREDIT NEFT", - "amount": 12000, - "date": "2026-04-15" - } - ] - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00201/bank_statement_mar_to_apr.pdf" - } - ] -} diff --git a/lending-poc/scripts/sample_case_needs_review.json b/lending-poc/scripts/sample_case_needs_review.json deleted file mode 100644 index 3cc4234..0000000 --- a/lending-poc/scripts/sample_case_needs_review.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "_test_design_notes": { - "purpose": "Borderline case: no mandatory field is missing (so the hard-fail short-circuit never triggers), but one check fails so the weighted score settles inside the NEEDS_REVIEW band (DECISION_FAIL_THRESHOLD 60.0 <= score < DECISION_PASS_THRESHOLD 90.0). Verified behavior (actually run through the pipeline): all identity checks (NAME, ADDRESS, AADHAAR, PAN, DOB) PASS -- ADDRESS_PROOF wording differs from Aadhaar (different locality: Kothrud vs Warje) but still clears the 0.55 embedding-similarity threshold at ~56.8, just above the line. The April slip's employer_name is shortened to 'Crestline' (missing 'Pvt Ltd') vs the bank narration 'CRESTLINE SALARY APR' -- this still scores EMPLOYER 100.0 via token_set_ratio (all of Crestline's tokens are a subset of the narration's), so shortening the name alone did NOT create a borderline EMPLOYER score as originally intended; left as-is since it demonstrates the matcher tolerates missing trailing suffix tokens when every present token matches. The actual driver of NEEDS_REVIEW is the May slip: no bank transaction anywhere near its window matches -> SALARY_DATE FAILS (0.0, reason=no_matching_credit_in_window), EMPLOYER FAILS as a consequence (nothing to verify against), and SALARY_CREDIT_COUNT drops to 2/3 (66.67).", - "expected_overall_decision": "NEEDS_REVIEW at ~86.17 overall score -- re-verify with: python scripts/run_demo.py scripts/sample_case_needs_review.json (or the inline pipeline invocation used to design this fixture)" - }, - "applicant_ref": "APP-2026-00202", - "documents": [ - { - "doc_type": "AADHAAR", - "extracted_fields": { - "name": "Anjali Ramesh Patil", - "address": "Flat 7, Silver Oak Society, Kothrud, Pune, Maharashtra 411038", - "aadhaar_number": "5566 7788 9900", - "date_of_birth": "1992-11-02" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00202/aadhaar_front.pdf" - }, - { - "doc_type": "PAN", - "extracted_fields": { - "name": "Anjali Ramesh Patil", - "pan_number": "ARPAT2025K" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00202/pan_card.pdf" - }, - { - "doc_type": "ADDRESS_PROOF", - "extracted_fields": { - "address": "Building B, Riverside Enclave, Warje, Pune, Maharashtra 411058" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00202/address_proof.pdf" - }, - { - "doc_type": "SALARY_SLIP", - "salary_slips": [ - { - "extracted_fields": { - "name": "Anjali Ramesh Patil", - "employer_name": "Crestline Systems Pvt Ltd", - "net_salary": 68000, - "salary_month": "2026-03" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00202/salary_slip_march.pdf" - }, - { - "extracted_fields": { - "name": "Anjali Ramesh Patil", - "employer_name": "Crestline Systems Pvt Ltd", - "net_salary": 68000, - "salary_month": "2026-04" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00202/salary_slip_april.pdf" - }, - { - "extracted_fields": { - "name": "Anjali Ramesh Patil", - "employer_name": "Crestline Systems Pvt Ltd", - "net_salary": 68000, - "salary_month": "2026-05" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00202/salary_slip_may.pdf" - } - ] - }, - { - "doc_type": "BANK_STATEMENT", - "extracted_fields": { - "name": "Anjali Ramesh Patil", - "transactions": [ - { - "narration": "Crestline Systems Pvt Ltd", - "amount": 68000, - "date": "2026-03-03" - }, - { - "narration": "Crestline Systems", - "amount": 68000, - "date": "2026-04-04" - }, - { - "narration": "Crestline Systems Pvt Ltd", - "amount": 68000, - "date": "2026-05-05" - }, - { - "narration": "ONLINE SHOPPING PAYMENT", - "amount": -3200, - "date": "2026-05-10" - } - ] - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00202/bank_statement_mar_to_may.pdf" - } - ] -} diff --git a/lending-poc/scripts/sample_case_pass.json b/lending-poc/scripts/sample_case_pass.json deleted file mode 100644 index a9a4642..0000000 --- a/lending-poc/scripts/sample_case_pass.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "_test_design_notes": { - "purpose": "Clean happy-path case: every identity document agrees with the Golden Record, every salary slip has a matching bank credit with a clearly-passing employer name, no mandatory fields missing. Verified behavior: every single check (NAME, ADDRESS, AADHAAR, PAN, DOB, EMPLOYER, SALARY_DATE, SALARY_CREDIT_COUNT) scores exactly 100.0.", - "expected_overall_decision": "PASS at 100.0 overall score, reasons=['score_meets_pass_threshold'] -- re-verify with: python scripts/run_demo.py scripts/sample_case_pass.json" - }, - "applicant_ref": "APP-2026-00200", - "documents": [ - { - "doc_type": "AADHAAR", - "extracted_fields": { - "name": "Rohit Kumar Sharma", - "address": "Flat 12, Sunrise Apartments, Andheri East, Mumbai, Maharashtra 400069", - "aadhaar_number": "1234 5678 9123", - "date_of_birth": "1990-06-21" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00200/aadhaar_front.pdf" - }, - { - "doc_type": "PAN", - "extracted_fields": { - "name": "Rohit Kumar Sharma", - "pan_number": "PQRSX9876Z" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00200/pan_card.pdf" - }, - { - "doc_type": "ADDRESS_PROOF", - "extracted_fields": { - "address": "Flat 12, Sunrise Apartments, Andheri East, Mumbai, Maharashtra 400069" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00200/address_proof.pdf" - }, - { - "doc_type": "SALARY_SLIP", - "salary_slips": [ - { - "extracted_fields": { - "name": "Rohit Kumar Sharma", - "employer_name": "Orion Software Pvt Ltd", - "net_salary": 90000, - "salary_month": "2026-03" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00200/salary_slip_march.pdf" - }, - { - "extracted_fields": { - "name": "Rohit Kumar Sharma", - "employer_name": "Orion Software Pvt Ltd", - "net_salary": 90000, - "salary_month": "2026-04" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00200/salary_slip_april.pdf" - }, - { - "extracted_fields": { - "name": "Rohit Kumar Sharma", - "employer_name": "Orion Software Pvt Ltd", - "net_salary": 92000, - "salary_month": "2026-05" - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00200/salary_slip_may.pdf" - } - ] - }, - { - "doc_type": "BANK_STATEMENT", - "extracted_fields": { - "name": "Rohit Kumar Sharma", - "transactions": [ - { - "narration": "ORION SOFTWARE PVT LTD SALARY MAR", - "amount": 90000, - "date": "2026-04-01" - }, - { - "narration": "HOUSE RENT EMI DEBIT", - "amount": -20000, - "date": "2026-04-03" - }, - { - "narration": "ORION SOFTWARE PVT LTD SALARY APR", - "amount": 90000, - "date": "2026-05-01" - }, - { - "narration": "GROCERY STORE PAYMENT", - "amount": -1500, - "date": "2026-05-06" - }, - { - "narration": "ORION SOFTWARE PVT LTD SALARY MAY", - "amount": 92000, - "date": "2026-06-01" - } - ] - }, - "source_file_ref": "s3://kyc-docs/APP-2026-00200/bank_statement_mar_to_jun.pdf" - } - ] -}