From 7d79f525adad2cc03ea4806e63b29b5d4a600a78 Mon Sep 17 00:00:00 2001 From: aircode610 Date: Mon, 22 Jun 2026 14:45:55 +0200 Subject: [PATCH 1/4] ci: add auto-label and auto-assign workflow for new issues Adds a GitHub Actions workflow that triggers on issue creation and automatically applies labels (type, component, priority, security) and assigns team members based on keyword matching in the issue title and body. Also creates 8 new repo labels for components, priorities, and security. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/issue-triage.yml | 208 +++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 .github/workflows/issue-triage.yml diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 0000000..7f8dd62 --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,208 @@ +name: Auto-label and assign issues + +on: + issues: + types: [opened] + +permissions: + issues: write + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - name: Triage issue + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issue = context.payload.issue; + const title = (issue.title || '').toLowerCase(); + const body = (issue.body || '').toLowerCase(); + const text = `${title} ${body}`; + + const labels = new Set(); + let assignees = new Set(); + + // ── 1. Issue-type labels ────────────────────────────────── + const bugPatterns = [ + 'bug', 'error', 'crash', 'fail', 'broken', 'not working', + 'exception', 'traceback', 'stacktrace', 'typeerror', + 'keyerror', 'valueerror', 'attributeerror', '500', + 'unexpected', 'regression', 'does not work', "doesn't work", + 'cannot', "can't", 'issue with' + ]; + const featurePatterns = [ + 'feature', 'enhancement', 'request', 'add support', + 'would be nice', 'proposal', 'improve', 'new option', + 'should support', 'wish', 'idea', 'suggestion' + ]; + const docPatterns = [ + 'documentation', 'docs', 'readme', 'typo in doc', + 'missing docs', 'unclear', 'example', 'tutorial', + 'how to', 'howto', 'instructions' + ]; + const questionPatterns = [ + 'question', 'how do i', 'how can i', 'is it possible', + 'what is the', 'help me', 'guidance', 'confused', + 'explain', 'clarification' + ]; + + if (bugPatterns.some(p => text.includes(p))) { + labels.add('bug'); + } + if (featurePatterns.some(p => text.includes(p))) { + labels.add('enhancement'); + } + if (docPatterns.some(p => text.includes(p))) { + labels.add('documentation'); + } + if (questionPatterns.some(p => text.includes(p))) { + labels.add('question'); + } + // If nothing matched above, default to bug for issues that + // mention errors, otherwise enhancement + if (labels.size === 0) { + labels.add('enhancement'); + } + + // ── 2. Component labels ─────────────────────────────────── + const frontendPatterns = [ + 'frontend', 'next.js', 'nextjs', 'react', 'typescript', + 'tailwind', 'css', 'ui', 'ux', 'component', 'page', + 'pwa', 'serwist', 'vercel', 'browser', 'layout', + 'responsive', 'mobile', 'dark mode', 'calendar view', + 'upload page', 'login page', 'signup page' + ]; + const backendPatterns = [ + 'backend', 'fastapi', 'api', 'endpoint', 'router', + 'database', 'sqlite', 'sqlmodel', 'auth', 'jwt', + 'cookie', 'session', 'cors', 'sse', 'server', + 'uvicorn', 'railway', 'render', '/letters', '/actions', + '/chat', '/health', 'migration', 'schema' + ]; + const aiAgentPatterns = [ + 'ocr', 'qwen-vl', 'react agent', 'langgraph', 'agent', + 'classification', 'risk score', 'tavily', 'search tool', + 'image processing', 'letter type', 'extraction', + 'qwen', 'llm', 'prompt', 'form fill', 'form_fill' + ]; + const aiRagPatterns = [ + 'rag', 'chromadb', 'chroma', 'embedding', 'retrieval', + 'legal text', 'corpus', 'ingestion', 'ingest', + 'citation', 'law', 'aufenthg', 'asylg', 'sgb', + 'bafög', 'bafoeg', 'response draft', 'response generation', + 'hallucination', 'grounding', 'checklist' + ]; + const infraPatterns = [ + 'ci', 'cd', 'ci/cd', 'deploy', 'docker', 'github action', + 'workflow', 'pipeline', 'env var', 'environment', + 'config', 'build', 'release', 'infra', 'devops' + ]; + + if (frontendPatterns.some(p => text.includes(p))) { + labels.add('component: frontend'); + assignees.add('saintnuno'); // Dev 1: Frontend + } + if (backendPatterns.some(p => text.includes(p))) { + labels.add('component: backend'); + assignees.add('aircode610'); // Dev 2: Backend + } + if (aiAgentPatterns.some(p => text.includes(p))) { + labels.add('component: ai-agent'); + assignees.add('danialeyz'); // Dev 4: OCR + ReAct + } + if (aiRagPatterns.some(p => text.includes(p))) { + labels.add('component: ai-rag'); + assignees.add('Alir3zag'); // Dev 3: RAG pipeline + } + if (infraPatterns.some(p => text.includes(p))) { + labels.add('component: infra'); + assignees.add('aircode610'); // Repo owner + } + + // ── 3. Priority labels ──────────────────────────────────── + const criticalPatterns = [ + 'critical', 'urgent', 'blocker', 'system down', + 'data loss', 'security vulnerability', 'production down', + 'p0', 'outage' + ]; + const highPatterns = [ + 'important', 'high priority', 'blocks', 'blocking', + 'severe', 'major', 'p1', 'asap', 'deadline missed', + 'cannot deploy' + ]; + const lowPatterns = [ + 'low priority', 'minor', 'cosmetic', 'nice to have', + 'nit', 'trivial', 'p3', 'when possible', 'polish' + ]; + + if (criticalPatterns.some(p => text.includes(p))) { + labels.add('priority: critical'); + } else if (highPatterns.some(p => text.includes(p))) { + labels.add('priority: high'); + } else if (lowPatterns.some(p => text.includes(p))) { + labels.add('priority: low'); + } else { + labels.add('priority: medium'); + } + + // ── 4. Security label ───────────────────────────────────── + const securityPatterns = [ + 'security', 'vulnerability', 'cve', 'xss', 'csrf', + 'injection', 'exploit', 'leak', 'exposed secret', + 'auth bypass', 'privilege escalation' + ]; + if (securityPatterns.some(p => text.includes(p))) { + labels.add('security'); + assignees.add('aircode610'); // Owner handles security + } + + // ── 5. Fallback assignment ──────────────────────────────── + // If no component was matched, assign to repo owner for triage + if (assignees.size === 0) { + assignees.add('aircode610'); + } + + // ── 6. Apply labels ─────────────────────────────────────── + const labelArray = [...labels]; + if (labelArray.length > 0) { + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + labels: labelArray, + }); + core.info(`Applied labels: ${labelArray.join(', ')}`); + } + + // ── 7. Assign users ─────────────────────────────────────── + const assigneeArray = [...assignees]; + if (assigneeArray.length > 0) { + await github.rest.issues.addAssignees({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + assignees: assigneeArray, + }); + core.info(`Assigned to: ${assigneeArray.join(', ')}`); + } + + // ── 8. Post triage comment ──────────────────────────────── + const commentBody = [ + `👋 Thanks for opening this issue!`, + ``, + `**Auto-triage summary:**`, + `- **Labels:** ${labelArray.map(l => '`' + l + '`').join(', ')}`, + `- **Assigned to:** ${assigneeArray.map(a => '@' + a).join(', ')}`, + ``, + `> This was automatically triaged based on the issue content. ` + + `If the labels or assignment look wrong, please update them manually.`, + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: commentBody, + }); From 011fecfe164d36c64bd99312853a96fae9da658f Mon Sep 17 00:00:00 2001 From: aircode610 Date: Mon, 22 Jun 2026 15:18:30 +0200 Subject: [PATCH 2/4] fix(letters): log extraction errors in /api/letters/{id}/extract (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generic except handler in extract_letter silently swallowed exceptions: it raised KlarHTTPException (caught by klar_exception_handler) but never logged the original traceback. The misleading comment claimed the error would be "Logged by unhandled_exception_handler" — but that handler is never reached. This made production debugging impossible for extraction failures on this endpoint. Added logger.exception() to the generic handler and logger.info() to the typed PdfRenderError/ExtractionError handlers, matching what public.py already does. Also added 3 regression tests for the /api/letters/{id}/extract path (24 total, all passing). Co-Authored-By: Claude Opus 4.6 (1M context) --- backend/app/routers/letters.py | 55 ++++++++++++--- backend/tests/test_scanned_pdf_graceful.py | 79 ++++++++++++++++++++++ 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/backend/app/routers/letters.py b/backend/app/routers/letters.py index 4812bbb..c004a74 100644 --- a/backend/app/routers/letters.py +++ b/backend/app/routers/letters.py @@ -1,5 +1,6 @@ """Letter upload + retrieval endpoints (auth-required, /api/letters/*).""" +import logging from uuid import UUID from fastapi import APIRouter, Depends, File, Query, UploadFile @@ -17,7 +18,12 @@ User, utcnow, ) -from app.schemas import ErrorResponse, LetterListItem, LetterResponse, LetterUploadResponse +from app.schemas import ( + ErrorResponse, + LetterListItem, + LetterResponse, + LetterUploadResponse, +) from app.services.extraction import ( ExtractionError, extract_from_letter_file, @@ -27,6 +33,8 @@ from app.services.persistence import persist_extraction from app.services.storage import detect_magic_mime, save_letter_file +logger = logging.getLogger("klar.letters") + router = APIRouter(prefix="/api/letters", tags=["letters"]) ACCEPTED_MIMES = { @@ -141,7 +149,8 @@ async def upload_letter( raise KlarHTTPException(415, ErrorCode.LETTER_CORRUPT_FILE) # Allow image/jpeg ↔ image/jpg variants; otherwise demand strict match. if actual_mime != file.content_type and not ( - actual_mime.startswith("image/") and file.content_type.startswith("image/") + actual_mime.startswith("image/") + and file.content_type.startswith("image/") and actual_mime.split("/")[-1] == file.content_type.split("/")[-1] ): raise KlarHTTPException( @@ -221,6 +230,7 @@ async def extract_letter( letter.original_file, mime, lang=letter.language ) except PdfRenderError as exc: + logger.info("PDF render failed for letter %s: %s", letter_id, exc) letter.status = LetterStatus.ERROR db.add(letter) db.commit() @@ -228,17 +238,27 @@ async def extract_letter( # actionable message ("try uploading it as an image instead"). raise KlarHTTPException(502, ErrorCode.PDF_RENDER_FAILED, message=str(exc)) except ExtractionError as exc: + logger.info("Extraction produced no text for letter %s: %s", letter_id, exc) letter.status = LetterStatus.ERROR db.add(letter) db.commit() # Scanned image-only PDF (no text layer) or malformed model output — # surface the typed, user-friendly message instead of a raw 500. raise KlarHTTPException(502, ErrorCode.EXTRACTION_FAILED, message=str(exc)) - except Exception: + except Exception as exc: + # Log the real error server-side so we can diagnose failures — the 502 + # response stays generic to avoid leaking provider details to the client. + # (Prior to this fix the exception was silently swallowed: the comment + # claimed "Logged by unhandled_exception_handler" but KlarHTTPException + # is routed to klar_exception_handler, so the original traceback was lost.) + logger.exception( + "Extraction failed for letter %s: %s", + letter_id, + exc, + ) letter.status = LetterStatus.ERROR db.add(letter) db.commit() - # Don't leak the raw exception. Logged by unhandled_exception_handler. raise KlarHTTPException(502, ErrorCode.EXTRACTION_FAILED) actions = persist_extraction(db, letter, extracted) @@ -267,7 +287,10 @@ async def extract_letter( ), responses={ 401: {"model": ErrorResponse, "description": "Not authenticated."}, - 422: {"model": ErrorResponse, "description": "Unknown status or category value."}, + 422: { + "model": ErrorResponse, + "description": "Unknown status or category value.", + }, }, ) def list_letters( @@ -287,8 +310,15 @@ def list_letters( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in LetterStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in LetterStatus), + } + ] + }, ) parsed_category: DocumentCategory | None = None if category: @@ -299,8 +329,15 @@ def list_letters( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown category: {category!r}.", - details={"errors": [{"field": "category", "message": "must be one of " - + ", ".join(c.value for c in DocumentCategory)}]}, + details={ + "errors": [ + { + "field": "category", + "message": "must be one of " + + ", ".join(c.value for c in DocumentCategory), + } + ] + }, ) stmt = select(Letter).where(Letter.user_id == user.id) diff --git a/backend/tests/test_scanned_pdf_graceful.py b/backend/tests/test_scanned_pdf_graceful.py index 4a07122..b0c2775 100644 --- a/backend/tests/test_scanned_pdf_graceful.py +++ b/backend/tests/test_scanned_pdf_graceful.py @@ -327,6 +327,85 @@ async def test_post_letter_unexpected_error_stays_generic(monkeypatch): assert "secrets" not in exc.message +# -------------------------------------------------------------------------- +# 7. /api/letters/{id}/extract (letters.py) — the sync extract endpoint must +# also yield typed 502 errors (not raw 500s) and log the failure. +# This was a gap in the original fix: the generic except handler silently +# swallowed exceptions without logging. +# -------------------------------------------------------------------------- + + +async def _call_extract_letter(monkeypatch, *, raise_exc): + """Drive letters.extract_letter with extraction stubbed to raise `raise_exc`. + + Returns the KlarHTTPException the handler raises (or None if it didn't). + """ + from app.routers import letters as letters_router + + async def _boom(*a, **k): + raise raise_exc + + monkeypatch.setattr(letters_router, "extract_from_letter_file", _boom) + + with Session(engine) as db: + user = _make_user(db) + letter = Letter( + user_id=user.id, + language="en", + status=LetterStatus.UPLOADED, + original_file="/tmp/scanned-no-text.pdf", + ) + db.add(letter) + db.commit() + db.refresh(letter) + try: + await letters_router.extract_letter( + letter_id=letter.id, db=db, user=user + ) + except Exception as exc: # noqa: BLE001 + return exc + return None + + +async def test_extract_letter_scanned_pdf_returns_extraction_failed(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_extract_letter( + monkeypatch, + raise_exc=ExtractionError("scanned image without readable content"), + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + assert exc.code == ErrorCode.EXTRACTION_FAILED + assert "readable content" in exc.message + + +async def test_extract_letter_corrupt_pdf_returns_pdf_render_failed(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_extract_letter( + monkeypatch, + raise_exc=PdfRenderError("Could not render this PDF. It may be corrupt."), + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + assert exc.code == ErrorCode.PDF_RENDER_FAILED + assert "corrupt" in exc.message + + +async def test_extract_letter_unexpected_error_stays_generic(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_extract_letter( + monkeypatch, raise_exc=RuntimeError("some provider 500 with secrets") + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + assert exc.code == ErrorCode.EXTRACTION_FAILED + # Raw provider error must NOT leak into the user-facing message. + assert "secrets" not in exc.message + + async def test_process_letter_stream_pdf_render_failure_emits_error(monkeypatch): _install_fake_agent_module() From b533812039064d8638847b28600311558992f438 Mon Sep 17 00:00:00 2001 From: aircode610 Date: Mon, 22 Jun 2026 15:24:37 +0200 Subject: [PATCH 3/4] ci: add lint and tests workflow, fix ruff formatting in test file The CI workflow (lint.yml) was missing from this branch, so PR #14 had zero status checks. Add it back with both the ruff lint job and a new tests job that runs pytest against backend/tests/. Also fix a minor ruff format violation in test_scanned_pdf_graceful.py. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/lint.yml | 45 ++++++++++++++++++++++ backend/tests/test_scanned_pdf_graceful.py | 4 +- 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..a36c103 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Lint Python + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install ruff + run: pip install ruff + + - name: Ruff check + run: ruff check backend/ ai/ + + - name: Ruff format check + run: ruff format --check backend/ ai/ + + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -r backend/requirements.txt pytest + + - name: Run tests + working-directory: backend + env: + DATABASE_URL: "sqlite:///test.db" + JWT_SECRET: "test-secret-test-secret-test-secret-32" + run: python -m pytest tests/ -v diff --git a/backend/tests/test_scanned_pdf_graceful.py b/backend/tests/test_scanned_pdf_graceful.py index b0c2775..b3b1d78 100644 --- a/backend/tests/test_scanned_pdf_graceful.py +++ b/backend/tests/test_scanned_pdf_graceful.py @@ -359,9 +359,7 @@ async def _boom(*a, **k): db.commit() db.refresh(letter) try: - await letters_router.extract_letter( - letter_id=letter.id, db=db, user=user - ) + await letters_router.extract_letter(letter_id=letter.id, db=db, user=user) except Exception as exc: # noqa: BLE001 return exc return None From 28d9ffd27d43e5e14d4fcc2ba51fdeb51a2b5475 Mon Sep 17 00:00:00 2001 From: aircode610 Date: Mon, 22 Jun 2026 15:41:11 +0200 Subject: [PATCH 4/4] fix(ci): resolve ruff lint/format errors and add pytest-asyncio to CI The CI workflow was failing for two reasons: 1. pytest-asyncio was not installed, causing all 14 async tests to fail with "async def functions are not natively supported" 2. ruff found 22 lint errors (unused imports, f-strings without placeholders, E402 import ordering, unused variables) in the merged code (PR branch + main) Adds pytest-asyncio to the CI install step, fixes all ruff lint errors (removing unused imports/variables, reordering imports in public.py), and applies ruff format across the codebase. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/lint.yml | 2 +- ai/form_fill.py | 8 +- ai/rag/generator.py | 14 +- ai/rag/ingest.py | 90 ++++++---- ai/rag/retrieval.py | 38 ++-- ai/rag/schemas.py | 4 +- ai/react_agent/agent.py | 48 +++-- ai/react_agent/ocr.py | 4 +- ai/schemas.py | 47 +++-- backend/app/auth/dependencies.py | 11 +- backend/app/auth/router.py | 2 - backend/app/auth/utils.py | 4 +- backend/app/config.py | 6 +- backend/app/database.py | 1 + backend/app/errors.py | 49 ++--- backend/app/main.py | 1 + backend/app/models.py | 38 ++-- backend/app/pipeline/orchestrator.py | 141 +++++++++++---- backend/app/rag/store.py | 1 + backend/app/routers/actions.py | 11 +- backend/app/routers/deadlines.py | 11 +- backend/app/routers/public.py | 112 ++++++------ backend/app/routers/rag.py | 2 +- backend/app/schemas.py | 20 ++- backend/app/services/__init__.py | 6 +- backend/app/services/ai_bridge.py | 257 ++++++++++++++------------- backend/app/services/extraction.py | 21 ++- backend/app/services/pdf_pages.py | 4 +- backend/app/services/persistence.py | 4 +- 29 files changed, 592 insertions(+), 365 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a36c103..823c361 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,7 +35,7 @@ jobs: python-version: "3.12" - name: Install dependencies - run: pip install -r backend/requirements.txt pytest + run: pip install -r backend/requirements.txt pytest pytest-asyncio - name: Run tests working-directory: backend diff --git a/ai/form_fill.py b/ai/form_fill.py index 6c3b868..50de5f3 100644 --- a/ai/form_fill.py +++ b/ai/form_fill.py @@ -11,9 +11,7 @@ import httpx DASHSCOPE_API_KEY = os.environ.get("DASHSCOPE_API_KEY", "") -DASHSCOPE_INTL_URL = ( - "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" -) +DASHSCOPE_INTL_URL = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" # Standard placeholder patterns for common German form fields PLACEHOLDER_MAP = { @@ -70,7 +68,9 @@ async def generate_filled_form( b64 = base64.b64encode(image_bytes).decode() ext = image_path.rsplit(".", 1)[-1].lower() - mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png"}.get(ext, "image/jpeg") + mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png"}.get( + ext, "image/jpeg" + ) data_url = f"data:{mime};base64,{b64}" field_instructions = _build_field_instructions(placeholders) diff --git a/ai/rag/generator.py b/ai/rag/generator.py index 11e64ff..fe78f81 100644 --- a/ai/rag/generator.py +++ b/ai/rag/generator.py @@ -14,8 +14,14 @@ QWEN_AGENT_MODEL = os.environ.get("QWEN_AGENT_MODEL", "qwen3.7-plus") LANGUAGE_NAMES = { - "en": "English", "de": "German", "tr": "Turkish", "ar": "Arabic", - "es": "Spanish", "fr": "French", "zh": "Chinese", "fa": "Persian", + "en": "English", + "de": "German", + "tr": "Turkish", + "ar": "Arabic", + "es": "Spanish", + "fr": "French", + "zh": "Chinese", + "fa": "Persian", } _model = ChatOpenAI( @@ -34,7 +40,9 @@ async def generate_response( language: str = "en", ) -> GenerationOutput: """Retrieve legal context from ChromaDB, inject into prompt, return structured output.""" - legal_context = retrieve_as_context(agent_result.letter_type, agent_result.consequence) + legal_context = retrieve_as_context( + agent_result.letter_type, agent_result.consequence + ) prompt = GENERATION_PROMPT.format( ocr_text=ocr_text[:3000], diff --git a/ai/rag/ingest.py b/ai/rag/ingest.py index fb26cff..c489210 100644 --- a/ai/rag/ingest.py +++ b/ai/rag/ingest.py @@ -26,31 +26,32 @@ # ── Paths ───────────────────────────────────────────────────────────────────── -ROOT = Path(__file__).resolve().parent.parent # ai/ -LAWS_DIR = ROOT / "data" / "laws" # ai/data/laws/ -CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ +ROOT = Path(__file__).resolve().parent.parent # ai/ +LAWS_DIR = ROOT / "data" / "laws" # ai/data/laws/ +CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ COLLECTION_NAME = "german_laws" # ── Law file → abbreviation map ─────────────────────────────────────────────── LAWS = { - "aufenthg.md": "AufenthG", - "aufenthv.md": "AufenthV", - "beschv.md": "BeschV", - "vwvfg.md": "VwVfG", - "bafoeg.md": "BAföG", - "asylg.md": "AsylG", - "asylblg.md": "AsylbLG", - "wogg.md": "WoGG", - "bmg.md": "BMG", - "intv.md": "IntV", - "owig.md": "OWiG", - "estg.md": "EStG", - "sgb5.md": "SGB V", + "aufenthg.md": "AufenthG", + "aufenthv.md": "AufenthV", + "beschv.md": "BeschV", + "vwvfg.md": "VwVfG", + "bafoeg.md": "BAföG", + "asylg.md": "AsylG", + "asylblg.md": "AsylbLG", + "wogg.md": "WoGG", + "bmg.md": "BMG", + "intv.md": "IntV", + "owig.md": "OWiG", + "estg.md": "EStG", + "sgb5.md": "SGB V", } # ── Qwen client ─────────────────────────────────────────────────────────────── + def get_qwen_client() -> OpenAI: api_key = os.getenv("DASHSCOPE_API_KEY") if not api_key: @@ -64,8 +65,8 @@ def get_qwen_client() -> OpenAI: # Qwen text-embedding-v3 limits -MAX_BATCH_SIZE = 10 # max texts per API call -MAX_CHARS = 6000 # conservative char limit (~8192 tokens safety margin) +MAX_BATCH_SIZE = 10 # max texts per API call +MAX_CHARS = 6000 # conservative char limit (~8192 tokens safety margin) def truncate(text: str) -> str: @@ -85,7 +86,7 @@ def embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]: texts = [truncate(t) for t in texts] for i in range(0, len(texts), MAX_BATCH_SIZE): - batch = texts[i:i + MAX_BATCH_SIZE] + batch = texts[i : i + MAX_BATCH_SIZE] response = client.embeddings.create( model="text-embedding-v3", input=batch, @@ -101,6 +102,7 @@ def embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]: # ── Chunking ────────────────────────────────────────────────────────────────── + def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: """ Split a law's markdown into one chunk per § paragraph. @@ -110,7 +112,7 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: id, text, paragraph, title, law """ # Match § headers at any heading level: ### § 1, #### § 4a, etc. - pattern = r'^#{1,4} (§ \d+[a-z]?\b.*?)$' + pattern = r"^#{1,4} (§ \d+[a-z]?\b.*?)$" matches = list(re.finditer(pattern, text, re.MULTILINE)) chunks = [] @@ -121,7 +123,7 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: end = matches[i + 1].start() if i + 1 < len(matches) else len(text) header = match.group(1).strip() - para_num_match = re.match(r'(§ \d+[a-z]?)', header) + para_num_match = re.match(r"(§ \d+[a-z]?)", header) para_num = para_num_match.group(1) if para_num_match else header body = text[start:end].strip() @@ -141,19 +143,22 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: counter += 1 seen_ids.add(unique_id) - chunks.append({ - "id": unique_id, - "text": body, - "paragraph": para_num, - "title": header, - "law": law_abbrev, - }) + chunks.append( + { + "id": unique_id, + "text": body, + "paragraph": para_num, + "title": header, + "law": law_abbrev, + } + ) return chunks # ── Main ────────────────────────────────────────────────────────────────────── + def ingest_all(): print("── Klar RAG Ingestion ──────────────────────────────────────") @@ -197,32 +202,39 @@ def ingest_all(): print(f" ⚠ No paragraphs parsed in {filename}") continue - print(f" Embedding {law_abbrev}: {len(chunks)} paragraphs ...", end=" ", flush=True) + print( + f" Embedding {law_abbrev}: {len(chunks)} paragraphs ...", + end=" ", + flush=True, + ) texts_to_embed = [c["text"] for c in chunks] embeddings = embed_texts(client, texts_to_embed) batch_size = 100 for i in range(0, len(chunks), batch_size): - batch_chunks = chunks[i:i + batch_size] - batch_embeddings = embeddings[i:i + batch_size] + batch_chunks = chunks[i : i + batch_size] + batch_embeddings = embeddings[i : i + batch_size] collection.add( ids=[c["id"] for c in batch_chunks], documents=[c["text"] for c in batch_chunks], - metadatas=[{ - "law": c["law"], - "paragraph": c["paragraph"], - "title": c["title"], - } for c in batch_chunks], + metadatas=[ + { + "law": c["law"], + "paragraph": c["paragraph"], + "title": c["title"], + } + for c in batch_chunks + ], embeddings=batch_embeddings, ) - print(f"✅") + print("✅") total_chunks += len(chunks) print() - print(f"── Done ────────────────────────────────────────────────────") + print("── Done ────────────────────────────────────────────────────") print(f" Total chunks ingested : {total_chunks}") print(f" ChromaDB saved to : {CHROMA_DIR.resolve()}") print() @@ -230,4 +242,4 @@ def ingest_all(): if __name__ == "__main__": - ingest_all() \ No newline at end of file + ingest_all() diff --git a/ai/rag/retrieval.py b/ai/rag/retrieval.py index aee8302..81c275c 100644 --- a/ai/rag/retrieval.py +++ b/ai/rag/retrieval.py @@ -25,20 +25,21 @@ # ── Paths ───────────────────────────────────────────────────────────────────── -ROOT = Path(__file__).resolve().parent.parent # ai/ -CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ +ROOT = Path(__file__).resolve().parent.parent # ai/ +CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ COLLECTION_NAME = "german_laws" # ── Schema ──────────────────────────────────────────────────────────────────── + @dataclass class LegalChunk: - section: str # e.g. "§ 81" - law: str # e.g. "AufenthG" - title: str # e.g. "§ 81 Beantragung des Aufenthaltstitels" - text: str # full paragraph text - citation: str # e.g. "§ 81 AufenthG" - score: float # cosine similarity, higher = more relevant + section: str # e.g. "§ 81" + law: str # e.g. "AufenthG" + title: str # e.g. "§ 81 Beantragung des Aufenthaltstitels" + text: str # full paragraph text + citation: str # e.g. "§ 81 AufenthG" + score: float # cosine similarity, higher = more relevant # ── Singleton clients ───────────────────────────────────────────────────────── @@ -87,6 +88,7 @@ def _embed_query(query: str) -> list[float]: # ── Core retrieval ──────────────────────────────────────────────────────────── + def retrieve_legal_context( letter_type: str, consequence: str, @@ -124,14 +126,16 @@ def retrieve_legal_context( results["distances"][0], ): section = meta.get("paragraph", meta.get("section", "Unknown")) - chunks.append(LegalChunk( - section=section, - law=meta["law"], - title=meta["title"], - text=doc, - citation=f"{section} {meta['law']}", - score=round(1 - distance, 4), - )) + chunks.append( + LegalChunk( + section=section, + law=meta["law"], + title=meta["title"], + text=doc, + citation=f"{section} {meta['law']}", + score=round(1 - distance, 4), + ) + ) return chunks @@ -156,4 +160,4 @@ def retrieve_as_context( text_preview = c.text[:1000] + "..." if len(c.text) > 1000 else c.text parts.append(f"[{c.citation}] {c.title}\n{text_preview}") - return "\n\n---\n\n".join(parts) \ No newline at end of file + return "\n\n---\n\n".join(parts) diff --git a/ai/rag/schemas.py b/ai/rag/schemas.py index edc7ba4..bcae2c2 100644 --- a/ai/rag/schemas.py +++ b/ai/rag/schemas.py @@ -8,7 +8,7 @@ @dataclass class RAGEvent: - type: str # "explanation" | "response_draft" | "checklist" | "citations" | "error" + type: str # "explanation" | "response_draft" | "checklist" | "citations" | "error" data: dict confidence: str = "high" # "high" if RAG matched well, "low" if no strong matches # data shapes per type: @@ -16,4 +16,4 @@ class RAGEvent: # response_draft: {"chunk": str} — streamed token by token # checklist: {"items": list[str]} — emitted once, complete # citations: {"items": list[dict]} — emitted once, [{section, text}, ...] - # error: {"message": str} \ No newline at end of file + # error: {"message": str} diff --git a/ai/react_agent/agent.py b/ai/react_agent/agent.py index 155c349..40a671a 100644 --- a/ai/react_agent/agent.py +++ b/ai/react_agent/agent.py @@ -43,9 +43,15 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: """Run the ReAct agent with structured output via LangGraph response_format.""" try: - result = await _agent.ainvoke({ - "messages": [HumanMessage(content=f"Analyze this German official letter:\n\n{ocr_text}")], - }) + result = await _agent.ainvoke( + { + "messages": [ + HumanMessage( + content=f"Analyze this German official letter:\n\n{ocr_text}" + ) + ], + } + ) analysis: AgentAnalysis = result["structured_response"] @@ -57,10 +63,22 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: except ValueError: pass - yield AgentEvent("classification", {"type": analysis.classification.type, "agency": analysis.classification.agency}) - yield AgentEvent("risk_score", {"score": analysis.risk_score.score, "label": analysis.risk_score.label}) + yield AgentEvent( + "classification", + { + "type": analysis.classification.type, + "agency": analysis.classification.agency, + }, + ) + yield AgentEvent( + "risk_score", + {"score": analysis.risk_score.score, "label": analysis.risk_score.label}, + ) if analysis.deadline.date: - yield AgentEvent("deadline", {"date": analysis.deadline.date, "days_remaining": days_remaining}) + yield AgentEvent( + "deadline", + {"date": analysis.deadline.date, "days_remaining": days_remaining}, + ) yield AgentEvent("consequence", {"text": analysis.consequence.text}) except Exception as e: @@ -70,9 +88,19 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: def get_last_agent_result(events: list[AgentEvent], ocr_text: str) -> AgentResult: """Reconstruct an AgentResult from collected events.""" data = {e.type: e.data for e in events} - c, d, r, q = data.get("classification", {}), data.get("deadline", {}), data.get("risk_score", {}), data.get("consequence", {}) + c, d, r, q = ( + data.get("classification", {}), + data.get("deadline", {}), + data.get("risk_score", {}), + data.get("consequence", {}), + ) return AgentResult( - ocr_text=ocr_text, letter_type=c.get("type", "Unknown"), agency=c.get("agency", "Unknown"), - deadline_date=d.get("date"), days_remaining=d.get("days_remaining"), - consequence=q.get("text", ""), risk_score=r.get("score", 3), risk_label=r.get("label", "Medium"), + ocr_text=ocr_text, + letter_type=c.get("type", "Unknown"), + agency=c.get("agency", "Unknown"), + deadline_date=d.get("date"), + days_remaining=d.get("days_remaining"), + consequence=q.get("text", ""), + risk_score=r.get("score", 3), + risk_label=r.get("label", "Medium"), ) diff --git a/ai/react_agent/ocr.py b/ai/react_agent/ocr.py index 8f047f7..e1359f1 100644 --- a/ai/react_agent/ocr.py +++ b/ai/react_agent/ocr.py @@ -42,7 +42,9 @@ def _parse_ocr_response(result: object) -> str: "image without readable content." ) - message = (choices[0] or {}).get("message") if isinstance(choices[0], dict) else None + message = ( + (choices[0] or {}).get("message") if isinstance(choices[0], dict) else None + ) content = (message or {}).get("content") if isinstance(message, dict) else None if not content or not str(content).strip(): diff --git a/ai/schemas.py b/ai/schemas.py index 0151249..17803a2 100644 --- a/ai/schemas.py +++ b/ai/schemas.py @@ -4,19 +4,32 @@ # --- Structured output models (used by LLM response_format) --- + class Classification(BaseModel): - type: str = Field(description="Letter type, e.g. 'Residence Permit - Document Request', 'Health Insurance - Tax ID Request'") - agency: str = Field(description="Sender agency name, e.g. 'Techniker Krankenkasse', 'Ausländerbehörde München'") + type: str = Field( + description="Letter type, e.g. 'Residence Permit - Document Request', 'Health Insurance - Tax ID Request'" + ) + agency: str = Field( + description="Sender agency name, e.g. 'Techniker Krankenkasse', 'Ausländerbehörde München'" + ) class Deadline(BaseModel): - date: str | None = Field(description="Deadline date in YYYY-MM-DD format, or null if no deadline") - days_remaining: int | None = Field(description="Days until deadline from today, or null") - source: str = Field(description="'letter' if read directly, 'calculated' if computed from letter date, 'searched' if from web, 'none' if no deadline applies") + date: str | None = Field( + description="Deadline date in YYYY-MM-DD format, or null if no deadline" + ) + days_remaining: int | None = Field( + description="Days until deadline from today, or null" + ) + source: str = Field( + description="'letter' if read directly, 'calculated' if computed from letter date, 'searched' if from web, 'none' if no deadline applies" + ) class Consequence(BaseModel): - text: str = Field(description="Detailed consequence description of what happens if deadline is missed or action not taken") + text: str = Field( + description="Detailed consequence description of what happens if deadline is missed or action not taken" + ) severity: str = Field(description="One-line severity summary") @@ -28,6 +41,7 @@ class RiskScore(BaseModel): class AgentAnalysis(BaseModel): """Structured output from the ReAct agent letter analysis.""" + classification: Classification deadline: Deadline consequence: Consequence @@ -35,23 +49,34 @@ class AgentAnalysis(BaseModel): class Citation(BaseModel): - section: str = Field(description="Legal paragraph reference, e.g. '§ 81 Abs. 4 AufenthG'") + section: str = Field( + description="Legal paragraph reference, e.g. '§ 81 Abs. 4 AufenthG'" + ) text: str = Field(description="Brief explanation of why this citation is relevant") class GenerationOutput(BaseModel): """Structured output from the response generation LLM.""" - explanation: str = Field(description="Clear plain-language explanation of the letter") + + explanation: str = Field( + description="Clear plain-language explanation of the letter" + ) response_draft: str = Field(description="Formal response letter in Behördendeutsch") - checklist: list[str] = Field(description="List of documents the user needs to prepare, with German terms in parentheses") - citations: list[Citation] = Field(default_factory=list, description="Legal § references that are relevant. Empty list if none found.") + checklist: list[str] = Field( + description="List of documents the user needs to prepare, with German terms in parentheses" + ) + citations: list[Citation] = Field( + default_factory=list, + description="Legal § references that are relevant. Empty list if none found.", + ) # --- Internal data transfer objects --- + @dataclass class AgentEvent: - type: str # "classification", "risk_score", "deadline", "consequence", "error" + type: str # "classification", "risk_score", "deadline", "consequence", "error" data: dict diff --git a/backend/app/auth/dependencies.py b/backend/app/auth/dependencies.py index 7659868..0c10f16 100644 --- a/backend/app/auth/dependencies.py +++ b/backend/app/auth/dependencies.py @@ -1,7 +1,6 @@ """FastAPI dependency that resolves the current user from the session cookie.""" import logging -from datetime import datetime from fastapi import Depends, Request, status from sqlmodel import Session as DBSession, select @@ -37,7 +36,8 @@ def _resolve_user(token: str | None, db: DBSession) -> User: logger.warning( "AUTH_SESSION_NOT_FOUND: no Session row for token=%s... " "(DB=%s; possible causes: db wiped, multiple workers, env mismatch)", - token_prefix, settings.database_url, + token_prefix, + settings.database_url, ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -47,7 +47,9 @@ def _resolve_user(token: str | None, db: DBSession) -> User: if session_row.expires_at < utcnow(): logger.info( "AUTH_SESSION_EXPIRED: token=%s... expired_at=%s now=%s", - token_prefix, session_row.expires_at, utcnow(), + token_prefix, + session_row.expires_at, + utcnow(), ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -59,7 +61,8 @@ def _resolve_user(token: str | None, db: DBSession) -> User: # Session row exists but the user it points to is gone — corrupt FK. logger.error( "AUTH: orphan Session row token=%s... user_id=%s has no User", - token_prefix, session_row.user_id, + token_prefix, + session_row.user_id, ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, diff --git a/backend/app/auth/router.py b/backend/app/auth/router.py index 151767b..67af069 100644 --- a/backend/app/auth/router.py +++ b/backend/app/auth/router.py @@ -1,7 +1,5 @@ """Authentication routes: signup, login, logout, me, forgot/reset password.""" -from datetime import datetime - from fastapi import APIRouter, Depends, Request, Response, status from pydantic import BaseModel, EmailStr, Field, field_validator from sqlmodel import Session as DBSession, select diff --git a/backend/app/auth/utils.py b/backend/app/auth/utils.py index 9d9a5f5..7198836 100644 --- a/backend/app/auth/utils.py +++ b/backend/app/auth/utils.py @@ -12,7 +12,9 @@ def hash_password(plain: str) -> str: """bcrypt with cost factor 12 — ~250ms on a modern laptop.""" - return bcrypt.hashpw(plain.encode("utf-8"), bcrypt.gensalt(rounds=12)).decode("utf-8") + return bcrypt.hashpw(plain.encode("utf-8"), bcrypt.gensalt(rounds=12)).decode( + "utf-8" + ) def verify_password(plain: str, password_hash: str) -> bool: diff --git a/backend/app/config.py b/backend/app/config.py index 15037ca..5a60bc6 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -46,7 +46,11 @@ def effective_llm_api_key(self) -> str: @property def effective_llm_base_url(self) -> str: - return self.qwen_api_base or self.llm_base_url or "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + return ( + self.qwen_api_base + or self.llm_base_url + or "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + ) @property def effective_llm_model(self) -> str: diff --git a/backend/app/database.py b/backend/app/database.py index f666408..836a132 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -21,6 +21,7 @@ def init_db() -> None: path = settings.database_url.replace("sqlite:///", "", 1) Path(path).parent.mkdir(parents=True, exist_ok=True) from app import models # noqa: F401 — populate SQLModel metadata + SQLModel.metadata.create_all(engine) diff --git a/backend/app/errors.py b/backend/app/errors.py index 20478a8..1240501 100644 --- a/backend/app/errors.py +++ b/backend/app/errors.py @@ -56,35 +56,43 @@ class ErrorCode(str, Enum): """ # --- generic --- - HTTP_ERROR = "HTTP_ERROR" # untyped fallback (legacy HTTPException) - INTERNAL_ERROR = "INTERNAL_ERROR" # unhandled exception - VALIDATION_ERROR = "VALIDATION_ERROR" # request body / query / path + HTTP_ERROR = "HTTP_ERROR" # untyped fallback (legacy HTTPException) + INTERNAL_ERROR = "INTERNAL_ERROR" # unhandled exception + VALIDATION_ERROR = "VALIDATION_ERROR" # request body / query / path # --- auth --- - AUTH_NOT_AUTHENTICATED = "AUTH_NOT_AUTHENTICATED" # no cookie at all - AUTH_SESSION_NOT_FOUND = "AUTH_SESSION_NOT_FOUND" # cookie present, but no Session row in DB - AUTH_SESSION_EXPIRED = "AUTH_SESSION_EXPIRED" # Session row exists but past expires_at + AUTH_NOT_AUTHENTICATED = "AUTH_NOT_AUTHENTICATED" # no cookie at all + AUTH_SESSION_NOT_FOUND = ( + "AUTH_SESSION_NOT_FOUND" # cookie present, but no Session row in DB + ) + AUTH_SESSION_EXPIRED = ( + "AUTH_SESSION_EXPIRED" # Session row exists but past expires_at + ) AUTH_INVALID_CREDENTIALS = "AUTH_INVALID_CREDENTIALS" # wrong email / password - AUTH_EMAIL_TAKEN = "AUTH_EMAIL_TAKEN" # signup with existing email - AUTH_INVALID_RESET_TOKEN = "AUTH_INVALID_RESET_TOKEN" # token unknown / already used + AUTH_EMAIL_TAKEN = "AUTH_EMAIL_TAKEN" # signup with existing email + AUTH_INVALID_RESET_TOKEN = ( + "AUTH_INVALID_RESET_TOKEN" # token unknown / already used + ) AUTH_RESET_TOKEN_EXPIRED = "AUTH_RESET_TOKEN_EXPIRED" # token past 15-min TTL # --- letters --- LETTER_NOT_FOUND = "LETTER_NOT_FOUND" - LETTER_FILE_MISSING = "LETTER_FILE_MISSING" # row exists but file gone + LETTER_FILE_MISSING = "LETTER_FILE_MISSING" # row exists but file gone LETTER_EMPTY_UPLOAD = "LETTER_EMPTY_UPLOAD" LETTER_TOO_LARGE = "LETTER_TOO_LARGE" LETTER_UNSUPPORTED_TYPE = "LETTER_UNSUPPORTED_TYPE" - LETTER_CORRUPT_FILE = "LETTER_CORRUPT_FILE" # magic-bytes mismatch - LETTER_MIME_MISMATCH = "LETTER_MIME_MISMATCH" # declared ≠ detected + LETTER_CORRUPT_FILE = "LETTER_CORRUPT_FILE" # magic-bytes mismatch + LETTER_MIME_MISMATCH = "LETTER_MIME_MISMATCH" # declared ≠ detected # --- actions --- ACTION_NOT_FOUND = "ACTION_NOT_FOUND" # --- pipeline / AI --- - EXTRACTION_FAILED = "EXTRACTION_FAILED" # SSE-only: model returned no tool call, parse error, etc. - LLM_PROVIDER_ERROR = "LLM_PROVIDER_ERROR" # network / 5xx from Qwen - PDF_RENDER_FAILED = "PDF_RENDER_FAILED" # pdf2image / poppler missing + EXTRACTION_FAILED = ( + "EXTRACTION_FAILED" # SSE-only: model returned no tool call, parse error, etc. + ) + LLM_PROVIDER_ERROR = "LLM_PROVIDER_ERROR" # network / 5xx from Qwen + PDF_RENDER_FAILED = "PDF_RENDER_FAILED" # pdf2image / poppler missing # User-facing default messages per code. Keep short, no jargon, no secrets. @@ -93,7 +101,6 @@ class ErrorCode(str, Enum): ErrorCode.HTTP_ERROR: "Something went wrong with that request.", ErrorCode.INTERNAL_ERROR: "Something went wrong on our end. Please try again.", ErrorCode.VALIDATION_ERROR: "Some fields in your request are invalid.", - ErrorCode.AUTH_NOT_AUTHENTICATED: "Please sign in to continue.", ErrorCode.AUTH_SESSION_NOT_FOUND: "Your session is no longer recognized. Please sign in again.", ErrorCode.AUTH_SESSION_EXPIRED: "Your session has expired. Please sign in again.", @@ -101,7 +108,6 @@ class ErrorCode(str, Enum): ErrorCode.AUTH_EMAIL_TAKEN: "An account with that email already exists.", ErrorCode.AUTH_INVALID_RESET_TOKEN: "This reset link is invalid or has already been used.", ErrorCode.AUTH_RESET_TOKEN_EXPIRED: "This reset link has expired. Please request a new one.", - ErrorCode.LETTER_NOT_FOUND: "That letter doesn't exist or you don't have access to it.", ErrorCode.LETTER_FILE_MISSING: "We can't find the uploaded file for this letter.", ErrorCode.LETTER_EMPTY_UPLOAD: "The uploaded file is empty.", @@ -109,9 +115,7 @@ class ErrorCode(str, Enum): ErrorCode.LETTER_UNSUPPORTED_TYPE: "We can only read JPEG, PNG, HEIC, WebP, or PDF letters.", ErrorCode.LETTER_CORRUPT_FILE: "The file looks corrupted or isn't the type it claims to be.", ErrorCode.LETTER_MIME_MISMATCH: "The file's content doesn't match its declared type.", - ErrorCode.ACTION_NOT_FOUND: "That action doesn't exist or you don't have access to it.", - ErrorCode.EXTRACTION_FAILED: "We couldn't read this letter. Try a clearer photo or PDF.", ErrorCode.LLM_PROVIDER_ERROR: "Our AI provider is having trouble. Please try again in a moment.", ErrorCode.PDF_RENDER_FAILED: "We couldn't open that PDF. Try uploading it as an image instead.", @@ -188,8 +192,9 @@ async def generic_http_exception_handler( # If detail is already a Klar envelope dict (from KlarHTTPException # routing through the default handler), pass it through. if isinstance(exc.detail, dict) and "code" in exc.detail: - return JSONResponse(status_code=exc.status_code, content=exc.detail, - headers=exc.headers) + return JSONResponse( + status_code=exc.status_code, content=exc.detail, headers=exc.headers + ) # Auth-shaped status codes get more specific codes by default. code = ErrorCode.HTTP_ERROR @@ -239,7 +244,9 @@ async def unhandled_exception_handler(req: Request, exc: Exception) -> JSONRespo """ logger.exception( "Unhandled exception on %s %s: %s", - req.method, req.url.path, exc, + req.method, + req.url.path, + exc, ) return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/backend/app/main.py b/backend/app/main.py index 04f428b..ef34d42 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -85,6 +85,7 @@ def _bridge_env_to_ai_team() -> None: value still wins. """ import os + if settings.effective_llm_api_key: os.environ.setdefault("DASHSCOPE_API_KEY", settings.effective_llm_api_key) if settings.effective_llm_base_url: diff --git a/backend/app/models.py b/backend/app/models.py index 69b4890..19404b4 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -55,20 +55,22 @@ class DocumentCategory(str, Enum): does not fit any defined bucket. """ - HEALTH_INSURANCE = "health_insurance" # AOK, TK, BARMER, private KV - OTHER_INSURANCE = "other_insurance" # Haftpflicht, Hausrat, KFZ, Leben - BANKING = "banking" # bank accounts, credit cards, SCHUFA - TAX = "tax" # Finanzamt - IMMIGRATION = "immigration" # Ausländerbehörde, residence/visa - EDUCATION = "education" # universities, BAföG, Studentenwerk - HOUSING = "housing" # landlord, property management - UTILITIES = "utilities" # Strom, Gas, Wasser, Internet, Mobilfunk - EMPLOYMENT = "employment" # Arbeitgeber, HR, Lohn - GOVERNMENT_BENEFITS = "government_benefits" # ALG I/II, Kindergeld, Elterngeld, Wohngeld - PENSION = "pension" # Deutsche Rentenversicherung - BROADCAST_FEE = "broadcast_fee" # Beitragsservice / Rundfunk - CIVIC = "civic" # Bürgeramt, Personalausweis, Pass - LEGAL_DEBT = "legal_debt" # Mahnbescheid, Inkasso, Bußgeld, Anwalt + HEALTH_INSURANCE = "health_insurance" # AOK, TK, BARMER, private KV + OTHER_INSURANCE = "other_insurance" # Haftpflicht, Hausrat, KFZ, Leben + BANKING = "banking" # bank accounts, credit cards, SCHUFA + TAX = "tax" # Finanzamt + IMMIGRATION = "immigration" # Ausländerbehörde, residence/visa + EDUCATION = "education" # universities, BAföG, Studentenwerk + HOUSING = "housing" # landlord, property management + UTILITIES = "utilities" # Strom, Gas, Wasser, Internet, Mobilfunk + EMPLOYMENT = "employment" # Arbeitgeber, HR, Lohn + GOVERNMENT_BENEFITS = ( + "government_benefits" # ALG I/II, Kindergeld, Elterngeld, Wohngeld + ) + PENSION = "pension" # Deutsche Rentenversicherung + BROADCAST_FEE = "broadcast_fee" # Beitragsservice / Rundfunk + CIVIC = "civic" # Bürgeramt, Personalausweis, Pass + LEGAL_DEBT = "legal_debt" # Mahnbescheid, Inkasso, Bußgeld, Anwalt OTHER = "other" @@ -114,15 +116,15 @@ class Letter(SQLModel, table=True): original_file: str = "" # Spec-flat structured fields (denormalized from ActionItem for /api/letters) - letter_type: str = "" # alias of document_type for spec compat - risk_score: int = 0 # denormalized highest action risk + letter_type: str = "" # alias of document_type for spec compat + risk_score: int = 0 # denormalized highest action risk deadline_date: Optional[date] = None # denormalized most-urgent action deadline # Rich Klar extras institution: str = "" document_type: str = "" category: DocumentCategory = DocumentCategory.OTHER - summary: str = "" # language matches Letter.language + summary: str = "" # language matches Letter.language language: str = "en" # OCR + long-form generation outputs @@ -132,7 +134,7 @@ class Letter(SQLModel, table=True): # "low confidence, get a human" prompt. Computed from extraction outputs. confidence: Optional[float] = None explanation: str = "" - response_draft: str = "" # ALWAYS German (formal reply to German institution) + response_draft: str = "" # ALWAYS German (formal reply to German institution) checklist: list[str] = Field(default_factory=list, sa_column=Column(JSON)) citations: list[dict] = Field(default_factory=list, sa_column=Column(JSON)) consequence: str = "" diff --git a/backend/app/pipeline/orchestrator.py b/backend/app/pipeline/orchestrator.py index 6048a0a..8896e95 100644 --- a/backend/app/pipeline/orchestrator.py +++ b/backend/app/pipeline/orchestrator.py @@ -40,7 +40,7 @@ import os import re import tempfile -from datetime import date, datetime +from datetime import date from typing import AsyncIterator from uuid import UUID @@ -166,10 +166,31 @@ async def _ocr_letter_file(path: str) -> str: # text for common German date patterns and pick the most-likely deadline. _GERMAN_MONTHS = { - "januar": 1, "jan": 1, "februar": 2, "feb": 2, "märz": 3, "mar": 3, "mrz": 3, - "april": 4, "apr": 4, "mai": 5, "juni": 6, "jun": 6, "juli": 7, "jul": 7, - "august": 8, "aug": 8, "september": 9, "sep": 9, "sept": 9, - "oktober": 10, "okt": 10, "november": 11, "nov": 11, "dezember": 12, "dez": 12, + "januar": 1, + "jan": 1, + "februar": 2, + "feb": 2, + "märz": 3, + "mar": 3, + "mrz": 3, + "april": 4, + "apr": 4, + "mai": 5, + "juni": 6, + "jun": 6, + "juli": 7, + "jul": 7, + "august": 8, + "aug": 8, + "september": 9, + "sep": 9, + "sept": 9, + "oktober": 10, + "okt": 10, + "november": 11, + "nov": 11, + "dezember": 12, + "dez": 12, } # Match: "28. Oktober 2021", "28 Oktober 2021", "den 28. Oktober 2021" @@ -269,16 +290,20 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str except Exception as exc: logger.exception( "AI team's modules failed to import — check env (DASHSCOPE_API_KEY, " - "TAVILY_API_KEY): %s", exc, + "TAVILY_API_KEY): %s", + exc, ) - yield sse_event("error", sse_error_payload( - ErrorCode.LLM_PROVIDER_ERROR, - message=( - "AI pipeline failed to initialize. Most likely cause: missing " - "DASHSCOPE_API_KEY or TAVILY_API_KEY env var on the backend " - "process. See server logs." + yield sse_event( + "error", + sse_error_payload( + ErrorCode.LLM_PROVIDER_ERROR, + message=( + "AI pipeline failed to initialize. Most likely cause: missing " + "DASHSCOPE_API_KEY or TAVILY_API_KEY env var on the backend " + "process. See server logs." + ), ), - )) + ) return out_lang = normalize_lang(lang) @@ -316,16 +341,18 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # STAGE 2 — ReAct agent (LangGraph + Tavily, ~5-15s) # ============================================================ agent_events_collected = [] - classification_data: dict | None = None risk_label = "Medium" async for ev in run_react_agent(ocr_text): agent_events_collected.append(ev) if ev.type == "classification": - classification_data = ev.data - category = ai_bridge.map_classification_to_category(ev.data.get("type", "")) - letter.document_type = ev.data.get("type", "") or letter.document_type + category = ai_bridge.map_classification_to_category( + ev.data.get("type", "") + ) + letter.document_type = ( + ev.data.get("type", "") or letter.document_type + ) letter.letter_type = letter.document_type letter.category = category letter.institution = ev.data.get("agency", "") or letter.institution @@ -364,17 +391,37 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str yield sse_event("consequence", {"text": consequence_text}) elif ev.type == "error": - logger.warning("ReAct agent emitted error: %s", ev.data.get("message")) + logger.warning( + "ReAct agent emitted error: %s", ev.data.get("message") + ) # Don't propagate immediately — try to continue with what we have. await asyncio.sleep(0.05) # Reconstruct AgentAnalysis-like dict from collected events - from ai.schemas import AgentAnalysis, Classification, Deadline, Consequence, RiskScore as TheirRiskScore - cls_data = next((e.data for e in agent_events_collected if e.type == "classification"), {}) - dl_data = next((e.data for e in agent_events_collected if e.type == "deadline"), {}) - rs_data = next((e.data for e in agent_events_collected if e.type == "risk_score"), {"score": 3, "label": "Medium", "reason": ""}) - cq_data = next((e.data for e in agent_events_collected if e.type == "consequence"), {"text": "", "severity": ""}) + from ai.schemas import ( + AgentAnalysis, + Classification, + Deadline, + Consequence, + RiskScore as TheirRiskScore, + ) + + cls_data = next( + (e.data for e in agent_events_collected if e.type == "classification"), + {}, + ) + dl_data = next( + (e.data for e in agent_events_collected if e.type == "deadline"), {} + ) + rs_data = next( + (e.data for e in agent_events_collected if e.type == "risk_score"), + {"score": 3, "label": "Medium", "reason": ""}, + ) + cq_data = next( + (e.data for e in agent_events_collected if e.type == "consequence"), + {"text": "", "severity": ""}, + ) # Fallback: if the agent didn't extract a deadline, scan the OCR # text with our German-date regex. Common for letters where the @@ -396,19 +443,35 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # Also emit a deadline SSE event so the frontend sees it live yield sse_event( "deadline", - {"date": agent_date_iso, "days_remaining": days_remaining, - "note": "Found via OCR text scan (agent missed it)"}, + { + "date": agent_date_iso, + "days_remaining": days_remaining, + "note": "Found via OCR text scan (agent missed it)", + }, ) analysis = AgentAnalysis( - classification=Classification(type=cls_data.get("type", "Unknown"), agency=cls_data.get("agency", "Unknown")), + classification=Classification( + type=cls_data.get("type", "Unknown"), + agency=cls_data.get("agency", "Unknown"), + ), deadline=Deadline( date=dl_data.get("date"), days_remaining=dl_data.get("days_remaining"), - source="letter" if (dl_data.get("date") and not fallback_date) else fallback_source if fallback_date else "none", + source="letter" + if (dl_data.get("date") and not fallback_date) + else fallback_source + if fallback_date + else "none", + ), + consequence=Consequence( + text=cq_data.get("text", ""), severity=cq_data.get("severity", "") + ), + risk_score=TheirRiskScore( + score=rs_data.get("score", 3), + label=rs_data.get("label", "Medium"), + reason=rs_data.get("reason", ""), ), - consequence=Consequence(text=cq_data.get("text", ""), severity=cq_data.get("severity", "")), - risk_score=TheirRiskScore(score=rs_data.get("score", 3), label=rs_data.get("label", "Medium"), reason=rs_data.get("reason", "")), ) unpacked = ai_bridge.unpack_agent_analysis(analysis) @@ -453,7 +516,11 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str if not dl_data.get("date"): yield sse_event( "deadline", - {"date": None, "days_remaining": None, "note": "No explicit deadline"}, + { + "date": None, + "days_remaining": None, + "note": "No explicit deadline", + }, ) # ============================================================ @@ -461,6 +528,7 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # ============================================================ try: from ai.rag.retrieval import retrieve_legal_context + # AI team's new signature (commit 61fd2b5): (letter_type, consequence, top_k) legal_chunks = retrieve_legal_context( letter_type=letter.document_type or "", @@ -468,14 +536,18 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str top_k=5, ) except Exception as e: - logger.warning("Legal retrieval failed: %s — continuing without citations", e) + logger.warning( + "Legal retrieval failed: %s — continuing without citations", e + ) legal_chunks = [] # ============================================================ # STAGE 4 — Grounded generation (~5-10s) # ============================================================ agent_result = ai_bridge.synthesize_agent_result(letter, action=action) - agent_result.risk_label = risk_label # use their qualitative label for grounding context + agent_result.risk_label = ( + risk_label # use their qualitative label for grounding context + ) generation = await ai_bridge.generate_grounded_response( ocr_text=ocr_text, @@ -483,7 +555,9 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str language=out_lang, legal_chunks=legal_chunks, ) - explanation, response_draft, checklist, citations = ai_bridge.unpack_generation_output(generation) + explanation, response_draft, checklist, citations = ( + ai_bridge.unpack_generation_output(generation) + ) # Stream explanation chunks for piece in _chunk_text_for_streaming(explanation, chunk_size=50): @@ -519,6 +593,7 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # Project to the same PublicLetter shape GET /letters/{id} returns. from app.routers.public import _public_letter + public = _public_letter(db, letter) yield sse_event( diff --git a/backend/app/rag/store.py b/backend/app/rag/store.py index cec1126..195b435 100644 --- a/backend/app/rag/store.py +++ b/backend/app/rag/store.py @@ -36,6 +36,7 @@ def init_chroma() -> None: coll = get_collection() if coll.count() == 0: from app.rag.seed import seed_corpus + seed_corpus(coll) diff --git a/backend/app/routers/actions.py b/backend/app/routers/actions.py index 4bdcb58..c152515 100644 --- a/backend/app/routers/actions.py +++ b/backend/app/routers/actions.py @@ -52,8 +52,15 @@ def list_actions( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( diff --git a/backend/app/routers/deadlines.py b/backend/app/routers/deadlines.py index c4dc27b..914f0dd 100644 --- a/backend/app/routers/deadlines.py +++ b/backend/app/routers/deadlines.py @@ -59,8 +59,15 @@ def list_deadlines( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( select(ActionItem, Letter) diff --git a/backend/app/routers/public.py b/backend/app/routers/public.py index 4b03459..18262d9 100644 --- a/backend/app/routers/public.py +++ b/backend/app/routers/public.py @@ -22,7 +22,6 @@ """ import logging -from datetime import datetime from uuid import UUID from fastapi import APIRouter, Depends, File, Query, UploadFile @@ -30,8 +29,6 @@ from sqlmodel import Session, select from app.auth.dependencies import get_current_user - -logger = logging.getLogger("klar.public") from app.database import get_session from app.errors import ErrorCode, KlarHTTPException from app.models import ( @@ -54,7 +51,6 @@ PublicLetter, ChatRequest, ChatResponse, - RagHit, RagQuery, RagResponse, ReplyDraft, @@ -65,13 +61,14 @@ from app.services.extraction import ( ExtractionError, extract_from_letter_file, - generate_reply_text, normalize_lang, ) from app.services.pdf_pages import PdfRenderError from app.services.persistence import persist_extraction from app.services.storage import detect_magic_mime, save_letter_file +logger = logging.getLogger("klar.public") + router = APIRouter(tags=["public"]) ACCEPTED_MIMES = { @@ -87,9 +84,7 @@ # ---------- shape projection: Letter + ActionItems → PublicLetter ---------- -def _public_action( - action: ActionItem, latest_risk: RiskScore | None -) -> PublicAction: +def _public_action(action: ActionItem, latest_risk: RiskScore | None) -> PublicAction: risk_breakdown = None if latest_risk is not None: risk_breakdown = RiskBreakdown( @@ -120,9 +115,7 @@ def _public_action( ) -def _load_risk_by_action( - db: Session, action_ids: list[UUID] -) -> dict[UUID, RiskScore]: +def _load_risk_by_action(db: Session, action_ids: list[UUID]) -> dict[UUID, RiskScore]: """Batch-load the most recent RiskScore per action — O(1) queries.""" if not action_ids: return {} @@ -140,9 +133,7 @@ def _load_risk_by_action( def _public_letter(db: Session, letter: Letter) -> PublicLetter: actions = list( - db.scalars( - select(ActionItem).where(ActionItem.letter_id == letter.id) - ).all() + db.scalars(select(ActionItem).where(ActionItem.letter_id == letter.id)).all() ) risk_by_action = _load_risk_by_action(db, [a.id for a in actions]) # Citations are stored as a list[dict] on the Letter row, but the public @@ -169,10 +160,7 @@ def _public_letter(db: Session, letter: Letter) -> PublicLetter: summary_en=letter.summary, # field renamed for frontend contract ocr_text=letter.ocr_text or None, confidence=letter.confidence, - actions=[ - _public_action(a, risk_by_action.get(a.id)) - for a in actions - ], + actions=[_public_action(a, risk_by_action.get(a.id)) for a in actions], extraction_warnings=letter.extraction_warnings or [], explanation=letter.explanation or "", consequence=letter.consequence or "", @@ -283,7 +271,8 @@ async def post_letter( # implementation details to the client. logger.exception( "Qwen extraction failed for letter %s: %s", - letter.id, exc, + letter.id, + exc, ) letter.status = LetterStatus.ERROR db.add(letter) @@ -362,8 +351,15 @@ def list_actions_public( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( @@ -456,7 +452,10 @@ def update_action_public( ), responses={ 401: {"model": ErrorResponse, "description": "Not authenticated."}, - 404: {"model": ErrorResponse, "description": "`LETTER_NOT_FOUND` or `ACTION_NOT_FOUND`."}, + 404: { + "model": ErrorResponse, + "description": "`LETTER_NOT_FOUND` or `ACTION_NOT_FOUND`.", + }, 502: {"model": ErrorResponse, "description": "`LLM_PROVIDER_ERROR`."}, }, ) @@ -483,16 +482,6 @@ async def generate_reply( action = db.get(ActionItem, action_uuid) if action is None or action.letter_id != letter.id: raise KlarHTTPException(404, ErrorCode.ACTION_NOT_FOUND) - action_titles = [action.title] - else: - actions = list( - db.scalars( - select(ActionItem).where(ActionItem.letter_id == letter.id) - ).all() - ) - # Prefer actions explicitly flagged reply_needed; fall back to all titles. - reply_actions = [a for a in actions if a.reply_needed] or actions - action_titles = [a.title for a in reply_actions] # 1) Retrieve real legal context from the AI team's law corpus try: @@ -532,12 +521,15 @@ async def generate_reply( except Exception as exc: logger.exception( "Reply generation failed for letter %s: %s", - letter_id, exc, + letter_id, + exc, ) raise KlarHTTPException(502, ErrorCode.LLM_PROVIDER_ERROR) # 4) Unpack + persist all 4 long-form fields - explanation, body_text, checklist, citations = ai_bridge.unpack_generation_output(generation) + explanation, body_text, checklist, citations = ai_bridge.unpack_generation_output( + generation + ) letter.explanation = explanation letter.response_draft = body_text letter.checklist = checklist @@ -604,7 +596,6 @@ async def chat_about_letter( db: Session = Depends(get_session), user: User = Depends(get_current_user), ): - import json import os from uuid import UUID as _UUID @@ -653,19 +644,23 @@ async def chat_about_letter( extra_body={"enable_thinking": False}, ) - response = await model.ainvoke([ - SystemMessage(content=system), - HumanMessage(content=payload.query), - ]) + response = await model.ainvoke( + [ + SystemMessage(content=system), + HumanMessage(content=payload.query), + ] + ) # Coerce raw dicts → CitationItem (same as _public_letter) clean_citations = [] for c in citations: if isinstance(c, dict) and c.get("section"): - clean_citations.append(CitationItem( - section=str(c.get("section", "")), - text=str(c.get("text", "")), - )) + clean_citations.append( + CitationItem( + section=str(c.get("section", "")), + text=str(c.get("text", "")), + ) + ) return ChatResponse(answer=response.content, citations=clean_citations) @@ -704,7 +699,7 @@ async def form_fill( placeholders = [] # From checklist (documents to prepare) - for item in (letter.checklist or []): + for item in letter.checklist or []: placeholders.append(str(item)) # From actions (steps the user needs to take) @@ -712,12 +707,25 @@ async def form_fill( db.scalars(select(ActionItem).where(ActionItem.letter_id == letter.id)).all() ) for action in actions: - for step in (action.steps or []): - if any(kw in step.lower() for kw in [ - "iban", "steuer", "name", "adresse", "address", "nummer", - "number", "unterschrift", "signature", "datum", "date", - "versichertennummer", "aktenzeichen", - ]): + for step in action.steps or []: + if any( + kw in step.lower() + for kw in [ + "iban", + "steuer", + "name", + "adresse", + "address", + "nummer", + "number", + "unterschrift", + "signature", + "datum", + "date", + "versichertennummer", + "aktenzeichen", + ] + ): placeholders.append(step) # Always include standard form fields — these match the PLACEHOLDER_MAP @@ -745,7 +753,9 @@ async def form_fill( placeholders=placeholders[:8], # Cap at 8 to keep prompt manageable ) except Exception as exc: - logger.exception("Form-fill generation failed for letter %s: %s", letter_id, exc) + logger.exception( + "Form-fill generation failed for letter %s: %s", letter_id, exc + ) raise KlarHTTPException(502, ErrorCode.LLM_PROVIDER_ERROR) return Response( diff --git a/backend/app/routers/rag.py b/backend/app/routers/rag.py index 419907a..b0a346b 100644 --- a/backend/app/routers/rag.py +++ b/backend/app/routers/rag.py @@ -5,7 +5,7 @@ from app.auth.dependencies import get_current_user from app.models import User from app.rag import store -from app.schemas import ErrorResponse, RagHit, RagQuery, RagResponse +from app.schemas import ErrorResponse, RagQuery, RagResponse from app.services import ai_bridge router = APIRouter(prefix="/api/rag", tags=["rag"]) diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 04a5558..c52b183 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -14,7 +14,7 @@ """ from datetime import date, datetime -from typing import Any, Literal, Optional +from typing import Literal, Optional from uuid import UUID from pydantic import BaseModel, EmailStr, Field @@ -185,7 +185,9 @@ class PublicAction(BaseModel): description="Full RiskScore breakdown — powers the 'why this risk' view.", ) deadline_confidence: Optional[float] = Field( - default=None, ge=0.0, le=1.0, + default=None, + ge=0.0, + le=1.0, description="0..1 confidence in the deadline value (null when unknown).", ) deadline_source: Optional[DeadlineSource] = Field( @@ -200,7 +202,8 @@ class PublicAction(BaseModel): ) reply_needed: bool = False amount_due_eur: Optional[float] = Field( - default=None, ge=0.0, + default=None, + ge=0.0, description=( "Outstanding amount the user must pay for this action, in EUR. " "Extracted from the OCR text by a regex pattern matcher." @@ -227,7 +230,9 @@ class PublicLetter(BaseModel): description="Verbatim German OCR text from the source. Never localized.", ) confidence: Optional[float] = Field( - default=None, ge=0.0, le=1.0, + default=None, + ge=0.0, + le=1.0, description="0..1 overall extraction confidence. <0.85 triggers a 'get a human' UI prompt.", ) actions: list[PublicAction] = Field(default_factory=list) @@ -283,7 +288,8 @@ class PublicActionListItem(BaseModel): status: ActionStatus reply_needed: bool amount_due_eur: Optional[float] = Field( - default=None, ge=0.0, + default=None, + ge=0.0, description=( "Outstanding EUR amount for this action, mirrored from the same " "field on PublicAction. Included on the list shape so the " @@ -414,7 +420,9 @@ class ErrorResponse(BaseModel): description="Stable machine-readable identifier — see docs/06-api-contract.md." ) message: str = Field(description="Localized, user-facing copy.") - detail: str = Field(description="Alias of `message` — for clients that expect FastAPI's default error shape.") + detail: str = Field( + description="Alias of `message` — for clients that expect FastAPI's default error shape." + ) details: Optional[ErrorDetails] = None diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py index 3818b10..0faa8ea 100644 --- a/backend/app/services/__init__.py +++ b/backend/app/services/__init__.py @@ -9,7 +9,11 @@ stream_explanation, stream_response_draft, ) -from app.services.pdf_pages import iter_data_urls, pdf_to_image_bytes, split_to_image_bytes +from app.services.pdf_pages import ( + iter_data_urls, + pdf_to_image_bytes, + split_to_image_bytes, +) from app.services.persistence import persist_extraction from app.services.risk import compute_risk from app.services.storage import detect_magic_mime, is_pdf, save_letter_file, user_dir diff --git a/backend/app/services/ai_bridge.py b/backend/app/services/ai_bridge.py index 0776499..e1ecd88 100644 --- a/backend/app/services/ai_bridge.py +++ b/backend/app/services/ai_bridge.py @@ -50,102 +50,89 @@ _CATEGORY_PATTERNS: list[tuple[str, DocumentCategory]] = [ # Most specific first - ("residence permit", DocumentCategory.IMMIGRATION), - ("aufenthaltstitel", DocumentCategory.IMMIGRATION), - ("ausländerbehörde", DocumentCategory.IMMIGRATION), - ("visa", DocumentCategory.IMMIGRATION), - ("aufenthalts", DocumentCategory.IMMIGRATION), - ("immigration", DocumentCategory.IMMIGRATION), - - ("health insurance", DocumentCategory.HEALTH_INSURANCE), - ("krankenkasse", DocumentCategory.HEALTH_INSURANCE), - ("krankenversicherung", DocumentCategory.HEALTH_INSURANCE), - ("aok", DocumentCategory.HEALTH_INSURANCE), - ("techniker krankenkasse", DocumentCategory.HEALTH_INSURANCE), - ("barmer", DocumentCategory.HEALTH_INSURANCE), - ("dak-gesundheit", DocumentCategory.HEALTH_INSURANCE), - - ("car insurance", DocumentCategory.OTHER_INSURANCE), - ("haftpflicht", DocumentCategory.OTHER_INSURANCE), - ("hausrat", DocumentCategory.OTHER_INSURANCE), - ("kfz-versicherung", DocumentCategory.OTHER_INSURANCE), - ("liability insurance", DocumentCategory.OTHER_INSURANCE), - - ("tax", DocumentCategory.TAX), - ("finanzamt", DocumentCategory.TAX), - ("steuer", DocumentCategory.TAX), - - ("university", DocumentCategory.EDUCATION), - ("universität", DocumentCategory.EDUCATION), - ("hochschule", DocumentCategory.EDUCATION), - ("immatrikulation", DocumentCategory.EDUCATION), - ("studentenwerk", DocumentCategory.EDUCATION), - ("bafög", DocumentCategory.EDUCATION), - ("rückmeldung", DocumentCategory.EDUCATION), - ("enrollment", DocumentCategory.EDUCATION), - - ("rent", DocumentCategory.HOUSING), - ("vermieter", DocumentCategory.HOUSING), - ("hausverwaltung", DocumentCategory.HOUSING), - ("mieterhöhung", DocumentCategory.HOUSING), - ("nebenkosten", DocumentCategory.HOUSING), - ("landlord", DocumentCategory.HOUSING), - - ("electricity", DocumentCategory.UTILITIES), - ("gas bill", DocumentCategory.UTILITIES), - ("internet", DocumentCategory.UTILITIES), - ("telekom", DocumentCategory.UTILITIES), - ("vodafone", DocumentCategory.UTILITIES), - ("stadtwerke", DocumentCategory.UTILITIES), - ("vattenfall", DocumentCategory.UTILITIES), - ("strom", DocumentCategory.UTILITIES), - - ("employer", DocumentCategory.EMPLOYMENT), - ("arbeitgeber", DocumentCategory.EMPLOYMENT), - ("lohn", DocumentCategory.EMPLOYMENT), - ("gehalt", DocumentCategory.EMPLOYMENT), - ("payroll", DocumentCategory.EMPLOYMENT), - ("arbeitsvertrag", DocumentCategory.EMPLOYMENT), - - ("unemployment", DocumentCategory.GOVERNMENT_BENEFITS), - ("kindergeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("elterngeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("wohngeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("arbeitslosengeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("bürgergeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("jobcenter", DocumentCategory.GOVERNMENT_BENEFITS), - ("familienkasse", DocumentCategory.GOVERNMENT_BENEFITS), - - ("pension", DocumentCategory.PENSION), - ("rentenversicherung", DocumentCategory.PENSION), - ("rente", DocumentCategory.PENSION), - - ("rundfunk", DocumentCategory.BROADCAST_FEE), - ("gez", DocumentCategory.BROADCAST_FEE), - ("beitragsservice", DocumentCategory.BROADCAST_FEE), - ("broadcasting fee", DocumentCategory.BROADCAST_FEE), - - ("bürgeramt", DocumentCategory.CIVIC), - ("einwohnermelde", DocumentCategory.CIVIC), - ("standesamt", DocumentCategory.CIVIC), - ("personalausweis", DocumentCategory.CIVIC), - ("reisepass", DocumentCategory.CIVIC), - ("meldebescheinigung", DocumentCategory.CIVIC), - - ("court", DocumentCategory.LEGAL_DEBT), - ("gericht", DocumentCategory.LEGAL_DEBT), - ("mahnbescheid", DocumentCategory.LEGAL_DEBT), - ("vollstreckung", DocumentCategory.LEGAL_DEBT), - ("inkasso", DocumentCategory.LEGAL_DEBT), - ("bußgeld", DocumentCategory.LEGAL_DEBT), - ("anwalt", DocumentCategory.LEGAL_DEBT), - ("debt collection", DocumentCategory.LEGAL_DEBT), - ("fine notice", DocumentCategory.LEGAL_DEBT), - - ("bank", DocumentCategory.BANKING), - ("sparkasse", DocumentCategory.BANKING), - ("schufa", DocumentCategory.BANKING), - ("kreditkarte", DocumentCategory.BANKING), + ("residence permit", DocumentCategory.IMMIGRATION), + ("aufenthaltstitel", DocumentCategory.IMMIGRATION), + ("ausländerbehörde", DocumentCategory.IMMIGRATION), + ("visa", DocumentCategory.IMMIGRATION), + ("aufenthalts", DocumentCategory.IMMIGRATION), + ("immigration", DocumentCategory.IMMIGRATION), + ("health insurance", DocumentCategory.HEALTH_INSURANCE), + ("krankenkasse", DocumentCategory.HEALTH_INSURANCE), + ("krankenversicherung", DocumentCategory.HEALTH_INSURANCE), + ("aok", DocumentCategory.HEALTH_INSURANCE), + ("techniker krankenkasse", DocumentCategory.HEALTH_INSURANCE), + ("barmer", DocumentCategory.HEALTH_INSURANCE), + ("dak-gesundheit", DocumentCategory.HEALTH_INSURANCE), + ("car insurance", DocumentCategory.OTHER_INSURANCE), + ("haftpflicht", DocumentCategory.OTHER_INSURANCE), + ("hausrat", DocumentCategory.OTHER_INSURANCE), + ("kfz-versicherung", DocumentCategory.OTHER_INSURANCE), + ("liability insurance", DocumentCategory.OTHER_INSURANCE), + ("tax", DocumentCategory.TAX), + ("finanzamt", DocumentCategory.TAX), + ("steuer", DocumentCategory.TAX), + ("university", DocumentCategory.EDUCATION), + ("universität", DocumentCategory.EDUCATION), + ("hochschule", DocumentCategory.EDUCATION), + ("immatrikulation", DocumentCategory.EDUCATION), + ("studentenwerk", DocumentCategory.EDUCATION), + ("bafög", DocumentCategory.EDUCATION), + ("rückmeldung", DocumentCategory.EDUCATION), + ("enrollment", DocumentCategory.EDUCATION), + ("rent", DocumentCategory.HOUSING), + ("vermieter", DocumentCategory.HOUSING), + ("hausverwaltung", DocumentCategory.HOUSING), + ("mieterhöhung", DocumentCategory.HOUSING), + ("nebenkosten", DocumentCategory.HOUSING), + ("landlord", DocumentCategory.HOUSING), + ("electricity", DocumentCategory.UTILITIES), + ("gas bill", DocumentCategory.UTILITIES), + ("internet", DocumentCategory.UTILITIES), + ("telekom", DocumentCategory.UTILITIES), + ("vodafone", DocumentCategory.UTILITIES), + ("stadtwerke", DocumentCategory.UTILITIES), + ("vattenfall", DocumentCategory.UTILITIES), + ("strom", DocumentCategory.UTILITIES), + ("employer", DocumentCategory.EMPLOYMENT), + ("arbeitgeber", DocumentCategory.EMPLOYMENT), + ("lohn", DocumentCategory.EMPLOYMENT), + ("gehalt", DocumentCategory.EMPLOYMENT), + ("payroll", DocumentCategory.EMPLOYMENT), + ("arbeitsvertrag", DocumentCategory.EMPLOYMENT), + ("unemployment", DocumentCategory.GOVERNMENT_BENEFITS), + ("kindergeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("elterngeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("wohngeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("arbeitslosengeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("bürgergeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("jobcenter", DocumentCategory.GOVERNMENT_BENEFITS), + ("familienkasse", DocumentCategory.GOVERNMENT_BENEFITS), + ("pension", DocumentCategory.PENSION), + ("rentenversicherung", DocumentCategory.PENSION), + ("rente", DocumentCategory.PENSION), + ("rundfunk", DocumentCategory.BROADCAST_FEE), + ("gez", DocumentCategory.BROADCAST_FEE), + ("beitragsservice", DocumentCategory.BROADCAST_FEE), + ("broadcasting fee", DocumentCategory.BROADCAST_FEE), + ("bürgeramt", DocumentCategory.CIVIC), + ("einwohnermelde", DocumentCategory.CIVIC), + ("standesamt", DocumentCategory.CIVIC), + ("personalausweis", DocumentCategory.CIVIC), + ("reisepass", DocumentCategory.CIVIC), + ("meldebescheinigung", DocumentCategory.CIVIC), + ("court", DocumentCategory.LEGAL_DEBT), + ("gericht", DocumentCategory.LEGAL_DEBT), + ("mahnbescheid", DocumentCategory.LEGAL_DEBT), + ("vollstreckung", DocumentCategory.LEGAL_DEBT), + ("inkasso", DocumentCategory.LEGAL_DEBT), + ("bußgeld", DocumentCategory.LEGAL_DEBT), + ("anwalt", DocumentCategory.LEGAL_DEBT), + ("debt collection", DocumentCategory.LEGAL_DEBT), + ("fine notice", DocumentCategory.LEGAL_DEBT), + ("bank", DocumentCategory.BANKING), + ("sparkasse", DocumentCategory.BANKING), + ("schufa", DocumentCategory.BANKING), + ("kreditkarte", DocumentCategory.BANKING), ] @@ -160,7 +147,9 @@ def map_classification_to_category(free_text_type: str | None) -> DocumentCatego for pattern, cat in _CATEGORY_PATTERNS: if pattern in needle: return cat - logger.debug("map_classification_to_category: no match for %r → OTHER", free_text_type) + logger.debug( + "map_classification_to_category: no match for %r → OTHER", free_text_type + ) return DocumentCategory.OTHER @@ -169,10 +158,10 @@ def map_classification_to_category(free_text_type: str | None) -> DocumentCatego # ============================================================ _LABEL_TO_SEVERITY: dict[str, Severity] = { - "critical": Severity.CRITICAL, - "high": Severity.HIGH, - "medium": Severity.MEDIUM, - "low": Severity.LOW, + "critical": Severity.CRITICAL, + "high": Severity.HIGH, + "medium": Severity.MEDIUM, + "low": Severity.LOW, "informational": Severity.LOW, } @@ -194,10 +183,10 @@ def map_their_severity_label(label: str | None) -> Severity: # ============================================================ _SOURCE_MAPPING: dict[str, DeadlineSource] = { - "letter": DeadlineSource.EXPLICIT, + "letter": DeadlineSource.EXPLICIT, "calculated": DeadlineSource.INFERRED, - "searched": DeadlineSource.INFERRED, # Tavily web search - "none": DeadlineSource.UNKNOWN, + "searched": DeadlineSource.INFERRED, # Tavily web search + "none": DeadlineSource.UNKNOWN, } @@ -218,6 +207,7 @@ def deadline_was_web_searched(source: str | None) -> bool: # Our risk_score (0-100) → their RiskScore.label (for generator input) # ============================================================ + def risk_label_from_score(score: int | None) -> str: """Map our 0-100 score → their label string (used when synthesizing an `AgentResult` to feed their `generate_response`).""" @@ -237,6 +227,7 @@ def risk_label_from_score(score: int | None) -> str: # Letter + (optional) ActionItem → AgentResult (their dataclass) # ============================================================ + def synthesize_agent_result( letter: Letter, action: ActionItem | None = None, @@ -274,6 +265,7 @@ def synthesize_agent_result( # Their LegalChunk → our RagHit (for /rag/search response) # ============================================================ + def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: """Map their `LegalChunk` from `ai.rag.retrieval` → our `RagHit` wire shape. @@ -285,9 +277,9 @@ def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: text=chunk.text, score=1.0, metadata={ - "section": chunk.section, - "law": chunk.law, - "title": chunk.title, + "section": chunk.section, + "law": chunk.law, + "title": chunk.title, "citation": chunk.citation, }, ) @@ -297,6 +289,7 @@ def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: # Their Citation → JSON dict (stored on Letter.citations column) # ============================================================ + def citation_to_dict(c: "Citation") -> dict[str, Any]: """Persist-shape for the `Letter.citations` JSON column. @@ -305,8 +298,8 @@ def citation_to_dict(c: "Citation") -> dict[str, Any]: """ return { "section": c.section, - "text": c.text, - "score": 1.0, # their structured citation doesn't carry a score + "text": c.text, + "score": 1.0, # their structured citation doesn't carry a score } @@ -318,6 +311,7 @@ def citations_to_dicts(cits: list["Citation"]) -> list[dict[str, Any]]: # Their GenerationOutput → (explanation, response_draft, checklist[], citations[dict]) # ============================================================ + def unpack_generation_output( out: "GenerationOutput", ) -> tuple[str, str, list[str], list[dict[str, Any]]]: @@ -336,6 +330,7 @@ def unpack_generation_output( # Their AgentAnalysis → (category, document_type, severity, deadline_date, ...) # ============================================================ + async def generate_grounded_response( ocr_text: str, agent_result: "AgentResult", @@ -369,8 +364,7 @@ async def generate_grounded_response( # Build the legal-context section from retrieved chunks if legal_chunks: legal_lines = [ - f"### {c.citation} — {c.title}\n{c.text}\n" - for c in legal_chunks + f"### {c.citation} — {c.title}\n{c.text}\n" for c in legal_chunks ] legal_context = "\n".join(legal_lines) else: @@ -403,13 +397,18 @@ async def generate_grounded_response( ), temperature=0, max_tokens=4096, - extra_body={"enable_thinking": False, "response_format": {"type": "json_object"}}, + extra_body={ + "enable_thinking": False, + "response_format": {"type": "json_object"}, + }, ) response = await raw_model.ainvoke([HumanMessage(content=prompt)]) raw_text = response.content if hasattr(response, "content") else str(response) if isinstance(raw_text, list): # Some langchain versions return content as a list of parts - raw_text = "".join(p.get("text", "") if isinstance(p, dict) else str(p) for p in raw_text) + raw_text = "".join( + p.get("text", "") if isinstance(p, dict) else str(p) for p in raw_text + ) payload = json.loads(raw_text) @@ -421,10 +420,12 @@ async def generate_grounded_response( # Bare "§ 81 AufenthG" — wrap as Citation with empty explanation cleaned_citations.append(Citation(section=c, text="")) elif isinstance(c, dict): - cleaned_citations.append(Citation( - section=str(c.get("section") or c.get("§") or "§"), - text=str(c.get("text") or c.get("explanation") or ""), - )) + cleaned_citations.append( + Citation( + section=str(c.get("section") or c.get("§") or "§"), + text=str(c.get("text") or c.get("explanation") or ""), + ) + ) else: logger.debug("Skipping unparseable citation: %r", c) @@ -454,17 +455,19 @@ def unpack_agent_analysis( try: parsed_deadline = _date.fromisoformat(deadline_iso) except ValueError: - logger.debug("Their agent returned non-ISO deadline %r — dropping", deadline_iso) + logger.debug( + "Their agent returned non-ISO deadline %r — dropping", deadline_iso + ) return { - "category": map_classification_to_category(analysis.classification.type), - "document_type": analysis.classification.type or "", - "institution": analysis.classification.agency or "", - "deadline": parsed_deadline, - "deadline_source": map_their_deadline_source(analysis.deadline.source), + "category": map_classification_to_category(analysis.classification.type), + "document_type": analysis.classification.type or "", + "institution": analysis.classification.agency or "", + "deadline": parsed_deadline, + "deadline_source": map_their_deadline_source(analysis.deadline.source), "deadline_was_searched": deadline_was_web_searched(analysis.deadline.source), - "consequence": analysis.consequence.text or "", - "severity": map_their_severity_label(analysis.risk_score.label), - "risk_label": analysis.risk_score.label or "Medium", - "risk_reason": analysis.risk_score.reason or "", + "consequence": analysis.consequence.text or "", + "severity": map_their_severity_label(analysis.risk_score.label), + "risk_label": analysis.risk_score.label or "Medium", + "risk_reason": analysis.risk_score.reason or "", } diff --git a/backend/app/services/extraction.py b/backend/app/services/extraction.py index 1c04fc6..d699d72 100644 --- a/backend/app/services/extraction.py +++ b/backend/app/services/extraction.py @@ -37,6 +37,7 @@ class ExtractionError(Exception): `EXTRACTION_FAILED` error instead of letting a raw exception become a 500. """ + # ISO 639-1 codes — matches the frontend's docs/06-frontend-integration-contract.md. # Qwen3.7-Plus handles all of these out of the box. Quality bar: # en/de: production-grade, the wedge languages @@ -316,7 +317,9 @@ async def extract_from_letter_file( json_parsed = json.loads(stripped) except json.JSONDecodeError: # Sometimes the model wraps the JSON in code fences - fence_match = re.search(r"```(?:json)?\s*(.*?)\s*```", stripped, re.DOTALL) + fence_match = re.search( + r"```(?:json)?\s*(.*?)\s*```", stripped, re.DOTALL + ) if fence_match: try: json_parsed = json.loads(fence_match.group(1).strip()) @@ -408,8 +411,12 @@ async def generate_reply_text( Used by POST /letters/{id}/reply (frontend contract §4.7). """ - actions_text = "\n".join(f"- {t}" for t in action_titles) or "- (keine spezifische Aktion)" - prompt = _response_prompt_from_letter(institution, document_type, actions_text, applicant) + actions_text = ( + "\n".join(f"- {t}" for t in action_titles) or "- (keine spezifische Aktion)" + ) + prompt = _response_prompt_from_letter( + institution, document_type, actions_text, applicant + ) client = _get_client() response = await client.chat.completions.create( @@ -464,7 +471,9 @@ async def _stream_text(prompt: str) -> AsyncIterator[str]: yield delta.content -async def stream_explanation(extracted: ExtractedLetter, lang: str) -> AsyncIterator[str]: +async def stream_explanation( + extracted: ExtractedLetter, lang: str +) -> AsyncIterator[str]: async for piece in _stream_text(_explanation_prompt(extracted, lang)): yield piece @@ -501,7 +510,9 @@ async def generate_checklist(extracted: ExtractedLetter, lang: str) -> list[str] # ---------- backwards-compat alias for the original entrypoint ---------- -async def extract_from_image(image_bytes: bytes, mime: str = "image/jpeg") -> ExtractedLetter: +async def extract_from_image( + image_bytes: bytes, mime: str = "image/jpeg" +) -> ExtractedLetter: """Legacy entrypoint: write bytes to a temp file then call the new API.""" import tempfile import os diff --git a/backend/app/services/pdf_pages.py b/backend/app/services/pdf_pages.py index 8f4ae61..fd0b810 100644 --- a/backend/app/services/pdf_pages.py +++ b/backend/app/services/pdf_pages.py @@ -20,7 +20,9 @@ class PdfRenderError(Exception): """ -def pdf_to_image_bytes(path: str, *, dpi: int = 200, max_pages: int = 12) -> list[bytes]: +def pdf_to_image_bytes( + path: str, *, dpi: int = 200, max_pages: int = 12 +) -> list[bytes]: """Render up to `max_pages` pages of `path` to PNG bytes. Imported lazily so callers that never touch a PDF don't pay the diff --git a/backend/app/services/persistence.py b/backend/app/services/persistence.py index eb28f81..049833b 100644 --- a/backend/app/services/persistence.py +++ b/backend/app/services/persistence.py @@ -34,7 +34,8 @@ def persist_extraction( # Overall confidence = min of available signals. Frontend uses <0.85 to # surface a "get a human" prompt. signals = [ - s for s in (extracted.language_confidence, extracted.category_confidence) + s + for s in (extracted.language_confidence, extracted.category_confidence) if s and s > 0 ] letter.confidence = min(signals) if signals else None @@ -84,6 +85,7 @@ def persist_extraction( # so totals across letters don't double-count. if letter_amount is not None and saved: from app.models import Severity as _Sev + sev_rank = {_Sev.CRITICAL: 4, _Sev.HIGH: 3, _Sev.MEDIUM: 2, _Sev.LOW: 1} amount_attached_to = max(saved, key=lambda x: sev_rank.get(x.severity, 0)) amount_attached_to.amount_due_eur = letter_amount