diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 0000000..7f8dd62 --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,208 @@ +name: Auto-label and assign issues + +on: + issues: + types: [opened] + +permissions: + issues: write + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - name: Triage issue + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issue = context.payload.issue; + const title = (issue.title || '').toLowerCase(); + const body = (issue.body || '').toLowerCase(); + const text = `${title} ${body}`; + + const labels = new Set(); + let assignees = new Set(); + + // ── 1. Issue-type labels ────────────────────────────────── + const bugPatterns = [ + 'bug', 'error', 'crash', 'fail', 'broken', 'not working', + 'exception', 'traceback', 'stacktrace', 'typeerror', + 'keyerror', 'valueerror', 'attributeerror', '500', + 'unexpected', 'regression', 'does not work', "doesn't work", + 'cannot', "can't", 'issue with' + ]; + const featurePatterns = [ + 'feature', 'enhancement', 'request', 'add support', + 'would be nice', 'proposal', 'improve', 'new option', + 'should support', 'wish', 'idea', 'suggestion' + ]; + const docPatterns = [ + 'documentation', 'docs', 'readme', 'typo in doc', + 'missing docs', 'unclear', 'example', 'tutorial', + 'how to', 'howto', 'instructions' + ]; + const questionPatterns = [ + 'question', 'how do i', 'how can i', 'is it possible', + 'what is the', 'help me', 'guidance', 'confused', + 'explain', 'clarification' + ]; + + if (bugPatterns.some(p => text.includes(p))) { + labels.add('bug'); + } + if (featurePatterns.some(p => text.includes(p))) { + labels.add('enhancement'); + } + if (docPatterns.some(p => text.includes(p))) { + labels.add('documentation'); + } + if (questionPatterns.some(p => text.includes(p))) { + labels.add('question'); + } + // If nothing matched above, default to bug for issues that + // mention errors, otherwise enhancement + if (labels.size === 0) { + labels.add('enhancement'); + } + + // ── 2. Component labels ─────────────────────────────────── + const frontendPatterns = [ + 'frontend', 'next.js', 'nextjs', 'react', 'typescript', + 'tailwind', 'css', 'ui', 'ux', 'component', 'page', + 'pwa', 'serwist', 'vercel', 'browser', 'layout', + 'responsive', 'mobile', 'dark mode', 'calendar view', + 'upload page', 'login page', 'signup page' + ]; + const backendPatterns = [ + 'backend', 'fastapi', 'api', 'endpoint', 'router', + 'database', 'sqlite', 'sqlmodel', 'auth', 'jwt', + 'cookie', 'session', 'cors', 'sse', 'server', + 'uvicorn', 'railway', 'render', '/letters', '/actions', + '/chat', '/health', 'migration', 'schema' + ]; + const aiAgentPatterns = [ + 'ocr', 'qwen-vl', 'react agent', 'langgraph', 'agent', + 'classification', 'risk score', 'tavily', 'search tool', + 'image processing', 'letter type', 'extraction', + 'qwen', 'llm', 'prompt', 'form fill', 'form_fill' + ]; + const aiRagPatterns = [ + 'rag', 'chromadb', 'chroma', 'embedding', 'retrieval', + 'legal text', 'corpus', 'ingestion', 'ingest', + 'citation', 'law', 'aufenthg', 'asylg', 'sgb', + 'bafög', 'bafoeg', 'response draft', 'response generation', + 'hallucination', 'grounding', 'checklist' + ]; + const infraPatterns = [ + 'ci', 'cd', 'ci/cd', 'deploy', 'docker', 'github action', + 'workflow', 'pipeline', 'env var', 'environment', + 'config', 'build', 'release', 'infra', 'devops' + ]; + + if (frontendPatterns.some(p => text.includes(p))) { + labels.add('component: frontend'); + assignees.add('saintnuno'); // Dev 1: Frontend + } + if (backendPatterns.some(p => text.includes(p))) { + labels.add('component: backend'); + assignees.add('aircode610'); // Dev 2: Backend + } + if (aiAgentPatterns.some(p => text.includes(p))) { + labels.add('component: ai-agent'); + assignees.add('danialeyz'); // Dev 4: OCR + ReAct + } + if (aiRagPatterns.some(p => text.includes(p))) { + labels.add('component: ai-rag'); + assignees.add('Alir3zag'); // Dev 3: RAG pipeline + } + if (infraPatterns.some(p => text.includes(p))) { + labels.add('component: infra'); + assignees.add('aircode610'); // Repo owner + } + + // ── 3. Priority labels ──────────────────────────────────── + const criticalPatterns = [ + 'critical', 'urgent', 'blocker', 'system down', + 'data loss', 'security vulnerability', 'production down', + 'p0', 'outage' + ]; + const highPatterns = [ + 'important', 'high priority', 'blocks', 'blocking', + 'severe', 'major', 'p1', 'asap', 'deadline missed', + 'cannot deploy' + ]; + const lowPatterns = [ + 'low priority', 'minor', 'cosmetic', 'nice to have', + 'nit', 'trivial', 'p3', 'when possible', 'polish' + ]; + + if (criticalPatterns.some(p => text.includes(p))) { + labels.add('priority: critical'); + } else if (highPatterns.some(p => text.includes(p))) { + labels.add('priority: high'); + } else if (lowPatterns.some(p => text.includes(p))) { + labels.add('priority: low'); + } else { + labels.add('priority: medium'); + } + + // ── 4. Security label ───────────────────────────────────── + const securityPatterns = [ + 'security', 'vulnerability', 'cve', 'xss', 'csrf', + 'injection', 'exploit', 'leak', 'exposed secret', + 'auth bypass', 'privilege escalation' + ]; + if (securityPatterns.some(p => text.includes(p))) { + labels.add('security'); + assignees.add('aircode610'); // Owner handles security + } + + // ── 5. Fallback assignment ──────────────────────────────── + // If no component was matched, assign to repo owner for triage + if (assignees.size === 0) { + assignees.add('aircode610'); + } + + // ── 6. Apply labels ─────────────────────────────────────── + const labelArray = [...labels]; + if (labelArray.length > 0) { + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + labels: labelArray, + }); + core.info(`Applied labels: ${labelArray.join(', ')}`); + } + + // ── 7. Assign users ─────────────────────────────────────── + const assigneeArray = [...assignees]; + if (assigneeArray.length > 0) { + await github.rest.issues.addAssignees({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + assignees: assigneeArray, + }); + core.info(`Assigned to: ${assigneeArray.join(', ')}`); + } + + // ── 8. Post triage comment ──────────────────────────────── + const commentBody = [ + `👋 Thanks for opening this issue!`, + ``, + `**Auto-triage summary:**`, + `- **Labels:** ${labelArray.map(l => '`' + l + '`').join(', ')}`, + `- **Assigned to:** ${assigneeArray.map(a => '@' + a).join(', ')}`, + ``, + `> This was automatically triaged based on the issue content. ` + + `If the labels or assignment look wrong, please update them manually.`, + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: commentBody, + }); diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..823c361 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Lint Python + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install ruff + run: pip install ruff + + - name: Ruff check + run: ruff check backend/ ai/ + + - name: Ruff format check + run: ruff format --check backend/ ai/ + + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -r backend/requirements.txt pytest pytest-asyncio + + - name: Run tests + working-directory: backend + env: + DATABASE_URL: "sqlite:///test.db" + JWT_SECRET: "test-secret-test-secret-test-secret-32" + run: python -m pytest tests/ -v diff --git a/ai/form_fill.py b/ai/form_fill.py index 6c3b868..50de5f3 100644 --- a/ai/form_fill.py +++ b/ai/form_fill.py @@ -11,9 +11,7 @@ import httpx DASHSCOPE_API_KEY = os.environ.get("DASHSCOPE_API_KEY", "") -DASHSCOPE_INTL_URL = ( - "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" -) +DASHSCOPE_INTL_URL = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" # Standard placeholder patterns for common German form fields PLACEHOLDER_MAP = { @@ -70,7 +68,9 @@ async def generate_filled_form( b64 = base64.b64encode(image_bytes).decode() ext = image_path.rsplit(".", 1)[-1].lower() - mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png"}.get(ext, "image/jpeg") + mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png"}.get( + ext, "image/jpeg" + ) data_url = f"data:{mime};base64,{b64}" field_instructions = _build_field_instructions(placeholders) diff --git a/ai/rag/generator.py b/ai/rag/generator.py index 11e64ff..fe78f81 100644 --- a/ai/rag/generator.py +++ b/ai/rag/generator.py @@ -14,8 +14,14 @@ QWEN_AGENT_MODEL = os.environ.get("QWEN_AGENT_MODEL", "qwen3.7-plus") LANGUAGE_NAMES = { - "en": "English", "de": "German", "tr": "Turkish", "ar": "Arabic", - "es": "Spanish", "fr": "French", "zh": "Chinese", "fa": "Persian", + "en": "English", + "de": "German", + "tr": "Turkish", + "ar": "Arabic", + "es": "Spanish", + "fr": "French", + "zh": "Chinese", + "fa": "Persian", } _model = ChatOpenAI( @@ -34,7 +40,9 @@ async def generate_response( language: str = "en", ) -> GenerationOutput: """Retrieve legal context from ChromaDB, inject into prompt, return structured output.""" - legal_context = retrieve_as_context(agent_result.letter_type, agent_result.consequence) + legal_context = retrieve_as_context( + agent_result.letter_type, agent_result.consequence + ) prompt = GENERATION_PROMPT.format( ocr_text=ocr_text[:3000], diff --git a/ai/rag/ingest.py b/ai/rag/ingest.py index fb26cff..c489210 100644 --- a/ai/rag/ingest.py +++ b/ai/rag/ingest.py @@ -26,31 +26,32 @@ # ── Paths ───────────────────────────────────────────────────────────────────── -ROOT = Path(__file__).resolve().parent.parent # ai/ -LAWS_DIR = ROOT / "data" / "laws" # ai/data/laws/ -CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ +ROOT = Path(__file__).resolve().parent.parent # ai/ +LAWS_DIR = ROOT / "data" / "laws" # ai/data/laws/ +CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ COLLECTION_NAME = "german_laws" # ── Law file → abbreviation map ─────────────────────────────────────────────── LAWS = { - "aufenthg.md": "AufenthG", - "aufenthv.md": "AufenthV", - "beschv.md": "BeschV", - "vwvfg.md": "VwVfG", - "bafoeg.md": "BAföG", - "asylg.md": "AsylG", - "asylblg.md": "AsylbLG", - "wogg.md": "WoGG", - "bmg.md": "BMG", - "intv.md": "IntV", - "owig.md": "OWiG", - "estg.md": "EStG", - "sgb5.md": "SGB V", + "aufenthg.md": "AufenthG", + "aufenthv.md": "AufenthV", + "beschv.md": "BeschV", + "vwvfg.md": "VwVfG", + "bafoeg.md": "BAföG", + "asylg.md": "AsylG", + "asylblg.md": "AsylbLG", + "wogg.md": "WoGG", + "bmg.md": "BMG", + "intv.md": "IntV", + "owig.md": "OWiG", + "estg.md": "EStG", + "sgb5.md": "SGB V", } # ── Qwen client ─────────────────────────────────────────────────────────────── + def get_qwen_client() -> OpenAI: api_key = os.getenv("DASHSCOPE_API_KEY") if not api_key: @@ -64,8 +65,8 @@ def get_qwen_client() -> OpenAI: # Qwen text-embedding-v3 limits -MAX_BATCH_SIZE = 10 # max texts per API call -MAX_CHARS = 6000 # conservative char limit (~8192 tokens safety margin) +MAX_BATCH_SIZE = 10 # max texts per API call +MAX_CHARS = 6000 # conservative char limit (~8192 tokens safety margin) def truncate(text: str) -> str: @@ -85,7 +86,7 @@ def embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]: texts = [truncate(t) for t in texts] for i in range(0, len(texts), MAX_BATCH_SIZE): - batch = texts[i:i + MAX_BATCH_SIZE] + batch = texts[i : i + MAX_BATCH_SIZE] response = client.embeddings.create( model="text-embedding-v3", input=batch, @@ -101,6 +102,7 @@ def embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]: # ── Chunking ────────────────────────────────────────────────────────────────── + def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: """ Split a law's markdown into one chunk per § paragraph. @@ -110,7 +112,7 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: id, text, paragraph, title, law """ # Match § headers at any heading level: ### § 1, #### § 4a, etc. - pattern = r'^#{1,4} (§ \d+[a-z]?\b.*?)$' + pattern = r"^#{1,4} (§ \d+[a-z]?\b.*?)$" matches = list(re.finditer(pattern, text, re.MULTILINE)) chunks = [] @@ -121,7 +123,7 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: end = matches[i + 1].start() if i + 1 < len(matches) else len(text) header = match.group(1).strip() - para_num_match = re.match(r'(§ \d+[a-z]?)', header) + para_num_match = re.match(r"(§ \d+[a-z]?)", header) para_num = para_num_match.group(1) if para_num_match else header body = text[start:end].strip() @@ -141,19 +143,22 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: counter += 1 seen_ids.add(unique_id) - chunks.append({ - "id": unique_id, - "text": body, - "paragraph": para_num, - "title": header, - "law": law_abbrev, - }) + chunks.append( + { + "id": unique_id, + "text": body, + "paragraph": para_num, + "title": header, + "law": law_abbrev, + } + ) return chunks # ── Main ────────────────────────────────────────────────────────────────────── + def ingest_all(): print("── Klar RAG Ingestion ──────────────────────────────────────") @@ -197,32 +202,39 @@ def ingest_all(): print(f" ⚠ No paragraphs parsed in {filename}") continue - print(f" Embedding {law_abbrev}: {len(chunks)} paragraphs ...", end=" ", flush=True) + print( + f" Embedding {law_abbrev}: {len(chunks)} paragraphs ...", + end=" ", + flush=True, + ) texts_to_embed = [c["text"] for c in chunks] embeddings = embed_texts(client, texts_to_embed) batch_size = 100 for i in range(0, len(chunks), batch_size): - batch_chunks = chunks[i:i + batch_size] - batch_embeddings = embeddings[i:i + batch_size] + batch_chunks = chunks[i : i + batch_size] + batch_embeddings = embeddings[i : i + batch_size] collection.add( ids=[c["id"] for c in batch_chunks], documents=[c["text"] for c in batch_chunks], - metadatas=[{ - "law": c["law"], - "paragraph": c["paragraph"], - "title": c["title"], - } for c in batch_chunks], + metadatas=[ + { + "law": c["law"], + "paragraph": c["paragraph"], + "title": c["title"], + } + for c in batch_chunks + ], embeddings=batch_embeddings, ) - print(f"✅") + print("✅") total_chunks += len(chunks) print() - print(f"── Done ────────────────────────────────────────────────────") + print("── Done ────────────────────────────────────────────────────") print(f" Total chunks ingested : {total_chunks}") print(f" ChromaDB saved to : {CHROMA_DIR.resolve()}") print() @@ -230,4 +242,4 @@ def ingest_all(): if __name__ == "__main__": - ingest_all() \ No newline at end of file + ingest_all() diff --git a/ai/rag/retrieval.py b/ai/rag/retrieval.py index aee8302..81c275c 100644 --- a/ai/rag/retrieval.py +++ b/ai/rag/retrieval.py @@ -25,20 +25,21 @@ # ── Paths ───────────────────────────────────────────────────────────────────── -ROOT = Path(__file__).resolve().parent.parent # ai/ -CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ +ROOT = Path(__file__).resolve().parent.parent # ai/ +CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ COLLECTION_NAME = "german_laws" # ── Schema ──────────────────────────────────────────────────────────────────── + @dataclass class LegalChunk: - section: str # e.g. "§ 81" - law: str # e.g. "AufenthG" - title: str # e.g. "§ 81 Beantragung des Aufenthaltstitels" - text: str # full paragraph text - citation: str # e.g. "§ 81 AufenthG" - score: float # cosine similarity, higher = more relevant + section: str # e.g. "§ 81" + law: str # e.g. "AufenthG" + title: str # e.g. "§ 81 Beantragung des Aufenthaltstitels" + text: str # full paragraph text + citation: str # e.g. "§ 81 AufenthG" + score: float # cosine similarity, higher = more relevant # ── Singleton clients ───────────────────────────────────────────────────────── @@ -87,6 +88,7 @@ def _embed_query(query: str) -> list[float]: # ── Core retrieval ──────────────────────────────────────────────────────────── + def retrieve_legal_context( letter_type: str, consequence: str, @@ -124,14 +126,16 @@ def retrieve_legal_context( results["distances"][0], ): section = meta.get("paragraph", meta.get("section", "Unknown")) - chunks.append(LegalChunk( - section=section, - law=meta["law"], - title=meta["title"], - text=doc, - citation=f"{section} {meta['law']}", - score=round(1 - distance, 4), - )) + chunks.append( + LegalChunk( + section=section, + law=meta["law"], + title=meta["title"], + text=doc, + citation=f"{section} {meta['law']}", + score=round(1 - distance, 4), + ) + ) return chunks @@ -156,4 +160,4 @@ def retrieve_as_context( text_preview = c.text[:1000] + "..." if len(c.text) > 1000 else c.text parts.append(f"[{c.citation}] {c.title}\n{text_preview}") - return "\n\n---\n\n".join(parts) \ No newline at end of file + return "\n\n---\n\n".join(parts) diff --git a/ai/rag/schemas.py b/ai/rag/schemas.py index edc7ba4..bcae2c2 100644 --- a/ai/rag/schemas.py +++ b/ai/rag/schemas.py @@ -8,7 +8,7 @@ @dataclass class RAGEvent: - type: str # "explanation" | "response_draft" | "checklist" | "citations" | "error" + type: str # "explanation" | "response_draft" | "checklist" | "citations" | "error" data: dict confidence: str = "high" # "high" if RAG matched well, "low" if no strong matches # data shapes per type: @@ -16,4 +16,4 @@ class RAGEvent: # response_draft: {"chunk": str} — streamed token by token # checklist: {"items": list[str]} — emitted once, complete # citations: {"items": list[dict]} — emitted once, [{section, text}, ...] - # error: {"message": str} \ No newline at end of file + # error: {"message": str} diff --git a/ai/react_agent/agent.py b/ai/react_agent/agent.py index 155c349..40a671a 100644 --- a/ai/react_agent/agent.py +++ b/ai/react_agent/agent.py @@ -43,9 +43,15 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: """Run the ReAct agent with structured output via LangGraph response_format.""" try: - result = await _agent.ainvoke({ - "messages": [HumanMessage(content=f"Analyze this German official letter:\n\n{ocr_text}")], - }) + result = await _agent.ainvoke( + { + "messages": [ + HumanMessage( + content=f"Analyze this German official letter:\n\n{ocr_text}" + ) + ], + } + ) analysis: AgentAnalysis = result["structured_response"] @@ -57,10 +63,22 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: except ValueError: pass - yield AgentEvent("classification", {"type": analysis.classification.type, "agency": analysis.classification.agency}) - yield AgentEvent("risk_score", {"score": analysis.risk_score.score, "label": analysis.risk_score.label}) + yield AgentEvent( + "classification", + { + "type": analysis.classification.type, + "agency": analysis.classification.agency, + }, + ) + yield AgentEvent( + "risk_score", + {"score": analysis.risk_score.score, "label": analysis.risk_score.label}, + ) if analysis.deadline.date: - yield AgentEvent("deadline", {"date": analysis.deadline.date, "days_remaining": days_remaining}) + yield AgentEvent( + "deadline", + {"date": analysis.deadline.date, "days_remaining": days_remaining}, + ) yield AgentEvent("consequence", {"text": analysis.consequence.text}) except Exception as e: @@ -70,9 +88,19 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: def get_last_agent_result(events: list[AgentEvent], ocr_text: str) -> AgentResult: """Reconstruct an AgentResult from collected events.""" data = {e.type: e.data for e in events} - c, d, r, q = data.get("classification", {}), data.get("deadline", {}), data.get("risk_score", {}), data.get("consequence", {}) + c, d, r, q = ( + data.get("classification", {}), + data.get("deadline", {}), + data.get("risk_score", {}), + data.get("consequence", {}), + ) return AgentResult( - ocr_text=ocr_text, letter_type=c.get("type", "Unknown"), agency=c.get("agency", "Unknown"), - deadline_date=d.get("date"), days_remaining=d.get("days_remaining"), - consequence=q.get("text", ""), risk_score=r.get("score", 3), risk_label=r.get("label", "Medium"), + ocr_text=ocr_text, + letter_type=c.get("type", "Unknown"), + agency=c.get("agency", "Unknown"), + deadline_date=d.get("date"), + days_remaining=d.get("days_remaining"), + consequence=q.get("text", ""), + risk_score=r.get("score", 3), + risk_label=r.get("label", "Medium"), ) diff --git a/ai/react_agent/ocr.py b/ai/react_agent/ocr.py index 8f047f7..e1359f1 100644 --- a/ai/react_agent/ocr.py +++ b/ai/react_agent/ocr.py @@ -42,7 +42,9 @@ def _parse_ocr_response(result: object) -> str: "image without readable content." ) - message = (choices[0] or {}).get("message") if isinstance(choices[0], dict) else None + message = ( + (choices[0] or {}).get("message") if isinstance(choices[0], dict) else None + ) content = (message or {}).get("content") if isinstance(message, dict) else None if not content or not str(content).strip(): diff --git a/ai/schemas.py b/ai/schemas.py index 0151249..17803a2 100644 --- a/ai/schemas.py +++ b/ai/schemas.py @@ -4,19 +4,32 @@ # --- Structured output models (used by LLM response_format) --- + class Classification(BaseModel): - type: str = Field(description="Letter type, e.g. 'Residence Permit - Document Request', 'Health Insurance - Tax ID Request'") - agency: str = Field(description="Sender agency name, e.g. 'Techniker Krankenkasse', 'Ausländerbehörde München'") + type: str = Field( + description="Letter type, e.g. 'Residence Permit - Document Request', 'Health Insurance - Tax ID Request'" + ) + agency: str = Field( + description="Sender agency name, e.g. 'Techniker Krankenkasse', 'Ausländerbehörde München'" + ) class Deadline(BaseModel): - date: str | None = Field(description="Deadline date in YYYY-MM-DD format, or null if no deadline") - days_remaining: int | None = Field(description="Days until deadline from today, or null") - source: str = Field(description="'letter' if read directly, 'calculated' if computed from letter date, 'searched' if from web, 'none' if no deadline applies") + date: str | None = Field( + description="Deadline date in YYYY-MM-DD format, or null if no deadline" + ) + days_remaining: int | None = Field( + description="Days until deadline from today, or null" + ) + source: str = Field( + description="'letter' if read directly, 'calculated' if computed from letter date, 'searched' if from web, 'none' if no deadline applies" + ) class Consequence(BaseModel): - text: str = Field(description="Detailed consequence description of what happens if deadline is missed or action not taken") + text: str = Field( + description="Detailed consequence description of what happens if deadline is missed or action not taken" + ) severity: str = Field(description="One-line severity summary") @@ -28,6 +41,7 @@ class RiskScore(BaseModel): class AgentAnalysis(BaseModel): """Structured output from the ReAct agent letter analysis.""" + classification: Classification deadline: Deadline consequence: Consequence @@ -35,23 +49,34 @@ class AgentAnalysis(BaseModel): class Citation(BaseModel): - section: str = Field(description="Legal paragraph reference, e.g. '§ 81 Abs. 4 AufenthG'") + section: str = Field( + description="Legal paragraph reference, e.g. '§ 81 Abs. 4 AufenthG'" + ) text: str = Field(description="Brief explanation of why this citation is relevant") class GenerationOutput(BaseModel): """Structured output from the response generation LLM.""" - explanation: str = Field(description="Clear plain-language explanation of the letter") + + explanation: str = Field( + description="Clear plain-language explanation of the letter" + ) response_draft: str = Field(description="Formal response letter in Behördendeutsch") - checklist: list[str] = Field(description="List of documents the user needs to prepare, with German terms in parentheses") - citations: list[Citation] = Field(default_factory=list, description="Legal § references that are relevant. Empty list if none found.") + checklist: list[str] = Field( + description="List of documents the user needs to prepare, with German terms in parentheses" + ) + citations: list[Citation] = Field( + default_factory=list, + description="Legal § references that are relevant. Empty list if none found.", + ) # --- Internal data transfer objects --- + @dataclass class AgentEvent: - type: str # "classification", "risk_score", "deadline", "consequence", "error" + type: str # "classification", "risk_score", "deadline", "consequence", "error" data: dict diff --git a/backend/app/auth/dependencies.py b/backend/app/auth/dependencies.py index 7659868..0c10f16 100644 --- a/backend/app/auth/dependencies.py +++ b/backend/app/auth/dependencies.py @@ -1,7 +1,6 @@ """FastAPI dependency that resolves the current user from the session cookie.""" import logging -from datetime import datetime from fastapi import Depends, Request, status from sqlmodel import Session as DBSession, select @@ -37,7 +36,8 @@ def _resolve_user(token: str | None, db: DBSession) -> User: logger.warning( "AUTH_SESSION_NOT_FOUND: no Session row for token=%s... " "(DB=%s; possible causes: db wiped, multiple workers, env mismatch)", - token_prefix, settings.database_url, + token_prefix, + settings.database_url, ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -47,7 +47,9 @@ def _resolve_user(token: str | None, db: DBSession) -> User: if session_row.expires_at < utcnow(): logger.info( "AUTH_SESSION_EXPIRED: token=%s... expired_at=%s now=%s", - token_prefix, session_row.expires_at, utcnow(), + token_prefix, + session_row.expires_at, + utcnow(), ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -59,7 +61,8 @@ def _resolve_user(token: str | None, db: DBSession) -> User: # Session row exists but the user it points to is gone — corrupt FK. logger.error( "AUTH: orphan Session row token=%s... user_id=%s has no User", - token_prefix, session_row.user_id, + token_prefix, + session_row.user_id, ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, diff --git a/backend/app/auth/router.py b/backend/app/auth/router.py index 151767b..67af069 100644 --- a/backend/app/auth/router.py +++ b/backend/app/auth/router.py @@ -1,7 +1,5 @@ """Authentication routes: signup, login, logout, me, forgot/reset password.""" -from datetime import datetime - from fastapi import APIRouter, Depends, Request, Response, status from pydantic import BaseModel, EmailStr, Field, field_validator from sqlmodel import Session as DBSession, select diff --git a/backend/app/auth/utils.py b/backend/app/auth/utils.py index 9d9a5f5..7198836 100644 --- a/backend/app/auth/utils.py +++ b/backend/app/auth/utils.py @@ -12,7 +12,9 @@ def hash_password(plain: str) -> str: """bcrypt with cost factor 12 — ~250ms on a modern laptop.""" - return bcrypt.hashpw(plain.encode("utf-8"), bcrypt.gensalt(rounds=12)).decode("utf-8") + return bcrypt.hashpw(plain.encode("utf-8"), bcrypt.gensalt(rounds=12)).decode( + "utf-8" + ) def verify_password(plain: str, password_hash: str) -> bool: diff --git a/backend/app/config.py b/backend/app/config.py index 15037ca..5a60bc6 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -46,7 +46,11 @@ def effective_llm_api_key(self) -> str: @property def effective_llm_base_url(self) -> str: - return self.qwen_api_base or self.llm_base_url or "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + return ( + self.qwen_api_base + or self.llm_base_url + or "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + ) @property def effective_llm_model(self) -> str: diff --git a/backend/app/database.py b/backend/app/database.py index f666408..836a132 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -21,6 +21,7 @@ def init_db() -> None: path = settings.database_url.replace("sqlite:///", "", 1) Path(path).parent.mkdir(parents=True, exist_ok=True) from app import models # noqa: F401 — populate SQLModel metadata + SQLModel.metadata.create_all(engine) diff --git a/backend/app/errors.py b/backend/app/errors.py index 20478a8..1240501 100644 --- a/backend/app/errors.py +++ b/backend/app/errors.py @@ -56,35 +56,43 @@ class ErrorCode(str, Enum): """ # --- generic --- - HTTP_ERROR = "HTTP_ERROR" # untyped fallback (legacy HTTPException) - INTERNAL_ERROR = "INTERNAL_ERROR" # unhandled exception - VALIDATION_ERROR = "VALIDATION_ERROR" # request body / query / path + HTTP_ERROR = "HTTP_ERROR" # untyped fallback (legacy HTTPException) + INTERNAL_ERROR = "INTERNAL_ERROR" # unhandled exception + VALIDATION_ERROR = "VALIDATION_ERROR" # request body / query / path # --- auth --- - AUTH_NOT_AUTHENTICATED = "AUTH_NOT_AUTHENTICATED" # no cookie at all - AUTH_SESSION_NOT_FOUND = "AUTH_SESSION_NOT_FOUND" # cookie present, but no Session row in DB - AUTH_SESSION_EXPIRED = "AUTH_SESSION_EXPIRED" # Session row exists but past expires_at + AUTH_NOT_AUTHENTICATED = "AUTH_NOT_AUTHENTICATED" # no cookie at all + AUTH_SESSION_NOT_FOUND = ( + "AUTH_SESSION_NOT_FOUND" # cookie present, but no Session row in DB + ) + AUTH_SESSION_EXPIRED = ( + "AUTH_SESSION_EXPIRED" # Session row exists but past expires_at + ) AUTH_INVALID_CREDENTIALS = "AUTH_INVALID_CREDENTIALS" # wrong email / password - AUTH_EMAIL_TAKEN = "AUTH_EMAIL_TAKEN" # signup with existing email - AUTH_INVALID_RESET_TOKEN = "AUTH_INVALID_RESET_TOKEN" # token unknown / already used + AUTH_EMAIL_TAKEN = "AUTH_EMAIL_TAKEN" # signup with existing email + AUTH_INVALID_RESET_TOKEN = ( + "AUTH_INVALID_RESET_TOKEN" # token unknown / already used + ) AUTH_RESET_TOKEN_EXPIRED = "AUTH_RESET_TOKEN_EXPIRED" # token past 15-min TTL # --- letters --- LETTER_NOT_FOUND = "LETTER_NOT_FOUND" - LETTER_FILE_MISSING = "LETTER_FILE_MISSING" # row exists but file gone + LETTER_FILE_MISSING = "LETTER_FILE_MISSING" # row exists but file gone LETTER_EMPTY_UPLOAD = "LETTER_EMPTY_UPLOAD" LETTER_TOO_LARGE = "LETTER_TOO_LARGE" LETTER_UNSUPPORTED_TYPE = "LETTER_UNSUPPORTED_TYPE" - LETTER_CORRUPT_FILE = "LETTER_CORRUPT_FILE" # magic-bytes mismatch - LETTER_MIME_MISMATCH = "LETTER_MIME_MISMATCH" # declared ≠ detected + LETTER_CORRUPT_FILE = "LETTER_CORRUPT_FILE" # magic-bytes mismatch + LETTER_MIME_MISMATCH = "LETTER_MIME_MISMATCH" # declared ≠ detected # --- actions --- ACTION_NOT_FOUND = "ACTION_NOT_FOUND" # --- pipeline / AI --- - EXTRACTION_FAILED = "EXTRACTION_FAILED" # SSE-only: model returned no tool call, parse error, etc. - LLM_PROVIDER_ERROR = "LLM_PROVIDER_ERROR" # network / 5xx from Qwen - PDF_RENDER_FAILED = "PDF_RENDER_FAILED" # pdf2image / poppler missing + EXTRACTION_FAILED = ( + "EXTRACTION_FAILED" # SSE-only: model returned no tool call, parse error, etc. + ) + LLM_PROVIDER_ERROR = "LLM_PROVIDER_ERROR" # network / 5xx from Qwen + PDF_RENDER_FAILED = "PDF_RENDER_FAILED" # pdf2image / poppler missing # User-facing default messages per code. Keep short, no jargon, no secrets. @@ -93,7 +101,6 @@ class ErrorCode(str, Enum): ErrorCode.HTTP_ERROR: "Something went wrong with that request.", ErrorCode.INTERNAL_ERROR: "Something went wrong on our end. Please try again.", ErrorCode.VALIDATION_ERROR: "Some fields in your request are invalid.", - ErrorCode.AUTH_NOT_AUTHENTICATED: "Please sign in to continue.", ErrorCode.AUTH_SESSION_NOT_FOUND: "Your session is no longer recognized. Please sign in again.", ErrorCode.AUTH_SESSION_EXPIRED: "Your session has expired. Please sign in again.", @@ -101,7 +108,6 @@ class ErrorCode(str, Enum): ErrorCode.AUTH_EMAIL_TAKEN: "An account with that email already exists.", ErrorCode.AUTH_INVALID_RESET_TOKEN: "This reset link is invalid or has already been used.", ErrorCode.AUTH_RESET_TOKEN_EXPIRED: "This reset link has expired. Please request a new one.", - ErrorCode.LETTER_NOT_FOUND: "That letter doesn't exist or you don't have access to it.", ErrorCode.LETTER_FILE_MISSING: "We can't find the uploaded file for this letter.", ErrorCode.LETTER_EMPTY_UPLOAD: "The uploaded file is empty.", @@ -109,9 +115,7 @@ class ErrorCode(str, Enum): ErrorCode.LETTER_UNSUPPORTED_TYPE: "We can only read JPEG, PNG, HEIC, WebP, or PDF letters.", ErrorCode.LETTER_CORRUPT_FILE: "The file looks corrupted or isn't the type it claims to be.", ErrorCode.LETTER_MIME_MISMATCH: "The file's content doesn't match its declared type.", - ErrorCode.ACTION_NOT_FOUND: "That action doesn't exist or you don't have access to it.", - ErrorCode.EXTRACTION_FAILED: "We couldn't read this letter. Try a clearer photo or PDF.", ErrorCode.LLM_PROVIDER_ERROR: "Our AI provider is having trouble. Please try again in a moment.", ErrorCode.PDF_RENDER_FAILED: "We couldn't open that PDF. Try uploading it as an image instead.", @@ -188,8 +192,9 @@ async def generic_http_exception_handler( # If detail is already a Klar envelope dict (from KlarHTTPException # routing through the default handler), pass it through. if isinstance(exc.detail, dict) and "code" in exc.detail: - return JSONResponse(status_code=exc.status_code, content=exc.detail, - headers=exc.headers) + return JSONResponse( + status_code=exc.status_code, content=exc.detail, headers=exc.headers + ) # Auth-shaped status codes get more specific codes by default. code = ErrorCode.HTTP_ERROR @@ -239,7 +244,9 @@ async def unhandled_exception_handler(req: Request, exc: Exception) -> JSONRespo """ logger.exception( "Unhandled exception on %s %s: %s", - req.method, req.url.path, exc, + req.method, + req.url.path, + exc, ) return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/backend/app/main.py b/backend/app/main.py index 04f428b..ef34d42 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -85,6 +85,7 @@ def _bridge_env_to_ai_team() -> None: value still wins. """ import os + if settings.effective_llm_api_key: os.environ.setdefault("DASHSCOPE_API_KEY", settings.effective_llm_api_key) if settings.effective_llm_base_url: diff --git a/backend/app/models.py b/backend/app/models.py index 69b4890..19404b4 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -55,20 +55,22 @@ class DocumentCategory(str, Enum): does not fit any defined bucket. """ - HEALTH_INSURANCE = "health_insurance" # AOK, TK, BARMER, private KV - OTHER_INSURANCE = "other_insurance" # Haftpflicht, Hausrat, KFZ, Leben - BANKING = "banking" # bank accounts, credit cards, SCHUFA - TAX = "tax" # Finanzamt - IMMIGRATION = "immigration" # Ausländerbehörde, residence/visa - EDUCATION = "education" # universities, BAföG, Studentenwerk - HOUSING = "housing" # landlord, property management - UTILITIES = "utilities" # Strom, Gas, Wasser, Internet, Mobilfunk - EMPLOYMENT = "employment" # Arbeitgeber, HR, Lohn - GOVERNMENT_BENEFITS = "government_benefits" # ALG I/II, Kindergeld, Elterngeld, Wohngeld - PENSION = "pension" # Deutsche Rentenversicherung - BROADCAST_FEE = "broadcast_fee" # Beitragsservice / Rundfunk - CIVIC = "civic" # Bürgeramt, Personalausweis, Pass - LEGAL_DEBT = "legal_debt" # Mahnbescheid, Inkasso, Bußgeld, Anwalt + HEALTH_INSURANCE = "health_insurance" # AOK, TK, BARMER, private KV + OTHER_INSURANCE = "other_insurance" # Haftpflicht, Hausrat, KFZ, Leben + BANKING = "banking" # bank accounts, credit cards, SCHUFA + TAX = "tax" # Finanzamt + IMMIGRATION = "immigration" # Ausländerbehörde, residence/visa + EDUCATION = "education" # universities, BAföG, Studentenwerk + HOUSING = "housing" # landlord, property management + UTILITIES = "utilities" # Strom, Gas, Wasser, Internet, Mobilfunk + EMPLOYMENT = "employment" # Arbeitgeber, HR, Lohn + GOVERNMENT_BENEFITS = ( + "government_benefits" # ALG I/II, Kindergeld, Elterngeld, Wohngeld + ) + PENSION = "pension" # Deutsche Rentenversicherung + BROADCAST_FEE = "broadcast_fee" # Beitragsservice / Rundfunk + CIVIC = "civic" # Bürgeramt, Personalausweis, Pass + LEGAL_DEBT = "legal_debt" # Mahnbescheid, Inkasso, Bußgeld, Anwalt OTHER = "other" @@ -114,15 +116,15 @@ class Letter(SQLModel, table=True): original_file: str = "" # Spec-flat structured fields (denormalized from ActionItem for /api/letters) - letter_type: str = "" # alias of document_type for spec compat - risk_score: int = 0 # denormalized highest action risk + letter_type: str = "" # alias of document_type for spec compat + risk_score: int = 0 # denormalized highest action risk deadline_date: Optional[date] = None # denormalized most-urgent action deadline # Rich Klar extras institution: str = "" document_type: str = "" category: DocumentCategory = DocumentCategory.OTHER - summary: str = "" # language matches Letter.language + summary: str = "" # language matches Letter.language language: str = "en" # OCR + long-form generation outputs @@ -132,7 +134,7 @@ class Letter(SQLModel, table=True): # "low confidence, get a human" prompt. Computed from extraction outputs. confidence: Optional[float] = None explanation: str = "" - response_draft: str = "" # ALWAYS German (formal reply to German institution) + response_draft: str = "" # ALWAYS German (formal reply to German institution) checklist: list[str] = Field(default_factory=list, sa_column=Column(JSON)) citations: list[dict] = Field(default_factory=list, sa_column=Column(JSON)) consequence: str = "" diff --git a/backend/app/pipeline/orchestrator.py b/backend/app/pipeline/orchestrator.py index 6048a0a..8896e95 100644 --- a/backend/app/pipeline/orchestrator.py +++ b/backend/app/pipeline/orchestrator.py @@ -40,7 +40,7 @@ import os import re import tempfile -from datetime import date, datetime +from datetime import date from typing import AsyncIterator from uuid import UUID @@ -166,10 +166,31 @@ async def _ocr_letter_file(path: str) -> str: # text for common German date patterns and pick the most-likely deadline. _GERMAN_MONTHS = { - "januar": 1, "jan": 1, "februar": 2, "feb": 2, "märz": 3, "mar": 3, "mrz": 3, - "april": 4, "apr": 4, "mai": 5, "juni": 6, "jun": 6, "juli": 7, "jul": 7, - "august": 8, "aug": 8, "september": 9, "sep": 9, "sept": 9, - "oktober": 10, "okt": 10, "november": 11, "nov": 11, "dezember": 12, "dez": 12, + "januar": 1, + "jan": 1, + "februar": 2, + "feb": 2, + "märz": 3, + "mar": 3, + "mrz": 3, + "april": 4, + "apr": 4, + "mai": 5, + "juni": 6, + "jun": 6, + "juli": 7, + "jul": 7, + "august": 8, + "aug": 8, + "september": 9, + "sep": 9, + "sept": 9, + "oktober": 10, + "okt": 10, + "november": 11, + "nov": 11, + "dezember": 12, + "dez": 12, } # Match: "28. Oktober 2021", "28 Oktober 2021", "den 28. Oktober 2021" @@ -269,16 +290,20 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str except Exception as exc: logger.exception( "AI team's modules failed to import — check env (DASHSCOPE_API_KEY, " - "TAVILY_API_KEY): %s", exc, + "TAVILY_API_KEY): %s", + exc, ) - yield sse_event("error", sse_error_payload( - ErrorCode.LLM_PROVIDER_ERROR, - message=( - "AI pipeline failed to initialize. Most likely cause: missing " - "DASHSCOPE_API_KEY or TAVILY_API_KEY env var on the backend " - "process. See server logs." + yield sse_event( + "error", + sse_error_payload( + ErrorCode.LLM_PROVIDER_ERROR, + message=( + "AI pipeline failed to initialize. Most likely cause: missing " + "DASHSCOPE_API_KEY or TAVILY_API_KEY env var on the backend " + "process. See server logs." + ), ), - )) + ) return out_lang = normalize_lang(lang) @@ -316,16 +341,18 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # STAGE 2 — ReAct agent (LangGraph + Tavily, ~5-15s) # ============================================================ agent_events_collected = [] - classification_data: dict | None = None risk_label = "Medium" async for ev in run_react_agent(ocr_text): agent_events_collected.append(ev) if ev.type == "classification": - classification_data = ev.data - category = ai_bridge.map_classification_to_category(ev.data.get("type", "")) - letter.document_type = ev.data.get("type", "") or letter.document_type + category = ai_bridge.map_classification_to_category( + ev.data.get("type", "") + ) + letter.document_type = ( + ev.data.get("type", "") or letter.document_type + ) letter.letter_type = letter.document_type letter.category = category letter.institution = ev.data.get("agency", "") or letter.institution @@ -364,17 +391,37 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str yield sse_event("consequence", {"text": consequence_text}) elif ev.type == "error": - logger.warning("ReAct agent emitted error: %s", ev.data.get("message")) + logger.warning( + "ReAct agent emitted error: %s", ev.data.get("message") + ) # Don't propagate immediately — try to continue with what we have. await asyncio.sleep(0.05) # Reconstruct AgentAnalysis-like dict from collected events - from ai.schemas import AgentAnalysis, Classification, Deadline, Consequence, RiskScore as TheirRiskScore - cls_data = next((e.data for e in agent_events_collected if e.type == "classification"), {}) - dl_data = next((e.data for e in agent_events_collected if e.type == "deadline"), {}) - rs_data = next((e.data for e in agent_events_collected if e.type == "risk_score"), {"score": 3, "label": "Medium", "reason": ""}) - cq_data = next((e.data for e in agent_events_collected if e.type == "consequence"), {"text": "", "severity": ""}) + from ai.schemas import ( + AgentAnalysis, + Classification, + Deadline, + Consequence, + RiskScore as TheirRiskScore, + ) + + cls_data = next( + (e.data for e in agent_events_collected if e.type == "classification"), + {}, + ) + dl_data = next( + (e.data for e in agent_events_collected if e.type == "deadline"), {} + ) + rs_data = next( + (e.data for e in agent_events_collected if e.type == "risk_score"), + {"score": 3, "label": "Medium", "reason": ""}, + ) + cq_data = next( + (e.data for e in agent_events_collected if e.type == "consequence"), + {"text": "", "severity": ""}, + ) # Fallback: if the agent didn't extract a deadline, scan the OCR # text with our German-date regex. Common for letters where the @@ -396,19 +443,35 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # Also emit a deadline SSE event so the frontend sees it live yield sse_event( "deadline", - {"date": agent_date_iso, "days_remaining": days_remaining, - "note": "Found via OCR text scan (agent missed it)"}, + { + "date": agent_date_iso, + "days_remaining": days_remaining, + "note": "Found via OCR text scan (agent missed it)", + }, ) analysis = AgentAnalysis( - classification=Classification(type=cls_data.get("type", "Unknown"), agency=cls_data.get("agency", "Unknown")), + classification=Classification( + type=cls_data.get("type", "Unknown"), + agency=cls_data.get("agency", "Unknown"), + ), deadline=Deadline( date=dl_data.get("date"), days_remaining=dl_data.get("days_remaining"), - source="letter" if (dl_data.get("date") and not fallback_date) else fallback_source if fallback_date else "none", + source="letter" + if (dl_data.get("date") and not fallback_date) + else fallback_source + if fallback_date + else "none", + ), + consequence=Consequence( + text=cq_data.get("text", ""), severity=cq_data.get("severity", "") + ), + risk_score=TheirRiskScore( + score=rs_data.get("score", 3), + label=rs_data.get("label", "Medium"), + reason=rs_data.get("reason", ""), ), - consequence=Consequence(text=cq_data.get("text", ""), severity=cq_data.get("severity", "")), - risk_score=TheirRiskScore(score=rs_data.get("score", 3), label=rs_data.get("label", "Medium"), reason=rs_data.get("reason", "")), ) unpacked = ai_bridge.unpack_agent_analysis(analysis) @@ -453,7 +516,11 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str if not dl_data.get("date"): yield sse_event( "deadline", - {"date": None, "days_remaining": None, "note": "No explicit deadline"}, + { + "date": None, + "days_remaining": None, + "note": "No explicit deadline", + }, ) # ============================================================ @@ -461,6 +528,7 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # ============================================================ try: from ai.rag.retrieval import retrieve_legal_context + # AI team's new signature (commit 61fd2b5): (letter_type, consequence, top_k) legal_chunks = retrieve_legal_context( letter_type=letter.document_type or "", @@ -468,14 +536,18 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str top_k=5, ) except Exception as e: - logger.warning("Legal retrieval failed: %s — continuing without citations", e) + logger.warning( + "Legal retrieval failed: %s — continuing without citations", e + ) legal_chunks = [] # ============================================================ # STAGE 4 — Grounded generation (~5-10s) # ============================================================ agent_result = ai_bridge.synthesize_agent_result(letter, action=action) - agent_result.risk_label = risk_label # use their qualitative label for grounding context + agent_result.risk_label = ( + risk_label # use their qualitative label for grounding context + ) generation = await ai_bridge.generate_grounded_response( ocr_text=ocr_text, @@ -483,7 +555,9 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str language=out_lang, legal_chunks=legal_chunks, ) - explanation, response_draft, checklist, citations = ai_bridge.unpack_generation_output(generation) + explanation, response_draft, checklist, citations = ( + ai_bridge.unpack_generation_output(generation) + ) # Stream explanation chunks for piece in _chunk_text_for_streaming(explanation, chunk_size=50): @@ -519,6 +593,7 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # Project to the same PublicLetter shape GET /letters/{id} returns. from app.routers.public import _public_letter + public = _public_letter(db, letter) yield sse_event( diff --git a/backend/app/rag/store.py b/backend/app/rag/store.py index cec1126..195b435 100644 --- a/backend/app/rag/store.py +++ b/backend/app/rag/store.py @@ -36,6 +36,7 @@ def init_chroma() -> None: coll = get_collection() if coll.count() == 0: from app.rag.seed import seed_corpus + seed_corpus(coll) diff --git a/backend/app/routers/actions.py b/backend/app/routers/actions.py index 4bdcb58..c152515 100644 --- a/backend/app/routers/actions.py +++ b/backend/app/routers/actions.py @@ -52,8 +52,15 @@ def list_actions( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( diff --git a/backend/app/routers/deadlines.py b/backend/app/routers/deadlines.py index c4dc27b..914f0dd 100644 --- a/backend/app/routers/deadlines.py +++ b/backend/app/routers/deadlines.py @@ -59,8 +59,15 @@ def list_deadlines( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( select(ActionItem, Letter) diff --git a/backend/app/routers/letters.py b/backend/app/routers/letters.py index 4812bbb..c004a74 100644 --- a/backend/app/routers/letters.py +++ b/backend/app/routers/letters.py @@ -1,5 +1,6 @@ """Letter upload + retrieval endpoints (auth-required, /api/letters/*).""" +import logging from uuid import UUID from fastapi import APIRouter, Depends, File, Query, UploadFile @@ -17,7 +18,12 @@ User, utcnow, ) -from app.schemas import ErrorResponse, LetterListItem, LetterResponse, LetterUploadResponse +from app.schemas import ( + ErrorResponse, + LetterListItem, + LetterResponse, + LetterUploadResponse, +) from app.services.extraction import ( ExtractionError, extract_from_letter_file, @@ -27,6 +33,8 @@ from app.services.persistence import persist_extraction from app.services.storage import detect_magic_mime, save_letter_file +logger = logging.getLogger("klar.letters") + router = APIRouter(prefix="/api/letters", tags=["letters"]) ACCEPTED_MIMES = { @@ -141,7 +149,8 @@ async def upload_letter( raise KlarHTTPException(415, ErrorCode.LETTER_CORRUPT_FILE) # Allow image/jpeg ↔ image/jpg variants; otherwise demand strict match. if actual_mime != file.content_type and not ( - actual_mime.startswith("image/") and file.content_type.startswith("image/") + actual_mime.startswith("image/") + and file.content_type.startswith("image/") and actual_mime.split("/")[-1] == file.content_type.split("/")[-1] ): raise KlarHTTPException( @@ -221,6 +230,7 @@ async def extract_letter( letter.original_file, mime, lang=letter.language ) except PdfRenderError as exc: + logger.info("PDF render failed for letter %s: %s", letter_id, exc) letter.status = LetterStatus.ERROR db.add(letter) db.commit() @@ -228,17 +238,27 @@ async def extract_letter( # actionable message ("try uploading it as an image instead"). raise KlarHTTPException(502, ErrorCode.PDF_RENDER_FAILED, message=str(exc)) except ExtractionError as exc: + logger.info("Extraction produced no text for letter %s: %s", letter_id, exc) letter.status = LetterStatus.ERROR db.add(letter) db.commit() # Scanned image-only PDF (no text layer) or malformed model output — # surface the typed, user-friendly message instead of a raw 500. raise KlarHTTPException(502, ErrorCode.EXTRACTION_FAILED, message=str(exc)) - except Exception: + except Exception as exc: + # Log the real error server-side so we can diagnose failures — the 502 + # response stays generic to avoid leaking provider details to the client. + # (Prior to this fix the exception was silently swallowed: the comment + # claimed "Logged by unhandled_exception_handler" but KlarHTTPException + # is routed to klar_exception_handler, so the original traceback was lost.) + logger.exception( + "Extraction failed for letter %s: %s", + letter_id, + exc, + ) letter.status = LetterStatus.ERROR db.add(letter) db.commit() - # Don't leak the raw exception. Logged by unhandled_exception_handler. raise KlarHTTPException(502, ErrorCode.EXTRACTION_FAILED) actions = persist_extraction(db, letter, extracted) @@ -267,7 +287,10 @@ async def extract_letter( ), responses={ 401: {"model": ErrorResponse, "description": "Not authenticated."}, - 422: {"model": ErrorResponse, "description": "Unknown status or category value."}, + 422: { + "model": ErrorResponse, + "description": "Unknown status or category value.", + }, }, ) def list_letters( @@ -287,8 +310,15 @@ def list_letters( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in LetterStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in LetterStatus), + } + ] + }, ) parsed_category: DocumentCategory | None = None if category: @@ -299,8 +329,15 @@ def list_letters( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown category: {category!r}.", - details={"errors": [{"field": "category", "message": "must be one of " - + ", ".join(c.value for c in DocumentCategory)}]}, + details={ + "errors": [ + { + "field": "category", + "message": "must be one of " + + ", ".join(c.value for c in DocumentCategory), + } + ] + }, ) stmt = select(Letter).where(Letter.user_id == user.id) diff --git a/backend/app/routers/public.py b/backend/app/routers/public.py index 4b03459..18262d9 100644 --- a/backend/app/routers/public.py +++ b/backend/app/routers/public.py @@ -22,7 +22,6 @@ """ import logging -from datetime import datetime from uuid import UUID from fastapi import APIRouter, Depends, File, Query, UploadFile @@ -30,8 +29,6 @@ from sqlmodel import Session, select from app.auth.dependencies import get_current_user - -logger = logging.getLogger("klar.public") from app.database import get_session from app.errors import ErrorCode, KlarHTTPException from app.models import ( @@ -54,7 +51,6 @@ PublicLetter, ChatRequest, ChatResponse, - RagHit, RagQuery, RagResponse, ReplyDraft, @@ -65,13 +61,14 @@ from app.services.extraction import ( ExtractionError, extract_from_letter_file, - generate_reply_text, normalize_lang, ) from app.services.pdf_pages import PdfRenderError from app.services.persistence import persist_extraction from app.services.storage import detect_magic_mime, save_letter_file +logger = logging.getLogger("klar.public") + router = APIRouter(tags=["public"]) ACCEPTED_MIMES = { @@ -87,9 +84,7 @@ # ---------- shape projection: Letter + ActionItems → PublicLetter ---------- -def _public_action( - action: ActionItem, latest_risk: RiskScore | None -) -> PublicAction: +def _public_action(action: ActionItem, latest_risk: RiskScore | None) -> PublicAction: risk_breakdown = None if latest_risk is not None: risk_breakdown = RiskBreakdown( @@ -120,9 +115,7 @@ def _public_action( ) -def _load_risk_by_action( - db: Session, action_ids: list[UUID] -) -> dict[UUID, RiskScore]: +def _load_risk_by_action(db: Session, action_ids: list[UUID]) -> dict[UUID, RiskScore]: """Batch-load the most recent RiskScore per action — O(1) queries.""" if not action_ids: return {} @@ -140,9 +133,7 @@ def _load_risk_by_action( def _public_letter(db: Session, letter: Letter) -> PublicLetter: actions = list( - db.scalars( - select(ActionItem).where(ActionItem.letter_id == letter.id) - ).all() + db.scalars(select(ActionItem).where(ActionItem.letter_id == letter.id)).all() ) risk_by_action = _load_risk_by_action(db, [a.id for a in actions]) # Citations are stored as a list[dict] on the Letter row, but the public @@ -169,10 +160,7 @@ def _public_letter(db: Session, letter: Letter) -> PublicLetter: summary_en=letter.summary, # field renamed for frontend contract ocr_text=letter.ocr_text or None, confidence=letter.confidence, - actions=[ - _public_action(a, risk_by_action.get(a.id)) - for a in actions - ], + actions=[_public_action(a, risk_by_action.get(a.id)) for a in actions], extraction_warnings=letter.extraction_warnings or [], explanation=letter.explanation or "", consequence=letter.consequence or "", @@ -283,7 +271,8 @@ async def post_letter( # implementation details to the client. logger.exception( "Qwen extraction failed for letter %s: %s", - letter.id, exc, + letter.id, + exc, ) letter.status = LetterStatus.ERROR db.add(letter) @@ -362,8 +351,15 @@ def list_actions_public( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( @@ -456,7 +452,10 @@ def update_action_public( ), responses={ 401: {"model": ErrorResponse, "description": "Not authenticated."}, - 404: {"model": ErrorResponse, "description": "`LETTER_NOT_FOUND` or `ACTION_NOT_FOUND`."}, + 404: { + "model": ErrorResponse, + "description": "`LETTER_NOT_FOUND` or `ACTION_NOT_FOUND`.", + }, 502: {"model": ErrorResponse, "description": "`LLM_PROVIDER_ERROR`."}, }, ) @@ -483,16 +482,6 @@ async def generate_reply( action = db.get(ActionItem, action_uuid) if action is None or action.letter_id != letter.id: raise KlarHTTPException(404, ErrorCode.ACTION_NOT_FOUND) - action_titles = [action.title] - else: - actions = list( - db.scalars( - select(ActionItem).where(ActionItem.letter_id == letter.id) - ).all() - ) - # Prefer actions explicitly flagged reply_needed; fall back to all titles. - reply_actions = [a for a in actions if a.reply_needed] or actions - action_titles = [a.title for a in reply_actions] # 1) Retrieve real legal context from the AI team's law corpus try: @@ -532,12 +521,15 @@ async def generate_reply( except Exception as exc: logger.exception( "Reply generation failed for letter %s: %s", - letter_id, exc, + letter_id, + exc, ) raise KlarHTTPException(502, ErrorCode.LLM_PROVIDER_ERROR) # 4) Unpack + persist all 4 long-form fields - explanation, body_text, checklist, citations = ai_bridge.unpack_generation_output(generation) + explanation, body_text, checklist, citations = ai_bridge.unpack_generation_output( + generation + ) letter.explanation = explanation letter.response_draft = body_text letter.checklist = checklist @@ -604,7 +596,6 @@ async def chat_about_letter( db: Session = Depends(get_session), user: User = Depends(get_current_user), ): - import json import os from uuid import UUID as _UUID @@ -653,19 +644,23 @@ async def chat_about_letter( extra_body={"enable_thinking": False}, ) - response = await model.ainvoke([ - SystemMessage(content=system), - HumanMessage(content=payload.query), - ]) + response = await model.ainvoke( + [ + SystemMessage(content=system), + HumanMessage(content=payload.query), + ] + ) # Coerce raw dicts → CitationItem (same as _public_letter) clean_citations = [] for c in citations: if isinstance(c, dict) and c.get("section"): - clean_citations.append(CitationItem( - section=str(c.get("section", "")), - text=str(c.get("text", "")), - )) + clean_citations.append( + CitationItem( + section=str(c.get("section", "")), + text=str(c.get("text", "")), + ) + ) return ChatResponse(answer=response.content, citations=clean_citations) @@ -704,7 +699,7 @@ async def form_fill( placeholders = [] # From checklist (documents to prepare) - for item in (letter.checklist or []): + for item in letter.checklist or []: placeholders.append(str(item)) # From actions (steps the user needs to take) @@ -712,12 +707,25 @@ async def form_fill( db.scalars(select(ActionItem).where(ActionItem.letter_id == letter.id)).all() ) for action in actions: - for step in (action.steps or []): - if any(kw in step.lower() for kw in [ - "iban", "steuer", "name", "adresse", "address", "nummer", - "number", "unterschrift", "signature", "datum", "date", - "versichertennummer", "aktenzeichen", - ]): + for step in action.steps or []: + if any( + kw in step.lower() + for kw in [ + "iban", + "steuer", + "name", + "adresse", + "address", + "nummer", + "number", + "unterschrift", + "signature", + "datum", + "date", + "versichertennummer", + "aktenzeichen", + ] + ): placeholders.append(step) # Always include standard form fields — these match the PLACEHOLDER_MAP @@ -745,7 +753,9 @@ async def form_fill( placeholders=placeholders[:8], # Cap at 8 to keep prompt manageable ) except Exception as exc: - logger.exception("Form-fill generation failed for letter %s: %s", letter_id, exc) + logger.exception( + "Form-fill generation failed for letter %s: %s", letter_id, exc + ) raise KlarHTTPException(502, ErrorCode.LLM_PROVIDER_ERROR) return Response( diff --git a/backend/app/routers/rag.py b/backend/app/routers/rag.py index 419907a..b0a346b 100644 --- a/backend/app/routers/rag.py +++ b/backend/app/routers/rag.py @@ -5,7 +5,7 @@ from app.auth.dependencies import get_current_user from app.models import User from app.rag import store -from app.schemas import ErrorResponse, RagHit, RagQuery, RagResponse +from app.schemas import ErrorResponse, RagQuery, RagResponse from app.services import ai_bridge router = APIRouter(prefix="/api/rag", tags=["rag"]) diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 04a5558..c52b183 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -14,7 +14,7 @@ """ from datetime import date, datetime -from typing import Any, Literal, Optional +from typing import Literal, Optional from uuid import UUID from pydantic import BaseModel, EmailStr, Field @@ -185,7 +185,9 @@ class PublicAction(BaseModel): description="Full RiskScore breakdown — powers the 'why this risk' view.", ) deadline_confidence: Optional[float] = Field( - default=None, ge=0.0, le=1.0, + default=None, + ge=0.0, + le=1.0, description="0..1 confidence in the deadline value (null when unknown).", ) deadline_source: Optional[DeadlineSource] = Field( @@ -200,7 +202,8 @@ class PublicAction(BaseModel): ) reply_needed: bool = False amount_due_eur: Optional[float] = Field( - default=None, ge=0.0, + default=None, + ge=0.0, description=( "Outstanding amount the user must pay for this action, in EUR. " "Extracted from the OCR text by a regex pattern matcher." @@ -227,7 +230,9 @@ class PublicLetter(BaseModel): description="Verbatim German OCR text from the source. Never localized.", ) confidence: Optional[float] = Field( - default=None, ge=0.0, le=1.0, + default=None, + ge=0.0, + le=1.0, description="0..1 overall extraction confidence. <0.85 triggers a 'get a human' UI prompt.", ) actions: list[PublicAction] = Field(default_factory=list) @@ -283,7 +288,8 @@ class PublicActionListItem(BaseModel): status: ActionStatus reply_needed: bool amount_due_eur: Optional[float] = Field( - default=None, ge=0.0, + default=None, + ge=0.0, description=( "Outstanding EUR amount for this action, mirrored from the same " "field on PublicAction. Included on the list shape so the " @@ -414,7 +420,9 @@ class ErrorResponse(BaseModel): description="Stable machine-readable identifier — see docs/06-api-contract.md." ) message: str = Field(description="Localized, user-facing copy.") - detail: str = Field(description="Alias of `message` — for clients that expect FastAPI's default error shape.") + detail: str = Field( + description="Alias of `message` — for clients that expect FastAPI's default error shape." + ) details: Optional[ErrorDetails] = None diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py index 3818b10..0faa8ea 100644 --- a/backend/app/services/__init__.py +++ b/backend/app/services/__init__.py @@ -9,7 +9,11 @@ stream_explanation, stream_response_draft, ) -from app.services.pdf_pages import iter_data_urls, pdf_to_image_bytes, split_to_image_bytes +from app.services.pdf_pages import ( + iter_data_urls, + pdf_to_image_bytes, + split_to_image_bytes, +) from app.services.persistence import persist_extraction from app.services.risk import compute_risk from app.services.storage import detect_magic_mime, is_pdf, save_letter_file, user_dir diff --git a/backend/app/services/ai_bridge.py b/backend/app/services/ai_bridge.py index 0776499..e1ecd88 100644 --- a/backend/app/services/ai_bridge.py +++ b/backend/app/services/ai_bridge.py @@ -50,102 +50,89 @@ _CATEGORY_PATTERNS: list[tuple[str, DocumentCategory]] = [ # Most specific first - ("residence permit", DocumentCategory.IMMIGRATION), - ("aufenthaltstitel", DocumentCategory.IMMIGRATION), - ("ausländerbehörde", DocumentCategory.IMMIGRATION), - ("visa", DocumentCategory.IMMIGRATION), - ("aufenthalts", DocumentCategory.IMMIGRATION), - ("immigration", DocumentCategory.IMMIGRATION), - - ("health insurance", DocumentCategory.HEALTH_INSURANCE), - ("krankenkasse", DocumentCategory.HEALTH_INSURANCE), - ("krankenversicherung", DocumentCategory.HEALTH_INSURANCE), - ("aok", DocumentCategory.HEALTH_INSURANCE), - ("techniker krankenkasse", DocumentCategory.HEALTH_INSURANCE), - ("barmer", DocumentCategory.HEALTH_INSURANCE), - ("dak-gesundheit", DocumentCategory.HEALTH_INSURANCE), - - ("car insurance", DocumentCategory.OTHER_INSURANCE), - ("haftpflicht", DocumentCategory.OTHER_INSURANCE), - ("hausrat", DocumentCategory.OTHER_INSURANCE), - ("kfz-versicherung", DocumentCategory.OTHER_INSURANCE), - ("liability insurance", DocumentCategory.OTHER_INSURANCE), - - ("tax", DocumentCategory.TAX), - ("finanzamt", DocumentCategory.TAX), - ("steuer", DocumentCategory.TAX), - - ("university", DocumentCategory.EDUCATION), - ("universität", DocumentCategory.EDUCATION), - ("hochschule", DocumentCategory.EDUCATION), - ("immatrikulation", DocumentCategory.EDUCATION), - ("studentenwerk", DocumentCategory.EDUCATION), - ("bafög", DocumentCategory.EDUCATION), - ("rückmeldung", DocumentCategory.EDUCATION), - ("enrollment", DocumentCategory.EDUCATION), - - ("rent", DocumentCategory.HOUSING), - ("vermieter", DocumentCategory.HOUSING), - ("hausverwaltung", DocumentCategory.HOUSING), - ("mieterhöhung", DocumentCategory.HOUSING), - ("nebenkosten", DocumentCategory.HOUSING), - ("landlord", DocumentCategory.HOUSING), - - ("electricity", DocumentCategory.UTILITIES), - ("gas bill", DocumentCategory.UTILITIES), - ("internet", DocumentCategory.UTILITIES), - ("telekom", DocumentCategory.UTILITIES), - ("vodafone", DocumentCategory.UTILITIES), - ("stadtwerke", DocumentCategory.UTILITIES), - ("vattenfall", DocumentCategory.UTILITIES), - ("strom", DocumentCategory.UTILITIES), - - ("employer", DocumentCategory.EMPLOYMENT), - ("arbeitgeber", DocumentCategory.EMPLOYMENT), - ("lohn", DocumentCategory.EMPLOYMENT), - ("gehalt", DocumentCategory.EMPLOYMENT), - ("payroll", DocumentCategory.EMPLOYMENT), - ("arbeitsvertrag", DocumentCategory.EMPLOYMENT), - - ("unemployment", DocumentCategory.GOVERNMENT_BENEFITS), - ("kindergeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("elterngeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("wohngeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("arbeitslosengeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("bürgergeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("jobcenter", DocumentCategory.GOVERNMENT_BENEFITS), - ("familienkasse", DocumentCategory.GOVERNMENT_BENEFITS), - - ("pension", DocumentCategory.PENSION), - ("rentenversicherung", DocumentCategory.PENSION), - ("rente", DocumentCategory.PENSION), - - ("rundfunk", DocumentCategory.BROADCAST_FEE), - ("gez", DocumentCategory.BROADCAST_FEE), - ("beitragsservice", DocumentCategory.BROADCAST_FEE), - ("broadcasting fee", DocumentCategory.BROADCAST_FEE), - - ("bürgeramt", DocumentCategory.CIVIC), - ("einwohnermelde", DocumentCategory.CIVIC), - ("standesamt", DocumentCategory.CIVIC), - ("personalausweis", DocumentCategory.CIVIC), - ("reisepass", DocumentCategory.CIVIC), - ("meldebescheinigung", DocumentCategory.CIVIC), - - ("court", DocumentCategory.LEGAL_DEBT), - ("gericht", DocumentCategory.LEGAL_DEBT), - ("mahnbescheid", DocumentCategory.LEGAL_DEBT), - ("vollstreckung", DocumentCategory.LEGAL_DEBT), - ("inkasso", DocumentCategory.LEGAL_DEBT), - ("bußgeld", DocumentCategory.LEGAL_DEBT), - ("anwalt", DocumentCategory.LEGAL_DEBT), - ("debt collection", DocumentCategory.LEGAL_DEBT), - ("fine notice", DocumentCategory.LEGAL_DEBT), - - ("bank", DocumentCategory.BANKING), - ("sparkasse", DocumentCategory.BANKING), - ("schufa", DocumentCategory.BANKING), - ("kreditkarte", DocumentCategory.BANKING), + ("residence permit", DocumentCategory.IMMIGRATION), + ("aufenthaltstitel", DocumentCategory.IMMIGRATION), + ("ausländerbehörde", DocumentCategory.IMMIGRATION), + ("visa", DocumentCategory.IMMIGRATION), + ("aufenthalts", DocumentCategory.IMMIGRATION), + ("immigration", DocumentCategory.IMMIGRATION), + ("health insurance", DocumentCategory.HEALTH_INSURANCE), + ("krankenkasse", DocumentCategory.HEALTH_INSURANCE), + ("krankenversicherung", DocumentCategory.HEALTH_INSURANCE), + ("aok", DocumentCategory.HEALTH_INSURANCE), + ("techniker krankenkasse", DocumentCategory.HEALTH_INSURANCE), + ("barmer", DocumentCategory.HEALTH_INSURANCE), + ("dak-gesundheit", DocumentCategory.HEALTH_INSURANCE), + ("car insurance", DocumentCategory.OTHER_INSURANCE), + ("haftpflicht", DocumentCategory.OTHER_INSURANCE), + ("hausrat", DocumentCategory.OTHER_INSURANCE), + ("kfz-versicherung", DocumentCategory.OTHER_INSURANCE), + ("liability insurance", DocumentCategory.OTHER_INSURANCE), + ("tax", DocumentCategory.TAX), + ("finanzamt", DocumentCategory.TAX), + ("steuer", DocumentCategory.TAX), + ("university", DocumentCategory.EDUCATION), + ("universität", DocumentCategory.EDUCATION), + ("hochschule", DocumentCategory.EDUCATION), + ("immatrikulation", DocumentCategory.EDUCATION), + ("studentenwerk", DocumentCategory.EDUCATION), + ("bafög", DocumentCategory.EDUCATION), + ("rückmeldung", DocumentCategory.EDUCATION), + ("enrollment", DocumentCategory.EDUCATION), + ("rent", DocumentCategory.HOUSING), + ("vermieter", DocumentCategory.HOUSING), + ("hausverwaltung", DocumentCategory.HOUSING), + ("mieterhöhung", DocumentCategory.HOUSING), + ("nebenkosten", DocumentCategory.HOUSING), + ("landlord", DocumentCategory.HOUSING), + ("electricity", DocumentCategory.UTILITIES), + ("gas bill", DocumentCategory.UTILITIES), + ("internet", DocumentCategory.UTILITIES), + ("telekom", DocumentCategory.UTILITIES), + ("vodafone", DocumentCategory.UTILITIES), + ("stadtwerke", DocumentCategory.UTILITIES), + ("vattenfall", DocumentCategory.UTILITIES), + ("strom", DocumentCategory.UTILITIES), + ("employer", DocumentCategory.EMPLOYMENT), + ("arbeitgeber", DocumentCategory.EMPLOYMENT), + ("lohn", DocumentCategory.EMPLOYMENT), + ("gehalt", DocumentCategory.EMPLOYMENT), + ("payroll", DocumentCategory.EMPLOYMENT), + ("arbeitsvertrag", DocumentCategory.EMPLOYMENT), + ("unemployment", DocumentCategory.GOVERNMENT_BENEFITS), + ("kindergeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("elterngeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("wohngeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("arbeitslosengeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("bürgergeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("jobcenter", DocumentCategory.GOVERNMENT_BENEFITS), + ("familienkasse", DocumentCategory.GOVERNMENT_BENEFITS), + ("pension", DocumentCategory.PENSION), + ("rentenversicherung", DocumentCategory.PENSION), + ("rente", DocumentCategory.PENSION), + ("rundfunk", DocumentCategory.BROADCAST_FEE), + ("gez", DocumentCategory.BROADCAST_FEE), + ("beitragsservice", DocumentCategory.BROADCAST_FEE), + ("broadcasting fee", DocumentCategory.BROADCAST_FEE), + ("bürgeramt", DocumentCategory.CIVIC), + ("einwohnermelde", DocumentCategory.CIVIC), + ("standesamt", DocumentCategory.CIVIC), + ("personalausweis", DocumentCategory.CIVIC), + ("reisepass", DocumentCategory.CIVIC), + ("meldebescheinigung", DocumentCategory.CIVIC), + ("court", DocumentCategory.LEGAL_DEBT), + ("gericht", DocumentCategory.LEGAL_DEBT), + ("mahnbescheid", DocumentCategory.LEGAL_DEBT), + ("vollstreckung", DocumentCategory.LEGAL_DEBT), + ("inkasso", DocumentCategory.LEGAL_DEBT), + ("bußgeld", DocumentCategory.LEGAL_DEBT), + ("anwalt", DocumentCategory.LEGAL_DEBT), + ("debt collection", DocumentCategory.LEGAL_DEBT), + ("fine notice", DocumentCategory.LEGAL_DEBT), + ("bank", DocumentCategory.BANKING), + ("sparkasse", DocumentCategory.BANKING), + ("schufa", DocumentCategory.BANKING), + ("kreditkarte", DocumentCategory.BANKING), ] @@ -160,7 +147,9 @@ def map_classification_to_category(free_text_type: str | None) -> DocumentCatego for pattern, cat in _CATEGORY_PATTERNS: if pattern in needle: return cat - logger.debug("map_classification_to_category: no match for %r → OTHER", free_text_type) + logger.debug( + "map_classification_to_category: no match for %r → OTHER", free_text_type + ) return DocumentCategory.OTHER @@ -169,10 +158,10 @@ def map_classification_to_category(free_text_type: str | None) -> DocumentCatego # ============================================================ _LABEL_TO_SEVERITY: dict[str, Severity] = { - "critical": Severity.CRITICAL, - "high": Severity.HIGH, - "medium": Severity.MEDIUM, - "low": Severity.LOW, + "critical": Severity.CRITICAL, + "high": Severity.HIGH, + "medium": Severity.MEDIUM, + "low": Severity.LOW, "informational": Severity.LOW, } @@ -194,10 +183,10 @@ def map_their_severity_label(label: str | None) -> Severity: # ============================================================ _SOURCE_MAPPING: dict[str, DeadlineSource] = { - "letter": DeadlineSource.EXPLICIT, + "letter": DeadlineSource.EXPLICIT, "calculated": DeadlineSource.INFERRED, - "searched": DeadlineSource.INFERRED, # Tavily web search - "none": DeadlineSource.UNKNOWN, + "searched": DeadlineSource.INFERRED, # Tavily web search + "none": DeadlineSource.UNKNOWN, } @@ -218,6 +207,7 @@ def deadline_was_web_searched(source: str | None) -> bool: # Our risk_score (0-100) → their RiskScore.label (for generator input) # ============================================================ + def risk_label_from_score(score: int | None) -> str: """Map our 0-100 score → their label string (used when synthesizing an `AgentResult` to feed their `generate_response`).""" @@ -237,6 +227,7 @@ def risk_label_from_score(score: int | None) -> str: # Letter + (optional) ActionItem → AgentResult (their dataclass) # ============================================================ + def synthesize_agent_result( letter: Letter, action: ActionItem | None = None, @@ -274,6 +265,7 @@ def synthesize_agent_result( # Their LegalChunk → our RagHit (for /rag/search response) # ============================================================ + def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: """Map their `LegalChunk` from `ai.rag.retrieval` → our `RagHit` wire shape. @@ -285,9 +277,9 @@ def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: text=chunk.text, score=1.0, metadata={ - "section": chunk.section, - "law": chunk.law, - "title": chunk.title, + "section": chunk.section, + "law": chunk.law, + "title": chunk.title, "citation": chunk.citation, }, ) @@ -297,6 +289,7 @@ def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: # Their Citation → JSON dict (stored on Letter.citations column) # ============================================================ + def citation_to_dict(c: "Citation") -> dict[str, Any]: """Persist-shape for the `Letter.citations` JSON column. @@ -305,8 +298,8 @@ def citation_to_dict(c: "Citation") -> dict[str, Any]: """ return { "section": c.section, - "text": c.text, - "score": 1.0, # their structured citation doesn't carry a score + "text": c.text, + "score": 1.0, # their structured citation doesn't carry a score } @@ -318,6 +311,7 @@ def citations_to_dicts(cits: list["Citation"]) -> list[dict[str, Any]]: # Their GenerationOutput → (explanation, response_draft, checklist[], citations[dict]) # ============================================================ + def unpack_generation_output( out: "GenerationOutput", ) -> tuple[str, str, list[str], list[dict[str, Any]]]: @@ -336,6 +330,7 @@ def unpack_generation_output( # Their AgentAnalysis → (category, document_type, severity, deadline_date, ...) # ============================================================ + async def generate_grounded_response( ocr_text: str, agent_result: "AgentResult", @@ -369,8 +364,7 @@ async def generate_grounded_response( # Build the legal-context section from retrieved chunks if legal_chunks: legal_lines = [ - f"### {c.citation} — {c.title}\n{c.text}\n" - for c in legal_chunks + f"### {c.citation} — {c.title}\n{c.text}\n" for c in legal_chunks ] legal_context = "\n".join(legal_lines) else: @@ -403,13 +397,18 @@ async def generate_grounded_response( ), temperature=0, max_tokens=4096, - extra_body={"enable_thinking": False, "response_format": {"type": "json_object"}}, + extra_body={ + "enable_thinking": False, + "response_format": {"type": "json_object"}, + }, ) response = await raw_model.ainvoke([HumanMessage(content=prompt)]) raw_text = response.content if hasattr(response, "content") else str(response) if isinstance(raw_text, list): # Some langchain versions return content as a list of parts - raw_text = "".join(p.get("text", "") if isinstance(p, dict) else str(p) for p in raw_text) + raw_text = "".join( + p.get("text", "") if isinstance(p, dict) else str(p) for p in raw_text + ) payload = json.loads(raw_text) @@ -421,10 +420,12 @@ async def generate_grounded_response( # Bare "§ 81 AufenthG" — wrap as Citation with empty explanation cleaned_citations.append(Citation(section=c, text="")) elif isinstance(c, dict): - cleaned_citations.append(Citation( - section=str(c.get("section") or c.get("§") or "§"), - text=str(c.get("text") or c.get("explanation") or ""), - )) + cleaned_citations.append( + Citation( + section=str(c.get("section") or c.get("§") or "§"), + text=str(c.get("text") or c.get("explanation") or ""), + ) + ) else: logger.debug("Skipping unparseable citation: %r", c) @@ -454,17 +455,19 @@ def unpack_agent_analysis( try: parsed_deadline = _date.fromisoformat(deadline_iso) except ValueError: - logger.debug("Their agent returned non-ISO deadline %r — dropping", deadline_iso) + logger.debug( + "Their agent returned non-ISO deadline %r — dropping", deadline_iso + ) return { - "category": map_classification_to_category(analysis.classification.type), - "document_type": analysis.classification.type or "", - "institution": analysis.classification.agency or "", - "deadline": parsed_deadline, - "deadline_source": map_their_deadline_source(analysis.deadline.source), + "category": map_classification_to_category(analysis.classification.type), + "document_type": analysis.classification.type or "", + "institution": analysis.classification.agency or "", + "deadline": parsed_deadline, + "deadline_source": map_their_deadline_source(analysis.deadline.source), "deadline_was_searched": deadline_was_web_searched(analysis.deadline.source), - "consequence": analysis.consequence.text or "", - "severity": map_their_severity_label(analysis.risk_score.label), - "risk_label": analysis.risk_score.label or "Medium", - "risk_reason": analysis.risk_score.reason or "", + "consequence": analysis.consequence.text or "", + "severity": map_their_severity_label(analysis.risk_score.label), + "risk_label": analysis.risk_score.label or "Medium", + "risk_reason": analysis.risk_score.reason or "", } diff --git a/backend/app/services/extraction.py b/backend/app/services/extraction.py index 1c04fc6..d699d72 100644 --- a/backend/app/services/extraction.py +++ b/backend/app/services/extraction.py @@ -37,6 +37,7 @@ class ExtractionError(Exception): `EXTRACTION_FAILED` error instead of letting a raw exception become a 500. """ + # ISO 639-1 codes — matches the frontend's docs/06-frontend-integration-contract.md. # Qwen3.7-Plus handles all of these out of the box. Quality bar: # en/de: production-grade, the wedge languages @@ -316,7 +317,9 @@ async def extract_from_letter_file( json_parsed = json.loads(stripped) except json.JSONDecodeError: # Sometimes the model wraps the JSON in code fences - fence_match = re.search(r"```(?:json)?\s*(.*?)\s*```", stripped, re.DOTALL) + fence_match = re.search( + r"```(?:json)?\s*(.*?)\s*```", stripped, re.DOTALL + ) if fence_match: try: json_parsed = json.loads(fence_match.group(1).strip()) @@ -408,8 +411,12 @@ async def generate_reply_text( Used by POST /letters/{id}/reply (frontend contract §4.7). """ - actions_text = "\n".join(f"- {t}" for t in action_titles) or "- (keine spezifische Aktion)" - prompt = _response_prompt_from_letter(institution, document_type, actions_text, applicant) + actions_text = ( + "\n".join(f"- {t}" for t in action_titles) or "- (keine spezifische Aktion)" + ) + prompt = _response_prompt_from_letter( + institution, document_type, actions_text, applicant + ) client = _get_client() response = await client.chat.completions.create( @@ -464,7 +471,9 @@ async def _stream_text(prompt: str) -> AsyncIterator[str]: yield delta.content -async def stream_explanation(extracted: ExtractedLetter, lang: str) -> AsyncIterator[str]: +async def stream_explanation( + extracted: ExtractedLetter, lang: str +) -> AsyncIterator[str]: async for piece in _stream_text(_explanation_prompt(extracted, lang)): yield piece @@ -501,7 +510,9 @@ async def generate_checklist(extracted: ExtractedLetter, lang: str) -> list[str] # ---------- backwards-compat alias for the original entrypoint ---------- -async def extract_from_image(image_bytes: bytes, mime: str = "image/jpeg") -> ExtractedLetter: +async def extract_from_image( + image_bytes: bytes, mime: str = "image/jpeg" +) -> ExtractedLetter: """Legacy entrypoint: write bytes to a temp file then call the new API.""" import tempfile import os diff --git a/backend/app/services/pdf_pages.py b/backend/app/services/pdf_pages.py index 8f4ae61..fd0b810 100644 --- a/backend/app/services/pdf_pages.py +++ b/backend/app/services/pdf_pages.py @@ -20,7 +20,9 @@ class PdfRenderError(Exception): """ -def pdf_to_image_bytes(path: str, *, dpi: int = 200, max_pages: int = 12) -> list[bytes]: +def pdf_to_image_bytes( + path: str, *, dpi: int = 200, max_pages: int = 12 +) -> list[bytes]: """Render up to `max_pages` pages of `path` to PNG bytes. Imported lazily so callers that never touch a PDF don't pay the diff --git a/backend/app/services/persistence.py b/backend/app/services/persistence.py index eb28f81..049833b 100644 --- a/backend/app/services/persistence.py +++ b/backend/app/services/persistence.py @@ -34,7 +34,8 @@ def persist_extraction( # Overall confidence = min of available signals. Frontend uses <0.85 to # surface a "get a human" prompt. signals = [ - s for s in (extracted.language_confidence, extracted.category_confidence) + s + for s in (extracted.language_confidence, extracted.category_confidence) if s and s > 0 ] letter.confidence = min(signals) if signals else None @@ -84,6 +85,7 @@ def persist_extraction( # so totals across letters don't double-count. if letter_amount is not None and saved: from app.models import Severity as _Sev + sev_rank = {_Sev.CRITICAL: 4, _Sev.HIGH: 3, _Sev.MEDIUM: 2, _Sev.LOW: 1} amount_attached_to = max(saved, key=lambda x: sev_rank.get(x.severity, 0)) amount_attached_to.amount_due_eur = letter_amount diff --git a/backend/tests/test_scanned_pdf_graceful.py b/backend/tests/test_scanned_pdf_graceful.py index 4a07122..b3b1d78 100644 --- a/backend/tests/test_scanned_pdf_graceful.py +++ b/backend/tests/test_scanned_pdf_graceful.py @@ -327,6 +327,83 @@ async def test_post_letter_unexpected_error_stays_generic(monkeypatch): assert "secrets" not in exc.message +# -------------------------------------------------------------------------- +# 7. /api/letters/{id}/extract (letters.py) — the sync extract endpoint must +# also yield typed 502 errors (not raw 500s) and log the failure. +# This was a gap in the original fix: the generic except handler silently +# swallowed exceptions without logging. +# -------------------------------------------------------------------------- + + +async def _call_extract_letter(monkeypatch, *, raise_exc): + """Drive letters.extract_letter with extraction stubbed to raise `raise_exc`. + + Returns the KlarHTTPException the handler raises (or None if it didn't). + """ + from app.routers import letters as letters_router + + async def _boom(*a, **k): + raise raise_exc + + monkeypatch.setattr(letters_router, "extract_from_letter_file", _boom) + + with Session(engine) as db: + user = _make_user(db) + letter = Letter( + user_id=user.id, + language="en", + status=LetterStatus.UPLOADED, + original_file="/tmp/scanned-no-text.pdf", + ) + db.add(letter) + db.commit() + db.refresh(letter) + try: + await letters_router.extract_letter(letter_id=letter.id, db=db, user=user) + except Exception as exc: # noqa: BLE001 + return exc + return None + + +async def test_extract_letter_scanned_pdf_returns_extraction_failed(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_extract_letter( + monkeypatch, + raise_exc=ExtractionError("scanned image without readable content"), + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + assert exc.code == ErrorCode.EXTRACTION_FAILED + assert "readable content" in exc.message + + +async def test_extract_letter_corrupt_pdf_returns_pdf_render_failed(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_extract_letter( + monkeypatch, + raise_exc=PdfRenderError("Could not render this PDF. It may be corrupt."), + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + assert exc.code == ErrorCode.PDF_RENDER_FAILED + assert "corrupt" in exc.message + + +async def test_extract_letter_unexpected_error_stays_generic(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_extract_letter( + monkeypatch, raise_exc=RuntimeError("some provider 500 with secrets") + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + assert exc.code == ErrorCode.EXTRACTION_FAILED + # Raw provider error must NOT leak into the user-facing message. + assert "secrets" not in exc.message + + async def test_process_letter_stream_pdf_render_failure_emits_error(monkeypatch): _install_fake_agent_module()