diff --git a/.gitignore b/.gitignore index 2393dbf..9cdf3fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .venv/ +venv/ .env .legalai/ manifest.csv @@ -12,4 +13,5 @@ detailed_gap_analysis.md .legalai/ # Database storage -qdrant_storage/ \ No newline at end of file +qdrant_storage/ +Extraction/venv \ No newline at end of file diff --git a/Extraction/main.py b/Extraction/main.py index 4263caf..ed530c5 100644 --- a/Extraction/main.py +++ b/Extraction/main.py @@ -64,6 +64,9 @@ # ── Utils ─────────────────────────────────────────────────────────────────── from Extraction.utils.helpers import dedup_assets, to_none, clean_rel_type +# ── Validation ────────────────────────────────────────────────────────────── +from Extraction.validate_llm import validate_entities, validate_case_llm + import requests from tenacity import retry, wait_exponential, stop_after_attempt @@ -190,6 +193,166 @@ def process_case(json_path: str, pdf_paths: list[str]) -> dict: result['missing_advocates'] = len(missing_advocates) result['judges_found'] = len(judges_data) + # ── Rule-based validation of LLM-extracted entities ────────────────── + judges_data, judges_dropped = validate_entities('judge', judges_data, result['cnr']) + raw_assets, assets_dropped = validate_entities('asset', raw_assets, result['cnr']) + validation_dropped = judges_dropped + assets_dropped + missing_log.extend( + f"[validation] {d['entity_type']}.{d['field']}={d['value']!r} failed rule check" + for d in validation_dropped + ) + + # ── LLM (Ollama) whole-case semantic validation ─────────────────────── + # Covers every free-text field with real semantic risk: persons.name/ + # address/role, judges.name/designation, lawyers.name (missing_advocates), + # organizations.name/address (new_parties of type organization), + # hearings.purpose/judge_designation/nature_of_disposal, and the + # case-level district/state pair. + # `address`/`role` map to PersonModel.address_text/role; + # `judge_designation` on hearings maps to HearingModel.judge_designation + # (a second, independent source of designation text from persons/judges). + # + # PRIMARY_FIELDS_BY_ENTITY: for these entity types, a flagged or missing + # `name` means the extracted value cannot be trusted to refer to a real + # entity at all — so the WHOLE entity is dropped rather than just the + # field being nulled. Every other checked field is secondary: only that + # field gets nulled, the entity itself survives. `case` has no primary + # field here (district/state are supplementary, the Case itself is + # never dropped by this check). + PRIMARY_FIELDS_BY_ENTITY = { + 'persons' : {'name'}, + 'judges' : {'name'}, + 'lawyers' : {'name'}, + 'organizations': {'name'}, + } + + lawyer_entries = [] + for adv in missing_advocates: + if isinstance(adv, str): + lawyer_entries.append({'name': to_none(adv)}) + else: + lawyer_entries.append({'name': to_none(adv.get('name'))}) + + organization_entries = [] + for np in new_parties: + if isinstance(np, str): + continue + if (np.get('type') or '').lower() != 'organization': + continue + organization_entries.append({ + 'name' : to_none(np.get('name')), + 'address': to_none(np.get('address')), + }) + + case_entities_for_llm = { + 'persons': [ + {'name': p.name, 'address': p.address_text, 'role': p.role} + for p in case.persons + ], + 'judges': [ + {'name': j.get('name'), 'designation': j.get('designation')} + for j in judges_data + ], + 'lawyers': lawyer_entries, + 'organizations': organization_entries, + 'hearings': [ + { + 'purpose': h.purpose, + 'judge_designation': h.judge_designation, + 'nature_of_disposal': h.diary_note.nature_of_disposal, + } + for h in case.hearings + ], + 'case': [ + {'district': case.district, 'state': case.state}, + ], + } + cleaned_case_entities, validation_dropped_llm = validate_case_llm( + case_entities_for_llm, result['cnr'], + ) + + # Fields flagged by the LLM as wrong-content for a PRIMARY field (see + # PRIMARY_FIELDS_BY_ENTITY above) mark their whole entity for removal; + # everything else is a secondary field and is simply nulled from the + # cleaned copy. persons.role has no valid "unknown" state either, so + # (like name) a flagged role is left as originally extracted. + dropped_by_primary: dict[str, set[int]] = { + etype: set() for etype in PRIMARY_FIELDS_BY_ENTITY + } + for d in validation_dropped_llm: + primary_fields = PRIMARY_FIELDS_BY_ENTITY.get(d['entity_type']) + if primary_fields and d['field'] in primary_fields: + dropped_by_primary[d['entity_type']].add(d['index']) + + for p, cleaned in zip(case.persons, cleaned_case_entities['persons']): + p.address_text = cleaned['address'] + for j, cleaned in zip(judges_data, cleaned_case_entities['judges']): + j['designation'] = cleaned['designation'] + for h, cleaned in zip(case.hearings, cleaned_case_entities['hearings']): + h.purpose = cleaned['purpose'] + h.judge_designation = cleaned['judge_designation'] + h.diary_note.nature_of_disposal = cleaned['nature_of_disposal'] + + # ── Drop whole entities whose primary field (name) was missing/flagged ── + persons_dropped_idx = { + i for i, p in enumerate(case.persons) if to_none(p.name) is None + } | dropped_by_primary['persons'] + if persons_dropped_idx: + case.persons = [ + p for i, p in enumerate(case.persons) if i not in persons_dropped_idx + ] + + judges_dropped_idx = { + i for i, j in enumerate(judges_data) if to_none(j.get('name')) is None + } | dropped_by_primary['judges'] + if judges_dropped_idx: + judges_data = [ + j for i, j in enumerate(judges_data) if i not in judges_dropped_idx + ] + + lawyers_dropped_idx = { + i for i, l in enumerate(lawyer_entries) if to_none(l.get('name')) is None + } | dropped_by_primary['lawyers'] + if lawyers_dropped_idx: + missing_advocates = [ + adv for i, adv in enumerate(missing_advocates) + if i not in lawyers_dropped_idx + ] + + orgs_dropped_idx = { + i for i, o in enumerate(organization_entries) if to_none(o.get('name')) is None + } | dropped_by_primary['organizations'] + if orgs_dropped_idx: + org_indices_in_new_parties = [ + i for i, np in enumerate(new_parties) + if not isinstance(np, str) and (np.get('type') or '').lower() == 'organization' + ] + drop_np_idx = {org_indices_in_new_parties[i] for i in orgs_dropped_idx} + new_parties = [ + np for i, np in enumerate(new_parties) if i not in drop_np_idx + ] + + result['dropped_entities'] = { + 'persons' : len(persons_dropped_idx), + 'judges' : len(judges_dropped_idx), + 'lawyers' : len(lawyers_dropped_idx), + 'organizations': len(orgs_dropped_idx), + } + missing_log.extend( + f"[validation-llm] dropped {etype}[{i}] — missing/invalid primary field 'name'" + for etype, idxs in ( + ('persons', persons_dropped_idx), ('judges', judges_dropped_idx), + ('lawyers', lawyers_dropped_idx), ('organizations', orgs_dropped_idx), + ) + for i in idxs + ) + + missing_log.extend( + f"[validation-llm] {d['entity_type']}.{d['field']}={d['value']!r} — {d['reason']}" + for d in validation_dropped_llm + ) + result['validation_dropped'] = validation_dropped + validation_dropped_llm + for asset in raw_assets: asset['_source_storage_id'] = next(iter(pdf_texts), None) deduped_assets = dedup_assets(raw_assets) diff --git a/Extraction/validate_llm/__init__.py b/Extraction/validate_llm/__init__.py new file mode 100644 index 0000000..53e50c6 --- /dev/null +++ b/Extraction/validate_llm/__init__.py @@ -0,0 +1,9 @@ +from Extraction.validate_llm.engine import ( + validate_entity, validate_entities, + validate_case_llm, +) + +__all__ = [ + 'validate_entity', 'validate_entities', + 'validate_case_llm', +] diff --git a/Extraction/validate_llm/engine.py b/Extraction/validate_llm/engine.py new file mode 100644 index 0000000..ba1199b --- /dev/null +++ b/Extraction/validate_llm/engine.py @@ -0,0 +1,124 @@ +""" +Extraction/validate_llm/engine.py +Rule-based validation engine. + +Walks a raw entity dict (as returned by the LLM extraction step) against +the declarative rules in field_rules.py. Any field that fails its rule(s) +is nulled out (kept in the dict as None) so the rest of the entity still +gets inserted — matching the codebase's existing to_none()/missing_data_log +convention for "no reliable value". + +Fields flagged here are also collected so a future LLM-review step can +pick them up (see FLAGGED_FOR_LLM below) without touching this module. +""" +import logging +from typing import Any + +from Extraction.utils.helpers import to_none +from Extraction.validate_llm.field_rules import ENTITY_RULES +from Extraction.validate_llm.llm_field_checks import check_case + +logger = logging.getLogger('pipeline') + + +def validate_entity(entity_type: str, data: dict, context: str = '') -> tuple[dict, list[dict]]: + """ + Validate one entity dict in place (returns a new dict; input is not mutated). + + Returns (cleaned_data, dropped) where `dropped` is a list of + {'entity_type', 'field', 'value', 'context'} describing what was nulled. + """ + rules = ENTITY_RULES.get(entity_type.lower()) + if not rules: + return data, [] + + cleaned = dict(data) + dropped: list[dict] = [] + + for field, checks in rules.items(): + if field not in cleaned: + continue + raw_value = to_none(cleaned.get(field)) + if raw_value is None: + continue + + if not all(check(raw_value) for check in checks): + dropped.append({ + 'entity_type': entity_type, + 'field' : field, + 'value' : raw_value, + 'context' : context, + }) + cleaned[field] = None + + if dropped: + for d in dropped: + logger.warning( + f"[validate_llm] dropped invalid field " + f"{d['entity_type']}.{d['field']}={d['value']!r} ({d['context']})" + ) + + return cleaned, dropped + + +def validate_entities(entity_type: str, items: list[dict], context: str = '') -> tuple[list[dict], list[dict]]: + """Validate a list of entity dicts (e.g. all judges_data). Returns (cleaned_items, all_dropped).""" + cleaned_items = [] + all_dropped: list[dict] = [] + for item in items: + cleaned, dropped = validate_entity(entity_type, item, context) + cleaned_items.append(cleaned) + all_dropped.extend(dropped) + return cleaned_items, all_dropped + + +def validate_case_llm(case_entities: dict[str, list[dict]], context: str = '') -> tuple[dict[str, list[dict]], list[dict]]: + """ + Run ONE Ollama call covering the whole case (see llm_field_checks.py) + — the model is asked to flag any field whose value looks semantically + wrong for its field name (generic check, not limited to a fixed set + of fields). Every flagged field is independently re-verified against + the input before being trusted (see check_case()'s hallucination + backstop) — nothing is dropped on the model's word alone. + + case_entities: {'persons': [dict, ...], 'judges': [dict, ...], ...} + (each inner dict must be a plain dict — not a Pydantic model — see + main.py for the persons->dict / dict->persons conversion at the call site) + + Returns two SEPARATE JSON-serializable structures: + cleaned_json — same shape as case_entities, with only verified-bad + fields nulled; everything else untouched. This is + what the next pipeline step / Neo4j insert should + read from. + dropped_json — a flat list of what was removed and why: + [{'entity_type', 'index', 'field', 'value', + 'reason', 'context'}, ...]. This is for logs/ + testing only — it never feeds back into the data + path. + """ + problems = check_case(case_entities, context) + if not problems: + return case_entities, [] + + cleaned = {etype: [dict(e) for e in items] for etype, items in case_entities.items()} + dropped_json: list[dict] = [] + + for p in problems: + entity = cleaned[p['entity_type']][p['index']] + field = p['field'] + value = entity.get(field) + dropped_json.append({ + 'entity_type': p['entity_type'], + 'index' : p['index'], + 'field' : field, + 'value' : value, + 'reason' : p['reason'], + 'context' : context, + }) + entity[field] = None + logger.warning( + f"[validate_llm] LLM dropped invalid field " + f"{p['entity_type']}[{p['index']}].{field}={value!r} ({context}) — {p['reason']}" + ) + + return cleaned, dropped_json diff --git a/Extraction/validate_llm/field_rules.py b/Extraction/validate_llm/field_rules.py new file mode 100644 index 0000000..56dcc76 --- /dev/null +++ b/Extraction/validate_llm/field_rules.py @@ -0,0 +1,93 @@ +""" +Extraction/validate_llm/field_rules.py +Declarative per-entity, per-field validation rules. + +Each entity maps field_name -> list of rule callables (see rules.py). +A field with no entry here is passed through unchecked (free-text fields +like `name`, `address`, `description`, `search_summary`, etc. have no +reliable rule and are left for the future LLM-review step). + +To add a rule: add/extend the field's list. To add an entity: add a new +top-level dict and register it in ENTITY_RULES. +""" +from Extraction.validate_llm.rules import ( + is_int, is_float, is_bool, in_range, regex, valid_date, one_of, +) + +CURRENT_YEAR = 2026 + +USER_RULES = { + 'aadhaar_no': [regex(r'\d{12}')], + 'pan_no' : [regex(r'[A-Za-z]{5}\d{4}[A-Za-z]')], + 'age' : [is_int, in_range(0, 120)], + 'gender' : [one_of('male', 'female', 'other')], +} + +JUDGE_RULES = { + 'heard_from_date': [valid_date], + 'heard_to_date' : [valid_date], + 'status' : [one_of('active', 'retired', 'transferred')], +} + +LAWYER_RULES = { + 'enrollment_date': [valid_date], +} + +CASE_RULES = { + 'filing_date' : [valid_date], + 'disposal_date' : [valid_date], + 'registration_date' : [valid_date], + 'first_hearing_date' : [valid_date], + 'last_hearing_date' : [valid_date], + 'next_hearing_date' : [valid_date], + 'decision_date' : [valid_date], + 'filing_year' : [is_int, in_range(1950, CURRENT_YEAR)], + 'in_favour_of' : [is_bool], + 'alleged_amount' : [is_float, in_range(0, float('inf'))], +} + +ORGANIZATION_RULES = { + 'cin' : [regex(r'[A-Za-z]\d{5}[A-Za-z]{2}\d{4}[A-Za-z]{3}\d{6}')], + 'gstin': [regex(r'\d{2}[A-Za-z]{5}\d{4}[A-Za-z]\d[A-Za-z\d]Z[A-Za-z\d]')], + 'pan' : [regex(r'[A-Za-z]{5}\d{4}[A-Za-z]')], +} + +COURT_RULES = { + 'hierarchy_level': [is_int, in_range(0, 10)], +} + +ACT_RULES = { + 'year': [is_int, in_range(1800, CURRENT_YEAR)], +} + +SECTION_RULES = { + 'bailable': [is_bool], +} + +CASE_HEARING_RULES = { + 'date' : [valid_date], + 'last_hearing_date' : [valid_date], + 'next_hearing_date' : [valid_date], +} + +ASSET_RULES = { + 'estimated_value_inr': [is_float, in_range(0, float('inf'))], +} + +DOCUMENT_RULES = { + 'order_date': [valid_date], +} + +ENTITY_RULES = { + 'user' : USER_RULES, + 'judge' : JUDGE_RULES, + 'lawyer' : LAWYER_RULES, + 'case' : CASE_RULES, + 'organization': ORGANIZATION_RULES, + 'court' : COURT_RULES, + 'act' : ACT_RULES, + 'section' : SECTION_RULES, + 'case_hearing': CASE_HEARING_RULES, + 'asset' : ASSET_RULES, + 'document' : DOCUMENT_RULES, +} diff --git a/Extraction/validate_llm/llm_field_checks.py b/Extraction/validate_llm/llm_field_checks.py new file mode 100644 index 0000000..1275141 --- /dev/null +++ b/Extraction/validate_llm/llm_field_checks.py @@ -0,0 +1,292 @@ +""" +Extraction/validate_llm/llm_field_checks.py +Whole-case LLM semantic validation: one Ollama call per case, given ONLY +a numbered list of the fields worth a semantic check, asked to return the +INVALID ones as JSON. + +The rule engine (field_rules.py) already handles anything checkable by +type/regex/range (IDs, dates, numeric ranges, etc.) before this ever +runs, so this step only has to judge free-text/semantic correctness: +does the *content* of a field actually match its *name* (e.g. a name +string sitting in an address field). + +Design choices that specifically target failure modes observed with 4B +local models (Qwen, Gemma): + 1. SEMANTIC_FIELDS WHITELIST — only fields with real semantic risk are + sent at all (case_number/dates/IDs etc. have no "wrong content" + failure mode worth an LLM call). Smaller prompt, fewer chances for + the model to invent a problem where none exists. + 2. NO FULL CASE JSON — earlier versions also embedded the entire + rule-cleaned case JSON "for reference", doubling context for no + benefit (the numbered list already has every value) and inviting + cross-field comparisons the prompt explicitly forbids. + 3. INVALID-ONLY JSON OUTPUT — asking for `n= verdict=valid` on every + single field forces the model to produce dozens of repetitive lines + it can lose count of. Asking only for the invalid ones, plus a + `checked` count the caller verifies against the true field count, + gets most of the robustness of "enumerate everything" at a fraction + of the output size. + 4. CLOSED, SMALL REASON VOCABULARY — few, non-overlapping codes are + easier for a small model to apply consistently than many similar + ones. + 5. TOLERANT PARSING — the model is not perfectly reliable about exact + key names/casing, so parsing normalizes minor variation instead of + failing closed. + +Every returned problem is independently re-verified against the original +input in check_case() before it is trusted (hallucination backstop) — +this verification is a correctness guarantee that holds regardless of +model or prompt wording. +""" +import json +import logging +import re + +from Extraction.validate_llm.ollama_client import ask_json + +logger = logging.getLogger('pipeline') + +# Only these fields carry real semantic risk (free text where the wrong +# *kind* of content could land) — everything else (IDs, dates, numbers, +# amounts) is already covered by field_rules.py and isn't worth a +# model call. +SEMANTIC_FIELDS = { + 'name', 'role', 'address', 'office_address', 'designation', + 'district', 'state', 'purpose', 'nature_of_disposal', +} + +# Closed vocabulary the model must pick from — no free-text reasons. +# Kept small and non-overlapping so a small model can apply it +# consistently (a name-shaped value in a name field and a role-label in +# a name field are both just "wrong_content", not two different things). +_REASON_CODES = { + 'wrong_content' : 'Value is the wrong kind of information for this field', + 'missing_location' : 'Address-like field has no building/street/locality/city/PIN at all', + 'missing_designation': 'Designation-like field has no judicial/professional title in it', + 'narrative_text' : 'Value is a full sentence/narrative where a short label was expected', +} +_REASON_CODES_TEXT = "\n".join(f" {code} — {desc}" for code, desc in _REASON_CODES.items()) + +def build_case_for_prompt(case_entities: dict[str, list[dict]]) -> dict[str, list[dict]]: + """ + case_entities: {'persons': [dict, ...], 'judges': [dict, ...], ...} + Returns a copy containing only entities/fields worth asking the LLM + about: fields in SEMANTIC_FIELDS that are still non-empty (nulled-out + fields, e.g. ones the rule engine already dropped, carry nothing to + judge; non-semantic fields like case_number/dates have no useful + check here) and only entities that have at least one such field. + """ + prepared: dict[str, list[dict]] = {} + for entity_type, entities in case_entities.items(): + prepared_entities = [] + for idx, entity in enumerate(entities): + fields = { + k: v for k, v in entity.items() + if k in SEMANTIC_FIELDS and v is not None and str(v).strip() != '' + } + if fields: + prepared_entities.append({'index': idx, **fields}) + if prepared_entities: + prepared[entity_type] = prepared_entities + return prepared + + +def _enumerate_fields(prepared_case: dict[str, list[dict]]) -> list[tuple[str, int, str, str]]: + """Flat, ordered list of every (entity_type, index, field, value) the model must judge.""" + out = [] + for entity_type, entities in prepared_case.items(): + for entity in entities: + idx = entity['index'] + for field, value in entity.items(): + if field == 'index': + continue + out.append((entity_type, idx, field, value)) + return out + +_PROMPT_TEMPLATE = """ +You are an expert validator for an Indian Court Case Management System. + +Judge whether each field below contains the correct KIND of information for +its field name. Do NOT judge spelling, capitalization, formatting, +abbreviations, or whether a value looks short/unusual — only whether it +belongs in that field at all. + +--- +## FIELD MEANINGS + +name (persons/judges/lawyers/organizations/courts) +Name of a person, company, bank, department, trust, court, or other +entity. Wrong only if clearly not a name (e.g. an address or a role word). + +role (persons/lawyers/organizations) +The entity's role in the case. Wrong only if it is clearly a name, +address, designation, court name, or date instead of a role. + +address / office_address (persons/lawyers/organizations/courts) +A physical/postal location. Wrong only if it has no location info at all. + +designation (judges/lawyers) +A judicial or professional title. May be long/multi-word, e.g. "Additional +District and Sessions Judge" — that is VALID. Wrong only if there is no +title/role word in it at all (e.g. it's just a name or a place). + +district +A judicial/administrative district name. Wrong only if it's clearly a +state, full address, court name, or person/org name instead. + +state +An Indian state or union territory name. Wrong only if it's clearly a +district, full address, court name, or person/org name instead. + +purpose (hearings) +The stage/procedural reason for a hearing. Often a single short word or +abbreviation, e.g. "Report", "Disposed", "SR/Objection" — that is VALID. +Wrong only if it is clearly a person's name, an address, or unrelated text. + +nature_of_disposal (hearings) +This field tells how the court case or hearing was concluded or what status/outcome was given to it by the court. +It is a short label describing the result of the hearing, not a description of what happened. +Valid examples: Adjourned, Dismissed, Dismissed In Default, Allowed,Rejected, Withdrawn, Settled, Disposed Of +Invalid examples: +- The case was dismissed because the petitioner did not appear before the court. +- Court gave time to the respondent to submit documents and fixed another hearing date. +- The petition was rejected after detailed examination of the evidence. + +--- +## RULES + +1. Judge only whether the value matches the meaning of its field. +2. Ignore spelling, capitalization, formatting, length, and writing style. +3. Judge every field independently — never compare fields to each other. +4. Only mark a field wrong if it is CLEARLY the wrong kind of information. +5. If in doubt, it is valid. + +--- +## REASON CODES + +Use exactly one of these for each invalid field — do not invent new ones: + +{reason_codes} + +--- +## FIELDS TO JUDGE + +{numbered_fields} + +--- +## OUTPUT FORMAT — STRICT JSON, NOTHING ELSE + +Return ONE JSON object, no other text, no markdown fences: + +{{"checked": {field_count}, "invalid": [{{"n": , "reason": ""}}, ...]}} + +- "checked" MUST equal {field_count} (the total number of fields listed above). +- "invalid" lists ONLY the fields that are wrong. Fields not listed are + assumed valid. +- If every field is valid, return {{"checked": {field_count}, "invalid": []}}. +""" + +def _build_prompt(prepared_case: dict[str, list[dict]], fields: list[tuple[str, int, str, str]]) -> str: + numbered = "\n".join( + f"{n} {et}[{idx}].{field} = {value!r}" + for n, (et, idx, field, value) in enumerate(fields, start=1) + ) + return _PROMPT_TEMPLATE.format( + reason_codes=_REASON_CODES_TEXT, + numbered_fields=numbered, + field_count=len(fields), + ) + + +def _normalize_invalid_entries(raw_invalid) -> dict[int, str]: + """Tolerant extraction of {n: verdict} from whatever shape the model gave 'invalid' as.""" + verdicts: dict[int, str] = {} + if not isinstance(raw_invalid, list): + return verdicts + for entry in raw_invalid: + if not isinstance(entry, dict): + continue + n = entry.get('n') + if n is None: + n = entry.get('index') or entry.get('field') or entry.get('number') + reason = entry.get('reason') or entry.get('verdict') or entry.get('code') + try: + n = int(n) + except (TypeError, ValueError): + continue + if not reason: + continue + verdicts[n] = str(reason).strip().lower() + return verdicts + + +def check_case(case_entities: dict[str, list[dict]], context: str = '') -> list[dict]: + """ + Run one Ollama call covering the whole case. Returns a list of + VERIFIED problems: [{'entity_type', 'index', 'field', 'reason'}]. + Every entry has been independently confirmed to correspond to a + real (entity_type, index, field) that was actually sent to the + model and still held a value — nothing here is applied on trust + alone. This function does not mutate anything; see engine.py for + how the caller turns these into the two output JSONs. + """ + prepared = build_case_for_prompt(case_entities) + if not prepared: + return [] + + fields = _enumerate_fields(prepared) + prompt = _build_prompt(prepared, fields) + + parsed = ask_json(prompt) + if parsed is None: + logger.warning(f"[validate_llm] whole-case LLM check unavailable for {context}") + return [] + + checked = parsed.get('checked') + if checked != len(fields): + # One retry — small models occasionally drop/duplicate a field on + # the first pass. If it still doesn't match, the response can't + # be trusted to be complete, so skip rather than risk silently + # missing (or hallucinating) a problem. + logger.warning( + f"[validate_llm] checked={checked!r} != expected={len(fields)} " + f"for {context} — retrying once" + ) + parsed = ask_json(prompt) + if parsed is None or parsed.get('checked') != len(fields): + logger.warning( + f"[validate_llm] whole-case LLM check count mismatch persisted " + f"for {context} — skipping" + ) + return [] + + verdicts = _normalize_invalid_entries(parsed.get('invalid')) + + problems = [] + for n, (entity_type, idx, field, _value) in enumerate(fields, start=1): + verdict = verdicts.get(n) + if verdict is None or verdict == 'valid': + continue + + # ── Closed-vocabulary backstop ─────────────────────────────── + # If the model invents a code outside our set, it did not + # correctly identify a real problem — treat the field as valid + # rather than flagging it. A hallucinated code can feed into + # whole-entity drops for primary fields (see main.py), so + # trusting it is too risky. + reason = _REASON_CODES.get(verdict) + if reason is None: + logger.warning( + f"[validate_llm] unknown verdict code {verdict!r} for " + f"{entity_type}[{idx}].{field} ({context}) — treating as valid" + ) + continue + + problems.append({ + 'entity_type': entity_type, + 'index': idx, + 'field': field, + 'reason': reason, + }) + + return problems diff --git a/Extraction/validate_llm/ollama_client.py b/Extraction/validate_llm/ollama_client.py new file mode 100644 index 0000000..ad4229d --- /dev/null +++ b/Extraction/validate_llm/ollama_client.py @@ -0,0 +1,133 @@ +""" +Extraction/validate_llm/ollama_client.py +Thin client for a local Ollama model, used as the LLM-review step for +field validation. Model/URL come from shared/config.py (VALIDATION_MODEL, +OLLAMA_URL) — swapping models is a .env change, not a code change. +""" +import json +import logging +import os +import re + +import requests +from tenacity import retry, wait_exponential, stop_after_attempt, before_sleep_log, retry_if_exception_type + +from shared.config import OLLAMA_URL, VALIDATION_MODEL + +logger = logging.getLogger('pipeline') + +GENERATE_URL = f"{OLLAMA_URL}/api/generate" + +_JSON_OBJECT_RE = re.compile(r'\{.*\}', re.DOTALL) + + +OLLAMA_TIMEOUT_SECONDS = int(os.environ.get('OLLAMA_TIMEOUT_SECONDS', '300')) + + +@retry( + wait=wait_exponential(multiplier=1, min=2, max=10), + stop=stop_after_attempt(2), + retry=retry_if_exception_type(Exception), + before_sleep=before_sleep_log(logger, logging.WARNING), + reraise=True, +) +def _call_ollama(prompt: str) -> str: + logger.info(f"[validate_llm] calling Ollama model={VALIDATION_MODEL!r} url={GENERATE_URL}") + try: + resp = requests.post( + GENERATE_URL, + json={ + 'model': VALIDATION_MODEL, + 'prompt': prompt, + 'stream': False, + 'think': False, + 'options': {'temperature': 0, 'seed': 42}, + }, + timeout=OLLAMA_TIMEOUT_SECONDS, + ) + resp.raise_for_status() + except requests.exceptions.HTTPError: + logger.warning( + f"[validate_llm] Ollama HTTP error: status={resp.status_code} " + f"body={resp.text[:500]!r}" + ) + raise + except requests.exceptions.RequestException as e: + logger.warning(f"[validate_llm] Ollama connection error: {e!r}") + raise + text = resp.json().get('response', '') + logger.info(f"[validate_llm] Ollama call succeeded model={VALIDATION_MODEL!r}") + logger.info(f"[validate_llm] Ollama raw response:\n{text}") + return text + + +def ask_valid(prompt: str) -> dict | None: + """ + Send a prompt to the validation model, expecting a JSON object + {"valid": bool, "reason": str} somewhere in the response. + Returns the parsed dict, or None if the call/parse failed + (caller should treat None as "skip — leave field as-is"). + """ + try: + raw = _call_ollama(prompt) + except Exception as e: + logger.warning(f"[validate_llm] Ollama call failed: {e}") + return None + + match = _JSON_OBJECT_RE.search(raw) + if not match: + logger.warning(f"[validate_llm] Ollama returned no JSON object: {raw!r}") + return None + + try: + parsed = json.loads(match.group(0)) + except json.JSONDecodeError: + logger.warning(f"[validate_llm] Ollama returned malformed JSON: {raw!r}") + return None + + if 'valid' not in parsed: + logger.warning(f"[validate_llm] Ollama JSON missing 'valid' key: {parsed!r}") + return None + + return parsed + + +def ask_json(prompt: str) -> dict | None: + """ + Send a prompt to the validation model, expecting a single JSON object + anywhere in the response (fenced or not). Returns the parsed dict, or + None if the call/parse failed (caller should treat None as "validator + unavailable — skip"). + """ + try: + raw = _call_ollama(prompt) + except Exception as e: + logger.warning(f"[validate_llm] Ollama call failed: {e}") + return None + + match = _JSON_OBJECT_RE.search(raw) + if not match: + logger.warning(f"[validate_llm] Ollama returned no JSON object: {raw!r}") + return None + + try: + return json.loads(match.group(0)) + except json.JSONDecodeError: + logger.warning(f"[validate_llm] Ollama returned malformed JSON: {raw!r}") + return None + + +def ask_lines(prompt: str) -> list[str] | None: + """ + Send a prompt to the validation model, expecting a plain-text + response of one finding per line (or the literal string NONE). + Returns the list of non-empty lines, or None if the call failed + (caller should treat None as "validator unavailable — skip"). + """ + try: + raw = _call_ollama(prompt) + except Exception as e: + logger.warning(f"[validate_llm] Ollama call failed: {e}") + return None + + return [line for line in raw.splitlines() if line.strip()] diff --git a/Extraction/validate_llm/rules.py b/Extraction/validate_llm/rules.py new file mode 100644 index 0000000..ff8b92a --- /dev/null +++ b/Extraction/validate_llm/rules.py @@ -0,0 +1,76 @@ +""" +Extraction/validate_llm/rules.py +Reusable rule primitives for field-level validation. + +Each rule is a callable: (value) -> bool (True = valid, False = drop). +`value` is never None here — the engine skips None/empty values before +calling rules, since "missing" is not the same failure as "wrong". +""" +import re +from Extraction.utils.helpers import parse_date + + +def is_int(v) -> bool: + if isinstance(v, bool): + return False + if isinstance(v, int): + return True + try: + int(str(v).strip()) + return True + except (TypeError, ValueError): + return False + + +def is_float(v) -> bool: + if isinstance(v, bool): + return False + try: + float(str(v).strip()) + return True + except (TypeError, ValueError): + return False + + +def is_bool(v) -> bool: + if isinstance(v, bool): + return True + return str(v).strip().lower() in ('true', 'false', 'yes', 'no') + + +def in_range(lo, hi): + def _check(v) -> bool: + if not is_float(v): + return False + return lo <= float(v) <= hi + return _check + + +def regex(pattern: str, flags=0): + compiled = re.compile(pattern, flags) + def _check(v) -> bool: + return bool(compiled.fullmatch(str(v).strip())) + return _check + + +def valid_date(v) -> bool: + return parse_date(v) is not None + + +def max_length(n: int): + def _check(v) -> bool: + return len(str(v).strip()) <= n + return _check + + +def one_of(*choices): + lowered = {c.lower() for c in choices} + def _check(v) -> bool: + return str(v).strip().lower() in lowered + return _check + + +def all_of(*checks): + def _check(v) -> bool: + return all(c(v) for c in checks) + return _check diff --git a/Extraction/validate_llm/test_prompt.py b/Extraction/validate_llm/test_prompt.py new file mode 100644 index 0000000..bece626 --- /dev/null +++ b/Extraction/validate_llm/test_prompt.py @@ -0,0 +1,132 @@ +""" +Extraction/validate_llm/test_prompt.py +Standalone tester for the whole-case LLM validation step. Calls the SAME +code the real pipeline uses (llm_field_checks.check_case) so results here +reflect production behavior exactly — no duplicated prompt logic to drift +out of sync. + +Usage: + python Extraction/validate_llm/test_prompt.py + VALIDATION_MODEL=qwen3:4b python Extraction/validate_llm/test_prompt.py + +To add a test case: add an entity to TEST_CASE (a case_entities dict) and +its expected-bad fields to EXPECTED_BAD below. +""" +import logging +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + +# shared/config.py requires a full .env (Neo4j, Qdrant, NVIDIA keys) that +# this standalone tester shouldn't need — stub the two values this test +# actually touches before importing anything that chains into config. +os.environ.setdefault("OLLAMA_URL", "http://localhost:11434") +if "MODEL" in os.environ: + os.environ["VALIDATION_MODEL"] = os.environ["MODEL"] +os.environ.setdefault("VALIDATION_MODEL", "reaperdoesntrun/Qwen3-0.6B-Distilled") +for _k in ("DATASET_ROOT", "NEO4J_URI", "NEO4J_USER", "NEO4J_PASSWORD", + "NEO4J_DATABASE", "QDRANT_URL", "QDRANT_COLLECTION", + "NVIDIA_API_KEY", "EXTRACTION_MODEL", "EMBEDDING_MODEL", "AGENT_MODEL"): + os.environ.setdefault(_k, "unused-for-this-test") + +from Extraction.validate_llm.llm_field_checks import check_case # noqa: E402 + +# ── Whole-case test fixture ────────────────────────────────────────────── +# Mirrors real case GJAH220268512019 plus extra entities/fields to check +# whether the model generalizes beyond address/designation without being +# told to, AND correctly leaves alone fields it has no basis to judge. +TEST_CASE = { + "persons": [ + {"name": "TATA CAPITAL FINANCIAL SERVICE LTD", + "address": "1) TATA CAPITAL FINANCIAL SERVICE LTD Advocate - D.N.GOSAI", + "role_in_case": "petitioner"}, + {"name": "VESTITI INDIA", + "address": "1) VESTITI INDIA", + "role_in_case": "respondent"}, + {"name": "HEMANT SANDHAVI", + "role_in_case": "respondent"}, + {"name": "Ramesh Kumar", + "address": "Flat No 4B, Green Valley Society, Nagpur", + "role_in_case": "petitioner"}, + ], + "judges": [ + {"designation": "ADDL. CHIEF METROPOLITAN MAGISTRATE"}, + {"designation": "Ramesh Kumar"}, + {"designation": "Civil Judge Senior Division"}, + ], + "lawyers": [ + {"name": "D.N.GOSAI"}, + # a field the model has no real basis to judge — should be left alone + {"name": "S. Mehta", "specialization": "Criminal"}, + ], + "case_hearings": [ + {"purpose": "PROCESS TO ACCUSED", "nature_of_disposal": None}, + {"purpose": "Disposed", "nature_of_disposal": "LOK ADALAT"}, + ], +} + +# (entity_type, index, field) -> should this be flagged as bad? +EXPECTED_BAD = { + ("persons", 0, "address"), + ("persons", 1, "address"), + ("judges", 1, "designation"), +} +# Everything else present in TEST_CASE is expected to be left alone, +# including fields the model has no grounds to judge (name, role_in_case, +# specialization, purpose, nature_of_disposal) and the valid address/ +# designation values. + + +def all_checked_fields(): + for entity_type, entities in TEST_CASE.items(): + for idx, entity in enumerate(entities): + for field, value in entity.items(): + if value is not None and str(value).strip() != '': + yield (entity_type, idx, field, value) + + +def main(): + model = os.environ["VALIDATION_MODEL"] + print(f"Model: {model}\n") + + problems = check_case(TEST_CASE, context="TEST-CASE") + flagged = {(p['entity_type'], p['index'], p['field']) for p in problems} + + print("── Flagged by model (post hallucination-check) ──") + if not problems: + print("(none)") + for p in problems: + print(f" {p['entity_type']}[{p['index']}].{p['field']} — {p['reason']}") + print() + + print("── Results ──") + passed = 0 + total = 0 + for entity_type, idx, field, value in all_checked_fields(): + total += 1 + key = (entity_type, idx, field) + expected_bad = key in EXPECTED_BAD + got_bad = key in flagged + ok = expected_bad == got_bad + passed += ok + status = "PASS" if ok else "FAIL" + print(f"[{status}] {entity_type}[{idx}].{field}={value!r} — " + f"expected {'INVALID' if expected_bad else 'valid'}, " + f"got {'INVALID' if got_bad else 'valid'}") + + # Any flagged item that isn't in our known-checked-fields set at all + # (shouldn't happen — check_case already verifies against its own + # input — but confirms the hallucination backstop end-to-end). + unexpected = flagged - {(e, i, f) for e, i, f, _ in all_checked_fields()} + if unexpected: + print(f"\nWARNING: flagged fields not in test fixture at all: {unexpected}") + + print(f"\n{passed}/{total} passed") + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index bdc90dc..28757b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,14 +57,14 @@ mpmath==1.3.0 multidict==6.7.1 murmurhash==1.0.15 neo4j==6.1.0 -networkx==3.6.1 -numpy==2.5.0 -onnxruntime==1.27.0 +networkx==3.4.2 +numpy==2.2.6 +onnxruntime==1.23.2 opencv-python==4.13.0.92 orjson==3.11.9 ormsgpack==1.12.2 packaging==26.2 -pandas==3.0.3 +pandas==2.3.3 paradict==0.0.16 pdf2image==1.17.0 pdfminer.six==20251230 diff --git a/shared/config.py b/shared/config.py index 8e8b1cf..062952d 100644 --- a/shared/config.py +++ b/shared/config.py @@ -47,6 +47,10 @@ "Content-Type" : "application/json", } +# ── Ollama / local validation LLM ─────────────────────────────────────────── +OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434") +VALIDATION_MODEL = os.environ.get("VALIDATION_MODEL", "reaperdoesntrun/Qwen3-0.6B-Distilled") + # ── Domain constants ─────────────────────────────────────────────────────── DISTRICT_OVERRIDES = { "mumbai cmm courts" : "Mumbai",