From 571582419405e24f3279e23aa520ad4d2df548a9 Mon Sep 17 00:00:00 2001 From: shipitdev Date: Wed, 4 Nov 2026 13:29:26 +0530 Subject: [PATCH 1/2] Add bounded RAGAS scoring and free evaluation reports --- .gitignore | 12 +++ scripts/eval_ragas.py | 90 +++++++++++++------ scripts/eval_suite.py | 184 +++++++++++++++++++++++++++++++++++++++ tests/test_eval_suite.py | 80 +++++++++++++++++ tests/test_ragas_eval.py | 20 ++++- 5 files changed, 358 insertions(+), 28 deletions(-) create mode 100644 scripts/eval_suite.py create mode 100644 tests/test_eval_suite.py diff --git a/.gitignore b/.gitignore index 09eecdd..2a23897 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,18 @@ !/tests/test_ingest_html.py !/tests/test_ragas_eval.py !/tests/test_retrieval.py +!/scripts/eval_suite.py +!/tests/test_eval_suite.py +!/evaluations/ +/evaluations/* +!/evaluations/overview.svg +!/evaluations/fixtures/ +/evaluations/fixtures/* +!/evaluations/fixtures/offline_demo.json +!/evaluations/results/ +/evaluations/results/* +!/evaluations/results/techqa_pilot_v1.json +!/evaluations/results/retrieval_baseline_v1.json !/.github/workflows/ !/frontend/src/ !/frontend/src/app/ diff --git a/scripts/eval_ragas.py b/scripts/eval_ragas.py index 3c88e75..c0d76e6 100644 --- a/scripts/eval_ragas.py +++ b/scripts/eval_ragas.py @@ -18,9 +18,28 @@ 'context_recall': ('user_input', 'reference', 'retrieved_contexts'), 'faithfulness': ('user_input', 'response', 'retrieved_contexts'), 'answer_relevancy': ('user_input', 'response'), + 'factual_correctness': ('response', 'reference'), + 'answer_completeness': ('response', 'reference'), } +class BudgetExceeded(RuntimeError): + pass + + +class RequestBudget: + def __init__(self, limit): + if limit < 0: + raise ValueError('Request budget must be nonnegative') + self.limit, self.sent, self.exhausted = limit, 0, False + + def consume(self): + if self.sent >= self.limit: + self.exhausted = True + raise BudgetExceeded('Evaluation HTTP request budget exhausted') + self.sent += 1 + + def fingerprint(value): return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode()).hexdigest() @@ -93,12 +112,13 @@ async def cached_judgment(generate, settings, prompt, response_model, directory, 'seconds': time.perf_counter() - started, 'http_calls': http_calls[offset:]}) -async def evaluate_case(case, scorers, *, timeout=300, journal=None): +async def evaluate_case(case, scorers, *, timeout=300, journal=None, budget=None): results = {} for name, scorer in scorers.items(): started = time.perf_counter() offset = len(journal) if journal is not None else 0 - if not case['retrieved_contexts'] or (name in ('faithfulness', 'answer_relevancy') and case['status'] != 'answered'): + if (('retrieved_contexts' in FIELDS[name] and not case['retrieved_contexts']) or + ('response' in FIELDS[name] and case['status'] != 'answered')): results[name] = {'state': 'skipped', 'score': None, 'reason': 'no_context_or_no_released_answer'} continue try: @@ -109,7 +129,8 @@ async def evaluate_case(case, scorers, *, timeout=300, journal=None): 'score': value if math.isfinite(value) else None} except Exception as exc: # Provider messages can contain inputs/headers; retain a safe category only. - results[name] = {'state': 'error', 'score': None, 'error_type': type(exc).__name__} + results[name] = {'state': 'budget_exhausted' if budget and budget.exhausted else 'error', + 'score': None, 'error_type': type(exc).__name__} results[name]['seconds'] = round(time.perf_counter() - started, 3) if journal is not None: results[name]['judge_calls'] = journal[offset:] @@ -123,7 +144,7 @@ def summarize(rows): values = [r['metrics'][name] for r in rows if name in r['metrics']] counts = Counter(v['state'] for v in values) scores = [v['score'] for v in values if v['state'] == 'success'] - summary['metrics'][name] = {state: counts[state] for state in ('success', 'error', 'undefined', 'skipped')} + summary['metrics'][name] = {state: counts[state] for state in ('success', 'error', 'undefined', 'skipped', 'budget_exhausted')} summary['metrics'][name]['mean'] = sum(scores) / len(scores) if scores else None return summary @@ -189,23 +210,28 @@ async def run(args): from ragas.llms import llm_factory from ragas.llms.base import InstructorBaseRagasLLM from ragas.embeddings.base import BaseRagasEmbedding - from ragas.metrics.collections import ContextPrecision, ContextRecall, Faithfulness, AnswerRelevancy + from ragas.metrics.collections import ContextPrecision, ContextRecall, Faithfulness, AnswerRelevancy, FactualCorrectness from rag_app.retriever import Retriever manifest = json.loads((args.cases / 'manifest.json').read_text()) - cases = [json.loads((args.cases / (qid + '.json')).read_text()) for qid in manifest['question_ids']] - if any(c['question_id'] != qid or not qid.startswith('TRAIN_') for c, qid in zip(cases, manifest['question_ids'])): + ids = manifest['question_ids'][:args.limit] + if len(set(ids)) != len(ids) or any(not qid.startswith('TRAIN_') or not qid.replace('_', '').isalnum() for qid in ids): + raise ValueError('Invalid training case IDs') + cases = [json.loads((args.cases / (qid + '.json')).read_text()) for qid in ids] + if not cases or any(c['question_id'] != qid for c, qid in zip(cases, ids)): raise ValueError('Case membership mismatch') key = os.getenv('GROQ_API_KEY') or os.getenv('GROK_API_KEY') - if not key: + if not key and args.max_requests: raise ValueError('Groq key missing') - retriever = Retriever() + retriever = Retriever() if 'answer_relevancy' in args.metrics else None + budget = RequestBudget(args.max_requests) settings = {'provider': 'groq', 'judge': args.judge, 'temperature': 0, 'reasoning_effort': 'low', 'transport': {'sdk_max_retries': 3, 'retry_after_buffer_seconds': 2, 'minimum_retry_seconds': 5}, 'max_tokens': 4096, 'strictness': 3, 'timeout_per_metric': args.timeout, 'packages': {p: importlib.metadata.version(p) for p in - ('ragas', 'openai', 'instructor', 'langchain-community', 'sentence-transformers', 'torch', 'transformers', 'torchvision')}, - 'implementation_sha256': sha256(Path(__file__)), 'embedding': retriever._gemma_policy(), + (('ragas', 'openai', 'instructor', 'langchain-community') + + (('sentence-transformers', 'torch', 'transformers', 'torchvision') if retriever else ()))}, + 'implementation_sha256': sha256(Path(__file__)), 'embedding': retriever._gemma_policy() if retriever else None, 'embedding_policy': 'same formatted Gemma query role for original/generated questions; normalized cosine', 'ragas_source_sha256': fingerprint([(str(p.relative_to(Path(__import__('ragas').__file__).parent)), sha256(p)) for p in sorted(Path(__import__('ragas').__file__).parent.rglob('*.py'))])} @@ -213,6 +239,7 @@ async def run(args): args.cache.mkdir(parents=True, exist_ok=True) async def request_hook(request): + budget.consume() # Includes SDK retries; blocks before any request is sent. request.extensions['eval_started'] = time.perf_counter() request.extensions['eval_call_index'] = len(http_calls) http_calls.append({'status': None, 'usage': {}, 'seconds': None}) @@ -232,22 +259,23 @@ async def response_hook(response): response.headers['retry-after'] = str(delay) print(f' Groq quota: waiting about {delay:g}s before retry', flush=True) - client = AsyncOpenAI(api_key=key, base_url='https://api.groq.com/openai/v1', timeout=60, max_retries=3, + client = AsyncOpenAI(api_key=key or 'cache-only-no-key', base_url='https://api.groq.com/openai/v1', timeout=60, max_retries=3, http_client=httpx.AsyncClient(event_hooks={'request': [request_hook], 'response': [response_hook]})) backend = llm_factory(args.judge, client=client, temperature=0, max_tokens=4096, reasoning_effort='low') class CachedJudge(InstructorBaseRagasLLM): - def __init__(self): + def __init__(self, draws_per_prompt=1): self.draws = Counter() + self.draws_per_prompt = draws_per_prompt def generate(self, prompt, response_model): return asyncio.run(self.agenerate(prompt, response_model)) async def agenerate(self, prompt, response_model): identity = fingerprint([prompt, response_model.model_json_schema()]) - draw = self.draws[identity] + draw = self.draws[identity] % self.draws_per_prompt self.draws[identity] += 1 - # Relevancy asks three identical prompts; cache each draw independently. + # Reuse identical grading work; only relevancy needs three separate draws. return await cached_judgment(backend.agenerate, settings, prompt, response_model, args.cache, journal, http_calls, draw=draw) @@ -261,27 +289,34 @@ async def aembed_text(self, text, **kwargs): return self.embed_text(text) judge = CachedJudge() - scorers = {'context_precision': ContextPrecision(llm=judge), 'context_recall': ContextRecall(llm=judge), - 'faithfulness': Faithfulness(llm=judge), - 'answer_relevancy': AnswerRelevancy(llm=judge, embeddings=LocalGemma(), strictness=3)} + factories = {'context_precision': lambda: ContextPrecision(llm=judge), + 'context_recall': lambda: ContextRecall(llm=judge), + 'faithfulness': lambda: Faithfulness(llm=judge), + 'answer_relevancy': lambda: AnswerRelevancy(llm=CachedJudge(3), embeddings=LocalGemma(), strictness=3), + 'factual_correctness': lambda: FactualCorrectness(llm=judge, mode='f1'), + 'answer_completeness': lambda: FactualCorrectness(llm=judge, mode='recall')} + scorers = {name: factories[name]() for name in dict.fromkeys(args.metrics)} rows = [] report = {'created_at': datetime.now(timezone.utc).isoformat(), 'settings': settings, - 'manifest': manifest, 'cases_sha256': {c['question_id']: sha256(args.cases / (c['question_id'] + '.json')) for c in cases}, + 'manifest': manifest, 'selected_question_ids': ids, 'selected_metrics': list(scorers), + 'cases_sha256': {c['question_id']: sha256(args.cases / (c['question_id'] + '.json')) for c in cases}, 'limitations': ['training tuning pilot, not final development', 'same model family writer/judge; manual review needed', 'answer metrics conditional on released answers; no global refusal accuracy', 'cost not calculated; actual token usage recorded']} started = time.perf_counter() try: for case in cases: print(case['question_id'] + ': RAGAS scoring', flush=True) - rows.append(await evaluate_case(case, scorers, timeout=args.timeout, journal=journal)) + rows.append(await evaluate_case(case, scorers, timeout=args.timeout, journal=journal, budget=budget)) report.update({'results': rows, 'summary': summarize(rows), 'evaluation_seconds': time.perf_counter() - started, 'judge_cache_hits': sum(x['cache_hit'] for x in journal), 'judge_cache_misses': sum(not x['cache_hit'] for x in journal), - 'http_calls': http_calls, 'embedding_cache': embedding_counts}) + 'http_calls': http_calls, 'embedding_cache': embedding_counts, + 'request_budget': {'limit': budget.limit, 'sent': budget.sent, 'exhausted': budget.exhausted}}) save(args.output, report) finally: await client.close() - retriever.close() + if retriever: + retriever.close() print(json.dumps(report['summary'], indent=2), flush=True) @@ -295,7 +330,10 @@ def main(): c.add_argument('--output', type=Path, default=ROOT / 'data/ragas/pilot_v1') r = subs.add_parser('run') r.add_argument('--cases', type=Path, default=ROOT / 'data/ragas/pilot_v1') - r.add_argument('--output', type=Path, default=ROOT / 'data/ragas/results/pilot_v1.json') + r.add_argument('--output', type=Path) + r.add_argument('--metrics', nargs='+', choices=tuple(FIELDS), required=True) + r.add_argument('--limit', type=int, default=3) + r.add_argument('--max-requests', type=int, default=12, help='Maximum HTTP attempts including retries; 0 is cache-only') r.add_argument('--cache', type=Path, default=ROOT / 'data/ragas/judge_cache') r.add_argument('--judge', default='openai/gpt-oss-120b') r.add_argument('--timeout', type=float, default=300) @@ -303,8 +341,10 @@ def main(): if args.command == 'collect': collect(args) else: - if args.timeout <= 0: - parser.error('timeout must be positive') + if args.timeout <= 0 or args.limit < 1 or args.max_requests < 0: + parser.error('timeout/limit must be positive; max-requests must be nonnegative') + if args.output is None: + args.output = ROOT / 'data/ragas/results' / ('run_' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ') + '.json') asyncio.run(run(args)) diff --git a/scripts/eval_suite.py b/scripts/eval_suite.py new file mode 100644 index 0000000..66d7dfd --- /dev/null +++ b/scripts/eval_suite.py @@ -0,0 +1,184 @@ +"""Free evaluation reports and paired regression comparisons over saved RAG cases.""" +import argparse +from collections import Counter, defaultdict +import json +import math +from pathlib import Path +import re + +from rag_app.generation import AnswerGenerator +from scripts.eval_ragas import fingerprint, save, summarize + +CONTRACT = 'saved_case_checks_v1' +VERSION = re.compile(r'\b([A-Za-z][A-Za-z0-9_-]{1,24})\s+v?(\d+(?:\.\d+){0,4})\b') + + +def load_cases(path): + path = Path(path) + if path.is_file(): + return json.loads(path.read_text())['cases'] + manifest = json.loads((path / 'manifest.json').read_text()) + ids = manifest['question_ids'] + if not ids or len(set(ids)) != len(ids) or any(not re.fullmatch(r'[A-Za-z0-9_-]+', i) for i in ids): + raise ValueError('Invalid case manifest membership') + rows = [json.loads((path / (qid + '.json')).read_text()) for qid in ids] + if [r['question_id'] for r in rows] != ids: + raise ValueError('Case IDs differ from manifest') + return rows + + +def judge_version_flags(case, result): + flags = {} + for metric, score in result.get('metrics', {}).items(): + original = case['reference'] if metric == 'context_recall' else case['response'] + if metric in ('factual_correctness', 'answer_completeness'): + original += '\n' + case['reference'] + allowed = defaultdict(set) + for label, version in VERSION.findall(original): + allowed[label.lower()].add(version) + for call in score.get('judge_calls', []): + decision = call.get('decision') or {} + claims = decision.get('statements', decision.get('classifications', decision.get('claims', []))) + for claim in claims: + text = claim if isinstance(claim, str) else claim.get('statement', '') + for label, version in VERSION.findall(text): + if label.lower() in allowed and version not in allowed[label.lower()]: + key = (metric, label.lower(), version) + flags[key] = {'metric': metric, 'label': label, 'introduced_version': version, + 'allowed_versions': sorted(allowed[label.lower()]), 'needs_review': True} + return list(flags.values()) + + +def build_report(cases, ragas=None): + if not cases or len({c['question_id'] for c in cases}) != len(cases): + raise ValueError('Expected nonempty unique cases') + for c in cases: + if c['status'] not in {'answered', 'insufficient_evidence', 'needs_clarification', 'generation_error'}: + raise ValueError('Unknown outcome') + if not all(isinstance(c[k], str) for k in ('question_id', 'user_input', 'reference', 'response')): + raise ValueError('Malformed case text') + ids = [s['id'] for s in c['sources']] + if len(set(ids)) != len(ids) or not all(isinstance(i, str) for i in ids): + raise ValueError('Duplicate or malformed source IDs') + judged = {} + if ragas is not None: + judged = {r['question_id']: r for r in ragas['results']} + if len(judged) != len(ragas['results']) or set(judged) != {c['question_id'] for c in cases}: + raise ValueError('RAGAS and case membership differ') + rows, citations, hits, seconds = [], [], [], [] + tokens = 0 + for c in cases: + hit = int(any(s['document_id'] == c['reference_provenance']['document_id'] for s in c['sources'])) + hits.append(hit) + citation = {'state': 'skipped', 'pass': None} + if c['status'] == 'answered': + error = AnswerGenerator._citation_error({'answer': c['response'], 'citations': c['citations']}, + {s['id'] for s in c['sources']}) + citation = {'state': 'checked', 'pass': error is None, 'reason': error} + citations.append(int(error is None)) + flags = judge_version_flags(c, judged.get(c['question_id'], {})) + metrics = {} + for name, m in judged.get(c['question_id'], {}).get('metrics', {}).items(): + value = m.get('score') + if m['state'] == 'success' and (not isinstance(value, (int, float)) or not math.isfinite(value)): + raise ValueError('Successful judge score must be finite') + metrics[name] = {'state': m['state'], 'score': value, + 'needs_review': any(f['metric'] == name for f in flags)} + rows.append({'question_id': c['question_id'], 'outcome': c['status'], + 'selected_gold_document_hit': hit, 'selected_passage_count': len(c['sources']), + 'citation_contract': citation, 'ragas': metrics, 'judge_integrity_flags': flags}) + gen = c.get('generation', {}) + elapsed = gen.get('seconds') + if isinstance(elapsed, (int, float)) and math.isfinite(elapsed): + seconds.append(elapsed) + tokens += sum(call.get('total_tokens') or 0 for call in gen.get('provider_calls', [])) + identity = sorted((c['question_id'], c['user_input'], c['reference'], c['reference_provenance']) + for c in cases) + report = {'schema': CONTRACT, 'cohort_id': fingerprint(identity), 'case_count': len(cases), + 'outcomes': dict(Counter(c['status'] for c in cases)), + 'metrics': {'answer_coverage': sum(c['status'] == 'answered' for c in cases) / len(cases), + 'selected_gold_document_hit_rate': sum(hits) / len(hits), + 'citation_contract_pass_rate': sum(citations) / len(citations) if citations else None}, + 'denominators': {'selected_gold_document_hit_rate': len(hits), 'citation_contract_pass_rate': len(citations)}, + 'operations': {'generation_seconds': seconds, 'generation_reported_tokens': tokens, + 'generation_usage_cases': sum('provider_calls' in c.get('generation', {}) for c in cases), + 'cost_usd': None}, 'cases': rows, + 'limits': ['Selected-document hit is not search Recall@K or semantic evidence recall.', + 'Citation validity does not establish claim support.', + 'Version flags are heuristics for review; raw scores are never adjusted.', + 'No style, toxicity/PII, live feedback or drift scores are computed.']} + if ragas is not None: + report['ragas_summary'] = summarize(ragas['results']) + for name, summary in report['ragas_summary']['metrics'].items(): + summary['flagged_for_review'] = sum(row['ragas'].get(name, {}).get('needs_review', False) for row in rows) + settings = ragas['settings'] + report['judge'] = {k: settings.get(k) for k in ('provider', 'judge', 'temperature', 'strictness', + 'max_tokens', 'embedding', 'embedding_policy', 'ragas_source_sha256', 'implementation_sha256')} + report['judge']['packages'] = settings.get('packages', {}) + report['judge_comparison_id'] = fingerprint(report['judge']) + report['operations']['evaluation'] = { + 'seconds': ragas.get('evaluation_seconds'), 'judge_cache_hits': ragas.get('judge_cache_hits'), + 'judge_cache_misses': ragas.get('judge_cache_misses'), 'http_attempts': len(ragas.get('http_calls', [])), + 'reported_tokens': sum((call.get('usage') or {}).get('total_tokens') or 0 for call in ragas.get('http_calls', [])), + 'scope': 'this recorded scoring pass only; earlier cache-populating passes cost extra'} + return report + + +def compare_reports(baseline, candidate): + if baseline['schema'] != candidate['schema'] or baseline['cohort_id'] != candidate['cohort_id']: + raise ValueError('Comparison requires identical cohort, references and metric contract') + old = {r['question_id']: r for r in baseline['cases']} + if set(old) != {r['question_id'] for r in candidate['cases']}: + raise ValueError('Comparison requires identical case membership') + semantic = bool(baseline.get('judge_comparison_id') and + baseline.get('judge_comparison_id') == candidate.get('judge_comparison_id')) + changes, outcomes, regressions = [], [], [] + for row in candidate['cases']: + previous = old[row['question_id']] + if previous['outcome'] != row['outcome']: + outcomes.append({'question_id': row['question_id'], 'before': previous['outcome'], 'after': row['outcome']}) + pairs = {'selected_gold_document_hit': (previous['selected_gold_document_hit'], row['selected_gold_document_hit'])} + if previous['citation_contract']['state'] == row['citation_contract']['state'] == 'checked': + pairs['citation_contract_pass'] = (int(previous['citation_contract']['pass']), int(row['citation_contract']['pass'])) + if semantic: + for name in previous['ragas'].keys() & row['ragas'].keys(): + a, b = previous['ragas'][name], row['ragas'][name] + if a['state'] == b['state'] == 'success' and not a['needs_review'] and not b['needs_review']: + pairs[name] = (a['score'], b['score']) + elif a['state'] != b['state']: + changes.append({'question_id': row['question_id'], 'metric': name, + 'before_state': a['state'], 'after_state': b['state']}) + for name, (a, b) in pairs.items(): + item = {'question_id': row['question_id'], 'metric': name, 'before': a, 'after': b, 'delta': b-a} + changes.append(item) + if b < a - 1e-9: + regressions.append(item) + return {'cohort_id': baseline['cohort_id'], 'changes': changes, 'outcome_changes': outcomes, + 'regressions': regressions, 'semantic_scores_comparable': semantic, + 'interpretation': 'Paired observed differences, not statistical significance or automatic release approval.'} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + subs = parser.add_subparsers(dest='command', required=True) + report = subs.add_parser('report') + report.add_argument('--cases', type=Path, required=True) + report.add_argument('--ragas', type=Path) + report.add_argument('--output', type=Path, required=True) + compare = subs.add_parser('compare') + compare.add_argument('--baseline', type=Path, required=True) + compare.add_argument('--candidate', type=Path, required=True) + compare.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if args.command == 'report': + raw = json.loads(args.ragas.read_text()) if args.ragas else None + output = build_report(load_cases(args.cases), raw) + else: + output = compare_reports(json.loads(args.baseline.read_text()), json.loads(args.candidate.read_text())) + save(args.output, output) + print(json.dumps({k: v for k, v in output.items() if k in ('case_count', 'metrics', 'outcomes', 'regressions')}, indent=2)) + print('Saved', args.output, '(no model calls)') + + +if __name__ == '__main__': + main() diff --git a/tests/test_eval_suite.py b/tests/test_eval_suite.py new file mode 100644 index 0000000..15a0488 --- /dev/null +++ b/tests/test_eval_suite.py @@ -0,0 +1,80 @@ +import copy +import json + +import pytest + +from scripts.eval_suite import build_report, compare_reports, judge_version_flags, load_cases + + +def cases(): + return [dict(question_id='DEMO_1', user_input='Which setting?', reference='Use TIP 2.1.', + reference_provenance={'document_id': 'gold', 'view': 'test', 'start': 0, 'end': 12}, + status='answered', response='Use TIP 2.1. [S1]', citations=['S1'], + sources=[{'id': 'S1', 'document_id': 'gold', 'text': 'Use TIP 2.1.'}], + generation={'seconds': 2, 'provider_calls': [{'total_tokens': 10}]})] + + +def test_free_report_uses_selected_evidence_not_search_recall_and_redacts_text(): + report = build_report(cases()) + assert report['metrics']['selected_gold_document_hit_rate'] == 1 + assert report['metrics']['answer_coverage'] == 1 + assert report['metrics']['citation_contract_pass_rate'] == 1 + assert report['operations']['generation_reported_tokens'] == 10 + assert 'Which setting?' not in json.dumps(report) + assert 'Use TIP 2.1.' not in json.dumps(report) + assert 'recall_at_10' not in report['metrics'] + + +def test_unknown_citations_and_withheld_cases_stay_visible(): + rows = cases() + rows[0]['response'] = 'Use TIP 2.1. [S9]' + rows += [dict(rows[0], question_id='DEMO_2', status='insufficient_evidence', + response='Cannot answer.', citations=[], sources=[])] + report = build_report(rows) + assert report['metrics']['answer_coverage'] == .5 + assert report['metrics']['citation_contract_pass_rate'] == 0 + assert report['outcomes']['insufficient_evidence'] == 1 + assert report['cases'][1]['citation_contract']['state'] == 'skipped' + + +def test_judge_version_guard_flags_changed_claim_but_not_reordering(): + raw = {'metrics': {'faithfulness': {'judge_calls': [ + {'decision': {'statements': ['TIP 1 is unsupported.']}}]}}} + flags = judge_version_flags(cases()[0], raw) + assert flags and flags[0]['metric'] == 'faithfulness' + raw['metrics']['faithfulness']['judge_calls'][0]['decision']['statements'] = ['For TIP 2.1, use the setting.'] + assert judge_version_flags(cases()[0], raw) == [] + + +def test_comparison_rejects_reference_changes_and_reports_per_case_regressions(): + original = cases(); candidate = copy.deepcopy(original) + candidate[0]['sources'][0]['document_id'] = 'different' + result = compare_reports(build_report(original), build_report(candidate)) + assert result['regressions'][0]['metric'] == 'selected_gold_document_hit' + candidate[0]['reference'] = 'A changed answer key' + with pytest.raises(ValueError, match='cohort'): + compare_reports(build_report(original), build_report(candidate)) + + +def test_duplicate_cases_and_ragas_membership_mismatches_are_rejected(): + with pytest.raises(ValueError): build_report(cases() * 2) + with pytest.raises(ValueError): + build_report(cases(), {'results': [{'question_id': 'OTHER', 'metrics': {}}]}) + + +def test_changed_embedding_identity_prevents_semantic_comparison_and_flagged_score_is_visible(): + raw = {'settings': {'judge': 'same', 'embedding': {'revision': 'first'}}, + 'results': [{'question_id': 'DEMO_1', 'status': 'answered', 'metrics': { + 'faithfulness': {'state': 'success', 'score': .5, 'judge_calls': [ + {'decision': {'statements': ['TIP 1 is unsupported.']}}]}}}]} + baseline = build_report(cases(), raw) + assert baseline['ragas_summary']['metrics']['faithfulness']['mean'] == .5 + assert baseline['ragas_summary']['metrics']['faithfulness']['flagged_for_review'] == 1 + raw['settings']['embedding']['revision'] = 'second' + candidate = build_report(cases(), raw) + assert not compare_reports(baseline, candidate)['semantic_scores_comparable'] + + +def test_case_loader_rejects_manifest_path_traversal(tmp_path): + (tmp_path / 'manifest.json').write_text(json.dumps({'question_ids': ['../outside']})) + with pytest.raises(ValueError): load_cases(tmp_path) diff --git a/tests/test_ragas_eval.py b/tests/test_ragas_eval.py index 3eef18a..9d626cd 100644 --- a/tests/test_ragas_eval.py +++ b/tests/test_ragas_eval.py @@ -5,7 +5,7 @@ import pytest -from scripts.eval_ragas import select_questions, make_case, metric_inputs, evaluate_case, summarize, cached_judgment, retry_delay +from scripts.eval_ragas import select_questions, make_case, metric_inputs, evaluate_case, summarize, cached_judgment, retry_delay, RequestBudget, BudgetExceeded def question(qid='TRAIN_Q1', answerable='Y'): @@ -50,6 +50,15 @@ def test_retry_after_has_buffer_for_fractional_token_refill(): assert retry_delay({'retry-after': 'nan'}) == 60 +def test_request_budget_counts_retries_and_cache_only_blocks_before_sending(): + budget = RequestBudget(2) + budget.consume(); budget.consume() + with pytest.raises(BudgetExceeded): budget.consume() + assert budget.sent == 2 and budget.exhausted + with pytest.raises(BudgetExceeded): RequestBudget(0).consume() + with pytest.raises(ValueError): RequestBudget(-1) + + def test_withheld_answers_keep_retrieval_scores_but_skip_answer_scores(): c = case(); c['status'] = 'insufficient_evidence' async def scorer(**kwargs): return SimpleNamespace(value=0.5) @@ -108,7 +117,7 @@ def test_real_ragas_aggregations_with_controlled_marking_not_network_calls(monke pytest.importorskip('ragas') from ragas.llms.base import InstructorBaseRagasLLM from ragas.embeddings.base import BaseRagasEmbedding - from ragas.metrics.collections import ContextPrecision, ContextRecall, Faithfulness, AnswerRelevancy + from ragas.metrics.collections import ContextPrecision, ContextRecall, Faithfulness, AnswerRelevancy, FactualCorrectness class Judge(InstructorBaseRagasLLM): def generate(self, prompt, response_model): @@ -120,6 +129,7 @@ async def agenerate(self, prompt, response_model): {'statement': 'One', 'reason': 'Supported', 'attributed': 1}, {'statement': 'Two', 'reason': 'Missing', 'attributed': 0}]}, 'StatementGeneratorOutput': {'statements': ['One', 'Two']}, + 'ClaimDecompositionOutput': {'claims': ['One', 'Two']}, 'NLIStatementOutput': {'statements': [ {'statement': 'One', 'reason': 'Supported', 'verdict': 1}, {'statement': 'Two', 'reason': 'Missing', 'verdict': 0}]}, @@ -135,8 +145,12 @@ async def aembed_text(self, text, **kwargs): return self.embed_text(text) results = asyncio.run(evaluate_case(case(), { 'context_precision': ContextPrecision(llm=judge), 'context_recall': ContextRecall(llm=judge), 'faithfulness': Faithfulness(llm=judge), - 'answer_relevancy': AnswerRelevancy(llm=judge, embeddings=Embeddings())}))['metrics'] + 'answer_relevancy': AnswerRelevancy(llm=judge, embeddings=Embeddings()), + 'factual_correctness': FactualCorrectness(llm=judge, mode='f1'), + 'answer_completeness': FactualCorrectness(llm=judge, mode='recall')}))['metrics'] assert results['context_precision']['score'] == pytest.approx(1) assert results['context_recall']['score'] == pytest.approx(.5) assert results['faithfulness']['score'] == pytest.approx(.5) assert results['answer_relevancy']['score'] == pytest.approx(1) + assert results['factual_correctness']['score'] == pytest.approx(.5) + assert results['answer_completeness']['score'] == pytest.approx(.5) From bdb7f47e26345968169e08c71934907faf56f943 Mon Sep 17 00:00:00 2001 From: shipitdev Date: Wed, 4 Nov 2026 13:29:51 +0530 Subject: [PATCH 2/2] Publish evaluation layers and measured TechQA results --- .github/workflows/ci.yml | 2 + README.md | 95 ++++- evaluations/fixtures/offline_demo.json | 57 +++ evaluations/overview.svg | 70 ++++ .../results/retrieval_baseline_v1.json | 134 +++++++ evaluations/results/techqa_pilot_v1.json | 331 ++++++++++++++++++ 6 files changed, 684 insertions(+), 5 deletions(-) create mode 100644 evaluations/fixtures/offline_demo.json create mode 100644 evaluations/overview.svg create mode 100644 evaluations/results/retrieval_baseline_v1.json create mode 100644 evaluations/results/techqa_pilot_v1.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4c2663..64a86b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: - run: python -m pip install -r requirements-test.txt - run: python -m pip check - run: python -m pytest -q + - name: Exercise the free evaluation report (synthetic cases, no API) + run: python -m scripts.eval_suite report --cases evaluations/fixtures/offline_demo.json --output /tmp/eval-demo.json frontend: runs-on: ubuntu-latest diff --git a/README.md b/README.md index e110c6d..35863ec 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A technical-support RAG workspace built around IBM TechQA. It retrieves source passages, writes a cited answer, and checks the answer before showing it. If the evidence is insufficient, it withholds the solution. -The repository contains application code, preparation/evaluation tools and offline tests. Datasets, model weights, vectors, credentials, evaluation outputs and working notes are kept outside Git. +The repository contains application code, preparation/evaluation tools and offline tests. Datasets, model weights, vectors, credentials, raw evaluation outputs and working notes are kept outside Git. A redacted evaluation summary and synthetic examples are published below. ## How it works @@ -19,6 +19,73 @@ Question → Gemma embedding → Qdrant exact search → BGE reranking - **Sessions:** recent conversation plus a summary, current-session memory, exact answer caching and an explicit regeneration bypass. - **Evaluation:** an offline RAGAS runner scores saved evidence and released answers. Judge decisions and local query embeddings are cached; withheld responses and evaluation errors stay visible. +## Evaluation suite + +![Evaluation layers: components, pipeline, application, regression and planned monitoring](evaluations/overview.svg) + +**Evaluate the parts, then their handoff, then the delivered answer.** RAGAS supplies semantic metrics; our Python checks cover source/citation contracts, recorded usage, judge integrity and paired regressions. Unit tests exercise failure handling with controlled fixtures. No single score is called “RAG accuracy.” + +| Layer | What we check | Implementation and status | +|---|---|---| +| Components | Retrieval Recall@K, MRR and nDCG against known article labels; writer/verifier behavior with supplied evidence | [Rank metrics](scripts/baseline.py), [retrieval tests](tests/test_app_retriever.py), [generator contract fixtures](tests/test_app_generation.py). Controlled fixtures are not a generator quality benchmark. | +| Pipeline | Selected context precision/recall, answer faithfulness and relevancy | [RAGAS runner](scripts/eval_ragas.py), measured on the pilot below. Reference text stays outside inference. | +| Application | Reference-based correctness and completeness; citation validity; withholding and layout contracts; recorded timing/tokens/cache use | Correctness = RAGAS `FactualCorrectness(mode="f1")`; completeness = its `mode="recall"` proxy. Both are opt-in and not yet benchmarked. [Free saved-case checks](scripts/eval_suite.py), [API contracts](tests/test_app_api.py), [layout checks](frontend/tests/genui-contract.test.mjs) are implemented. | +| Regression | Re-run a fixed case set, then inspect per-case differences | [Comparison CLI](scripts/eval_suite.py) rejects changed cohorts/references. Semantic comparisons also require matching judge settings and exclude scores flagged for review. | +| Online | Sampled live judging, feedback, drift and failure-to-test-set promotion | **Planned.** Content-free operational logs exist; live quality scoring, Langfuse integration and automated feedback loops do not. | + +Style rubrics and broad toxicity/PII/jailbreak evaluation are also planned. Existing origin/session, withholding and generated-component checks cover specific application contracts; they are not a comprehensive safety benchmark. Optional semantic metrics compare against available references, which may omit valid alternative answers. + +### Component measurements already recorded + +A separate [offline retrieval replay](evaluations/results/retrieval_baseline_v1.json) used **97 known-positive training tuning questions**. Article Recall@5 was **83/97 (85.57%)** and Recall@10 **87/97 (89.69%)**. The packed-context reference-word-run proxy passed **80/97 (82.47%)**; that proxy is not semantic context recall. + +A 120-query local Qdrant search check recorded **24.41 ms p50 / 38.54 ms p95**, using saved query vectors. Those timings exclude query encoding, reranking and generation. These historical component runs have different conditions and denominators from the three-case pipeline pilot below. + +### Measured pilot, with its limitations + +The pilot used the **first three sorted answerable IDs** in the saved training tuning split. It produced **one answer and two withheld responses**. Final development was not evaluated. + +| Case | Outcome | Context precision | Context recall | Faithfulness | Answer relevancy | +|---|---|---:|---:|---:|---:| +| TRAIN_Q007 | Withheld | 0.756 | 0.000 | Skipped | Skipped | +| TRAIN_Q008 | Withheld | 0.887 | 1.000 | Skipped | Skipped | +| TRAIN_Q013 | Answered | 0.792 | 1.000 | **0.500 — judge error flagged** | 0.979 | +| Raw mean | 1 answered / 3 attempted | **0.812 (n=3)** | **0.667 (n=3)** | **0.500 (n=1)** | **0.979 (n=1)** | + +Eight metric results succeeded, four were deliberately skipped, and the completed pass had zero metric errors. These are integration observations, not final accuracy or a measured improvement. + +The failure review matters as much as the averages: + +- **Q007:** the selected evidence missed the reference information. Related MQ passages could score as useful while context recall remained zero. +- **Q008:** the selected evidence supported the reference, but the pipeline withheld. Inspect the release path, rather than assume retrieval failed. +- **Q013:** the judge changed **“TIP 2.1” to “TIP 1”** while extracting claims, then penalized its altered claim. The raw faithfulness score is retained and flagged; it is not reliable evidence of an unsupported application answer. The new version-preservation heuristic catches this for review without rewriting the score. + +The [published, redacted result](evaluations/results/techqa_pilot_v1.json) includes per-case outcomes, denominators, settings, source-artifact hashes and review flags. Questions, references, source passages, credentials and full judge traces remain local. The summary records the November 3 run; today's export and additional contract checks made **zero new live judge calls**. + +| Recorded operation | Observation | Scope | +|---|---:|---| +| Answer coverage | 1/3 | Reported separately from conditional answer scores | +| Selected gold-article hit | 2/3 | Final evidence only; **not search Recall@10** or semantic recall | +| Citation contract | 1/1 released answer | IDs/format only; does not prove factual support | +| Completed resumed scoring pass | 48.90 s; 17,135 reported tokens | 25 cached judgments, 8 fresh judgments; earlier passes cost extra | +| Historical cache replay | 33 hits, 0 misses, **0 API calls** | Same raw scores; 0.022 s scoring loop excludes startup/imports | +| Correctness / completeness | Not measured | Available as opt-in RAGAS checks | +| Dollar cost / online drift | Not measured | Pricing and online monitoring not configured | + +### Try the suite without a model or dataset + +After installing `requirements-test.txt`, run the [synthetic demonstration](evaluations/fixtures/offline_demo.json): + +```bash +python -m scripts.eval_suite report \ + --cases evaluations/fixtures/offline_demo.json --output /tmp/eval-demo.json +python -m scripts.eval_suite compare \ + --baseline /tmp/eval-demo.json --candidate /tmp/eval-demo.json \ + --output /tmp/eval-diff.json +``` + +The report checks selected-document hits, citation contracts and outcomes. It does not run an LLM. The comparison prints paired changes; an unchanged demo should have no regressions. CI exercises this path, the budget controls and actual RAGAS aggregation with mocked judge decisions. Synthetic test results are kept separate from the TechQA pilot. + ## Requirements Python 3.13, Node.js 22, npm, uv, Docker Compose, a Groq API key, and prepared TechQA/model artifacts. The frontend and API run locally; Qdrant runs in Docker. @@ -114,14 +181,32 @@ Open `http://127.0.0.1:3000`. The example `APP_ORIGIN` matches that address. For uv venv --python 3.13 .venv-ragas uv pip install --python .venv-ragas/bin/python -r requirements-ragas.txt .venv-ingest/bin/python -m scripts.eval_ragas collect --limit 3 -.venv-ragas/bin/python -m scripts.eval_ragas run +.venv-ragas/bin/python -m scripts.eval_ragas run \ + --metrics context_precision context_recall --limit 3 --max-requests 12 ``` Collection additionally requires `data/splits/techqa_v1.json` and the original training question/document files under `data/raw/techqa/TechQA/training_and_dev/`. References are checked against original answer spans and never passed to inference. -Collection saves standalone canonical answers, without session memory or UI generation. Scoring runs context precision, context recall, faithfulness and answer relevancy sequentially. It reuses local Gemma for relevancy embeddings. Results, per-call marking decisions, usage and caches stay under `data/ragas/`. Use a new output directory for a changed case set; see `--help` for paths and judge/timeout options. +Collection saves standalone canonical answers, without session memory or UI generation. Scoring runs only the explicitly selected metrics, sequentially. Only answer relevancy loads local Gemma; the other judge metrics do not require local embedding weights. Results, per-call marking decisions, usage and caches stay under `data/ragas/`. Use a new output directory for a changed case set or serving-code version; see `--help` for paths and judge/timeout options. + +Choose metrics for the change you are investigating: + +- Retrieval/context change: `--metrics context_precision context_recall`. +- Answer-writing change: `--metrics faithfulness answer_relevancy`. +- Reference-based application review: `--metrics factual_correctness answer_completeness` (opt-in; no pilot scores yet). + +The default scoring limit is three cases and **12 HTTP attempts, including retries**. Set `--max-requests 0` for cache-only execution; missing decisions become `budget_exhausted`, not zero quality. Use a fresh `--output` for a comparison; omitted output names are timestamped so earlier reports survive. These are request caps, **not a dollar/token spending guarantee**. Inspect actual usage before raising them; one metric can make several calls. + +Create a free report from actual local artifacts: + +```bash +python -m scripts.eval_suite report --cases data/ragas/pilot_v1 \ + --ragas data/ragas/results/pilot_v1.json --output /tmp/pilot-review.json +``` + +Compare two reports with the `compare` command above. Cohort and reference identities must match. Judge/embedding settings must also match before semantic score deltas are reported; flagged scores stay visible but are excluded from those deltas. A negative delta is an observed regression to inspect, not statistical proof. -The current runner defaults to all four metrics and has **no hard token/dollar budget**. Use small case limits and inspect quota/usage before expanding. Failed evaluations are not zero scores; answer metrics are skipped on withheld cases. LLM judges can alter facts during claim extraction, so inspect their decisions before using scores as quality gates. +Failed evaluations are not zero scores; answer metrics are skipped on withheld cases. LLM judges can alter facts during claim extraction, so inspect their decisions before using scores as quality gates. ## Operating limits @@ -131,4 +216,4 @@ The current runner defaults to all four metrics and has **no hard token/dollar b - Verified components appear progressively after checking; unverified draft tokens are not streamed to users. - Backup restoration, broader answer-quality evaluation and production load testing remain separate work. -The initial history is a reconstructed import of local build stages from October 28–November 3, 2026. Author and committer dates identify those reconstructed stages; the repository was first published on November 3, 2026. It is not an archive of the exact working tree at each historical moment. +The initial history is a reconstructed import of local build stages from October 28–November 3, 2026. Author and committer dates identify those reconstructed stages; the repository was first published on November 3, 2026. It is not an archive of the exact working tree at each historical moment. Subsequent commits record ongoing development normally. diff --git a/evaluations/fixtures/offline_demo.json b/evaluations/fixtures/offline_demo.json new file mode 100644 index 0000000..40f6fb6 --- /dev/null +++ b/evaluations/fixtures/offline_demo.json @@ -0,0 +1,57 @@ +{ + "scope": "synthetic demonstration, not TechQA quality evidence", + "cases": [ + { + "question_id": "DEMO_ANSWER", + "user_input": "Which file stores the Beacon URL, and what follows an edit?", + "reference": "Edit beacon.ini, then restart the Beacon connector.", + "reference_provenance": { + "document_id": "beacon", + "view": "synthetic_v1", + "start": 0, + "end": 51 + }, + "status": "answered", + "response": "Edit beacon.ini, then restart the connector. [S1]", + "citations": [ + "S1" + ], + "sources": [ + { + "id": "S1", + "document_id": "beacon", + "text": "Edit beacon.ini, then restart the Beacon connector." + } + ], + "generation": { + "seconds": 0, + "provider_calls": [] + } + }, + { + "question_id": "DEMO_WITHHELD", + "user_input": "How do I enable a setting missing from the supplied manual?", + "reference": "Use the documented setting.", + "reference_provenance": { + "document_id": "settings", + "view": "synthetic_v1", + "start": 0, + "end": 27 + }, + "status": "insufficient_evidence", + "response": "The available sources do not support a complete answer.", + "citations": [], + "sources": [ + { + "id": "S1", + "document_id": "unrelated", + "text": "A different product has a different setting." + } + ], + "generation": { + "seconds": 0, + "provider_calls": [] + } + } + ] +} diff --git a/evaluations/overview.svg b/evaluations/overview.svg new file mode 100644 index 0000000..c096569 --- /dev/null +++ b/evaluations/overview.svg @@ -0,0 +1,70 @@ + +TechQA RAG evaluation suite: components, pipeline, application, regression and planned monitoring +Implemented Python contract checks and RAGAS pipeline evaluation, optional factual correctness and completeness judges, paired regression reports, and planned sampled online monitoring. A three-case pilot is integration evidence, not a final benchmark. + + + +TECHQA RAG / EVALUATION SUITE +Measure each handoff. +RAGAS for semantic judgments. Python for contracts, operations and comparisons. + +SHIPPED + +OPT-IN / NOT BENCHMARKED + +PLANNED + +01 Components — test one part at a time + +Retriever +Recall@K · MRR · nDCG +Labeled rankings · programmatic checks + +Generator with supplied evidence +Writer / verifier / repair fixtures +Controlled contracts; quality benchmark pending + + +02 Pipeline — evaluate the real retrieved context + +SHIPPED +Context precision → context recall → faithfulness → answer relevancy +Saved evidence + released answer + reference; withheld cases and errors stay visible. +Selected metrics · case limits · request caps · cached marking decisions + + +03 Application — answer quality, contracts and operating cost + + + +Quality + +OPT-IN +Factual correctness (F1) +Completeness (recall mode) +Style / broad safety: planned +Contracts +Citations · withholding · layouts +Judge version-preservation flags +Human review for flagged scores +Operations +Time · tokens · cache use +HTTP attempts and quota errors +Dollar pricing not configured + + +Regression is a comparison, not another metric +Same cases + same references → paired outcome and score changes → inspect regressions. + + +04 Online — sampled production monitoring + +PLANNED +Random samples + flagged requests · calibrated judges · feedback · drift monitoring +Operational logs exist today; Langfuse, online scoring and automated feedback do not. + + +Review failures → curate training cases → rerun the fixed suite +Manual review is active. Automatic case promotion and self-improvement are planned. +Evidence so far: 3-case tuning pilot. Final held-out quality and production readiness are not established. + diff --git a/evaluations/results/retrieval_baseline_v1.json b/evaluations/results/retrieval_baseline_v1.json new file mode 100644 index 0000000..cc01645 --- /dev/null +++ b/evaluations/results/retrieval_baseline_v1.json @@ -0,0 +1,134 @@ +{ + "scope": "Historical offline component replay; saved query vectors, known-positive training tuning cohort; not generated-answer quality", + "source_result_sha256": "f91f050ca8431e6e301759d1b48a8507c1fffc90dfc44971d43e82f5a65a0fc1", + "query_search_result_sha256": "ea19e2badbe539e5e594c6ce6d4da2780165fbfe629eb0fee27e9a411cbe608c", + "method": "Qdrant local exact + frozen query vectors + existing pinned full-question reranker scores plus new CPU scores for boundary changes; no new embeddings or API calls", + "case_ids": [ + "TRAIN_Q007", + "TRAIN_Q008", + "TRAIN_Q013", + "TRAIN_Q019", + "TRAIN_Q037", + "TRAIN_Q041", + "TRAIN_Q045", + "TRAIN_Q046", + "TRAIN_Q063", + "TRAIN_Q073", + "TRAIN_Q090", + "TRAIN_Q091", + "TRAIN_Q093", + "TRAIN_Q097", + "TRAIN_Q106", + "TRAIN_Q111", + "TRAIN_Q117", + "TRAIN_Q126", + "TRAIN_Q128", + "TRAIN_Q139", + "TRAIN_Q140", + "TRAIN_Q152", + "TRAIN_Q155", + "TRAIN_Q163", + "TRAIN_Q165", + "TRAIN_Q188", + "TRAIN_Q190", + "TRAIN_Q193", + "TRAIN_Q195", + "TRAIN_Q209", + "TRAIN_Q222", + "TRAIN_Q225", + "TRAIN_Q230", + "TRAIN_Q240", + "TRAIN_Q247", + "TRAIN_Q252", + "TRAIN_Q256", + "TRAIN_Q260", + "TRAIN_Q267", + "TRAIN_Q270", + "TRAIN_Q277", + "TRAIN_Q278", + "TRAIN_Q285", + "TRAIN_Q310", + "TRAIN_Q312", + "TRAIN_Q322", + "TRAIN_Q334", + "TRAIN_Q341", + "TRAIN_Q345", + "TRAIN_Q346", + "TRAIN_Q353", + "TRAIN_Q360", + "TRAIN_Q361", + "TRAIN_Q363", + "TRAIN_Q365", + "TRAIN_Q371", + "TRAIN_Q376", + "TRAIN_Q379", + "TRAIN_Q382", + "TRAIN_Q383", + "TRAIN_Q386", + "TRAIN_Q388", + "TRAIN_Q396", + "TRAIN_Q399", + "TRAIN_Q402", + "TRAIN_Q414", + "TRAIN_Q417", + "TRAIN_Q425", + "TRAIN_Q429", + "TRAIN_Q442", + "TRAIN_Q452", + "TRAIN_Q455", + "TRAIN_Q458", + "TRAIN_Q465", + "TRAIN_Q488", + "TRAIN_Q489", + "TRAIN_Q492", + "TRAIN_Q497", + "TRAIN_Q511", + "TRAIN_Q513", + "TRAIN_Q519", + "TRAIN_Q521", + "TRAIN_Q522", + "TRAIN_Q525", + "TRAIN_Q533", + "TRAIN_Q538", + "TRAIN_Q544", + "TRAIN_Q548", + "TRAIN_Q556", + "TRAIN_Q560", + "TRAIN_Q565", + "TRAIN_Q572", + "TRAIN_Q574", + "TRAIN_Q576", + "TRAIN_Q585", + "TRAIN_Q593", + "TRAIN_Q599" + ], + "metrics": { + "article_recall_at_5": { + "numerator": 83, + "denominator": 97, + "score": 0.8556701030927835 + }, + "article_recall_at_10": { + "numerator": 87, + "denominator": 97, + "score": 0.8969072164948454 + }, + "reference_word_run_proxy_pass": { + "numerator": 80, + "denominator": 97, + "score": 0.8247422680412371 + } + }, + "proxy_definition": "At least 90% longest contiguous reference-word run in the packed context, against URL-normalized known gold article; not semantic context recall.", + "search_timing": { + "queries": 120, + "search_p50_ms": 24.40745849889936, + "search_p95_ms": 38.54311635077465, + "setup": "local Qdrant Docker v1.16.3 via Colima 2CPU/2GiB; saved T4 query vectors; exact cosine; no live encoder/reranker/generation" + }, + "limits": [ + "Frozen-query search timing excludes live query encoding, reranking and generation.", + "Different cohort and runtime conditions from three-case RAGAS pilot; do not compare the metric values directly.", + "No document text, question text, answer labels or model weights distributed." + ] +} diff --git a/evaluations/results/techqa_pilot_v1.json b/evaluations/results/techqa_pilot_v1.json new file mode 100644 index 0000000..102ad1b --- /dev/null +++ b/evaluations/results/techqa_pilot_v1.json @@ -0,0 +1,331 @@ +{ + "schema": "saved_case_checks_v1", + "cohort_id": "d5ca7576b283e2a2c1c12983f77e19c0aae28f02bec832fdc8dbd7f2526758e7", + "case_count": 3, + "outcomes": { + "insufficient_evidence": 2, + "answered": 1 + }, + "metrics": { + "answer_coverage": 0.3333333333333333, + "selected_gold_document_hit_rate": 0.6666666666666666, + "citation_contract_pass_rate": 1.0 + }, + "denominators": { + "selected_gold_document_hit_rate": 3, + "citation_contract_pass_rate": 1 + }, + "operations": { + "generation_seconds": [ + 68.74449349999486, + 121.2497846250044, + 66.348813083001 + ], + "generation_reported_tokens": 40295, + "generation_usage_cases": 3, + "cost_usd": null, + "evaluation": { + "seconds": 48.90108508300182, + "judge_cache_hits": 25, + "judge_cache_misses": 8, + "http_attempts": 10, + "reported_tokens": 17135, + "scope": "this recorded scoring pass only; earlier cache-populating passes cost extra" + } + }, + "cases": [ + { + "question_id": "TRAIN_Q007", + "outcome": "insufficient_evidence", + "selected_gold_document_hit": 0, + "selected_passage_count": 5, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_precision": { + "state": "success", + "score": 0.7555555555303703, + "needs_review": false + }, + "context_recall": { + "state": "success", + "score": 0.0, + "needs_review": false + }, + "faithfulness": { + "state": "skipped", + "score": null, + "needs_review": false + }, + "answer_relevancy": { + "state": "skipped", + "score": null, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q008", + "outcome": "insufficient_evidence", + "selected_gold_document_hit": 1, + "selected_passage_count": 12, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_precision": { + "state": "success", + "score": 0.8874999999778125, + "needs_review": false + }, + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + }, + "faithfulness": { + "state": "skipped", + "score": null, + "needs_review": false + }, + "answer_relevancy": { + "state": "skipped", + "score": null, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q013", + "outcome": "answered", + "selected_gold_document_hit": 1, + "selected_passage_count": 8, + "citation_contract": { + "state": "checked", + "pass": true, + "reason": null + }, + "ragas": { + "context_precision": { + "state": "success", + "score": 0.7916666666402777, + "needs_review": false + }, + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": true + }, + "faithfulness": { + "state": "success", + "score": 0.5, + "needs_review": true + }, + "answer_relevancy": { + "state": "success", + "score": 0.9793743376624086, + "needs_review": false + } + }, + "judge_integrity_flags": [ + { + "metric": "context_recall", + "label": "TIP", + "introduced_version": "1", + "allowed_versions": [ + "2.1", + "2.2", + "2.2.0.1" + ], + "needs_review": true + }, + { + "metric": "faithfulness", + "label": "TIP", + "introduced_version": "1", + "allowed_versions": [ + "2.1", + "2.2", + "2.2.0.1" + ], + "needs_review": true + } + ] + } + ], + "limits": [ + "Selected-document hit is not search Recall@K or semantic evidence recall.", + "Citation validity does not establish claim support.", + "Version flags are heuristics for review; raw scores are never adjusted.", + "No style, toxicity/PII, live feedback or drift scores are computed." + ], + "ragas_summary": { + "attempted": 3, + "outcomes": { + "insufficient_evidence": 2, + "answered": 1 + }, + "metrics": { + "context_precision": { + "success": 3, + "error": 0, + "undefined": 0, + "skipped": 0, + "budget_exhausted": 0, + "mean": 0.8115740740494868, + "flagged_for_review": 0 + }, + "context_recall": { + "success": 3, + "error": 0, + "undefined": 0, + "skipped": 0, + "budget_exhausted": 0, + "mean": 0.6666666666666666, + "flagged_for_review": 1 + }, + "faithfulness": { + "success": 1, + "error": 0, + "undefined": 0, + "skipped": 2, + "budget_exhausted": 0, + "mean": 0.5, + "flagged_for_review": 1 + }, + "answer_relevancy": { + "success": 1, + "error": 0, + "undefined": 0, + "skipped": 2, + "budget_exhausted": 0, + "mean": 0.9793743376624086, + "flagged_for_review": 0 + } + } + }, + "judge": { + "provider": "groq", + "judge": "openai/gpt-oss-120b", + "temperature": 0, + "strictness": 3, + "max_tokens": 4096, + "embedding": { + "model": "google/embeddinggemma-2", + "revision": "f9972ac22e696a105c482202646979e33e4b5a1e", + "assets": [ + [ + "1_Pooling/config.json", + "2730fc1de168630e474c4066987f46d091de2189" + ], + [ + "2_Normalize/config.json", + "8babba92405167e8718211dc51f44fb117e423e4" + ], + [ + "config.json", + "dc524b2cbef566d66d8533c67dc68a6a8f5d340a" + ], + [ + "config_sentence_transformers.json", + "ba7dd4d5481cbaf271d5e8d1ad4271c5a45d2fd7" + ], + [ + "model.safetensors", + "197a32965d4b1105faf060417baa899e193fb73cd401f42ec9295234d5553d79" + ], + [ + "modules.json", + "1ab20c8ab29b8338488b415e5c09af14b742fe3d" + ], + [ + "preprocessor_config.json", + "4adff307a29874c969eda7edb5d3f07faaef879d" + ], + [ + "processor_config.json", + "63043a5a2fd2e25a2fc5336fcdaf24e515d12289" + ], + [ + "sentence_bert_config.json", + "b3b98f8308641875fca1e4ddbfae5f38996bb54f" + ], + [ + "tokenizer.json", + "4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4" + ], + [ + "tokenizer.model", + "e594c8a90eb08d8bda498ff4747977dc827ae0c3c56b5c0d41a605a22d02ef03" + ], + [ + "tokenizer_config.json", + "a9afd12fb3ca2c418fd405320ace85ad46748d03" + ] + ], + "device": "cpu", + "dtype": "float32", + "attention": "eager", + "query_prefix": "task: question answering | query: ", + "software": { + "torch": "2.15.0", + "transformers": "5.22.1", + "sentence-transformers": "6.2.0" + } + }, + "embedding_policy": "same formatted Gemma query role for original/generated questions; normalized cosine", + "ragas_source_sha256": "3e1a68a9cbef46834c672b716266089b1764d5d7bd424f997e92c8d92083e160", + "implementation_sha256": "42ad58ae92c93bea40d69ed39c1524353646c87016b8438c1440030166b7d769", + "packages": { + "ragas": "0.4.3", + "openai": "2.54.0", + "instructor": "1.17.0", + "langchain-community": "0.4.1", + "sentence-transformers": "6.2.0", + "torch": "2.15.0", + "transformers": "5.22.1", + "torchvision": "0.29.2" + } + }, + "judge_comparison_id": "bb152ffb496a60bcb2a9dfb65358b12dbaf8795fb8ea4e02c2af10b3456bd34d", + "provenance": { + "dataset": "IBM TechQA", + "split": "training tuning; first three sorted answerable IDs", + "run_date": "2026-11-03", + "purpose": "integration pilot, not final benchmark", + "source_run_sha256": "ba1613d7bcf47e85d13be3b40b79945a55cd7157550a1297f37dda57eef8c9e4", + "collection_manifest_sha256": "4bc1e40c648b98fa5e3980e41103aaa472fad296396cbffb1797c368b4f66b5b", + "private_artifacts": "Original questions, source passages, answers, references and full judge traces are not distributed.", + "export_date": "2026-11-04", + "new_live_judge_calls_for_export": 0 + }, + "cache_replay": { + "same_scores": true, + "judge_cache_hits": 33, + "judge_cache_misses": 0, + "http_attempts": 0, + "scoring_loop_seconds": 0.022332082997309044, + "scope": "historical replay; excludes process startup and imports" + }, + "review_notes": { + "TRAIN_Q007": "Known answer article absent from selected sources; semantic reference recall zero.", + "TRAIN_Q008": "Selected evidence supports reference but pipeline withheld; investigate release path.", + "TRAIN_Q013": "Judge changed TIP 2.1 to TIP 1 while extracting claims. Raw faithfulness is flagged, not adjusted." + }, + "available_not_benchmarked": [ + "factual_correctness", + "answer_completeness" + ], + "planned": [ + "style rubric", + "broad toxicity/PII/jailbreak evaluation", + "sampled online judging", + "feedback/drift monitoring", + "automated promotion into golden sets" + ] +}