diff --git a/README.md b/README.md index 5c71a8b..2af3d47 100644 --- a/README.md +++ b/README.md @@ -76,24 +76,24 @@ Style rubrics and broad toxicity/PII/jailbreak evaluation are also planned. Exis ### Wider training run with prompt version 5 -The [saved result](evaluations/results/techqa_wide_v5.json) covers **15 of 30 planned training questions**. Generation stopped on a provider rate limit. The run produced **8 released answers, 2 withheld responses, 3 citation-format failures, 1 local quota-wait deadline and 1 provider rate-limit failure**. The selected evidence included the labeled article for **14/15** questions. All **8 released answers** passed citation-format checks. +The [saved result](evaluations/results/techqa_wide_v5.json) covers **all 30 planned training questions**. It produced **15 released answers, 6 withheld responses, 1 clarification request and 8 generation failures**. Seven failures omitted inline citations even after repair. One earlier failure reached a local quota-wait deadline. The selected evidence included the labeled article for **25/30** questions. All **15 released answers** passed citation-format checks. -RAGAS used `openai/gpt-oss-20b` after a supported/unsupported faithfulness check scored 1.0/0.0. Answer generation used `openai/gpt-oss-120b`. Samples were selected separately for each metric to bound judging cost. +RAGAS used `openai/gpt-oss-20b` after a supported/unsupported faithfulness check scored 1.0/0.0. Answer generation used `openai/gpt-oss-120b`. Samples follow the fixed question order, with answer metrics conditional on released answers. | RAGAS metric | Raw mean | Scored cases | Review flags | |---|---:|---:|---:| -| Context recall | 90.9% | 11 | 0 | -| Context precision | 90.2% | 2 | 0 | -| Faithfulness | 97.5% | 5 | 0 | +| Context recall | 83.3% | 30 | 0 | +| Context precision | 87.9% | 4 | 0 | +| Faithfulness | 94.4% | 10 | 0 | | Answer relevancy | 93.0% | 3 | 0 | | Factual correctness against the reference | 42.6% | 5 | 1 | | Completeness against the reference | 53.4% | 5 | 1 | -Three additional context-recall cases reached the judge request cap. Raw means retain flagged values. LangSmith leaves those flagged values pending review, so its displayed averages use fewer cases. Source inspection also found a reference-scope issue where a question asks for a bulletin link but its reference contains the full bulletin text. Correctness and completeness scores require review before being used as release criteria. +Context recall scored the full cohort. Precision scored four of five attempted cases. Expanded correctness/completeness stopped when the provider requested a 527-second wait, beyond the metric deadline. Relevancy retains its earlier three-case sample. Raw means retain flagged values. LangSmith leaves flagged values pending review, so its displayed averages use fewer cases. Source inspection also found a reference-scope issue where a question asks for a bulletin link but its reference contains the full bulletin text. Correctness and completeness need review before use as release criteria. -Generation used **41 HTTP attempts / 206,804 reported tokens**. Judging used **120 attempts / 172,079 tokens**, plus **4 calibration calls / 3,757 tokens**. The runner reused **29 cached judge decisions**. Median answered-case batch time was **128.1 seconds**, including quota waits and local retrieval. These timings describe a serial batch. Reserved development questions remain available for a later release evaluation. +Generation used **77 HTTP attempts / 396,211 reported tokens**, including the archived rate-limited attempt. Judging used **228 attempts / 384,342 tokens**, plus **4 calibration calls / 3,757 tokens**. The runner reused **108 cached judge decisions**. Median answered-case batch time was **129.7 seconds**, including quota waits and local retrieval. These timings describe a serial batch. Reserved development questions remain available for a later release evaluation. -The collector initially stopped on a local request deadline. That condition was corrected, existing cases were reused, and collection continued until a provider HTTP 429 response. Retryable failed attempts are now archived before replacement. +The collector initially stopped on a local request deadline. That condition was corrected, existing cases were reused, and all 30 questions finished after a provider-key update. Retryable failed attempts are archived before replacement. The pipeline stayed fixed during the evaluation. ### Component measurements already recorded diff --git a/evaluations/results/techqa_wide_v5.json b/evaluations/results/techqa_wide_v5.json index c4842f2..1b9fdb1 100644 --- a/evaluations/results/techqa_wide_v5.json +++ b/evaluations/results/techqa_wide_v5.json @@ -2,73 +2,74 @@ "scope": "Seeded answerable training tuning cohort. Standalone canonical retrieval, writing, verification and repair.", "selection": { "planned": 30, - "collected": 15, - "pending": 15, + "collected": 30, + "pending": 0, "seed": 20261110 }, "model": "openai/gpt-oss-120b", "judge": "openai/gpt-oss-20b", "prompt_version": "strict_v5_message_roles", "outcomes": { - "generation_error": 5, - "insufficient_evidence": 2, - "answered": 8 + "generation_error": 8, + "insufficient_evidence": 6, + "answered": 15, + "needs_clarification": 1 }, "generation_errors": { - "invalid_citations": 3, - "quota_deadline": 1, - "http_429": 1 + "invalid_citations": 7, + "quota_deadline": 1 }, "metrics": { - "answer_coverage": 0.5333333333333333, - "selected_gold_document_hit_rate": 0.9333333333333333, + "answer_coverage": 0.5, + "selected_gold_document_hit_rate": 0.8333333333333334, "citation_contract_pass_rate": 1.0 }, "denominators": { - "selected_gold_document_hit_rate": 15, - "citation_contract_pass_rate": 8 + "selected_gold_document_hit_rate": 30, + "citation_contract_pass_rate": 15 }, "ragas_raw": { - "attempted": 15, + "attempted": 30, "outcomes": { - "generation_error": 5, - "insufficient_evidence": 2, - "answered": 8 + "generation_error": 8, + "insufficient_evidence": 6, + "answered": 15, + "needs_clarification": 1 }, "metrics": { "context_recall": { - "success": 11, + "success": 30, "error": 0, "undefined": 0, "skipped": 0, - "budget_exhausted": 3, - "mean": 0.9090909090909091, + "budget_exhausted": 0, + "mean": 0.8333333333333334, "flagged_for_review": 0 }, "faithfulness": { - "success": 5, + "success": 10, "error": 0, "undefined": 0, "skipped": 3, "budget_exhausted": 0, - "mean": 0.975, + "mean": 0.9441666666666666, "flagged_for_review": 0 }, "context_precision": { - "success": 2, - "error": 0, + "success": 4, + "error": 1, "undefined": 0, "skipped": 0, "budget_exhausted": 0, - "mean": 0.9020833333069618, + "mean": 0.8785416666421532, "flagged_for_review": 0 }, "factual_correctness": { "success": 5, - "error": 0, + "error": 1, "undefined": 0, "skipped": 0, - "budget_exhausted": 0, + "budget_exhausted": 4, "mean": 0.426, "flagged_for_review": 1 }, @@ -77,7 +78,7 @@ "error": 0, "undefined": 0, "skipped": 0, - "budget_exhausted": 0, + "budget_exhausted": 5, "mean": 0.534, "flagged_for_review": 1 }, @@ -93,45 +94,150 @@ } }, "sampling": { - "core_initial_ids": [ - "TRAIN_Q252", - "TRAIN_Q425", - "TRAIN_Q376", - "TRAIN_Q013", - "TRAIN_Q414" - ], - "precision_ids": [ - "TRAIN_Q252", - "TRAIN_Q425" - ], - "answer_quality_ids": [ - "TRAIN_Q376", - "TRAIN_Q013", - "TRAIN_Q117", - "TRAIN_Q548", - "TRAIN_Q310" - ], - "relevance_ids": [ - "TRAIN_Q376", - "TRAIN_Q013", - "TRAIN_Q117" - ], - "wider_recall": "remaining budget after other metrics" + "original": { + "core_initial_ids": [ + "TRAIN_Q252", + "TRAIN_Q425", + "TRAIN_Q376", + "TRAIN_Q013", + "TRAIN_Q414" + ], + "precision_ids": [ + "TRAIN_Q252", + "TRAIN_Q425" + ], + "answer_quality_ids": [ + "TRAIN_Q376", + "TRAIN_Q013", + "TRAIN_Q117", + "TRAIN_Q548", + "TRAIN_Q310" + ], + "relevance_ids": [ + "TRAIN_Q376", + "TRAIN_Q013", + "TRAIN_Q117" + ], + "wider_recall": "remaining budget after other metrics" + }, + "resume": [ + [ + "recall", + [ + "TRAIN_Q252", + "TRAIN_Q425", + "TRAIN_Q376", + "TRAIN_Q013", + "TRAIN_Q414", + "TRAIN_Q488", + "TRAIN_Q361", + "TRAIN_Q117", + "TRAIN_Q548", + "TRAIN_Q310", + "TRAIN_Q230", + "TRAIN_Q353", + "TRAIN_Q599", + "TRAIN_Q365", + "TRAIN_Q388", + "TRAIN_Q417", + "TRAIN_Q556", + "TRAIN_Q093", + "TRAIN_Q152", + "TRAIN_Q046", + "TRAIN_Q045", + "TRAIN_Q106", + "TRAIN_Q188", + "TRAIN_Q090", + "TRAIN_Q560", + "TRAIN_Q007", + "TRAIN_Q334", + "TRAIN_Q260", + "TRAIN_Q465", + "TRAIN_Q128" + ], + [ + "context_recall" + ] + ], + [ + "faithfulness", + [ + "TRAIN_Q376", + "TRAIN_Q013", + "TRAIN_Q117", + "TRAIN_Q548", + "TRAIN_Q310", + "TRAIN_Q230", + "TRAIN_Q353", + "TRAIN_Q599", + "TRAIN_Q417", + "TRAIN_Q093" + ], + [ + "faithfulness" + ] + ], + [ + "precision", + [ + "TRAIN_Q252", + "TRAIN_Q425", + "TRAIN_Q376", + "TRAIN_Q013", + "TRAIN_Q414" + ], + [ + "context_precision" + ] + ], + [ + "quality", + [ + "TRAIN_Q376", + "TRAIN_Q013", + "TRAIN_Q117", + "TRAIN_Q548", + "TRAIN_Q310", + "TRAIN_Q230", + "TRAIN_Q353", + "TRAIN_Q599", + "TRAIN_Q417", + "TRAIN_Q093" + ], + [ + "factual_correctness", + "answer_completeness" + ] + ], + [ + "relevancy", + [ + "TRAIN_Q376", + "TRAIN_Q013", + "TRAIN_Q117", + "TRAIN_Q548", + "TRAIN_Q310" + ], + [ + "answer_relevancy" + ] + ] + ] }, "review_notes": { "TRAIN_Q376": "Reference scope needs review. A source-link answer is compared with the full bulletin text.", "TRAIN_Q310": "Heuristic flag needs review. The as-5 match concerns initial connection counts while as-60 concerns cache timeout. Both numbers appear in the original response." }, "operations": { - "generation_http_attempts": 41, - "generation_reported_tokens": 206804, - "judge_http_attempts": 120, - "judge_reported_tokens": 172079, + "generation_http_attempts": 77, + "generation_reported_tokens": 396211, + "judge_http_attempts": 228, + "judge_reported_tokens": 384342, "calibration_http_attempts": 4, "calibration_reported_tokens": 3757, - "judge_cache_hits": 29, - "judge_cache_misses": 69, - "answered_batch_seconds_median": 128.05175710400363, + "judge_cache_hits": 108, + "judge_cache_misses": 135, + "answered_batch_seconds_median": 129.65495645900955, "answered_batch_seconds_max": 141.51168862500344, "timing_scope": "Includes retrieval, model calls and quota pacing in a serial batch.", "cost_usd": null @@ -146,9 +252,13 @@ { "limit": 52, "sent": 3 + }, + { + "limit": 49, + "sent": 36 } ], - "judge_cap": 120, + "judge_cap": 240, "calibration_cap": 8 }, "cases": [ @@ -243,6 +353,11 @@ "state": "success", "score": 0.9170371340678626, "needs_review": false + }, + "context_precision": { + "state": "success", + "score": 0.8766666666491334, + "needs_review": false } }, "judge_integrity_flags": [] @@ -282,6 +397,11 @@ "state": "success", "score": 0.9793743376624086, "needs_review": false + }, + "context_precision": { + "state": "success", + "score": 0.8333333333055556, + "needs_review": false } }, "judge_integrity_flags": [] @@ -305,6 +425,11 @@ "state": "skipped", "score": null, "needs_review": false + }, + "context_precision": { + "state": "error", + "score": null, + "needs_review": false } }, "judge_integrity_flags": [] @@ -486,6 +611,21 @@ "state": "success", "score": 0.0, "needs_review": false + }, + "faithfulness": { + "state": "success", + "score": 0.9, + "needs_review": false + }, + "factual_correctness": { + "state": "error", + "score": null, + "needs_review": false + }, + "answer_completeness": { + "state": "budget_exhausted", + "score": null, + "needs_review": false } }, "judge_integrity_flags": [] @@ -502,6 +642,21 @@ }, "ragas": { "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + }, + "faithfulness": { + "state": "success", + "score": 1.0, + "needs_review": false + }, + "factual_correctness": { + "state": "budget_exhausted", + "score": null, + "needs_review": false + }, + "answer_completeness": { "state": "budget_exhausted", "score": null, "needs_review": false @@ -521,6 +676,21 @@ }, "ragas": { "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + }, + "faithfulness": { + "state": "success", + "score": 0.6666666666666666, + "needs_review": false + }, + "factual_correctness": { + "state": "budget_exhausted", + "score": null, + "needs_review": false + }, + "answer_completeness": { "state": "budget_exhausted", "score": null, "needs_review": false @@ -539,6 +709,58 @@ }, "ragas": { "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q388", + "outcome": "insufficient_evidence", + "selected_gold_document_hit": 1, + "selected_passage_count": 12, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q417", + "outcome": "answered", + "selected_gold_document_hit": 1, + "selected_passage_count": 8, + "citation_contract": { + "state": "checked", + "pass": true, + "reason": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + }, + "faithfulness": { + "state": "success", + "score": 1.0, + "needs_review": false + }, + "factual_correctness": { + "state": "budget_exhausted", + "score": null, + "needs_review": false + }, + "answer_completeness": { "state": "budget_exhausted", "score": null, "needs_review": false @@ -547,7 +769,206 @@ "judge_integrity_flags": [] }, { - "question_id": "TRAIN_Q388", + "question_id": "TRAIN_Q556", + "outcome": "insufficient_evidence", + "selected_gold_document_hit": 1, + "selected_passage_count": 9, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q093", + "outcome": "answered", + "selected_gold_document_hit": 1, + "selected_passage_count": 12, + "citation_contract": { + "state": "checked", + "pass": true, + "reason": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + }, + "faithfulness": { + "state": "success", + "score": 1.0, + "needs_review": false + }, + "factual_correctness": { + "state": "budget_exhausted", + "score": null, + "needs_review": false + }, + "answer_completeness": { + "state": "budget_exhausted", + "score": null, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q152", + "outcome": "generation_error", + "selected_gold_document_hit": 1, + "selected_passage_count": 9, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q046", + "outcome": "needs_clarification", + "selected_gold_document_hit": 1, + "selected_passage_count": 12, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q045", + "outcome": "insufficient_evidence", + "selected_gold_document_hit": 0, + "selected_passage_count": 8, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 0.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q106", + "outcome": "answered", + "selected_gold_document_hit": 1, + "selected_passage_count": 8, + "citation_contract": { + "state": "checked", + "pass": true, + "reason": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q188", + "outcome": "answered", + "selected_gold_document_hit": 1, + "selected_passage_count": 21, + "citation_contract": { + "state": "checked", + "pass": true, + "reason": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q090", + "outcome": "generation_error", + "selected_gold_document_hit": 1, + "selected_passage_count": 11, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q560", + "outcome": "insufficient_evidence", + "selected_gold_document_hit": 0, + "selected_passage_count": 16, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 0.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q007", + "outcome": "answered", + "selected_gold_document_hit": 0, + "selected_passage_count": 6, + "citation_contract": { + "state": "checked", + "pass": true, + "reason": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 0.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q334", "outcome": "generation_error", "selected_gold_document_hit": 1, "selected_passage_count": 12, @@ -555,55 +976,84 @@ "state": "skipped", "pass": null }, - "ragas": {}, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q260", + "outcome": "answered", + "selected_gold_document_hit": 1, + "selected_passage_count": 8, + "citation_contract": { + "state": "checked", + "pass": true, + "reason": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q465", + "outcome": "generation_error", + "selected_gold_document_hit": 0, + "selected_passage_count": 8, + "citation_contract": { + "state": "skipped", + "pass": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 0.0, + "needs_review": false + } + }, + "judge_integrity_flags": [] + }, + { + "question_id": "TRAIN_Q128", + "outcome": "answered", + "selected_gold_document_hit": 1, + "selected_passage_count": 9, + "citation_contract": { + "state": "checked", + "pass": true, + "reason": null + }, + "ragas": { + "context_recall": { + "state": "success", + "score": 1.0, + "needs_review": false + } + }, "judge_integrity_flags": [] } ], "artifact_sha256": { "collection_manifest": "a49bf986219036e536a5395b8450c13167316f6595a774d16307f239f55aba00", "raw_ragas": "cf0e07a38ea351304aefc6f62d5e7a8d16caa0fa271c3c158277208ca54e0b48", - "final_composed_ragas": "b4f32bad9144a5db2b33570c50e7c42812d64fc47a10b9453ff59d612aa3ad2f" + "final_composed_ragas": "16f1e9017e4c6aa1e3e655f24b2b3e9670f464c9afc9388548628b43cda9f81b" }, "limitations": [ - "Provider rate limiting stopped generation with 15 planned cases pending.", - "Three context recall scores exhausted the judge request budget.", + "All 30 generation cases finished. One historical local quota deadline remains a failed case. A provider quota reset exceeded the judge deadline during expansion.", + "Context recall scored all 30 cases. Precision scored four of five attempted cases. Expanded correctness/completeness hit provider limits and relevancy retains its earlier three-case sample.", "Metrics use different sample sizes. Answer metrics are conditional on released answers.", "Raw means retain one case with heuristic review flags. LangSmith withholds its flagged metric values.", "Reference agreement depends on answer scope and reference completeness. Scores require review.", "The judge differs from earlier runs. This run establishes a fresh baseline.", "Reserved development questions were not used." - ], - "judge_config": { - "temperature": 0, - "reasoning_effort": "low", - "max_tokens": 2048, - "strictness": 3, - "packages": { - "ragas": "0.4.3", - "openai": "2.54.0", - "instructor": "1.17.0", - "langchain-community": "0.4.1", - "sentence-transformers": "6.2.0", - "torch": "2.15.0", - "transformers": "5.22.1", - "torchvision": "0.29.2" - }, - "ragas_source_sha256": "3e1a68a9cbef46834c672b716266089b1764d5d7bd424f997e92c8d92083e160", - "implementation_sha256": "f39f2b56005861d92b8df4e2fc33004dc2fed6abff256f428c916c470bc009fe" - }, - "inference_source_sha256": { - "rag_app/__init__.py": "2fdce6dc6a2613b920f0df2e2dcaffbf3aa3775fa03b3b2b35294f6f2b609431", - "rag_app/api.py": "194c7657b566f0330ce2e463ab7772ecbb4e7aeee3ef2feddaff1b77a03d8553", - "rag_app/feedback.py": "61ef4c889fb31d58e73ceba58426f9747f6a7ae7b563f3334f950391eb4bdb49", - "rag_app/generation.py": "9afb38360c6374db464e6bbc7feb0a10f9e5786a7162f5a9180ad9b9e39be135", - "rag_app/indexes.py": "bbd9759f62a68b3a57380c587f772018421e00d918542257232c4eb8cfbf0a5c", - "rag_app/observability.py": "eec2d67e7a02622391d6f4b8d0ef0fe86b53c9f444c6d226b4f2f9d4948e023a", - "rag_app/presentation.py": "f3d01624ef8cdb8961041517a8712ebec745e350e4aef03dcd99895205ac33d2", - "rag_app/retriever.py": "c49ed1d3376044f656d20d632f54761d89461c5d7e4ab8ab3d2a831e81459940", - "rag_app/state.py": "201aba9117c53f6b34a9a4455a413953a5aede2b6baa78129cb5e1698f75fc45", - "rag_app/storage.py": "7220891a37d005ad734440a56bea69c1161ffab807e194b4aca69363fdaedb1a", - "rag_app/tracing.py": "7ccd4f804b359adbd77e168178db9eedf73eac4f8ca7cac639fa76032f4adc92" - }, - "split_sha256": "9b3c6d289f9570e6d7230dec90b15c769b0fc56ceb609bb2c3f44dc73a9f977a", - "inference_snapshot_matches_current": true + ] }