From 8e8089b84d8cfaa58c6feb5014bd96317e30452e Mon Sep 17 00:00:00 2001
From: shipitdev <harshsinghrajpoot07@gmail.com>
Date: Tue, 10 Nov 2026 19:43:47 +0530
Subject: [PATCH 1/2] Collect run feedback and require reviewed references for
 regression cases

---
 .gitignore                             |   6 ++
 README.md                              |  18 +++-
 frontend/src/app/api/feedback/route.ts |  22 ++++
 frontend/src/app/globals.css           |  11 ++
 frontend/src/components/workspace.tsx  |  58 ++++++++++
 rag_app/api.py                         |  35 +++++-
 rag_app/feedback.py                    |  61 +++++++++++
 scripts/eval_ragas.py                  |  26 +++--
 scripts/eval_suite.py                  |   8 +-
 scripts/feedback_cases.py              | 119 +++++++++++++++++++++
 scripts/publish_langsmith.py           |  14 ++-
 tests/test_feedback.py                 | 141 +++++++++++++++++++++++++
 tests/test_ragas_eval.py               |  11 ++
 13 files changed, 514 insertions(+), 16 deletions(-)
 create mode 100644 frontend/src/app/api/feedback/route.ts
 create mode 100644 rag_app/feedback.py
 create mode 100644 scripts/feedback_cases.py
 create mode 100644 tests/test_feedback.py

diff --git a/.gitignore b/.gitignore
index 59613b8..adb2746 100644
--- a/.gitignore
+++ b/.gitignore
@@ -134,3 +134,9 @@
 !/tests/test_langsmith_export.py
 
 !/evaluations/results/observability_v1.json
+
+!/rag_app/feedback.py
+!/tests/test_feedback.py
+!/scripts/feedback_cases.py
+!/frontend/src/app/api/feedback/
+!/frontend/src/app/api/feedback/route.ts
diff --git a/README.md b/README.md
index e880cea..228b346 100644
--- a/README.md
+++ b/README.md
@@ -42,6 +42,22 @@ Publishing replays saved outputs into a LangSmith experiment with response statu
 
 The [November 10 observability batch](evaluations/results/observability_v1.json) selected **12 training questions**. Retrieval selected the labeled article for **11/12**. **3 answers** were released and passed citation-format checks. **9 provider quota failures** prevented completion. The run used **15 HTTP attempts and 33,230 reported tokens**. This batch establishes dashboard visibility and exposes a quota-handling issue, which is now covered by a stop-on-quota regression test. New semantic RAGAS scores await available provider quota.
 
+## Feedback and future evaluation cases
+
+Every completed response offers Helpful and Not helpful, with an optional explanation. Sending feedback saves the question, resolved follow-up, answer, sources, private diagnostics and pipeline versions in `data/app/feedback/`. These private records persist after the session ends. A stable feedback record also attaches the rating and comment to the LangSmith trace. If cloud synchronization fails, the local record remains available and `python -m scripts.feedback_cases sync` retries it.
+
+Review the saved case against its sources, write a checked reference answer in a local text file, then approve and export it.
+
+```bash
+python -m scripts.feedback_cases approve --run-id RUN_UUID \
+  --reference-file /path/to/checked-answer.txt --notes "Verified against the original source"
+python -m scripts.feedback_cases export --output data/ragas/feedback_review_v1
+python -m scripts.eval_ragas run --cases data/ragas/feedback_review_v1 \
+  --metrics faithfulness factual_correctness answer_completeness --limit 5 --max-requests 20
+```
+
+Approval and review notes are required for feedback-based RAGAS evaluation. Use these cases as a regression set alongside the fixed training sample. Human votes and comments identify examples for review, while checked references support correctness scoring. Human ratings can also help assess and improve judge agreement. Each export uses a new directory to preserve the earlier evaluation version.
+
 ## Evaluation suite
 
 ![Evaluation layers: components, pipeline, application, regression and planned monitoring](evaluations/overview.svg)
@@ -54,7 +70,7 @@ The [November 10 observability batch](evaluations/results/observability_v1.json)
 | Pipeline | Selected context precision/recall, answer faithfulness and relevancy | [RAGAS runner](scripts/eval_ragas.py), measured on the pilot below. Reference text stays outside inference. |
 | Application | Reference-based correctness and completeness; citation validity; withholding and layout contracts; recorded timing/tokens/cache use | Correctness = RAGAS `FactualCorrectness(mode="f1")`; completeness = its `mode="recall"` proxy. Both are opt-in and not yet benchmarked. [Free saved-case checks](scripts/eval_suite.py), [API contracts](tests/test_app_api.py), [layout checks](frontend/tests/genui-contract.test.mjs) are implemented. |
 | Regression | Re-run a fixed case set, then inspect per-case differences | [Comparison CLI](scripts/eval_suite.py) rejects changed cohorts/references. Semantic comparisons also require matching judge settings and exclude scores flagged for review. |
-| Online | Sampled live judging, feedback, drift and failure-to-test-set promotion | **Planned.** Content-free operational logs exist; live quality scoring, Langfuse integration and automated feedback loops do not. |
+| Online | User feedback, reviewed regression cases, sampled judging and drift | Helpful/unhelpful votes and optional comments save the run for review and sync to LangSmith. Reviewed references can be exported for RAGAS. Sampled live judging and drift monitoring remain planned. |
 
 Style rubrics and broad toxicity/PII/jailbreak evaluation are also planned. Existing origin/session, withholding and generated-component checks cover specific application contracts; they are not a comprehensive safety benchmark. Optional semantic metrics compare against available references, which may omit valid alternative answers.
 
diff --git a/frontend/src/app/api/feedback/route.ts b/frontend/src/app/api/feedback/route.ts
new file mode 100644
index 0000000..a86346a
--- /dev/null
+++ b/frontend/src/app/api/feedback/route.ts
@@ -0,0 +1,22 @@
+import { isAllowedOrigin } from "@/lib/request-origin";
+
+export async function POST(request: Request) {
+  if (!isAllowedOrigin(request)) return Response.json({ message: "Cross-origin requests are not allowed." }, { status: 403 });
+  if (Number(request.headers.get("content-length") || 0) > 16_000) return Response.json({ message: "Feedback is too large." }, { status: 413 });
+  let body: string;
+  try { body = await request.text(); }
+  catch { return Response.json({ message: "Feedback could not be read." }, { status: 400 }); }
+  if (body.length > 16_000) return Response.json({ message: "Feedback is too large." }, { status: 413 });
+  try {
+    const upstream = await fetch(new URL("feedback", process.env.RAG_API_URL || "http://127.0.0.1:8000/ask"), {
+      method: "POST",
+      headers: { "content-type": "application/json", ...(request.headers.get("cookie") ? { cookie: request.headers.get("cookie")! } : {}) },
+      body,
+      cache: "no-store",
+      signal: AbortSignal.timeout(10_000),
+    });
+    return new Response(upstream.body, { status: upstream.status, headers: { "content-type": "application/json", "cache-control": "no-store" } });
+  } catch {
+    return Response.json({ message: "Feedback could not be saved. Please retry." }, { status: 502 });
+  }
+}
diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css
index 180dd3a..f2c7669 100644
--- a/frontend/src/app/globals.css
+++ b/frontend/src/app/globals.css
@@ -238,3 +238,14 @@ body { --body: var(--font-geist), Arial, sans-serif; --mono: var(--font-geist-mo
 @media (min-width: 901px) {
   .rail-label { position: absolute; width: 1px; height: 1px; margin: -1px; padding: 0; overflow: hidden; clip: rect(0, 0, 0, 0); white-space: nowrap; }
 }
+.answer-feedback { margin-top: 28px; padding-top: 20px; border-top: 1px solid var(--border); font-size: 13px; }
+.answer-feedback fieldset { min-width: 0; border: 0; margin: 0; padding: 0; }
+.answer-feedback legend { margin-bottom: 12px; font-weight: 500; }
+.feedback-choices { display: flex; gap: 8px; margin-bottom: 16px; }
+.feedback-choices button { min-height: 36px; padding: 0 12px; border: 1px solid #d4d4d4; border-radius: 6px; background: white; }
+.feedback-choices button[aria-pressed="true"] { border-color: #141414; background: #eee; }
+.answer-feedback label { display: block; margin-bottom: 8px; }
+.answer-feedback label span, .feedback-note { color: #626262; }
+.answer-feedback textarea { width: 100%; padding: 10px 12px; border: 1px solid #d4d4d4; border-radius: 6px; resize: vertical; background: white; }
+.feedback-note { font-size: 12px; }
+.answer-feedback button:disabled { opacity: .6; cursor: default; }
diff --git a/frontend/src/components/workspace.tsx b/frontend/src/components/workspace.tsx
index 5adc215..1ad0522 100644
--- a/frontend/src/components/workspace.tsx
+++ b/frontend/src/components/workspace.tsx
@@ -19,6 +19,14 @@ type FinalEvent = {
 type Result = FinalEvent & { question: string; runId: number; genUiValid?: boolean };
 type ServiceStatus = { api: string; presentation_status: string };
 
+type FeedbackState = {
+  choice: boolean | null;
+  comment: string;
+  state: "idle" | "saving" | "saved" | "error";
+  message: string;
+};
+const emptyFeedback: FeedbackState = { choice: null, comment: "", state: "idle", message: "" };
+
 function validSource(value: unknown): value is VerifiedSource {
   if (!value || typeof value !== "object") return false;
   const source = value as Record<string, unknown>;
@@ -102,6 +110,7 @@ async function* readEvents(response: Response) {
 }
 
 export default function Workspace() {
+  const [feedback, setFeedback] = useState<FeedbackState>(emptyFeedback);
   const [elapsed, setElapsed] = useState(0);
   const [question, setQuestion] = useState("");
   const [busy, setBusy] = useState(false);
@@ -129,6 +138,7 @@ export default function Workspace() {
       runCounter.current += 1;
       setQuestion("");
       setResult(null);
+    setFeedback(emptyFeedback);
       setStages([]);
       setRetrievedSources([]);
       setError("");
@@ -162,6 +172,7 @@ export default function Workspace() {
     setFailedRun(null);
     setOpenUiMessage("");
     setResult(null);
+    setFeedback(emptyFeedback);
     setStages([]);
     setRetrievedSources([]);
     setQuestion(submitted);
@@ -228,6 +239,25 @@ export default function Workspace() {
     }
   }
 
+  async function submitFeedback(event: FormEvent<HTMLFormElement>) {
+    event.preventDefault();
+    if (!result?.request_id || feedback.choice === null || feedback.state === "saving") return;
+    const runId = result.runId;
+    setFeedback((current) => ({ ...current, state: "saving", message: "" }));
+    try {
+      const response = await fetch("/api/feedback", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ request_id: result.request_id, helpful: feedback.choice, comment: feedback.comment }),
+        signal: AbortSignal.timeout(10_000),
+      });
+      if (!response.ok) throw new Error("Feedback could not be saved. Please retry.");
+      if (runCounter.current === runId) setFeedback((current) => ({ ...current, state: "saved", message: "Feedback saved for review." }));
+    } catch {
+      if (runCounter.current === runId) setFeedback((current) => ({ ...current, state: "error", message: "Feedback could not be saved. Please retry." }));
+    }
+  }
+
   function stopRequest(event: MouseEvent<HTMLButtonElement>) {
     // Cancellation can switch this same button back to submit during the click.
     event.preventDefault();
@@ -248,6 +278,7 @@ export default function Workspace() {
       runCounter.current += 1;
       setQuestion("");
       setResult(null);
+    setFeedback(emptyFeedback);
       setStages([]);
       setRetrievedSources([]);
       setError("");
@@ -379,6 +410,33 @@ export default function Workspace() {
                     <p>{result.status === "needs_clarification" ? "Refine your question with the missing system, version, or context and ask again." : "Try a narrower question or check that the relevant documents are in this knowledge base."}</p>
                   </div>
                 )}
+                {result.request_id && (
+                  <form className="answer-feedback" onSubmit={submitFeedback} aria-busy={feedback.state === "saving"}>
+                    <fieldset disabled={feedback.state === "saving"}>
+                      <legend>Was this answer helpful?</legend>
+                      <div className="feedback-choices">
+                        {[true, false].map((choice) => (
+                          <button key={String(choice)} type="button" aria-pressed={feedback.choice === choice}
+                            onClick={() => setFeedback((current) => ({ ...current, choice, state: "idle", message: "" }))}>
+                            {choice ? "Helpful" : "Not helpful"}
+                          </button>
+                        ))}
+                      </div>
+                      {feedback.choice !== null && (
+                        <>
+                          <label htmlFor="feedback-comment">{feedback.choice ? "Anything we should improve?" : "What went wrong?"} <span>(optional)</span></label>
+                          <textarea id="feedback-comment" maxLength={2000} rows={3} value={feedback.comment}
+                            onChange={(event) => setFeedback((current) => ({ ...current, comment: event.target.value, state: "idle", message: "" }))} />
+                          <p className="feedback-note">Sending feedback saves this question, answer, sources and run details for review.</p>
+                          <button type="submit" className="secondary-button" disabled={feedback.state === "saved"}>
+                            {feedback.state === "saving" ? "Saving…" : "Send feedback"}
+                          </button>
+                        </>
+                      )}
+                    </fieldset>
+                    {feedback.message && <p role={feedback.state === "error" ? "alert" : "status"}>{feedback.message}</p>}
+                  </form>
+                )}
               </article>
             )}
 
diff --git a/rag_app/api.py b/rag_app/api.py
index 603b867..0c782b3 100644
--- a/rag_app/api.py
+++ b/rag_app/api.py
@@ -13,7 +13,7 @@
 from collections import defaultdict, deque
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.responses import StreamingResponse
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ConfigDict
 from dotenv import load_dotenv
 from rag_app.state import SessionStore, AnswerCache
 from rag_app.observability import persist_run
@@ -28,14 +28,23 @@ class Ask(BaseModel):
     regenerate: bool = False
 
 
+class Feedback(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    request_id: uuid.UUID
+    helpful: bool = Field(strict=True)
+    comment: str = Field(default='', max_length=2000)
+
+
 def create_app(retriever=None, generator=None, resolver=None):
     app = FastAPI(title='TechQA evidence workspace')
     serial = threading.Lock()
     admission = threading.BoundedSemaphore(5)
     sessions = SessionStore()
     answers = AnswerCache()
+    feedback_runs = AnswerCache(ttl_seconds=1800, max_entries=256, max_bytes=32 * 1024**2)
     limits = defaultdict(deque)
     limiter_lock = threading.Lock()
+    feedback_lock = threading.Lock()
     runtime = {'retriever': retriever, 'generator': generator, 'resolver': resolver}
 
     def pipeline_version():
@@ -110,12 +119,29 @@ def end(request: Request):
         if origin and origin != allowed:
             raise HTTPException(403, 'Origin not allowed')
         sid = request.cookies.get('techqa_session', '')
-        sessions.end(sid); answers.clear_session(sid)
+        sessions.end(sid); answers.clear_session(sid); feedback_runs.clear_session(sid)
         from fastapi.responses import JSONResponse
         response = JSONResponse({'status': 'cleared'})
         response.delete_cookie('techqa_session')
         return response
 
+    @app.post('/feedback')
+    def feedback(body: Feedback, request: Request):
+        origin = request.headers.get('origin')
+        if origin and origin != os.getenv('APP_ORIGIN', 'http://localhost:3000'):
+            raise HTTPException(403, 'Origin not allowed')
+        session = sessions.get(request.cookies.get('techqa_session', ''))
+        snapshot = feedback_runs.get(session.id, str(body.request_id), session_expires_at=session.expires_at) if session else None
+        if snapshot is None:
+            raise HTTPException(404, 'This run is unavailable in the current session')
+        from rag_app.feedback import record_feedback
+        try:
+            with feedback_lock:
+                record = record_feedback(ROOT, snapshot, body.helpful, body.comment.strip())
+        except OSError:
+            raise HTTPException(503, 'Feedback could not be saved. Please retry.') from None
+        return {'status': 'saved', 'review_status': record['review_status']}
+
     @app.post('/ask')
     async def ask(body: Ask, request: Request):
         if not body.question.strip():
@@ -268,6 +294,11 @@ def run_worker():
                         pass
                     result['metrics']['total_ms'] = round((time.monotonic() - started) * 1000, 2)
                     root_run.end(outputs=result)
+                feedback_runs.put(session.id, rid, {
+                    'question': body.question, 'resolved_question': question, 'result': result,
+                    'pipeline_version': identity['pipeline'], 'prompt_version': PROMPT_VERSION,
+                    'model': MODEL, 'index': identity['index'], 'private_trace': private_trace,
+                }, session_expires_at=session.expires_at)
                 emit('final', result)
                 write_metrics({'request_id': rid, 'status': result['status'], 'metrics': result['metrics']},private_trace)
                 completed = True
diff --git a/rag_app/feedback.py b/rag_app/feedback.py
new file mode 100644
index 0000000..40890cf
--- /dev/null
+++ b/rag_app/feedback.py
@@ -0,0 +1,61 @@
+"""Durable, private user feedback and its associated run snapshot."""
+
+import logging
+import os
+from pathlib import Path
+import time
+import uuid
+
+from langsmith.utils import LangSmithNotFoundError
+from rag_app.tracing import get_client
+from rag_app.indexes import atomic_json
+
+
+def save_feedback(root, request_id, record):
+    directory = Path(root) / "data/app/feedback"
+    directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+    path = directory / (str(uuid.UUID(request_id)) + ".json")
+    atomic_json(path, record)
+    path.chmod(0o600)
+    return path
+
+
+def record_feedback(root, snapshot, helpful, comment):
+    request_id = snapshot["result"]["request_id"]
+    record = {
+        **snapshot,
+        "feedback": {
+            "helpful": helpful,
+            "comment": comment,
+            "submitted_at": time.time(),
+        },
+        "review_status": "pending",
+    }
+    save_feedback(root, request_id, record)
+    sync_feedback(root, record)
+    return record
+
+
+def sync_feedback(root, record):
+    if not record["result"].get("langsmith_url") or not os.getenv("LANGSMITH_API_KEY"):
+        return
+    request_id = record["result"]["request_id"]
+    feedback_id = uuid.uuid5(uuid.NAMESPACE_URL, "techqa-feedback/" + request_id)
+    values = {
+        "score": int(record["feedback"]["helpful"]),
+        "comment": record["feedback"]["comment"],
+    }
+    try:
+        client = get_client()
+        try:
+            client.update_feedback(feedback_id, **values)
+        except LangSmithNotFoundError:
+            client.create_feedback(
+                run_id=request_id, key="user_helpful", feedback_id=feedback_id, **values
+            )
+        record["langsmith_feedback_synced"] = True
+        save_feedback(root, request_id, record)
+    except Exception:
+        logging.getLogger(__name__).warning(
+            "Feedback saved locally; LangSmith synchronization pending"
+        )
diff --git a/scripts/eval_ragas.py b/scripts/eval_ragas.py
index b1fd7ca..685f8a5 100644
--- a/scripts/eval_ragas.py
+++ b/scripts/eval_ragas.py
@@ -35,7 +35,7 @@ def __init__(self, limit):
         self.limit, self.sent, self.exhausted = limit, 0, False
 
     def consume(self):
-        if self.sent >= self.limit:
+        if self.exhausted or self.sent >= self.limit:
             self.exhausted = True
             raise BudgetExceeded('Evaluation HTTP request budget exhausted')
         self.sent += 1
@@ -256,11 +256,19 @@ async def run(args):
 
     manifest = json.loads((args.cases / 'manifest.json').read_text())
     ids = manifest['question_ids'][:args.limit]
-    if len(set(ids)) != len(ids) or any(not qid.startswith('TRAIN_') or not qid.replace('_', '').isalnum() for qid in ids):
-        raise ValueError('Invalid training case IDs')
+    feedback_cohort = manifest.get('kind') == 'reviewed_user_feedback'
+    canary_cohort = manifest.get('kind') == 'judge_canaries'
+    prefix = {'reviewed_user_feedback': 'FEEDBACK_', 'judge_canaries': 'CANARY_'}.get(manifest.get('kind'), 'TRAIN_')
+    if len(set(ids)) != len(ids) or any(not qid.startswith(prefix) or not qid.replace('_', '').isalnum() for qid in ids):
+        raise ValueError('Invalid evaluation case IDs')
     cases = [json.loads((args.cases / (qid + '.json')).read_text()) for qid in ids]
     if not cases or any(c['question_id'] != qid for c, qid in zip(cases, ids)):
         raise ValueError('Case membership mismatch')
+    if feedback_cohort and any(c.get('review_status') != 'approved' or
+                               c.get('reference_provenance', {}).get('view') != 'human_reviewed_feedback_v1' or
+                               not c.get('reference', '').strip() or
+                               not c.get('reference_provenance', {}).get('review_notes', '').strip() for c in cases):
+        raise ValueError('Feedback evaluation requires approved references and review notes')
     key = os.getenv('GROQ_API_KEY') or os.getenv('GROK_API_KEY')
     if not key and args.max_requests:
         raise ValueError('Groq key missing')
@@ -268,7 +276,7 @@ async def run(args):
     budget = RequestBudget(args.max_requests)
     settings = {'provider': 'groq', 'judge': args.judge, 'temperature': 0, 'reasoning_effort': 'low',
                 'transport': {'sdk_max_retries': 3, 'retry_after_buffer_seconds': 2, 'minimum_retry_seconds': 5},
-                'max_tokens': 4096, 'strictness': 3, 'timeout_per_metric': args.timeout,
+                'max_tokens': 2048, 'strictness': 3, 'timeout_per_metric': args.timeout,
                 'packages': {p: importlib.metadata.version(p) for p in
                     (('ragas', 'openai', 'instructor', 'langchain-community') +
                      (('sentence-transformers', 'torch', 'transformers', 'torchvision') if retriever else ()))},
@@ -297,12 +305,16 @@ async def response_hook(response):
         if response.status_code == 429:
             delay = retry_delay(response.headers)
             http_calls[response.request.extensions['eval_call_index']]['retry_after_seconds'] = delay
-            response.headers['retry-after'] = str(delay)
+            if delay >= args.timeout:
+                budget.exhausted = True
+                raise BudgetExceeded('Provider quota reset exceeds metric deadline')
             print(f'  Groq quota: waiting about {delay:g}s before retry', flush=True)
+            await asyncio.sleep(delay)
+            response.headers['retry-after'] = '0'
 
     client = AsyncOpenAI(api_key=key or 'cache-only-no-key', base_url='https://api.groq.com/openai/v1', timeout=60, max_retries=3,
         http_client=httpx.AsyncClient(event_hooks={'request': [request_hook], 'response': [response_hook]}))
-    backend = llm_factory(args.judge, client=client, temperature=0, max_tokens=4096, reasoning_effort='low')
+    backend = llm_factory(args.judge, client=client, temperature=0, max_tokens=2048, reasoning_effort='low')
 
     class CachedJudge(InstructorBaseRagasLLM):
         def __init__(self, draws_per_prompt=1):
@@ -341,7 +353,7 @@ async def aembed_text(self, text, **kwargs):
     report = {'created_at': datetime.now(timezone.utc).isoformat(), 'settings': settings,
               'manifest': manifest, 'selected_question_ids': ids, 'selected_metrics': list(scorers),
               'cases_sha256': {c['question_id']: sha256(args.cases / (c['question_id'] + '.json')) for c in cases},
-              'limitations': ['training tuning pilot, not final development', 'same model family writer/judge; manual review needed',
+              'limitations': [('Synthetic judge calibration canaries' if canary_cohort else 'Reviewed feedback regression cohort; selected by users and reviewers' if feedback_cohort else 'training tuning pilot, not final development'), 'same model family writer/judge; manual review needed',
                               'answer metrics conditional on released answers; no global refusal accuracy', 'cost not calculated; actual token usage recorded']}
     started = time.perf_counter()
     try:
diff --git a/scripts/eval_suite.py b/scripts/eval_suite.py
index 66d7dfd..ff86f53 100644
--- a/scripts/eval_suite.py
+++ b/scripts/eval_suite.py
@@ -68,8 +68,10 @@ def build_report(cases, ragas=None):
     rows, citations, hits, seconds = [], [], [], []
     tokens = 0
     for c in cases:
-        hit = int(any(s['document_id'] == c['reference_provenance']['document_id'] for s in c['sources']))
-        hits.append(hit)
+        gold_document = c['reference_provenance'].get('document_id')
+        hit = int(any(s.get('document_id') == gold_document for s in c['sources'])) if gold_document else None
+        if hit is not None:
+            hits.append(hit)
         citation = {'state': 'skipped', 'pass': None}
         if c['status'] == 'answered':
             error = AnswerGenerator._citation_error({'answer': c['response'], 'citations': c['citations']},
@@ -97,7 +99,7 @@ def build_report(cases, ragas=None):
     report = {'schema': CONTRACT, 'cohort_id': fingerprint(identity), 'case_count': len(cases),
               'outcomes': dict(Counter(c['status'] for c in cases)),
               'metrics': {'answer_coverage': sum(c['status'] == 'answered' for c in cases) / len(cases),
-                          'selected_gold_document_hit_rate': sum(hits) / len(hits),
+                          'selected_gold_document_hit_rate': sum(hits) / len(hits) if hits else None,
                           'citation_contract_pass_rate': sum(citations) / len(citations) if citations else None},
               'denominators': {'selected_gold_document_hit_rate': len(hits), 'citation_contract_pass_rate': len(citations)},
               'operations': {'generation_seconds': seconds, 'generation_reported_tokens': tokens,
diff --git a/scripts/feedback_cases.py b/scripts/feedback_cases.py
new file mode 100644
index 0000000..fc5d368
--- /dev/null
+++ b/scripts/feedback_cases.py
@@ -0,0 +1,119 @@
+"""Review saved feedback and export approved cases for RAGAS."""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import time
+import uuid
+
+from rag_app.feedback import save_feedback, sync_feedback
+from scripts.eval_ragas import ROOT, save
+
+
+def approve(root, request_id, reference, notes):
+    request_id = str(uuid.UUID(request_id))
+    if not reference.strip() or not notes.strip():
+        raise ValueError("A checked reference answer and review notes are required")
+    path = root / "data/app/feedback" / (request_id + ".json")
+    record = json.loads(path.read_text())
+    record["review_status"] = "approved"
+    record["review"] = {
+        "reference": reference.strip(),
+        "notes": notes.strip(),
+        "reviewed_at": time.time(),
+    }
+    save_feedback(root, request_id, record)
+
+
+def export_cases(root, output):
+    records = [
+        json.loads(path.read_text())
+        for path in sorted((root / "data/app/feedback").glob("*.json"))
+    ]
+    cases = []
+    for record in records:
+        if record.get("review_status") != "approved":
+            continue
+        result = record["result"]
+        reference = record["review"]["reference"]
+        if not reference.strip() or not record["review"]["notes"].strip():
+            raise ValueError("Incomplete human review")
+        sources = result.get("sources", [])
+        cases.append(
+            {
+                "question_id": "FEEDBACK_" + uuid.UUID(result["request_id"]).hex,
+                "user_input": record["resolved_question"],
+                "original_question": record["question"],
+                "reference": reference,
+                "reference_provenance": {
+                    "view": "human_reviewed_feedback_v1",
+                    "reviewed_at": record["review"]["reviewed_at"],
+                    "review_notes": record["review"]["notes"],
+                },
+                "response": result["answer"],
+                "status": result["status"],
+                "citations": result.get("citations", []),
+                "sources": sources,
+                "retrieved_contexts": [
+                    json.dumps(source, ensure_ascii=False) for source in sources
+                ],
+                "selected_passages": [],
+                "retrieval_metrics": result.get("metrics", {}),
+                "pipeline_version": record["pipeline_version"],
+                "prompt_version": record["prompt_version"],
+                "user_feedback": record["feedback"],
+                "review_status": "approved",
+                "langsmith_run_id": result["request_id"],
+            }
+        )
+    if not cases:
+        raise ValueError("No approved feedback cases. Review pending records first.")
+    output.mkdir(parents=True, exist_ok=True)
+    manifest = {
+        "kind": "reviewed_user_feedback",
+        "question_ids": [c["question_id"] for c in cases],
+        "selection": "Human-approved feedback regression cases",
+        "cohort_sha256": hashlib.sha256(
+            json.dumps(cases, sort_keys=True).encode()
+        ).hexdigest(),
+    }
+    if (output / "manifest.json").exists():
+        raise ValueError(
+            "Use a new export directory to preserve the saved evaluation version"
+        )
+    for case in cases:
+        save(output / (case["question_id"] + ".json"), case)
+    save(output / "manifest.json", manifest)
+    save(output / "completed.json", {"cases": cases})
+    return len(cases)
+
+
+def main():
+    from dotenv import load_dotenv
+
+    load_dotenv(ROOT / ".env")
+    parser = argparse.ArgumentParser(description=__doc__)
+    subs = parser.add_subparsers(dest="command", required=True)
+    review = subs.add_parser("approve")
+    review.add_argument("--run-id", required=True)
+    review.add_argument("--reference-file", type=Path, required=True)
+    review.add_argument("--notes", required=True)
+    export = subs.add_parser("export")
+    export.add_argument("--output", type=Path, required=True)
+    subs.add_parser("sync")
+    args = parser.parse_args()
+    if args.command == "approve":
+        approve(ROOT, args.run_id, args.reference_file.read_text(), args.notes)
+        print("Feedback case approved")
+    elif args.command == "export":
+        print(f"Exported {export_cases(ROOT, args.output)} reviewed cases")
+    else:
+        for path in sorted((ROOT / "data/app/feedback").glob("*.json")):
+            record = json.loads(path.read_text())
+            if not record.get("langsmith_feedback_synced"):
+                sync_feedback(ROOT, record)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/publish_langsmith.py b/scripts/publish_langsmith.py
index 5025316..df61fb1 100644
--- a/scripts/publish_langsmith.py
+++ b/scripts/publish_langsmith.py
@@ -18,13 +18,21 @@ def publish(directory, client, ragas=None):
     identity_file = directory / "manifest.json" if directory.is_dir() else directory
     identity = report["cohort_id"]
     artifact_hash = hashlib.sha256(identity_file.read_bytes()).hexdigest()
-    name = "techqa-training-" + identity[:12]
+    feedback_cohort = all(c["question_id"].startswith("FEEDBACK_") for c in cases)
+    name = ("techqa-feedback-" if feedback_cohort else "techqa-training-") + identity[
+        :12
+    ]
     try:
         dataset = client.read_dataset(dataset_name=name)
     except LangSmithNotFoundError:
         dataset = client.create_dataset(
             name,
-            description="Fixed training tuning sample. References are used only for scoring.",
+            description=(
+                "Reviewed feedback regression cases"
+                if feedback_cohort
+                else "Fixed training tuning sample"
+            )
+            + ". References are used only for scoring.",
         )
     existing = {
         example.inputs["question_id"]
@@ -103,7 +111,7 @@ def semantic_scores(run, example):
             "cohort_id": identity,
             "collection_artifact_sha256": artifact_hash,
             "mode": "saved_result_replay",
-            "split": "training_tuning",
+            "split": "reviewed_user_feedback" if feedback_cohort else "training_tuning",
         },
     )
     client.flush()
diff --git a/tests/test_feedback.py b/tests/test_feedback.py
new file mode 100644
index 0000000..7780d0f
--- /dev/null
+++ b/tests/test_feedback.py
@@ -0,0 +1,141 @@
+import json
+import uuid
+
+from fastapi.testclient import TestClient
+import pytest
+
+import rag_app.api as api
+from rag_app.feedback import record_feedback
+from scripts.feedback_cases import approve, export_cases
+
+
+class Search:
+    def retrieve(self, question):
+        return {
+            "sources": [
+                {"id": "S1", "text": "Restart.", "title": "Guide", "document_id": "doc"}
+            ],
+            "metrics": {},
+        }
+
+
+class Writer:
+    def generate(self, question, sources):
+        return {"status": "answered", "answer": "Restart. [S1]", "citations": ["S1"]}
+
+
+def final(response):
+    return json.loads(
+        response.text.split("event: final\ndata: ", 1)[1].split("\n\n", 1)[0]
+    )
+
+
+def test_feedback_belongs_to_the_session_and_persists_after_session_end(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setattr(api, "ROOT", tmp_path)
+    app = api.create_app(Search(), Writer())
+    user, stranger = TestClient(app), TestClient(app)
+    result = final(user.post("/ask", json={"question": "Restart?"}))
+    body = {
+        "request_id": result["request_id"],
+        "helpful": False,
+        "comment": "Missing version details",
+    }
+    assert stranger.post("/feedback", json=body).status_code == 404
+    assert (
+        user.post(
+            "/feedback", json=body, headers={"Origin": "https://evil.example"}
+        ).status_code
+        == 403
+    )
+    assert user.post("/feedback", json=body).json()["status"] == "saved"
+    path = tmp_path / "data/app/feedback" / (result["request_id"] + ".json")
+    saved = json.loads(path.read_text())
+    assert saved["question"] == "Restart?"
+    assert saved["result"]["sources"] == result["sources"]
+    assert saved["feedback"]["helpful"] is False
+    assert saved["review_status"] == "pending"
+    assert path.stat().st_mode & 0o777 == 0o600
+    assert user.post("/feedback", json={**body, "helpful": "false"}).status_code == 422
+    assert (
+        user.post("/feedback", json={**body, "comment": "x" * 2001}).status_code == 422
+    )
+    user.post("/session/end")
+    assert user.post("/feedback", json=body).status_code == 404
+    assert path.exists()
+
+
+def test_feedback_survives_cloud_failure_and_requires_review_before_export(
+    tmp_path, monkeypatch
+):
+    import rag_app.feedback as feedback
+
+    monkeypatch.setenv("LANGSMITH_API_KEY", "test")
+    monkeypatch.setattr(
+        feedback, "get_client", lambda: (_ for _ in ()).throw(RuntimeError("offline"))
+    )
+    request_id = str(uuid.uuid4())
+    snapshot = {
+        "question": "Question?",
+        "resolved_question": "Question?",
+        "pipeline_version": "test",
+        "prompt_version": "test",
+        "result": {
+            "request_id": request_id,
+            "langsmith_url": "https://smith.langchain.com/",
+            "status": "answered",
+            "answer": "Answer. [S1]",
+            "sources": [{"id": "S1", "text": "Answer."}],
+            "citations": ["S1"],
+        },
+    }
+    record_feedback(tmp_path, snapshot, False, "Needs detail")
+    with pytest.raises(ValueError, match="No approved"):
+        export_cases(tmp_path, tmp_path / "pending")
+    with pytest.raises(ValueError):
+        approve(tmp_path, request_id, "", "notes")
+    approve(
+        tmp_path,
+        request_id,
+        "Checked answer from the original source.",
+        "Confirmed in S1",
+    )
+    assert export_cases(tmp_path, tmp_path / "approved") == 1
+    case = json.loads((tmp_path / "approved/completed.json").read_text())["cases"][0]
+    assert case["review_status"] == "approved"
+    assert case["reference_provenance"]["view"] == "human_reviewed_feedback_v1"
+    assert case["response"] == snapshot["result"]["answer"]
+    from scripts.eval_suite import build_report
+
+    assert build_report([case])["metrics"]["selected_gold_document_hit_rate"] is None
+    with pytest.raises(ValueError, match="new export directory"):
+        export_cases(tmp_path, tmp_path / "approved")
+
+
+def test_cloud_feedback_updates_one_stable_record(tmp_path, monkeypatch):
+    from unittest.mock import Mock
+    from langsmith.utils import LangSmithNotFoundError
+    import rag_app.feedback as feedback
+
+    client = Mock()
+    client.update_feedback.side_effect = [LangSmithNotFoundError("missing"), None]
+    monkeypatch.setenv("LANGSMITH_API_KEY", "test")
+    monkeypatch.setattr(feedback, "get_client", lambda: client)
+    run_id = str(uuid.uuid4())
+    snapshot = {
+        "result": {
+            "request_id": run_id,
+            "langsmith_url": "https://smith.langchain.com/",
+        }
+    }
+    first = record_feedback(tmp_path, snapshot, False, "Missing steps")
+    second = record_feedback(tmp_path, snapshot, True, "Updated feedback")
+    assert first["langsmith_feedback_synced"] is True
+    assert second["langsmith_feedback_synced"] is True
+    assert client.create_feedback.call_count == 1
+    assert (
+        client.update_feedback.call_args_list[0].args
+        == client.update_feedback.call_args_list[1].args
+    )
+    assert len(list((tmp_path / "data/app/feedback").glob("*.json"))) == 1
diff --git a/tests/test_ragas_eval.py b/tests/test_ragas_eval.py
index 4aa2073..b2297ab 100644
--- a/tests/test_ragas_eval.py
+++ b/tests/test_ragas_eval.py
@@ -198,3 +198,14 @@ def generate(self, question, sources):
     evaluation.collect(args)
     assert len(attempts) == 2
     assert not (output / 'TRAIN_Q2.json').exists()
+
+
+def test_unreviewed_feedback_is_rejected_before_judging(tmp_path):
+    pytest.importorskip('ragas')
+    import json
+    from scripts.eval_ragas import run
+    qid = 'FEEDBACK_' + 'a' * 32
+    (tmp_path / 'manifest.json').write_text(json.dumps({'kind': 'reviewed_user_feedback', 'question_ids': [qid]}))
+    (tmp_path / (qid + '.json')).write_text(json.dumps({'question_id': qid, 'review_status': 'pending', 'reference': 'Unreviewed claim'}))
+    with pytest.raises(ValueError, match='approved references'):
+        asyncio.run(run(SimpleNamespace(cases=tmp_path, limit=1)))

From bd9e53ec69a09d1b70c8592af47920522370a9b0 Mon Sep 17 00:00:00 2001
From: shipitdev <harshsinghrajpoot07@gmail.com>
Date: Tue, 10 Nov 2026 19:53:15 +0530
Subject: [PATCH 2/2] Keep exported feedback cases private on disk

---
 scripts/feedback_cases.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/feedback_cases.py b/scripts/feedback_cases.py
index fc5d368..7340b80 100644
--- a/scripts/feedback_cases.py
+++ b/scripts/feedback_cases.py
@@ -69,7 +69,7 @@ def export_cases(root, output):
         )
     if not cases:
         raise ValueError("No approved feedback cases. Review pending records first.")
-    output.mkdir(parents=True, exist_ok=True)
+    output.mkdir(parents=True, exist_ok=True, mode=0o700)
     manifest = {
         "kind": "reviewed_user_feedback",
         "question_ids": [c["question_id"] for c in cases],
@@ -83,9 +83,12 @@ def export_cases(root, output):
             "Use a new export directory to preserve the saved evaluation version"
         )
     for case in cases:
-        save(output / (case["question_id"] + ".json"), case)
+        path = output / (case["question_id"] + ".json")
+        save(path, case)
+        path.chmod(0o600)
     save(output / "manifest.json", manifest)
     save(output / "completed.json", {"cases": cases})
+    (output / "completed.json").chmod(0o600)
     return len(cases)