Collect user feedback and export reviewed RAGAS regression cases

.gitignore

@@ -134,3 +134,9 @@
 !/tests/test_langsmith_export.py
 
 !/evaluations/results/observability_v1.json
+
+!/rag_app/feedback.py
+!/tests/test_feedback.py
+!/scripts/feedback_cases.py
+!/frontend/src/app/api/feedback/
+!/frontend/src/app/api/feedback/route.ts

README.md

@@ -42,6 +42,22 @@ Publishing replays saved outputs into a LangSmith experiment with response statu
 
 The [November 10 observability batch](evaluations/results/observability_v1.json) selected **12 training questions**. Retrieval selected the labeled article for **11/12**. **3 answers** were released and passed citation-format checks. **9 provider quota failures** prevented completion. The run used **15 HTTP attempts and 33,230 reported tokens**. This batch establishes dashboard visibility and exposes a quota-handling issue, which is now covered by a stop-on-quota regression test. New semantic RAGAS scores await available provider quota.
 
+## Feedback and future evaluation cases
+
+Every completed response offers Helpful and Not helpful, with an optional explanation. Sending feedback saves the question, resolved follow-up, answer, sources, private diagnostics and pipeline versions in `data/app/feedback/`. These private records persist after the session ends. A stable feedback record also attaches the rating and comment to the LangSmith trace. If cloud synchronization fails, the local record remains available and `python -m scripts.feedback_cases sync` retries it.
+
+Review the saved case against its sources, write a checked reference answer in a local text file, then approve and export it.
+
+```bash
+python -m scripts.feedback_cases approve --run-id RUN_UUID \
+  --reference-file /path/to/checked-answer.txt --notes "Verified against the original source"
+python -m scripts.feedback_cases export --output data/ragas/feedback_review_v1
+python -m scripts.eval_ragas run --cases data/ragas/feedback_review_v1 \
+  --metrics faithfulness factual_correctness answer_completeness --limit 5 --max-requests 20
+```
+
+Approval and review notes are required for feedback-based RAGAS evaluation. Use these cases as a regression set alongside the fixed training sample. Human votes and comments identify examples for review, while checked references support correctness scoring. Human ratings can also help assess and improve judge agreement. Each export uses a new directory to preserve the earlier evaluation version.
+
 ## Evaluation suite
 
 ![Evaluation layers: components, pipeline, application, regression and planned monitoring](evaluations/overview.svg)
@@ -54,7 +70,7 @@ The [November 10 observability batch](evaluations/results/observability_v1.json)
 | Pipeline | Selected context precision/recall, answer faithfulness and relevancy | [RAGAS runner](scripts/eval_ragas.py), measured on the pilot below. Reference text stays outside inference. |
 | Application | Reference-based correctness and completeness; citation validity; withholding and layout contracts; recorded timing/tokens/cache use | Correctness = RAGAS `FactualCorrectness(mode="f1")`; completeness = its `mode="recall"` proxy. Both are opt-in and not yet benchmarked. [Free saved-case checks](scripts/eval_suite.py), [API contracts](tests/test_app_api.py), [layout checks](frontend/tests/genui-contract.test.mjs) are implemented. |
 | Regression | Re-run a fixed case set, then inspect per-case differences | [Comparison CLI](scripts/eval_suite.py) rejects changed cohorts/references. Semantic comparisons also require matching judge settings and exclude scores flagged for review. |
-| Online | Sampled live judging, feedback, drift and failure-to-test-set promotion | **Planned.** Content-free operational logs exist; live quality scoring, Langfuse integration and automated feedback loops do not. |
+| Online | User feedback, reviewed regression cases, sampled judging and drift | Helpful/unhelpful votes and optional comments save the run for review and sync to LangSmith. Reviewed references can be exported for RAGAS. Sampled live judging and drift monitoring remain planned. |
 
 Style rubrics and broad toxicity/PII/jailbreak evaluation are also planned. Existing origin/session, withholding and generated-component checks cover specific application contracts; they are not a comprehensive safety benchmark. Optional semantic metrics compare against available references, which may omit valid alternative answers.
 

frontend/src/app/api/feedback/route.ts

@@ -0,0 +1,22 @@
+import { isAllowedOrigin } from "@/lib/request-origin";
+
+export async function POST(request: Request) {
+  if (!isAllowedOrigin(request)) return Response.json({ message: "Cross-origin requests are not allowed." }, { status: 403 });
+  if (Number(request.headers.get("content-length") || 0) > 16_000) return Response.json({ message: "Feedback is too large." }, { status: 413 });
+  let body: string;
+  try { body = await request.text(); }
+  catch { return Response.json({ message: "Feedback could not be read." }, { status: 400 }); }
+  if (body.length > 16_000) return Response.json({ message: "Feedback is too large." }, { status: 413 });
+  try {
+    const upstream = await fetch(new URL("feedback", process.env.RAG_API_URL || "http://127.0.0.1:8000/ask"), {
+      method: "POST",
+      headers: { "content-type": "application/json", ...(request.headers.get("cookie") ? { cookie: request.headers.get("cookie")! } : {}) },
+      body,
+      cache: "no-store",
+      signal: AbortSignal.timeout(10_000),
+    });
+    return new Response(upstream.body, { status: upstream.status, headers: { "content-type": "application/json", "cache-control": "no-store" } });
+  } catch {
+    return Response.json({ message: "Feedback could not be saved. Please retry." }, { status: 502 });
+  }
+}

frontend/src/app/globals.css

@@ -238,3 +238,14 @@ body { --body: var(--font-geist), Arial, sans-serif; --mono: var(--font-geist-mo
 @media (min-width: 901px) {
   .rail-label { position: absolute; width: 1px; height: 1px; margin: -1px; padding: 0; overflow: hidden; clip: rect(0, 0, 0, 0); white-space: nowrap; }
 }
+.answer-feedback { margin-top: 28px; padding-top: 20px; border-top: 1px solid var(--border); font-size: 13px; }
+.answer-feedback fieldset { min-width: 0; border: 0; margin: 0; padding: 0; }
+.answer-feedback legend { margin-bottom: 12px; font-weight: 500; }
+.feedback-choices { display: flex; gap: 8px; margin-bottom: 16px; }
+.feedback-choices button { min-height: 36px; padding: 0 12px; border: 1px solid #d4d4d4; border-radius: 6px; background: white; }
+.feedback-choices button[aria-pressed="true"] { border-color: #141414; background: #eee; }
+.answer-feedback label { display: block; margin-bottom: 8px; }
+.answer-feedback label span, .feedback-note { color: #626262; }
+.answer-feedback textarea { width: 100%; padding: 10px 12px; border: 1px solid #d4d4d4; border-radius: 6px; resize: vertical; background: white; }
+.feedback-note { font-size: 12px; }
+.answer-feedback button:disabled { opacity: .6; cursor: default; }

frontend/src/components/workspace.tsx

@@ -19,6 +19,14 @@ type FinalEvent = {
 type Result = FinalEvent & { question: string; runId: number; genUiValid?: boolean };
 type ServiceStatus = { api: string; presentation_status: string };
 
+type FeedbackState = {
+  choice: boolean | null;
+  comment: string;
+  state: "idle" | "saving" | "saved" | "error";
+  message: string;
+};
+const emptyFeedback: FeedbackState = { choice: null, comment: "", state: "idle", message: "" };
+
 function validSource(value: unknown): value is VerifiedSource {
   if (!value || typeof value !== "object") return false;
   const source = value as Record<string, unknown>;
@@ -102,6 +110,7 @@ async function* readEvents(response: Response) {
 }
 
 export default function Workspace() {
+  const [feedback, setFeedback] = useState<FeedbackState>(emptyFeedback);
   const [elapsed, setElapsed] = useState(0);
   const [question, setQuestion] = useState("");
   const [busy, setBusy] = useState(false);
@@ -129,6 +138,7 @@ export default function Workspace() {
       runCounter.current += 1;
       setQuestion("");
       setResult(null);
+    setFeedback(emptyFeedback);
       setStages([]);
       setRetrievedSources([]);
       setError("");
@@ -162,6 +172,7 @@ export default function Workspace() {
     setFailedRun(null);
     setOpenUiMessage("");
     setResult(null);
+    setFeedback(emptyFeedback);
     setStages([]);
     setRetrievedSources([]);
     setQuestion(submitted);
@@ -228,6 +239,25 @@ export default function Workspace() {
     }
   }
 
+  async function submitFeedback(event: FormEvent<HTMLFormElement>) {
+    event.preventDefault();
+    if (!result?.request_id || feedback.choice === null || feedback.state === "saving") return;
+    const runId = result.runId;
+    setFeedback((current) => ({ ...current, state: "saving", message: "" }));
+    try {
+      const response = await fetch("/api/feedback", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ request_id: result.request_id, helpful: feedback.choice, comment: feedback.comment }),
+        signal: AbortSignal.timeout(10_000),
+      });
+      if (!response.ok) throw new Error("Feedback could not be saved. Please retry.");
+      if (runCounter.current === runId) setFeedback((current) => ({ ...current, state: "saved", message: "Feedback saved for review." }));
+    } catch {
+      if (runCounter.current === runId) setFeedback((current) => ({ ...current, state: "error", message: "Feedback could not be saved. Please retry." }));
+    }
+  }
+
   function stopRequest(event: MouseEvent<HTMLButtonElement>) {
     // Cancellation can switch this same button back to submit during the click.
     event.preventDefault();
@@ -248,6 +278,7 @@ export default function Workspace() {
       runCounter.current += 1;
       setQuestion("");
       setResult(null);
+    setFeedback(emptyFeedback);
       setStages([]);
       setRetrievedSources([]);
       setError("");
@@ -379,6 +410,33 @@ export default function Workspace() {
                     <p>{result.status === "needs_clarification" ? "Refine your question with the missing system, version, or context and ask again." : "Try a narrower question or check that the relevant documents are in this knowledge base."}</p>
                   </div>
                 )}
+                {result.request_id && (
+                  <form className="answer-feedback" onSubmit={submitFeedback} aria-busy={feedback.state === "saving"}>
+                    <fieldset disabled={feedback.state === "saving"}>
+                      <legend>Was this answer helpful?</legend>
+                      <div className="feedback-choices">
+                        {[true, false].map((choice) => (
+                          <button key={String(choice)} type="button" aria-pressed={feedback.choice === choice}
+                            onClick={() => setFeedback((current) => ({ ...current, choice, state: "idle", message: "" }))}>
+                            {choice ? "Helpful" : "Not helpful"}
+                          </button>
+                        ))}
+                      </div>
+                      {feedback.choice !== null && (
+                        <>
+                          <label htmlFor="feedback-comment">{feedback.choice ? "Anything we should improve?" : "What went wrong?"} <span>(optional)</span></label>
+                          <textarea id="feedback-comment" maxLength={2000} rows={3} value={feedback.comment}
+                            onChange={(event) => setFeedback((current) => ({ ...current, comment: event.target.value, state: "idle", message: "" }))} />
+                          <p className="feedback-note">Sending feedback saves this question, answer, sources and run details for review.</p>
+                          <button type="submit" className="secondary-button" disabled={feedback.state === "saved"}>
+                            {feedback.state === "saving" ? "Saving…" : "Send feedback"}
+                          </button>
+                        </>
+                      )}
+                    </fieldset>
+                    {feedback.message && <p role={feedback.state === "error" ? "alert" : "status"}>{feedback.message}</p>}
+                  </form>
+                )}
               </article>
             )}
 

rag_app/api.py

@@ -13,7 +13,7 @@
 from collections import defaultdict, deque
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.responses import StreamingResponse
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ConfigDict
 from dotenv import load_dotenv
 from rag_app.state import SessionStore, AnswerCache
 from rag_app.observability import persist_run
@@ -28,14 +28,23 @@ class Ask(BaseModel):
     regenerate: bool = False
 
 
+class Feedback(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    request_id: uuid.UUID
+    helpful: bool = Field(strict=True)
+    comment: str = Field(default='', max_length=2000)
+
+
 def create_app(retriever=None, generator=None, resolver=None):
     app = FastAPI(title='TechQA evidence workspace')
     serial = threading.Lock()
     admission = threading.BoundedSemaphore(5)
     sessions = SessionStore()
     answers = AnswerCache()
+    feedback_runs = AnswerCache(ttl_seconds=1800, max_entries=256, max_bytes=32 * 1024**2)
     limits = defaultdict(deque)
     limiter_lock = threading.Lock()
+    feedback_lock = threading.Lock()
     runtime = {'retriever': retriever, 'generator': generator, 'resolver': resolver}
 
     def pipeline_version():
@@ -110,12 +119,29 @@ def end(request: Request):
         if origin and origin != allowed:
             raise HTTPException(403, 'Origin not allowed')
         sid = request.cookies.get('techqa_session', '')
-        sessions.end(sid); answers.clear_session(sid)
+        sessions.end(sid); answers.clear_session(sid); feedback_runs.clear_session(sid)
         from fastapi.responses import JSONResponse
         response = JSONResponse({'status': 'cleared'})
         response.delete_cookie('techqa_session')
         return response
 
+    @app.post('/feedback')
+    def feedback(body: Feedback, request: Request):
+        origin = request.headers.get('origin')
+        if origin and origin != os.getenv('APP_ORIGIN', 'http://localhost:3000'):
+            raise HTTPException(403, 'Origin not allowed')
+        session = sessions.get(request.cookies.get('techqa_session', ''))
+        snapshot = feedback_runs.get(session.id, str(body.request_id), session_expires_at=session.expires_at) if session else None
+        if snapshot is None:
+            raise HTTPException(404, 'This run is unavailable in the current session')
+        from rag_app.feedback import record_feedback
+        try:
+            with feedback_lock:
+                record = record_feedback(ROOT, snapshot, body.helpful, body.comment.strip())
+        except OSError:
+            raise HTTPException(503, 'Feedback could not be saved. Please retry.') from None
+        return {'status': 'saved', 'review_status': record['review_status']}
+
     @app.post('/ask')
     async def ask(body: Ask, request: Request):
         if not body.question.strip():
@@ -268,6 +294,11 @@ def run_worker():
                         pass
                     result['metrics']['total_ms'] = round((time.monotonic() - started) * 1000, 2)
                     root_run.end(outputs=result)
+                feedback_runs.put(session.id, rid, {
+                    'question': body.question, 'resolved_question': question, 'result': result,
+                    'pipeline_version': identity['pipeline'], 'prompt_version': PROMPT_VERSION,
+                    'model': MODEL, 'index': identity['index'], 'private_trace': private_trace,
+                }, session_expires_at=session.expires_at)
                 emit('final', result)
                 write_metrics({'request_id': rid, 'status': result['status'], 'metrics': result['metrics']},private_trace)
                 completed = True

rag_app/feedback.py

@@ -0,0 +1,61 @@
+"""Durable, private user feedback and its associated run snapshot."""
+
+import logging
+import os
+from pathlib import Path
+import time
+import uuid
+
+from langsmith.utils import LangSmithNotFoundError
+from rag_app.tracing import get_client
+from rag_app.indexes import atomic_json
+
+
+def save_feedback(root, request_id, record):
+    directory = Path(root) / "data/app/feedback"
+    directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+    path = directory / (str(uuid.UUID(request_id)) + ".json")
+    atomic_json(path, record)
+    path.chmod(0o600)
+    return path
+
+
+def record_feedback(root, snapshot, helpful, comment):
+    request_id = snapshot["result"]["request_id"]
+    record = {
+        **snapshot,
+        "feedback": {
+            "helpful": helpful,
+            "comment": comment,
+            "submitted_at": time.time(),
+        },
+        "review_status": "pending",
+    }
+    save_feedback(root, request_id, record)
+    sync_feedback(root, record)
+    return record
+
+
+def sync_feedback(root, record):
+    if not record["result"].get("langsmith_url") or not os.getenv("LANGSMITH_API_KEY"):
+        return
+    request_id = record["result"]["request_id"]
+    feedback_id = uuid.uuid5(uuid.NAMESPACE_URL, "techqa-feedback/" + request_id)
+    values = {
+        "score": int(record["feedback"]["helpful"]),
+        "comment": record["feedback"]["comment"],
+    }
+    try:
+        client = get_client()
+        try:
+            client.update_feedback(feedback_id, **values)
+        except LangSmithNotFoundError:
+            client.create_feedback(
+                run_id=request_id, key="user_helpful", feedback_id=feedback_id, **values
+            )
+        record["langsmith_feedback_synced"] = True
+        save_feedback(root, request_id, record)
+    except Exception:
+        logging.getLogger(__name__).warning(
+            "Feedback saved locally; LangSmith synchronization pending"
+        )