Grade benchmark mechanics with verified reference resolution

Author: icecrasher321

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-detail.tsx

@@ -143,7 +143,7 @@ function BenchmarkEditor({
       <BenchmarkStep
         number={3}
         title='Fill in the blanks'
-        description='A fresh reader fills the blanks by reading or searching the generated plan. It has no enterprise access or prior memory.'
+        description='A fresh reader fills the blanks from the generated plan and may look up references already named there. Lookups cannot supply missing workflow rules. The reader never sees expected answers or the completed workspace.'
         pending={stage === 'reconstruct'}
         action={
           <Chip

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-results.tsx

@@ -43,6 +43,14 @@ export function BenchmarkResults({
                   {review ? ' · Human reviewed' : ''}
                 </span>
               )}
+              {!result && answer?.evidenceError && (
+                <div>
+                  <dt className='text-[var(--text-muted)]'>Evidence needs review</dt>
+                  <dd className='mt-1 whitespace-pre-wrap break-words text-[var(--text-body)]'>
+                    {answer.evidenceError}
+                  </dd>
+                </div>
+              )}
             </div>
             <dl className='flex flex-col gap-3 text-small'>
               {showGrade && (
@@ -54,7 +62,7 @@ export function BenchmarkResults({
                 </div>
               )}
               <div>
-                <dt className='text-[var(--text-muted)]'>Answer from the generated spec</dt>
+                <dt className='text-[var(--text-muted)]'>Recovered answer</dt>
                 <dd className='mt-1 whitespace-pre-wrap break-words text-[var(--text-body)]'>
                   {answer?.answer || 'No answer returned'}
                 </dd>
@@ -67,10 +75,37 @@ export function BenchmarkResults({
                   {answer?.support || 'No supporting passage'}
                 </dd>
               </div>
+              {answer?.sources?.map((source, index) => (
+                <div key={`${source.citationId}-${index}`}>
+                  <dt className='text-[var(--text-muted)]'>
+                    Resolved reference ·{' '}
+                    {source.url ? (
+                      <a
+                        href={source.url}
+                        target='_blank'
+                        rel='noopener noreferrer'
+                        className='underline'
+                      >
+                        {source.title || source.citationId}
+                      </a>
+                    ) : (
+                      source.title || source.citationId
+                    )}
+                  </dt>
+                  <dd className='mt-1 whitespace-pre-wrap break-words text-[var(--text-body)]'>
+                    {source.quote}
+                  </dd>
+                </div>
+              ))}
               {result && (
                 <div>
                   <dt className='text-[var(--text-muted)]'>
                     AI assessment
+                    {result.basis === 'missing'
+                      ? ' · Missing from plan'
+                      : result.basis === 'reference'
+                        ? ' · Reference resolved'
+                        : ''}
                     {review ? ` · ${result.correct ? 'Recovered' : 'Not recovered'}` : ''}
                   </dt>
                   <dd className='mt-1 whitespace-pre-wrap break-words text-[var(--text-body)]'>

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-run-comparison.tsx

@@ -30,6 +30,11 @@ function RunSnapshot({
           <p className='mt-1 text-[var(--text-muted)] text-small tabular-nums'>
             {Math.round((run.correct / run.total) * 100)}% · {run.correct} of {run.total} recovered
           </p>
+          <p className='mt-1 text-[var(--text-muted)] text-small'>
+            {run.artifacts.recoveryMode === 'references'
+              ? 'Spec with reference resolution'
+              : 'Spec only'}
+          </p>
           {run.reviewedCount > 0 && (
             <p className='mt-1 text-[var(--text-muted)] text-small'>
               {run.reviewedCount} human overrides · AI score{' '}
@@ -108,7 +113,7 @@ export function BenchmarkRunComparison({
         <p role='status' className='text-[var(--text-body)] text-small'>
           {comparable
             ? `${delta > 0 ? '+' : ''}${Number(delta.toFixed(1))} percentage points vs baseline · ${improved} details improved · ${regressed} regressed`
-            : 'These runs used different task briefs or reference details. Their scores are not directly comparable.'}
+            : 'These runs used different inputs or recovery methods. Their scores are not directly comparable.'}
         </p>
       )}
       {(run.reviewedCount > 0 || (baseline?.reviewedCount ?? 0) > 0) && (

apps/sim/lib/benchmarks/application/run-stage.ts

@@ -16,6 +16,7 @@ import {
 } from '@/lib/benchmarks/artifacts'
 import { getBenchmarkMothershipUrl } from '@/lib/benchmarks/config'
 import { gradeReconstruction, validateReconstruction } from '@/lib/benchmarks/evaluation'
+import { verifyRecoveryEvidence } from '@/lib/benchmarks/evidence'
 import {
   distillationMessages,
   gradingMessages,
@@ -35,6 +36,7 @@ import {
   benchmarkBriefSchema,
   benchmarkGradeSchema,
   benchmarkReconstructionSchema,
+  benchmarkSourceSchema,
   benchmarkSpecSchema,
 } from '@/lib/benchmarks/types'
 import { executeBenchmarkJson, executeBenchmarkPlan } from '@/lib/benchmarks/worker'
@@ -58,9 +60,19 @@ const redactionSchema = z
   })
   .strict()
 const reconstructionSchema = z
-  .object({ answers: z.array(benchmarkReconstructionSchema).min(1) })
+  .object({
+    answers: z
+      .array(
+        benchmarkReconstructionSchema.pick({ id: true, answer: true, support: true }).extend({
+          sources: z.array(benchmarkSourceSchema.pick({ citationId: true, quote: true })),
+        })
+      )
+      .min(1),
+  })
+  .strict()
+const gradingSchema = z
+  .object({ judgments: z.array(benchmarkGradeSchema.required({ basis: true })).min(1) })
   .strict()
-const gradingSchema = z.object({ judgments: z.array(benchmarkGradeSchema).min(1) }).strict()
 
 interface RunBenchmarkStageInput {
   organizationId: string
@@ -97,7 +109,7 @@ async function performStage(
   const artifacts = benchmark.artifacts
   switch (stage) {
     case 'distill': {
-      const result = await executeBenchmarkJson({
+      const { data: result } = await executeBenchmarkJson({
         principal,
         benchmark,
         signal,
@@ -108,7 +120,7 @@ async function performStage(
       return { artifacts: applyBenchmarkPatch(artifacts, result), plannerChatId: null }
     }
     case 'redact': {
-      const result = await executeBenchmarkJson({
+      const { data: result } = await executeBenchmarkJson({
         principal,
         benchmark,
         signal,
@@ -135,19 +147,26 @@ async function performStage(
       }
     }
     case 'reconstruct': {
-      const result = await executeBenchmarkJson({
+      const { data: result, toolCalls } = await executeBenchmarkJson({
         principal,
         benchmark,
         signal,
         schema: reconstructionSchema,
         messages: reconstructionMessages(artifacts.redactedSpec),
-        profile: { stage: 'reconstruct', spec: artifacts.generatedSpec! },
+        profile: { stage: 'resolve', spec: artifacts.generatedSpec! },
       })
       validateReconstruction(artifacts.blanks, result.answers)
-      return { artifacts: { ...artifacts, reconstruction: result.answers, grade: null } }
+      const reconstruction = verifyRecoveryEvidence(
+        artifacts.generatedSpec!,
+        result.answers,
+        toolCalls
+      )
+      return {
+        artifacts: { ...artifacts, recoveryMode: 'references', reconstruction, grade: null },
+      }
     }
     case 'grade': {
-      const result = await executeBenchmarkJson({
+      const { data: result } = await executeBenchmarkJson({
         principal,
         benchmark,
         signal,

apps/sim/lib/benchmarks/evaluation.test.ts

@@ -43,6 +43,19 @@ describe('benchmark reconstruction grading', () => {
     expect(result.every(({ correct }) => !correct)).toBe(true)
   })
 
+  it('keeps an answer available for review without crediting unverified source evidence', () => {
+    const result = gradeReconstruction({
+      blanks,
+      reconstruction: reconstructed.map((answer) => ({
+        ...answer,
+        evidenceError: 'Source quote was not retrieved.',
+      })),
+      generatedSpec: 'Support owns the escalation.',
+      judgments: blanks.map(({ id }) => ({ id, correct: true, reason: 'Equivalent answer.' })),
+    })
+    expect(result[0]).toMatchObject({ correct: false, reason: 'Source quote was not retrieved.' })
+  })
+
   it('rejects an incomplete judge response instead of silently passing ungraded answers', () => {
     expect(() =>
       gradeReconstruction({
@@ -53,4 +66,29 @@ describe('benchmark reconstruction grading', () => {
       })
     ).toThrow()
   })
+
+  it('rejects a missing mechanic even when an external source supplies the expected answer', () => {
+    const spec = 'Use the Sim repository for issue intake.'
+    const result = gradeReconstruction({
+      blanks: [{ id: 'handoff', answer: 'Engineering accepts the case' }],
+      reconstruction: [
+        {
+          id: 'handoff',
+          answer: 'Engineering accepts the case',
+          support: spec,
+          sources: [{ citationId: 'repo', quote: 'Engineering accepts the case' }],
+        },
+      ],
+      generatedSpec: spec,
+      judgments: [
+        {
+          id: 'handoff',
+          correct: true,
+          basis: 'missing',
+          reason: 'The repository document describes the handoff, but the plan omits it.',
+        },
+      ],
+    })
+    expect(result[0].correct).toBe(false)
+  })
 })

apps/sim/lib/benchmarks/evaluation.ts

@@ -25,7 +25,7 @@ export function validateReconstruction(
   requireMatchingIds(blanks, values)
 }
 
-/** Correct guesses earn credit only when the reader quotes the submitted plan as evidence. */
+/** Source lookups may resolve references, but cannot supply mechanics absent from the plan. */
 export function gradeReconstruction(input: {
   blanks: BenchmarkArtifacts['blanks']
   reconstruction: Reconstruction
@@ -39,6 +39,7 @@ export function gradeReconstruction(input: {
   return input.blanks.map(({ id }) => {
     const answer = answers.get(id)!
     const judgment = judgments.get(id)!
+    if (answer.evidenceError) return { ...judgment, correct: false, reason: answer.evidenceError }
     if (
       !answer.answer.trim() ||
       !answer.support.trim() ||
@@ -50,6 +51,14 @@ export function gradeReconstruction(input: {
         reason: 'No exact supporting passage was provided from the generated spec.',
       }
     }
+    if (judgment.basis === 'missing') return { ...judgment, correct: false }
+    if (judgment.basis === 'reference' && !answer.sources?.length) {
+      return {
+        ...judgment,
+        correct: false,
+        reason: 'No verified source evidence was provided for this reference.',
+      }
+    }
     return judgment
   })
 }

apps/sim/lib/benchmarks/evidence.test.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it } from 'vitest'
+import { verifyRecoveryEvidence } from '@/lib/benchmarks/evidence'
+import type { ToolCallSummary } from '@/lib/mothership/request/types'
+
+describe('benchmark source provenance', () => {
+  const support = 'Use the Sim repository for issue intake.'
+  const answer = {
+    id: 'repository',
+    answer: 'simstudioai/sim',
+    support,
+    sources: [{ citationId: 'repo', quote: 'simstudioai/sim' }],
+  }
+  const call: ToolCallSummary = {
+    id: 'lookup',
+    name: 'search_workspace',
+    status: 'success',
+    params: { specPassage: support, query: 'Sim repository' },
+    result: {
+      success: true,
+      data: {
+        results: [
+          {
+            citationId: 'repo',
+            content: 'Repository simstudioai/sim',
+            citationUrl: 'https://github.com/simstudioai/sim',
+            documentName: 'Sim',
+          },
+        ],
+      },
+    },
+  }
+
+  it('attaches the actual source URL only to quotations retrieved for the same spec passage', () => {
+    const [verified] = verifyRecoveryEvidence(support, [answer], [call])
+    expect(verified.sources).toEqual([
+      { ...answer.sources[0], url: 'https://github.com/simstudioai/sim', title: 'Sim' },
+    ])
+  })
+
+  it.each([
+    { ...call, status: 'error' as const },
+    { ...call, name: 'sim_cli' },
+    {
+      ...call,
+      params: { specPassage: 'Use the Mothership repository.', query: 'Mothership repository' },
+    },
+    {
+      ...call,
+      result: {
+        success: true,
+        data: { results: [{ citationId: 'different', content: 'simstudioai/sim' }] },
+      },
+    },
+    {
+      ...call,
+      result: {
+        success: true,
+        data: { results: [{ citationId: 'repo', content: 'simstudioai/mothership' }] },
+      },
+    },
+  ])(
+    'preserves the answer for review but rejects failed, unrelated, or fabricated evidence',
+    (toolCall) => {
+      const [verified] = verifyRecoveryEvidence(support, [answer], [toolCall])
+      expect(verified.answer).toBe(answer.answer)
+      expect(verified.sources).toEqual([])
+      expect(verified.evidenceError).toContain('repo')
+    }
+  )
+
+  it('does not turn an unsafe retrieved URL into a clickable source', () => {
+    const [verified] = verifyRecoveryEvidence(
+      support,
+      [answer],
+      [
+        {
+          ...call,
+          result: {
+            success: true,
+            data: {
+              results: [
+                {
+                  citationId: 'repo',
+                  content: 'simstudioai/sim',
+                  citationUrl: 'javascript:alert(1)',
+                },
+              ],
+            },
+          },
+        },
+      ]
+    )
+    expect(verified.sources?.[0].url).toBeUndefined()
+  })
+})