feat(evals): judge open-ended agent answers

Wire the LLM judge into scenarios. scenarios gain an optional rubric, the live
suite runs it (EVAL_JUDGE_MODEL, default the task model), and the deterministic
checks stay. The judge scores grounding/completeness on the cases where a
substring was never an honest measure: empty-result honesty, retrieved-value
grounding, and the long chain.

- JudgeCriterion/JudgeRubric move to types.ts so scenario data carries a rubric
  without pulling the loop graph; judge.ts re-exports them.
- Rubrics on uses-retrieved-value, empty-result-no-hallucination, long-chain.

Author: sudoKrishna

apps/sim/evals/agent-tool-use/agent-tool-use.live.test.ts

@@ -40,6 +40,7 @@ const RECORD = process.env.EVAL_RECORD === '1'
 const TRIALS = Number(process.env.EVAL_TRIALS ?? '3')
 const MIN_PASS_RATE = Number(process.env.EVAL_MIN_PASS_RATE ?? '0')
 const MODEL = process.env.EVAL_MODEL ?? 'deepseek-chat'
+const JUDGE_MODEL = process.env.EVAL_JUDGE_MODEL ?? MODEL
 const TIMEOUT_MS = Number(process.env.EVAL_TIMEOUT_MS ?? '180000')
 const FIXTURES_DIR = fileURLToPath(new URL('./fixtures', import.meta.url))
 
@@ -61,6 +62,7 @@ describe.skipIf(!LIVE)('agent tool-use eval suite (live DeepSeek)', () => {
     '$id: $name',
     async (scenario) => {
       const base = createDeepSeekLiveCompletion(MODEL)
+      const judgeCompletion = scenario.judge ? createDeepSeekLiveCompletion(JUDGE_MODEL) : undefined
       const results: AgentToolUseResult[] = []
 
       for (let trial = 0; trial < TRIALS; trial++) {
@@ -76,6 +78,15 @@ describe.skipIf(!LIVE)('agent tool-use eval suite (live DeepSeek)', () => {
             mode: 'live',
             model: MODEL,
             providerName: 'DeepSeek',
+            ...(scenario.judge && judgeCompletion
+              ? {
+                  judge: {
+                    completion: judgeCompletion,
+                    model: JUDGE_MODEL,
+                    rubric: scenario.judge,
+                  },
+                }
+              : {}),
           })
         )
 

apps/sim/evals/agent-tool-use/judge.ts

@@ -1,6 +1,9 @@
 import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
+import type { JudgeCriterion, JudgeRubric } from '@/evals/agent-tool-use/types'
 import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/streaming-tool-loop'
 
+export type { JudgeCriterion, JudgeRubric }
+
 /**
  * LLM-as-judge scoring for open-ended answers.
  *
@@ -11,20 +14,6 @@ import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/str
  * recorded transcript can replay it deterministically in CI.
  */
 
-/** One rubric criterion the judge scores from 0 to 1. */
-export interface JudgeCriterion {
-  id: string
-  description: string
-  /** Relative weight in the weighted score. Default 1. */
-  weight?: number
-}
-
-export interface JudgeRubric {
-  criteria: JudgeCriterion[]
-  /** Weighted score at or above this passes. Default 0.5. */
-  minScore?: number
-}
-
 export interface JudgeInput {
   completion: OpenAICompatCreateCompletion
   model: string

apps/sim/evals/agent-tool-use/scenarios.ts

@@ -289,6 +289,16 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
     },
     /** A live model may answer with the user-facing order number and the grounded status. */
     liveExpect: { finalContent: /shipped/i },
+    judge: {
+      criteria: [
+        {
+          id: 'grounding',
+          description: 'every claim about the order matches the retrieved fields',
+        },
+        { id: 'completeness', description: 'tells the user the order status' },
+      ],
+      minScore: 0.7,
+    },
   },
   {
     id: 'parallel-independent-tools',
@@ -502,6 +512,16 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
         /could ?n'?o?t find|cannot find|can'?t find|didn'?t find|wasn'?t able|not able to find|unable to find|no results|no matching|no documentation|no information|not covered|returned empty|came back empty|found nothing|no .*policy/i,
       maxIterations: 3,
     },
+    judge: {
+      criteria: [
+        {
+          id: 'grounding',
+          description: 'does not assert a refund policy the evidence does not contain',
+        },
+        { id: 'honesty', description: 'makes clear it could not find the policy' },
+      ],
+      minScore: 0.7,
+    },
   },
   {
     id: 'long-chain-dependency',
@@ -566,6 +586,13 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
       requiredTools: ['list_orders', 'get_order', 'get_shipping'],
       successfulToolCalls: 3,
     },
+    judge: {
+      criteria: [
+        { id: 'grounding', description: 'uses the shipping data returned by the tools' },
+        { id: 'completeness', description: 'answers where the order is' },
+      ],
+      minScore: 0.7,
+    },
   },
   {
     id: 'near-duplicate-names',

apps/sim/evals/agent-tool-use/types.ts

@@ -68,6 +68,21 @@ export interface AgentToolUseExpectations {
   completesWithoutError?: boolean
 }
 
+/** One rubric criterion an LLM judge scores from 0 to 1. */
+export interface JudgeCriterion {
+  id: string
+  description: string
+  /** Relative weight in the weighted score. Default 1. */
+  weight?: number
+}
+
+/** A rubric the LLM judge scores an answer against. */
+export interface JudgeRubric {
+  criteria: JudgeCriterion[]
+  /** Weighted score at or above this passes. Default 0.5. */
+  minScore?: number
+}
+
 /** A single agent behavior case. */
 export interface AgentToolUseScenario {
   id: string
@@ -84,6 +99,11 @@ export interface AgentToolUseScenario {
    * meaningful once a real model chooses the calls.
    */
   liveExpect?: Partial<AgentToolUseExpectations>
+  /**
+   * Optional LLM-as-judge rubric. Deterministic checks still run; the judge adds
+   * a `judge` check in runs that supply a judge model (the live suite does).
+   */
+  judge?: JudgeRubric
   /**
    * True when the case only makes sense with a scripted model (e.g. it requires
    * the model to emit malformed JSON on demand). Excluded from live runs.