feat(evals): record judge identity and refuse cross-identity comparisons

A judge score is only comparable when the evaluator is the same. Every verdict
now carries { model, rubricDigest, parserVersion, temperature }, rubricDigest
canonicalizes the rubric, and compareJudgeIdentities reports which fields differ
so a delta across a changed evaluator is insufficient evidence, not improvement.

- judge.ts: JudgeIdentity/JudgeScore/JudgeVerdict split, rubricDigest, compare
- judge.test.ts: digest stability/change + comparison guard + identity in verdict
- harness.ts: the judge check shows the judge model and rubric digest

Author: sudoKrishna

apps/sim/evals/agent-tool-use/harness.ts

@@ -494,7 +494,7 @@ export async function runScenario(
       checks.push({
         name: 'judge',
         passed: verdict.passed,
-        detail: `weighted ${verdict.weightedScore.toFixed(2)}; ${verdict.rationale}`,
+        detail: `weighted ${verdict.weightedScore.toFixed(2)} [judge ${verdict.identity.model} ${verdict.identity.rubricDigest}]; ${verdict.rationale}`,
       })
     } catch (error) {
       checks.push({ name: 'judge', passed: false, detail: `judge failed: ${String(error)}` })

apps/sim/evals/agent-tool-use/judge.test.ts

@@ -1,6 +1,11 @@
 import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
 import { describe, expect, it } from 'vitest'
-import { judgeAnswer, parseJudgeVerdict } from '@/evals/agent-tool-use/judge'
+import {
+  compareJudgeIdentities,
+  judgeAnswer,
+  parseJudgeVerdict,
+  rubricDigest,
+} from '@/evals/agent-tool-use/judge'
 import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/streaming-tool-loop'
 
 function completionReturning(text: string): OpenAICompatCreateCompletion {
@@ -96,3 +101,56 @@ describe('parseJudgeVerdict', () => {
     expect(() => parseJudgeVerdict('I think it is fine.', rubric)).toThrow('JSON object')
   })
 })
+
+describe('judge identity', () => {
+  it('records the model, parser version, and a digest of the rubric', async () => {
+    const verdict = await judgeAnswer({
+      completion: completionReturning(
+        '{"scores":{"grounding":1,"completeness":1},"rationale":"good"}'
+      ),
+      model: 'deepseek-chat',
+      userMessage: 'q',
+      answer: 'a',
+      rubric,
+    })
+    expect(verdict.identity).toMatchObject({ model: 'deepseek-chat', parserVersion: '1' })
+    expect(verdict.identity.rubricDigest).toBe(rubricDigest(rubric))
+  })
+
+  it('digests the rubric independently of key order', () => {
+    const reordered = {
+      minScore: 0.7,
+      criteria: [
+        { description: 'every claim is supported by the evidence', id: 'grounding' },
+        { description: 'answers the user request', id: 'completeness' },
+      ],
+    }
+    expect(rubricDigest(reordered)).toBe(rubricDigest(rubric))
+  })
+
+  it('changes the digest when a criterion changes', () => {
+    const changed = {
+      criteria: [
+        { id: 'grounding', description: 'changed' },
+        { id: 'completeness', description: 'answers the user request' },
+      ],
+    }
+    expect(rubricDigest(changed)).not.toBe(rubricDigest(rubric))
+  })
+
+  it('refuses comparison when a material identity field differs', () => {
+    const base = { model: 'deepseek-chat', rubricDigest: 'sha256:abc', parserVersion: '1' }
+    expect(compareJudgeIdentities(base, { ...base })).toEqual({
+      comparable: true,
+      differingFields: [],
+    })
+    expect(compareJudgeIdentities(base, { ...base, model: 'deepseek-reasoner' })).toEqual({
+      comparable: false,
+      differingFields: ['model'],
+    })
+    expect(compareJudgeIdentities(base, { ...base, parserVersion: '2' })).toEqual({
+      comparable: false,
+      differingFields: ['parserVersion'],
+    })
+  })
+})

apps/sim/evals/agent-tool-use/judge.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto'
 import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
 import type { JudgeCriterion, JudgeRubric } from '@/evals/agent-tool-use/types'
 import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/streaming-tool-loop'
@@ -24,13 +25,58 @@ export interface JudgeInput {
   evidence?: string
 }
 
-export interface JudgeVerdict {
+/** Parser/schema revision, bumped when the response contract changes. */
+export const JUDGE_PARSER_VERSION = '1'
+
+/**
+ * Who judged and how. Two scores are only comparable when this matches; a delta
+ * across differing identity is not evidence of improvement.
+ */
+export interface JudgeIdentity {
+  model: string
+  rubricDigest: string
+  parserVersion: string
+  temperature?: number
+}
+
+export interface JudgeScore {
   scores: Record<string, number>
   rationale: string
   weightedScore: number
   passed: boolean
 }
 
+export interface JudgeVerdict extends JudgeScore {
+  identity: JudgeIdentity
+}
+
+/** Stable digest of the rubric's grading contract, independent of key order. */
+export function rubricDigest(rubric: JudgeRubric): string {
+  const canonical = JSON.stringify(
+    rubric.criteria.map((criterion) => ({
+      id: criterion.id,
+      description: criterion.description,
+      weight: criterion.weight ?? 1,
+    }))
+  )
+  return `sha256:${createHash('sha256').update(canonical).digest('hex').slice(0, 16)}`
+}
+
+/** Whether two verdicts may be compared, and which identity fields differ. */
+export function compareJudgeIdentities(
+  baseline: JudgeIdentity,
+  candidate: JudgeIdentity
+): { comparable: boolean; differingFields: string[] } {
+  const fields: Array<keyof JudgeIdentity> = [
+    'model',
+    'rubricDigest',
+    'parserVersion',
+    'temperature',
+  ]
+  const differingFields = fields.filter((field) => baseline[field] !== candidate[field])
+  return { comparable: differingFields.length === 0, differingFields }
+}
+
 const JUDGE_SYSTEM_PROMPT =
   'You are a strict, literal evaluator of assistant answers. Score each criterion independently. ' +
   'Do not reward fluency or confidence; reward only what the answer actually establishes. ' +
@@ -86,7 +132,7 @@ function clampScore(value: unknown): number | undefined {
 }
 
 /** Parses and validates a judge response against the rubric. */
-export function parseJudgeVerdict(raw: string, rubric: JudgeRubric): JudgeVerdict {
+export function parseJudgeVerdict(raw: string, rubric: JudgeRubric): JudgeScore {
   const parsed = JSON.parse(extractJsonObject(raw)) as {
     scores?: Record<string, unknown>
     rationale?: unknown
@@ -129,5 +175,14 @@ export async function judgeAnswer(input: JudgeInput): Promise<JudgeVerdict> {
       { role: 'user', content: buildJudgePrompt(input) },
     ],
   })
-  return parseJudgeVerdict(await collectContent(iterable), input.rubric)
+  const score = parseJudgeVerdict(await collectContent(iterable), input.rubric)
+  return {
+    ...score,
+    identity: {
+      model: input.model,
+      rubricDigest: rubricDigest(input.rubric),
+      parserVersion: JUDGE_PARSER_VERSION,
+      ...(input.temperature !== undefined ? { temperature: input.temperature } : {}),
+    },
+  }
 }