fix(eval): distinguish pending and completed Judge results

.github/workflows/ci.yml

@@ -83,6 +83,10 @@ jobs:
           YOLO_SKILL_BROWSER_TEST: '1'
           YOLO_SKILL_BROWSER_ARTIFACT_DIR: artifacts/skill-browser
         run: node --test tests/skillCatalog.browser.test.js
+      - name: Browser eval judgment status acceptance
+        env:
+          YOLO_EVAL_BROWSER_TEST: '1'
+        run: node --test tests/evalJudge.browser.test.js
       - uses: actions/upload-artifact@v7
         if: always()
         with:

docs/architecture.md

@@ -662,9 +662,18 @@ flowchart LR
 
 配套的可量化评测（`scripts/run-eval.mjs`，`GET /api/eval` 看板）把“95% 准确率”变成三层回归指标：
 `L1` 契约级（门禁）、`L2` 结果级（`eval/resultSim.mjs` 在夹具表上模拟聚合校验行数/列/总额）、
-`L3` 答案级（LLM-as-Judge 反思；未配置 Judge 时标记 `judgePending`，不造假通过）。基线按 level 记录于
+`L3` 答案级（LLM-as-Judge 反思；Judge 缺失、调用失败或返回无法解析时均标记 `judgePending`）。基线按 level 记录于
 `eval/baseline.json`，看板展示通过率、门禁分布与回归 diff。
 
+评测报告中的 `passed` / `passRate` / `overall.accuracy` 仍表示夹具断言符合预期的比例，
+不是 Judge 判定通过率；例如正确识别出数字错误的负向夹具也算通过。
+L3 每例和 level `totals` 新增 `judgeAttempted`（实际发起调用，包括异常）和
+`judgeCompleted`（取得可解析且未降级的判定，包括负向判定）；每例为布尔值，汇总为数量。
+这里的“完成”沿用生产反思解析器的接受规则，不额外执行完整字段或类型校验。
+无数据而跳过策略检查的用例不计为尝试、完成或待判定。`judgePending` 表示仍需有效判定的降级用例。
+兼容旧报告消费者，`judgeAvailable` 和 `totals.judgeRan` 保留原有 `reflection.judgeUsed` 语义：
+返回坏 JSON 仍计入，调用抛异常不计入，因此它们不能表示有效判定已完成。看板使用 `judgeCompleted`。
+
 ## 6. 端到端问数时序
 
 ```mermaid

eval/answerCase.mjs

@@ -0,0 +1,47 @@
+// Eval status is separate from production reflection and fixture pass/fail.
+import { reflectOnAnswer } from '../src/reflection.js';
+
+export async function runL3Case(evalCase, judge) {
+  let judgeAttempted = false;
+  const trackedJudge = typeof judge === 'function' ? (...args) => {
+    judgeAttempted = true;
+    return judge(...args);
+  } : null;
+  const reflection = await reflectOnAnswer({
+    question: evalCase.question,
+    answer: evalCase.answer,
+    result: evalCase.result,
+    judge: trackedJudge,
+  });
+  const expect = evalCase.expectReflect ?? {};
+  const checks = [];
+  if (expect.shouldPass !== undefined) {
+    checks.push({
+      assertion: 'shouldPass',
+      ok: reflection.passed === expect.shouldPass,
+      expected: expect.shouldPass,
+      actual: reflection.passed,
+    });
+  }
+  if (expect.needsReplan !== undefined) {
+    checks.push({
+      assertion: 'needsReplan',
+      ok: reflection.needsReplan === expect.needsReplan,
+      expected: expect.needsReplan,
+      actual: reflection.needsReplan,
+    });
+  }
+  const judgePending = reflection.degraded;
+  const judgeCompleted = reflection.judgeUsed && !reflection.degraded;
+  return {
+    id: evalCase.id,
+    description: evalCase.description ?? '',
+    passed: checks.every((check) => check.ok),
+    judgeAvailable: reflection.judgeUsed,
+    judgePending,
+    judgeAttempted,
+    judgeCompleted,
+    checks,
+    reflection,
+  };
+}

public/js/pages/eval.js

@@ -9,7 +9,7 @@ const LEVELS = ['L1', 'L2', 'L3'];
 const LEVEL_HINT = {
   L1: '契约级：编译查询契约并校验门禁（确定性，CI 可跑）',
   L2: '结果级：在夹具表上模拟聚合，校验行数/列/总额（确定性）',
-  L3: '答案级：LLM-as-Judge 反思回答；未配置 Judge 时标记为「待 Judge」，不造假',
+  L3: '答案级：LLM-as-Judge 反思回答；Judge 缺失、调用失败或返回无法解析时标记为「待有效判定」',
 };
 
 function pct(rate) {
@@ -26,10 +26,10 @@ function levelCard(level, report) {
   const judgeTags = [];
   if (level === 'L3') {
     if (t.judgePending > 0) {
-      judgeTags.push(tag(`Judge 待配置 ${t.judgePending}`, 'amber'));
+      judgeTags.push(tag(`Judge 待有效判定 ${t.judgePending}`, 'amber'));
     }
-    if (t.judgeRan > 0) {
-      judgeTags.push(tag(`Judge 已运行 ${t.judgeRan}`, 'teal'));
+    if (t.judgeCompleted > 0) {
+      judgeTags.push(tag(`Judge 已完成判定 ${t.judgeCompleted}`, 'teal'));
     }
   }
   const gateBlocks = Object.entries(report?.gateBlockDistribution ?? {})
@@ -97,7 +97,7 @@ function renderReport(report) {
       <div class="section-head">
         <div>
           <h2>评测看板</h2>
-          <p>把「95% 准确率」变成可量化指标：L1 契约级 + L2 结果级 + L3 答案级（LLM-as-Judge）三层回归，随每次取数路径变更持续跟踪。</p>
+          <p>用例通过率衡量夹具断言是否符合预期；Judge 判定完成情况单独展示。L1 契约级 + L2 结果级 + L3 答案级（LLM-as-Judge）三层回归，随每次取数路径变更持续跟踪。</p>
         </div>
         <div class="page-toolbar">
           <button class="btn btn-quiet" type="button" data-eval-refresh>${icon('refresh', '重新评测')}重新评测</button>
@@ -107,7 +107,7 @@ function renderReport(report) {
       <div class="section-band">
         <div class="kpi-grid">
           <div class="kpi-item">
-            <span>总体准确率</span>
+            <span>总体用例通过率</span>
             <strong>${pct(overall.accuracy)}</strong>
             <span class="muted">${overall.passed}/${overall.cases} 通过</span>
             <span>${generatedAt ? tag(`生成于 ${generatedAt.toLocaleString('zh-CN')}`, 'blue') : ''}</span>

scripts/run-eval.mjs

@@ -17,7 +17,7 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { compileEvalCase } from '../eval/fixtures.mjs';
 import { simulateContractResult, checkL2Result } from '../eval/resultSim.mjs';
-import { reflectOnAnswer } from '../src/reflection.js';
+import { runL3Case } from '../eval/answerCase.mjs';
 
 const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const evalRoot = path.join(projectRoot, 'eval');
@@ -128,47 +128,6 @@ function runL2Case(evalCase) {
   };
 }
 
-// ---------------------------------------------------------------------------
-// L3
-// ---------------------------------------------------------------------------
-
-async function runL3Case(evalCase, judge) {
-  const reflection = await reflectOnAnswer({
-    question: evalCase.question,
-    answer: evalCase.answer,
-    result: evalCase.result,
-    judge,
-  });
-  const expect = evalCase.expectReflect ?? {};
-  const checks = [];
-  if (expect.shouldPass !== undefined) {
-    checks.push({
-      assertion: 'shouldPass',
-      ok: reflection.passed === expect.shouldPass,
-      expected: expect.shouldPass,
-      actual: reflection.passed,
-    });
-  }
-  if (expect.needsReplan !== undefined) {
-    checks.push({
-      assertion: 'needsReplan',
-      ok: reflection.needsReplan === expect.needsReplan,
-      expected: expect.needsReplan,
-      actual: reflection.needsReplan,
-    });
-  }
-  const judgePending = reflection.degraded && !reflection.judgeUsed;
-  return {
-    id: evalCase.id,
-    description: evalCase.description ?? '',
-    passed: checks.every((check) => check.ok),
-    judgeAvailable: reflection.judgeUsed,
-    judgePending,
-    checks,
-    reflection,
-  };
-}
-
 // ---------------------------------------------------------------------------
 // baseline / diff
 // ---------------------------------------------------------------------------
@@ -234,7 +193,10 @@ export async function runLevel(level, { judge = null } = {}) {
   const baseline = readBaseline();
   const baselineDiff = diffBaseline(baseline, level, results);
   const judgePending = results.filter((result) => result.judgePending).length;
+  // Legacy judgeRan follows reflection.judgeUsed; it does not imply a usable verdict.
   const judgeRan = results.filter((result) => result.judgeAvailable).length;
+  const judgeAttempted = results.filter((result) => result.judgeAttempted).length;
+  const judgeCompleted = results.filter((result) => result.judgeCompleted).length;
 
   return {
     level,
@@ -245,6 +207,8 @@ export async function runLevel(level, { judge = null } = {}) {
       passRate: results.length > 0 ? passed / results.length : 0,
       judgePending,
       judgeRan,
+      judgeAttempted,
+      judgeCompleted,
     },
     gateBlockDistribution,
     results,
@@ -348,7 +312,8 @@ async function main() {
     const r = allReport.levels[lvl];
     console.log(`[eval] ${lvl}: ${r.totals.passed}/${r.totals.cases} passed (pass rate ${(r.totals.passRate * 100).toFixed(1)}%)`
       + (r.totals.judgePending ? `, judgePending=${r.totals.judgePending}` : '')
-      + (r.totals.judgeRan ? `, judgeRan=${r.totals.judgeRan}` : ''));
+      + (r.totals.judgeAttempted ? `, judgeAttempted=${r.totals.judgeAttempted}` : '')
+      + (r.totals.judgeCompleted ? `, judgeCompleted=${r.totals.judgeCompleted}` : ''));
     if (Object.keys(r.gateBlockDistribution).length > 0) {
       console.log(`[eval] ${lvl} gate blocks:`, JSON.stringify(r.gateBlockDistribution));
     }
@@ -365,7 +330,7 @@ async function main() {
       }
     }
   }
-  console.log(`[eval] overall accuracy: ${(report.overall.accuracy * 100).toFixed(1)}% (${report.overall.passed}/${report.overall.cases})`);
+  console.log(`[eval] overall fixture pass rate: ${(report.overall.accuracy * 100).toFixed(1)}% (${report.overall.passed}/${report.overall.cases})`);
   console.log(`[eval] report -> ${path.relative(projectRoot, reportPath)}`);
 
   if (updateBaseline) {

tests/evalJudge.browser.test.js

@@ -0,0 +1,47 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { runAllLevels } from '../scripts/run-eval.mjs';
+import { TEST_PASSWORD, withSkillServer } from './fixtures/skillServer.js';
+
+test('browser: eval shows degraded judges as pending and only valid judgments as completed', {
+  skip: !process.env.YOLO_EVAL_BROWSER_TEST, timeout: 90000,
+}, async () => {
+  const { chromium, expect } = await import('@playwright/test');
+  await withSkillServer(async ({ baseUrl }) => {
+    const browser = await chromium.launch({ headless: true, channel: process.env.YOLO_BROWSER_CHANNEL });
+    try {
+      const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
+      const errors = [];
+      page.on('pageerror', (error) => errors.push(error.message));
+      let report = await runAllLevels({ judge: async () => 'not-json' });
+      await page.route('**/api/eval', (route) => route.fulfill({ json: report }));
+      await page.goto(`${baseUrl}/eval`);
+      await page.locator('#loginUsername').fill('admin');
+      await page.locator('#loginPassword').fill(TEST_PASSWORD);
+      await page.getByRole('button', { name: '登录', exact: true }).click();
+      const pending = page.locator('.tag-amber', { hasText: 'Judge 待有效判定' });
+      const completed = page.locator('.tag-teal', { hasText: 'Judge 已完成判定' });
+      await expect(pending).toHaveText('Judge 待有效判定 2');
+      await expect(completed).toHaveCount(0);
+      await expect(page.getByText('总体用例通过率', { exact: true })).toBeVisible();
+
+      let calls = 0;
+      report = await runAllLevels({ judge: async () => ++calls === 1
+        ? JSON.stringify({ answersQuestion: true, consistentWithData: true, needsReplan: false, reason: 'ok', hint: '' })
+        : 'not-json' });
+      await page.locator('[data-eval-refresh]').click();
+      await expect(pending).toHaveText('Judge 待有效判定 1');
+      await expect(completed).toHaveText('Judge 已完成判定 1');
+
+      report = await runAllLevels({ judge: async () => JSON.stringify({
+        answersQuestion: true, consistentWithData: true, needsReplan: false, reason: 'ok', hint: '',
+      }) });
+      await page.locator('[data-eval-refresh]').click();
+      await expect(pending).toHaveCount(0);
+      await expect(completed).toHaveText('Judge 已完成判定 2');
+      assert.deepEqual(errors, []);
+    } finally {
+      await browser.close();
+    }
+  });
+});

tests/evalJudge.test.mjs

@@ -0,0 +1,89 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { runLevel, runAllLevels } from '../scripts/run-eval.mjs';
+import { runL3Case } from '../eval/answerCase.mjs';
+
+const validVerdict = JSON.stringify({
+  answersQuestion: true, consistentWithData: true, needsReplan: false,
+  reason: 'Matches the supplied data', hint: '',
+});
+
+for (const { name, judge, attempted, legacyRan } of [
+  { name: 'missing', judge: null, attempted: 0, legacyRan: 0 },
+  { name: 'throws', judge: async () => { throw new Error('offline'); }, attempted: 2, legacyRan: 0 },
+  { name: 'malformed', judge: async () => 'not-json', attempted: 2, legacyRan: 2 },
+  { name: 'empty', judge: async () => '', attempted: 2, legacyRan: 2 },
+  { name: 'null response', judge: async () => null, attempted: 2, legacyRan: 2 },
+  { name: 'unrelated JSON', judge: async () => '{}', attempted: 2, legacyRan: 2 },
+]) {
+  test(`L3 keeps ${name} judgments pending while retaining deterministic fixture checks`, async () => {
+    const report = await runLevel('L3', { judge });
+    assert.equal(report.totals.judgePending, 2);
+    assert.equal(report.totals.judgeAttempted, attempted);
+    assert.equal(report.totals.judgeCompleted, 0);
+    assert.equal(report.totals.judgeRan, legacyRan);
+    assert.equal(report.totals.cases, 2);
+    assert.equal(report.totals.passed, 2);
+    assert.equal(report.totals.failed, 0);
+    assert.equal(report.totals.passRate, 1);
+    assert.deepEqual(report.results.map((r) => r.reflection.passed), [true, false]);
+    for (const result of report.results) {
+      assert.equal(result.judgePending, true);
+      assert.equal(result.judgeAttempted, attempted > 0);
+      assert.equal(result.judgeCompleted, false);
+      assert.equal(result.judgeAvailable, legacyRan > 0);
+      assert.equal(result.reflection.degraded, true);
+    }
+  });
+}
+
+test('a valid judge completes L3 judgments without masking a deterministic numeric failure', async () => {
+  const report = await runLevel('L3', { judge: async () => validVerdict });
+  assert.equal(report.totals.judgePending, 0);
+  assert.equal(report.totals.judgeAttempted, 2);
+  assert.equal(report.totals.judgeCompleted, 2);
+  assert.equal(report.totals.judgeRan, 2);
+  assert.equal(report.totals.passed, 2);
+  assert.deepEqual(report.results.map((r) => r.reflection.passed), [true, false]);
+  assert.ok(report.results.every((r) => r.judgeCompleted && !r.judgePending));
+});
+
+test('a no-data fixture skips judgment without becoming pending or attempted', async () => {
+  const result = await runL3Case({
+    id: 'no-data', question: '销售额', answer: '请补充时间范围',
+    result: { columns: [], rows: [] },
+    expectReflect: { shouldPass: true, needsReplan: false },
+  }, async () => { throw new Error('no-data fixtures must skip the judge'); });
+  assert.equal(result.passed, true);
+  assert.equal(result.judgePending, false);
+  assert.equal(result.judgeAttempted, false);
+  assert.equal(result.judgeCompleted, false);
+  assert.equal(result.judgeAvailable, false);
+});
+
+test('valid negative judgments count as completed and still fail mismatching fixture expectations', async () => {
+  const report = await runLevel('L3', { judge: async () => JSON.stringify({
+    answersQuestion: false, consistentWithData: false, needsReplan: true,
+    reason: 'Off topic', hint: 'Change the query dimension',
+  }) });
+  assert.equal(report.totals.judgeCompleted, 2);
+  assert.equal(report.totals.judgePending, 0);
+  assert.equal(report.totals.passed, 0);
+  assert.equal(report.totals.failed, 2);
+  assert.ok(report.results.every((r) => r.reflection.needsReplan));
+});
+
+test('all-level report keeps fixture totals consistent while L3 judgments are pending', async () => {
+  const report = await runAllLevels({ judge: async () => 'not-json' });
+  assert.equal(report.levels.L3.totals.judgePending, 2);
+  assert.equal(report.levels.L3.totals.judgeCompleted, 0);
+  for (const level of ['L1', 'L2']) {
+    assert.equal(report.levels[level].totals.judgeAttempted, 0);
+    assert.equal(report.levels[level].totals.judgeCompleted, 0);
+    assert.equal(report.levels[level].totals.judgePending, 0);
+  }
+  assert.equal(report.overall.cases, Object.values(report.levels).reduce((sum, r) => sum + r.totals.cases, 0));
+  assert.equal(report.overall.passed, report.overall.cases);
+  assert.equal(report.overall.failed, 0);
+  assert.equal(report.overall.accuracy, 1);
+});