diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc5465..78e3bc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,10 @@ jobs: YOLO_SKILL_BROWSER_TEST: '1' YOLO_SKILL_BROWSER_ARTIFACT_DIR: artifacts/skill-browser run: node --test tests/skillCatalog.browser.test.js + - name: Browser eval judgment status acceptance + env: + YOLO_EVAL_BROWSER_TEST: '1' + run: node --test tests/evalJudge.browser.test.js - uses: actions/upload-artifact@v7 if: always() with: diff --git a/docs/architecture.md b/docs/architecture.md index d45b911..359487a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -662,9 +662,18 @@ flowchart LR 配套的可量化评测（`scripts/run-eval.mjs`，`GET /api/eval` 看板）把“95% 准确率”变成三层回归指标： `L1` 契约级（门禁）、`L2` 结果级（`eval/resultSim.mjs` 在夹具表上模拟聚合校验行数/列/总额）、 -`L3` 答案级（LLM-as-Judge 反思；未配置 Judge 时标记 `judgePending`，不造假通过）。基线按 level 记录于 +`L3` 答案级（LLM-as-Judge 反思；Judge 缺失、调用失败或返回无法解析时均标记 `judgePending`）。基线按 level 记录于 `eval/baseline.json`，看板展示通过率、门禁分布与回归 diff。 +评测报告中的 `passed` / `passRate` / `overall.accuracy` 仍表示夹具断言符合预期的比例， +不是 Judge 判定通过率；例如正确识别出数字错误的负向夹具也算通过。 +L3 每例和 level `totals` 新增 `judgeAttempted`（实际发起调用，包括异常）和 +`judgeCompleted`（取得可解析且未降级的判定，包括负向判定）；每例为布尔值，汇总为数量。 +这里的“完成”沿用生产反思解析器的接受规则，不额外执行完整字段或类型校验。 +无数据而跳过策略检查的用例不计为尝试、完成或待判定。`judgePending` 表示仍需有效判定的降级用例。 +兼容旧报告消费者，`judgeAvailable` 和 `totals.judgeRan` 保留原有 `reflection.judgeUsed` 语义： +返回坏 JSON 仍计入，调用抛异常不计入，因此它们不能表示有效判定已完成。看板使用 `judgeCompleted`。 + ## 6. 端到端问数时序 ```mermaid diff --git a/eval/answerCase.mjs b/eval/answerCase.mjs new file mode 100644 index 0000000..e141fde --- /dev/null +++ b/eval/answerCase.mjs @@ -0,0 +1,47 @@ +// Eval status is separate from production reflection and fixture pass/fail. +import { reflectOnAnswer } from '../src/reflection.js'; + +export async function runL3Case(evalCase, judge) { + let judgeAttempted = false; + const trackedJudge = typeof judge === 'function' ? (...args) => { + judgeAttempted = true; + return judge(...args); + } : null; + const reflection = await reflectOnAnswer({ + question: evalCase.question, + answer: evalCase.answer, + result: evalCase.result, + judge: trackedJudge, + }); + const expect = evalCase.expectReflect ?? {}; + const checks = []; + if (expect.shouldPass !== undefined) { + checks.push({ + assertion: 'shouldPass', + ok: reflection.passed === expect.shouldPass, + expected: expect.shouldPass, + actual: reflection.passed, + }); + } + if (expect.needsReplan !== undefined) { + checks.push({ + assertion: 'needsReplan', + ok: reflection.needsReplan === expect.needsReplan, + expected: expect.needsReplan, + actual: reflection.needsReplan, + }); + } + const judgePending = reflection.degraded; + const judgeCompleted = reflection.judgeUsed && !reflection.degraded; + return { + id: evalCase.id, + description: evalCase.description ?? '', + passed: checks.every((check) => check.ok), + judgeAvailable: reflection.judgeUsed, + judgePending, + judgeAttempted, + judgeCompleted, + checks, + reflection, + }; +} diff --git a/public/js/pages/eval.js b/public/js/pages/eval.js index c12dc89..abcf128 100644 --- a/public/js/pages/eval.js +++ b/public/js/pages/eval.js @@ -9,7 +9,7 @@ const LEVELS = ['L1', 'L2', 'L3']; const LEVEL_HINT = { L1: '契约级：编译查询契约并校验门禁（确定性，CI 可跑）', L2: '结果级：在夹具表上模拟聚合，校验行数/列/总额（确定性）', - L3: '答案级：LLM-as-Judge 反思回答；未配置 Judge 时标记为「待 Judge」，不造假', + L3: '答案级：LLM-as-Judge 反思回答；Judge 缺失、调用失败或返回无法解析时标记为「待有效判定」', }; function pct(rate) { @@ -26,10 +26,10 @@ function levelCard(level, report) { const judgeTags = []; if (level === 'L3') { if (t.judgePending > 0) { - judgeTags.push(tag(`Judge 待配置 ${t.judgePending}`, 'amber')); + judgeTags.push(tag(`Judge 待有效判定 ${t.judgePending}`, 'amber')); } - if (t.judgeRan > 0) { - judgeTags.push(tag(`Judge 已运行 ${t.judgeRan}`, 'teal')); + if (t.judgeCompleted > 0) { + judgeTags.push(tag(`Judge 已完成判定 ${t.judgeCompleted}`, 'teal')); } } const gateBlocks = Object.entries(report?.gateBlockDistribution ?? {}) @@ -97,7 +97,7 @@ function renderReport(report) {

评测看板

-

把「95% 准确率」变成可量化指标：L1 契约级 + L2 结果级 + L3 答案级（LLM-as-Judge）三层回归，随每次取数路径变更持续跟踪。

+

用例通过率衡量夹具断言是否符合预期；Judge 判定完成情况单独展示。L1 契约级 + L2 结果级 + L3 答案级（LLM-as-Judge）三层回归，随每次取数路径变更持续跟踪。

@@ -107,7 +107,7 @@ function renderReport(report) {
- 总体准确率 + 总体用例通过率 ${pct(overall.accuracy)} ${overall.passed}/${overall.cases} 通过 ${generatedAt ? tag(`生成于 ${generatedAt.toLocaleString('zh-CN')}`, 'blue') : ''} diff --git a/scripts/run-eval.mjs b/scripts/run-eval.mjs index b7ce514..d9baf83 100644 --- a/scripts/run-eval.mjs +++ b/scripts/run-eval.mjs @@ -17,7 +17,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { compileEvalCase } from '../eval/fixtures.mjs'; import { simulateContractResult, checkL2Result } from '../eval/resultSim.mjs'; -import { reflectOnAnswer } from '../src/reflection.js'; +import { runL3Case } from '../eval/answerCase.mjs'; const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const evalRoot = path.join(projectRoot, 'eval'); @@ -128,47 +128,6 @@ function runL2Case(evalCase) { }; } -// --------------------------------------------------------------------------- -// L3 -// --------------------------------------------------------------------------- - -async function runL3Case(evalCase, judge) { - const reflection = await reflectOnAnswer({ - question: evalCase.question, - answer: evalCase.answer, - result: evalCase.result, - judge, - }); - const expect = evalCase.expectReflect ?? {}; - const checks = []; - if (expect.shouldPass !== undefined) { - checks.push({ - assertion: 'shouldPass', - ok: reflection.passed === expect.shouldPass, - expected: expect.shouldPass, - actual: reflection.passed, - }); - } - if (expect.needsReplan !== undefined) { - checks.push({ - assertion: 'needsReplan', - ok: reflection.needsReplan === expect.needsReplan, - expected: expect.needsReplan, - actual: reflection.needsReplan, - }); - } - const judgePending = reflection.degraded && !reflection.judgeUsed; - return { - id: evalCase.id, - description: evalCase.description ?? '', - passed: checks.every((check) => check.ok), - judgeAvailable: reflection.judgeUsed, - judgePending, - checks, - reflection, - }; -} - // --------------------------------------------------------------------------- // baseline / diff // --------------------------------------------------------------------------- @@ -234,7 +193,10 @@ export async function runLevel(level, { judge = null } = {}) { const baseline = readBaseline(); const baselineDiff = diffBaseline(baseline, level, results); const judgePending = results.filter((result) => result.judgePending).length; + // Legacy judgeRan follows reflection.judgeUsed; it does not imply a usable verdict. const judgeRan = results.filter((result) => result.judgeAvailable).length; + const judgeAttempted = results.filter((result) => result.judgeAttempted).length; + const judgeCompleted = results.filter((result) => result.judgeCompleted).length; return { level, @@ -245,6 +207,8 @@ export async function runLevel(level, { judge = null } = {}) { passRate: results.length > 0 ? passed / results.length : 0, judgePending, judgeRan, + judgeAttempted, + judgeCompleted, }, gateBlockDistribution, results, @@ -348,7 +312,8 @@ async function main() { const r = allReport.levels[lvl]; console.log(`[eval] ${lvl}: ${r.totals.passed}/${r.totals.cases} passed (pass rate ${(r.totals.passRate * 100).toFixed(1)}%)` + (r.totals.judgePending ? `, judgePending=${r.totals.judgePending}` : '') - + (r.totals.judgeRan ? `, judgeRan=${r.totals.judgeRan}` : '')); + + (r.totals.judgeAttempted ? `, judgeAttempted=${r.totals.judgeAttempted}` : '') + + (r.totals.judgeCompleted ? `, judgeCompleted=${r.totals.judgeCompleted}` : '')); if (Object.keys(r.gateBlockDistribution).length > 0) { console.log(`[eval] ${lvl} gate blocks:`, JSON.stringify(r.gateBlockDistribution)); } @@ -365,7 +330,7 @@ async function main() { } } } - console.log(`[eval] overall accuracy: ${(report.overall.accuracy * 100).toFixed(1)}% (${report.overall.passed}/${report.overall.cases})`); + console.log(`[eval] overall fixture pass rate: ${(report.overall.accuracy * 100).toFixed(1)}% (${report.overall.passed}/${report.overall.cases})`); console.log(`[eval] report -> ${path.relative(projectRoot, reportPath)}`); if (updateBaseline) { diff --git a/tests/evalJudge.browser.test.js b/tests/evalJudge.browser.test.js new file mode 100644 index 0000000..20250a9 --- /dev/null +++ b/tests/evalJudge.browser.test.js @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { runAllLevels } from '../scripts/run-eval.mjs'; +import { TEST_PASSWORD, withSkillServer } from './fixtures/skillServer.js'; + +test('browser: eval shows degraded judges as pending and only valid judgments as completed', { + skip: !process.env.YOLO_EVAL_BROWSER_TEST, timeout: 90000, +}, async () => { + const { chromium, expect } = await import('@playwright/test'); + await withSkillServer(async ({ baseUrl }) => { + const browser = await chromium.launch({ headless: true, channel: process.env.YOLO_BROWSER_CHANNEL }); + try { + const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); + const errors = []; + page.on('pageerror', (error) => errors.push(error.message)); + let report = await runAllLevels({ judge: async () => 'not-json' }); + await page.route('**/api/eval', (route) => route.fulfill({ json: report })); + await page.goto(`${baseUrl}/eval`); + await page.locator('#loginUsername').fill('admin'); + await page.locator('#loginPassword').fill(TEST_PASSWORD); + await page.getByRole('button', { name: '登录', exact: true }).click(); + const pending = page.locator('.tag-amber', { hasText: 'Judge 待有效判定' }); + const completed = page.locator('.tag-teal', { hasText: 'Judge 已完成判定' }); + await expect(pending).toHaveText('Judge 待有效判定 2'); + await expect(completed).toHaveCount(0); + await expect(page.getByText('总体用例通过率', { exact: true })).toBeVisible(); + + let calls = 0; + report = await runAllLevels({ judge: async () => ++calls === 1 + ? JSON.stringify({ answersQuestion: true, consistentWithData: true, needsReplan: false, reason: 'ok', hint: '' }) + : 'not-json' }); + await page.locator('[data-eval-refresh]').click(); + await expect(pending).toHaveText('Judge 待有效判定 1'); + await expect(completed).toHaveText('Judge 已完成判定 1'); + + report = await runAllLevels({ judge: async () => JSON.stringify({ + answersQuestion: true, consistentWithData: true, needsReplan: false, reason: 'ok', hint: '', + }) }); + await page.locator('[data-eval-refresh]').click(); + await expect(pending).toHaveCount(0); + await expect(completed).toHaveText('Judge 已完成判定 2'); + assert.deepEqual(errors, []); + } finally { + await browser.close(); + } + }); +}); diff --git a/tests/evalJudge.test.mjs b/tests/evalJudge.test.mjs new file mode 100644 index 0000000..4f94004 --- /dev/null +++ b/tests/evalJudge.test.mjs @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { runLevel, runAllLevels } from '../scripts/run-eval.mjs'; +import { runL3Case } from '../eval/answerCase.mjs'; + +const validVerdict = JSON.stringify({ + answersQuestion: true, consistentWithData: true, needsReplan: false, + reason: 'Matches the supplied data', hint: '', +}); + +for (const { name, judge, attempted, legacyRan } of [ + { name: 'missing', judge: null, attempted: 0, legacyRan: 0 }, + { name: 'throws', judge: async () => { throw new Error('offline'); }, attempted: 2, legacyRan: 0 }, + { name: 'malformed', judge: async () => 'not-json', attempted: 2, legacyRan: 2 }, + { name: 'empty', judge: async () => '', attempted: 2, legacyRan: 2 }, + { name: 'null response', judge: async () => null, attempted: 2, legacyRan: 2 }, + { name: 'unrelated JSON', judge: async () => '{}', attempted: 2, legacyRan: 2 }, +]) { + test(`L3 keeps ${name} judgments pending while retaining deterministic fixture checks`, async () => { + const report = await runLevel('L3', { judge }); + assert.equal(report.totals.judgePending, 2); + assert.equal(report.totals.judgeAttempted, attempted); + assert.equal(report.totals.judgeCompleted, 0); + assert.equal(report.totals.judgeRan, legacyRan); + assert.equal(report.totals.cases, 2); + assert.equal(report.totals.passed, 2); + assert.equal(report.totals.failed, 0); + assert.equal(report.totals.passRate, 1); + assert.deepEqual(report.results.map((r) => r.reflection.passed), [true, false]); + for (const result of report.results) { + assert.equal(result.judgePending, true); + assert.equal(result.judgeAttempted, attempted > 0); + assert.equal(result.judgeCompleted, false); + assert.equal(result.judgeAvailable, legacyRan > 0); + assert.equal(result.reflection.degraded, true); + } + }); +} + +test('a valid judge completes L3 judgments without masking a deterministic numeric failure', async () => { + const report = await runLevel('L3', { judge: async () => validVerdict }); + assert.equal(report.totals.judgePending, 0); + assert.equal(report.totals.judgeAttempted, 2); + assert.equal(report.totals.judgeCompleted, 2); + assert.equal(report.totals.judgeRan, 2); + assert.equal(report.totals.passed, 2); + assert.deepEqual(report.results.map((r) => r.reflection.passed), [true, false]); + assert.ok(report.results.every((r) => r.judgeCompleted && !r.judgePending)); +}); + +test('a no-data fixture skips judgment without becoming pending or attempted', async () => { + const result = await runL3Case({ + id: 'no-data', question: '销售额', answer: '请补充时间范围', + result: { columns: [], rows: [] }, + expectReflect: { shouldPass: true, needsReplan: false }, + }, async () => { throw new Error('no-data fixtures must skip the judge'); }); + assert.equal(result.passed, true); + assert.equal(result.judgePending, false); + assert.equal(result.judgeAttempted, false); + assert.equal(result.judgeCompleted, false); + assert.equal(result.judgeAvailable, false); +}); + +test('valid negative judgments count as completed and still fail mismatching fixture expectations', async () => { + const report = await runLevel('L3', { judge: async () => JSON.stringify({ + answersQuestion: false, consistentWithData: false, needsReplan: true, + reason: 'Off topic', hint: 'Change the query dimension', + }) }); + assert.equal(report.totals.judgeCompleted, 2); + assert.equal(report.totals.judgePending, 0); + assert.equal(report.totals.passed, 0); + assert.equal(report.totals.failed, 2); + assert.ok(report.results.every((r) => r.reflection.needsReplan)); +}); + +test('all-level report keeps fixture totals consistent while L3 judgments are pending', async () => { + const report = await runAllLevels({ judge: async () => 'not-json' }); + assert.equal(report.levels.L3.totals.judgePending, 2); + assert.equal(report.levels.L3.totals.judgeCompleted, 0); + for (const level of ['L1', 'L2']) { + assert.equal(report.levels[level].totals.judgeAttempted, 0); + assert.equal(report.levels[level].totals.judgeCompleted, 0); + assert.equal(report.levels[level].totals.judgePending, 0); + } + assert.equal(report.overall.cases, Object.values(report.levels).reduce((sum, r) => sum + r.totals.cases, 0)); + assert.equal(report.overall.passed, report.overall.cases); + assert.equal(report.overall.failed, 0); + assert.equal(report.overall.accuracy, 1); +});