Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -669,7 +669,9 @@ flowchart LR
不是 Judge 判定通过率;例如正确识别出数字错误的负向夹具也算通过。
L3 每例和 level `totals` 新增 `judgeAttempted`(实际发起调用,包括异常)和
`judgeCompleted`(取得可解析且未降级的判定,包括负向判定);每例为布尔值,汇总为数量。
这里的“完成”沿用生产反思解析器的接受规则,不额外执行完整字段或类型校验。
有效判定要求 `answersQuestion`、`consistentWithData`、`needsReplan` 三项均为 JSON 布尔值;
缺项或使用字符串等其他类型均降级并保留待判定状态,不触发 Judge 策略重规划;独立的数字检查和有数据拒答检查仍生效。
这里的“完成”沿用生产反思解析器的接受规则;`reason`、`hint` 仍为可选说明,额外属性不影响判定。
无数据而跳过策略检查的用例不计为尝试、完成或待判定。`judgePending` 表示仍需有效判定的降级用例。
兼容旧报告消费者,`judgeAvailable` 和 `totals.judgeRan` 保留原有 `reflection.judgeUsed` 语义:
返回坏 JSON 仍计入,调用抛异常不计入,因此它们不能表示有效判定已完成。看板使用 `judgeCompleted`。
Expand Down
16 changes: 8 additions & 8 deletions src/reflection.js
Original file line number Diff line number Diff line change
Expand Up @@ -272,17 +272,17 @@ function parseJudgeVerdict(text) {
}
try {
const obj = JSON.parse(match[0]);
// Require at least one genuine verdict field so an unrelated JSON payload
// (e.g. a presentation contract) is never mistaken for a verdict.
const hasVerdictField = ['answersQuestion', 'consistentWithData', 'needsReplan']
.some((key) => typeof obj[key] === 'boolean');
if (!hasVerdictField) {
// Require complete boolean decisions; partial or coerced verdicts cannot
// safely determine whether an answer passes or needs a strategy replan.
const hasValidDecisions = ['answersQuestion', 'consistentWithData', 'needsReplan']
.every((key) => typeof obj[key] === 'boolean');
if (!hasValidDecisions) {
return null;
}
return {
answersQuestion: Boolean(obj.answersQuestion),
consistentWithData: Boolean(obj.consistentWithData),
needsReplan: Boolean(obj.needsReplan),
answersQuestion: obj.answersQuestion,
consistentWithData: obj.consistentWithData,
needsReplan: obj.needsReplan,
reason: String(obj.reason ?? ''),
hint: String(obj.hint ?? ''),
};
Expand Down
3 changes: 3 additions & 0 deletions tests/evalJudge.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ for (const { name, judge, attempted, legacyRan } of [
{ name: 'empty', judge: async () => '', attempted: 2, legacyRan: 2 },
{ name: 'null response', judge: async () => null, attempted: 2, legacyRan: 2 },
{ name: 'unrelated JSON', judge: async () => '{}', attempted: 2, legacyRan: 2 },
{ name: 'partial verdict', judge: async () => '{"answersQuestion":true}', attempted: 2, legacyRan: 2 },
{ name: 'string consistency', judge: async () => '{"answersQuestion":true,"consistentWithData":"false","needsReplan":false}', attempted: 2, legacyRan: 2 },
{ name: 'string replan', judge: async () => '{"answersQuestion":true,"consistentWithData":true,"needsReplan":"false"}', attempted: 2, legacyRan: 2 },
]) {
test(`L3 keeps ${name} judgments pending while retaining deterministic fixture checks`, async () => {
const report = await runLevel('L3', { judge });
Expand Down
84 changes: 84 additions & 0 deletions tests/reflection.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,90 @@ test('③ strategy sound uses judge verdict (needsReplan triggers hint)', async
assert.equal(r.hint, '应改按 customer_name 维度拆分而非整体求和');
});

for (const field of ['answersQuestion', 'consistentWithData', 'needsReplan']) {
for (const [name, value] of [
['missing', undefined], ['null', null], ['string true', 'true'],
['string false', 'false'], ['number', 1], ['object', {}], ['array', []],
]) {
test(`③ strategy sound degrades without replanning when ${field} is ${name}`, async () => {
const verdict = { answersQuestion: true, consistentWithData: true, needsReplan: false };
verdict[field] = value;
const r = await checkStrategySound({
question: '本月销售额', answer: '本月销售额为123.45万元。', result: RESULT,
judge: async () => JSON.stringify(verdict),
});
assert.equal(r.degraded, true);
assert.equal(r.judgeUsed, true);
assert.equal(r.pass, true);
assert.equal(r.needsReplan, false);
assert.equal(r.hint, '');
assert.equal(r.reason, '');
});
}
}

for (const { name, text, pass, needsReplan, reason, hint } of [
{
name: 'positive without optional text',
text: '{"answersQuestion":true,"consistentWithData":true,"needsReplan":false}',
pass: true, needsReplan: false, reason: '', hint: '',
},
{
name: 'negative without optional text',
text: '{"answersQuestion":false,"consistentWithData":false,"needsReplan":true}',
pass: false, needsReplan: true, reason: '', hint: '',
},
{
name: 'fenced JSON with prose, optional text and extra properties',
text: 'Verdict follows:\n```json\n{"answersQuestion":true,"consistentWithData":false,"needsReplan":true,"reason":"数字不符","hint":"重查销售额","extra":"allowed"}\n```\nDone.',
pass: false, needsReplan: true, reason: '数字不符', hint: '重查销售额',
},
{
name: 'optional text coercion',
text: '{"answersQuestion":true,"consistentWithData":true,"needsReplan":false,"reason":123,"hint":null}',
pass: true, needsReplan: false, reason: '123', hint: '',
},
]) {
test(`③ strategy sound accepts a complete boolean verdict: ${name}`, async () => {
const r = await checkStrategySound({
question: '本月销售额', answer: '本月销售额为123.45万元。', result: RESULT,
judge: async () => text,
});
assert.equal(r.degraded, false);
assert.equal(r.judgeUsed, true);
assert.equal(r.pass, pass);
assert.equal(r.needsReplan, needsReplan);
assert.equal(r.reason, reason);
assert.equal(r.hint, hint);
});
}

test('reflectOnAnswer: malformed judge retains numeric failure without triggering a strategy replan', async () => {
const r = await reflectOnAnswer({
question: '本月销售额', answer: '本月总销售额高达 1.23 亿元。', result: RESULT,
judge: async () => '{"answersQuestion":true,"consistentWithData":true,"needsReplan":"false"}',
});
assert.equal(r.degraded, true);
assert.equal(r.judgeUsed, true);
assert.equal(r.passed, false);
assert.equal(r.checks.find((c) => c.name === 'numeric_consistency').pass, false);
assert.equal(r.needsReplan, false);
assert.equal(r.hint, null);
});

test('reflectOnAnswer: malformed judge retains refusal-with-data replanning', async () => {
const r = await reflectOnAnswer({
question: '本月销售额', answer: '抱歉,我无法确定您要查询的口径。', result: RESULT,
judge: async () => '{"answersQuestion":true}',
});
assert.equal(r.degraded, true);
assert.equal(r.judgeUsed, true);
assert.equal(r.passed, false);
assert.equal(r.checks.find((c) => c.name === 'answerable').refusalWithData, true);
assert.equal(r.needsReplan, true);
assert.ok(r.hint.includes('不要反问或歉意式拒答'));
});

test('reflectOnAnswer: clean answer passes without replan (no judge)', async () => {
const r = await reflectOnAnswer({
question: '本月销售额前10客户',
Expand Down
Loading