diff --git a/docs/adr/ADR-0057-darwin-bound-guard.md b/docs/adr/ADR-0057-darwin-bound-guard.md new file mode 100644 index 0000000..6345533 --- /dev/null +++ b/docs/adr/ADR-0057-darwin-bound-guard.md @@ -0,0 +1,33 @@ +# ADR-0057: Fail-visible Darwin bound guard in the evaluation-adapter layer + +- Date: 2026-09-07 +- Status: Proposed (draft PR `dream/evaluation-adapters-20260907`) +- Companion to: ADR-2024 (deterministic evaluator veto) + +## Context + +Nightly pipeline step 10 bounds every Darwin run: generations <= 3, +candidates per generation <= 4, promoted lineages <= 1. On 2026-09-07 +(run `ab4ced4e48b76e83`, commit `9f4d8a9a`) the REQUIRED darwin evaluator +returned `outcome=PASSED` while its leaderboard listed five candidates +in generation 2 (`g2_v0..g2_v4`). The bound existed only as prose; +nothing on the evaluator path detected the breach. + +## Decision + +The adapter layer gains a pure, dependency-free bound checker +(`packages/cli/src/darwinBounds.ts`) that parses leaderboard output, +counts candidates per generation label, and reports violations. +Parsing is fail-visible: zero rows parsed => `unparsable`, never `ok`. +Policy constants live in `DARWIN_BOUNDS` for one-line adjustment. + +## Consequences + ++ Silent bound breaches become machine-detectable; tonight's real + leaderboard is pinned as a regression fixture in tests. ++ Fail-visible parsing guards against leaderboard format drift. +- Hard-wiring the checker into the live evaluator gate (auto-veto) is + deliberately deferred to a human-reviewed follow-up. + +Numbering: 0057 follows known-highest ADR-056; renumber on merge if the +sequence has advanced. diff --git a/packages/cli/src/darwinBounds.test.ts b/packages/cli/src/darwinBounds.test.ts new file mode 100644 index 0000000..7b7eecb --- /dev/null +++ b/packages/cli/src/darwinBounds.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; +import { + checkDarwinBounds, + checkDarwinBoundsFromStdout, + parseLeaderboardRows, +} from './darwinBounds'; + +// Fixture: shape of the real 2026-09-07 REQUIRED-evaluator leaderboard +// (run ab4ced4e48b76e83), which PASSED with g2 holding five candidates. +const LB_2026_09_07 = [ + 'Darwin Mode — leaderboard', + ' 0.765 baseline [planner] safety=1.00 pass=0.60 ◀ winner', + ' 0.765 g1_v0 [planner] safety=1.00 pass=0.60', + ' 0.765 g1_v1 [toolPolicy] safety=1.00 pass=0.60', + ' 0.765 g1_v2 [reviewer] safety=1.00 pass=0.60', + ' 0.765 g1_v3 [toolPolicy] safety=1.00 pass=0.60', + ' 0.765 g2_v0 [reviewer] safety=1.00 pass=0.60', + ' 0.765 g2_v1 [planner] safety=1.00 pass=0.60', + ' 0.765 g2_v2 [contextBuilder] safety=1.00 pass=0.60', + ' 0.765 g2_v3 [toolPolicy] safety=1.00 pass=0.60', + ' 0.765 g2_v4 [scorePolicy] safety=1.00 pass=0.60', + '', + 'Winner: baseline', + 'Lineage: baseline', + 'Delta over baseline: +0.000', +].join('\n'); + +const LB_COMPLIANT = LB_2026_09_07.split('\n') + .filter((line) => !line.includes('g2_v4')) + .join('\n'); + +describe('parseLeaderboardRows', () => { + it('parses the real 2026-09-07 leaderboard: 1 baseline + 9 mutants', () => { + const rows = parseLeaderboardRows(LB_2026_09_07); + expect(rows).toHaveLength(10); + expect(rows.filter((r) => r.id === 'baseline')).toHaveLength(1); + }); + + it('returns [] when no leaderboard lines are present', () => { + expect(parseLeaderboardRows('nothing to see here')).toEqual([]); + }); +}); + +describe('checkDarwinBounds', () => { + it('flags the real 2026-09-07 run: g2 held 5 candidates (> 4)', () => { + const report = checkDarwinBoundsFromStdout(LB_2026_09_07, 0); + expect(report.parseStatus).toBe('ok'); + expect(report.candidatesPerGeneration).toEqual({ 1: 4, 2: 5 }); + expect(report.maxCandidatesPerGeneration).toBe(5); + expect(report.ok).toBe(false); + expect(report.violations.join('\n')).toContain('g2 candidates=5 > 4'); + }); + + it('accepts the same run with g2_v4 removed (compliant)', () => { + const report = checkDarwinBoundsFromStdout(LB_COMPLIANT, 0); + expect(report.ok).toBe(true); + expect(report.violations).toEqual([]); + expect(report.maxCandidatesPerGeneration).toBe(4); + }); + + it('never counts baseline rows as generation candidates', () => { + const report = checkDarwinBoundsFromStdout(LB_2026_09_07, 0); + expect(report.candidatesPerGeneration[0]).toBeUndefined(); + }); + + it('flags more than 3 generations', () => { + const rows: { id: string; generation: number }[] = []; + for (let g = 1; g <= 4; g++) { + for (let v = 0; v < 2; v++) { + rows.push({ id: `g${g}_v${v}`, generation: g }); + } + } + const report = checkDarwinBounds(rows, 0); + expect(report.ok).toBe(false); + expect(report.violations.some((v) => v.startsWith('generations=4'))).toBe(true); + }); + + it('flags more than 1 promoted lineage', () => { + const rows = parseLeaderboardRows(LB_COMPLIANT); + const report = checkDarwinBounds(rows, 2); + expect(report.violations).toContain('promotedLineages=2 > 1'); + }); +}); + +describe('checkDarwinBoundsFromStdout', () => { + it('is fail-visible when no leaderboard rows can be parsed', () => { + const report = checkDarwinBoundsFromStdout('', 0); + expect(report.parseStatus).toBe('unparsable'); + expect(report.ok).toBe(false); + expect(report.violations[0]).toBe('leaderboard unparsable: 0 rows read'); + }); +}); diff --git a/packages/cli/src/darwinBounds.ts b/packages/cli/src/darwinBounds.ts new file mode 100644 index 0000000..b69da0a --- /dev/null +++ b/packages/cli/src/darwinBounds.ts @@ -0,0 +1,108 @@ +/** + * Darwin bound policy — nightly pipeline step 10: + * generations ≤ 3, candidates/generation ≤ 4, promoted lineages ≤ 1. + * + * Motivation (2026-09-07, run ab4ced4e48b76e83): the REQUIRED darwin + * evaluator returned outcome=PASSED while its leaderboard listed five + * candidates in generation 2 (g2_v0..g2_v4) — a silent bound breach. + * This module makes the bound machine-checkable: additive-only and + * dependency-free (pattern proven on 2026-09-06). + */ + +export interface DarwinLeaderboardRow { + id: string; + generation: number; +} + +export interface DarwinBoundsReport { + ok: boolean; + parseStatus: 'ok' | 'unparsable'; + generations: number; + candidatesPerGeneration: Record; + maxCandidatesPerGeneration: number; + promotedLineages: number; + violations: string[]; +} + +export const DARWIN_BOUNDS = { + maxGenerations: 3, + maxCandidatesPerGeneration: 4, + maxPromotedLineages: 1, +} as const; + +const ROW_RE = /^\s*[\d.]+\s+(baseline|g(\d+)_v(\d+))\s/; + +export function parseLeaderboardRows(stdout: string): DarwinLeaderboardRow[] { + const rows: DarwinLeaderboardRow[] = []; + for (const line of stdout.split('\n')) { + const m = ROW_RE.exec(line); + if (!m) continue; + rows.push( + m[1] === 'baseline' + ? { id: 'baseline', generation: 0 } + : { id: m[1], generation: Number(m[2]) }, + ); + } + return rows; +} + +export function checkDarwinBounds( + rows: DarwinLeaderboardRow[], + promotedLineages: number, +): DarwinBoundsReport { + const perGen = new Map(); + for (const row of rows) { + if (row.id === 'baseline') continue; + perGen.set(row.generation, (perGen.get(row.generation) ?? 0) + 1); + } + + const violations: string[] = []; + if (perGen.size > DARWIN_BOUNDS.maxGenerations) { + violations.push(`generations=${perGen.size} > ${DARWIN_BOUNDS.maxGenerations}`); + } + let maxCandidates = 0; + for (const [gen, count] of perGen) { + maxCandidates = Math.max(maxCandidates, count); + if (count > DARWIN_BOUNDS.maxCandidatesPerGeneration) { + violations.push( + `g${gen} candidates=${count} > ${DARWIN_BOUNDS.maxCandidatesPerGeneration}`, + ); + } + } + if (promotedLineages > DARWIN_BOUNDS.maxPromotedLineages) { + violations.push( + `promotedLineages=${promotedLineages} > ${DARWIN_BOUNDS.maxPromotedLineages}`, + ); + } + + const candidatesPerGeneration: Record = {}; + for (const [gen, count] of perGen) { + candidatesPerGeneration[gen] = count; + } + + return { + ok: violations.length === 0, + parseStatus: rows.length === 0 ? 'unparsable' : 'ok', + generations: perGen.size, + candidatesPerGeneration, + maxCandidatesPerGeneration: maxCandidates, + promotedLineages, + violations, + }; +} + +export function checkDarwinBoundsFromStdout( + stdout: string, + promotedLineages: number, +): DarwinBoundsReport { + const rows = parseLeaderboardRows(stdout); + const report = checkDarwinBounds(rows, promotedLineages); + if (rows.length === 0) { + return { + ...report, + ok: false, + violations: ['leaderboard unparsable: 0 rows read', ...report.violations], + }; + } + return report; +}