dream(evaluation-adapters): luator entrypoint at commit `9f4d8a9aadae67735273f124a4d73c5

docs/adr/ADR-0057-darwin-bound-guard.md

@@ -0,0 +1,33 @@
+# ADR-0057: Fail-visible Darwin bound guard in the evaluation-adapter layer
+
+- Date: 2026-09-07
+- Status: Proposed (draft PR `dream/evaluation-adapters-20260907`)
+- Companion to: ADR-2024 (deterministic evaluator veto)
+
+## Context
+
+Nightly pipeline step 10 bounds every Darwin run: generations <= 3,
+candidates per generation <= 4, promoted lineages <= 1. On 2026-09-07
+(run `ab4ced4e48b76e83`, commit `9f4d8a9a`) the REQUIRED darwin evaluator
+returned `outcome=PASSED` while its leaderboard listed five candidates
+in generation 2 (`g2_v0..g2_v4`). The bound existed only as prose;
+nothing on the evaluator path detected the breach.
+
+## Decision
+
+The adapter layer gains a pure, dependency-free bound checker
+(`packages/cli/src/darwinBounds.ts`) that parses leaderboard output,
+counts candidates per generation label, and reports violations.
+Parsing is fail-visible: zero rows parsed => `unparsable`, never `ok`.
+Policy constants live in `DARWIN_BOUNDS` for one-line adjustment.
+
+## Consequences
+
++ Silent bound breaches become machine-detectable; tonight's real
+  leaderboard is pinned as a regression fixture in tests.
++ Fail-visible parsing guards against leaderboard format drift.
+- Hard-wiring the checker into the live evaluator gate (auto-veto) is
+  deliberately deferred to a human-reviewed follow-up.
+
+Numbering: 0057 follows known-highest ADR-056; renumber on merge if the
+sequence has advanced.

packages/cli/src/darwinBounds.test.ts

@@ -0,0 +1,92 @@
+import { describe, expect, it } from 'vitest';
+import {
+  checkDarwinBounds,
+  checkDarwinBoundsFromStdout,
+  parseLeaderboardRows,
+} from './darwinBounds';
+
+// Fixture: shape of the real 2026-09-07 REQUIRED-evaluator leaderboard
+// (run ab4ced4e48b76e83), which PASSED with g2 holding five candidates.
+const LB_2026_09_07 = [
+  'Darwin Mode — leaderboard',
+  '  0.765  baseline  [planner]  safety=1.00  pass=0.60 ◀ winner',
+  '  0.765  g1_v0  [planner]  safety=1.00  pass=0.60',
+  '  0.765  g1_v1  [toolPolicy]  safety=1.00  pass=0.60',
+  '  0.765  g1_v2  [reviewer]  safety=1.00  pass=0.60',
+  '  0.765  g1_v3  [toolPolicy]  safety=1.00  pass=0.60',
+  '  0.765  g2_v0  [reviewer]  safety=1.00  pass=0.60',
+  '  0.765  g2_v1  [planner]  safety=1.00  pass=0.60',
+  '  0.765  g2_v2  [contextBuilder]  safety=1.00  pass=0.60',
+  '  0.765  g2_v3  [toolPolicy]  safety=1.00  pass=0.60',
+  '  0.765  g2_v4  [scorePolicy]  safety=1.00  pass=0.60',
+  '',
+  'Winner: baseline',
+  'Lineage: baseline',
+  'Delta over baseline: +0.000',
+].join('\n');
+
+const LB_COMPLIANT = LB_2026_09_07.split('\n')
+  .filter((line) => !line.includes('g2_v4'))
+  .join('\n');
+
+describe('parseLeaderboardRows', () => {
+  it('parses the real 2026-09-07 leaderboard: 1 baseline + 9 mutants', () => {
+    const rows = parseLeaderboardRows(LB_2026_09_07);
+    expect(rows).toHaveLength(10);
+    expect(rows.filter((r) => r.id === 'baseline')).toHaveLength(1);
+  });
+
+  it('returns [] when no leaderboard lines are present', () => {
+    expect(parseLeaderboardRows('nothing to see here')).toEqual([]);
+  });
+});
+
+describe('checkDarwinBounds', () => {
+  it('flags the real 2026-09-07 run: g2 held 5 candidates (> 4)', () => {
+    const report = checkDarwinBoundsFromStdout(LB_2026_09_07, 0);
+    expect(report.parseStatus).toBe('ok');
+    expect(report.candidatesPerGeneration).toEqual({ 1: 4, 2: 5 });
+    expect(report.maxCandidatesPerGeneration).toBe(5);
+    expect(report.ok).toBe(false);
+    expect(report.violations.join('\n')).toContain('g2 candidates=5 > 4');
+  });
+
+  it('accepts the same run with g2_v4 removed (compliant)', () => {
+    const report = checkDarwinBoundsFromStdout(LB_COMPLIANT, 0);
+    expect(report.ok).toBe(true);
+    expect(report.violations).toEqual([]);
+    expect(report.maxCandidatesPerGeneration).toBe(4);
+  });
+
+  it('never counts baseline rows as generation candidates', () => {
+    const report = checkDarwinBoundsFromStdout(LB_2026_09_07, 0);
+    expect(report.candidatesPerGeneration[0]).toBeUndefined();
+  });
+
+  it('flags more than 3 generations', () => {
+    const rows: { id: string; generation: number }[] = [];
+    for (let g = 1; g <= 4; g++) {
+      for (let v = 0; v < 2; v++) {
+        rows.push({ id: `g${g}_v${v}`, generation: g });
+      }
+    }
+    const report = checkDarwinBounds(rows, 0);
+    expect(report.ok).toBe(false);
+    expect(report.violations.some((v) => v.startsWith('generations=4'))).toBe(true);
+  });
+
+  it('flags more than 1 promoted lineage', () => {
+    const rows = parseLeaderboardRows(LB_COMPLIANT);
+    const report = checkDarwinBounds(rows, 2);
+    expect(report.violations).toContain('promotedLineages=2 > 1');
+  });
+});
+
+describe('checkDarwinBoundsFromStdout', () => {
+  it('is fail-visible when no leaderboard rows can be parsed', () => {
+    const report = checkDarwinBoundsFromStdout('', 0);
+    expect(report.parseStatus).toBe('unparsable');
+    expect(report.ok).toBe(false);
+    expect(report.violations[0]).toBe('leaderboard unparsable: 0 rows read');
+  });
+});

packages/cli/src/darwinBounds.ts

@@ -0,0 +1,108 @@
+/**
+ * Darwin bound policy — nightly pipeline step 10:
+ *   generations ≤ 3, candidates/generation ≤ 4, promoted lineages ≤ 1.
+ *
+ * Motivation (2026-09-07, run ab4ced4e48b76e83): the REQUIRED darwin
+ * evaluator returned outcome=PASSED while its leaderboard listed five
+ * candidates in generation 2 (g2_v0..g2_v4) — a silent bound breach.
+ * This module makes the bound machine-checkable: additive-only and
+ * dependency-free (pattern proven on 2026-09-06).
+ */
+
+export interface DarwinLeaderboardRow {
+  id: string;
+  generation: number;
+}
+
+export interface DarwinBoundsReport {
+  ok: boolean;
+  parseStatus: 'ok' | 'unparsable';
+  generations: number;
+  candidatesPerGeneration: Record<number, number>;
+  maxCandidatesPerGeneration: number;
+  promotedLineages: number;
+  violations: string[];
+}
+
+export const DARWIN_BOUNDS = {
+  maxGenerations: 3,
+  maxCandidatesPerGeneration: 4,
+  maxPromotedLineages: 1,
+} as const;
+
+const ROW_RE = /^\s*[\d.]+\s+(baseline|g(\d+)_v(\d+))\s/;
+
+export function parseLeaderboardRows(stdout: string): DarwinLeaderboardRow[] {
+  const rows: DarwinLeaderboardRow[] = [];
+  for (const line of stdout.split('\n')) {
+    const m = ROW_RE.exec(line);
+    if (!m) continue;
+    rows.push(
+      m[1] === 'baseline'
+        ? { id: 'baseline', generation: 0 }
+        : { id: m[1], generation: Number(m[2]) },
+    );
+  }
+  return rows;
+}
+
+export function checkDarwinBounds(
+  rows: DarwinLeaderboardRow[],
+  promotedLineages: number,
+): DarwinBoundsReport {
+  const perGen = new Map<number, number>();
+  for (const row of rows) {
+    if (row.id === 'baseline') continue;
+    perGen.set(row.generation, (perGen.get(row.generation) ?? 0) + 1);
+  }
+
+  const violations: string[] = [];
+  if (perGen.size > DARWIN_BOUNDS.maxGenerations) {
+    violations.push(`generations=${perGen.size} > ${DARWIN_BOUNDS.maxGenerations}`);
+  }
+  let maxCandidates = 0;
+  for (const [gen, count] of perGen) {
+    maxCandidates = Math.max(maxCandidates, count);
+    if (count > DARWIN_BOUNDS.maxCandidatesPerGeneration) {
+      violations.push(
+        `g${gen} candidates=${count} > ${DARWIN_BOUNDS.maxCandidatesPerGeneration}`,
+      );
+    }
+  }
+  if (promotedLineages > DARWIN_BOUNDS.maxPromotedLineages) {
+    violations.push(
+      `promotedLineages=${promotedLineages} > ${DARWIN_BOUNDS.maxPromotedLineages}`,
+    );
+  }
+
+  const candidatesPerGeneration: Record<number, number> = {};
+  for (const [gen, count] of perGen) {
+    candidatesPerGeneration[gen] = count;
+  }
+
+  return {
+    ok: violations.length === 0,
+    parseStatus: rows.length === 0 ? 'unparsable' : 'ok',
+    generations: perGen.size,
+    candidatesPerGeneration,
+    maxCandidatesPerGeneration: maxCandidates,
+    promotedLineages,
+    violations,
+  };
+}
+
+export function checkDarwinBoundsFromStdout(
+  stdout: string,
+  promotedLineages: number,
+): DarwinBoundsReport {
+  const rows = parseLeaderboardRows(stdout);
+  const report = checkDarwinBounds(rows, promotedLineages);
+  if (rows.length === 0) {
+    return {
+      ...report,
+      ok: false,
+      violations: ['leaderboard unparsable: 0 rows read', ...report.violations],
+    };
+  }
+  return report;
+}