diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 48c84a00..4c9d57bb 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -62,3 +62,13 @@ {"skill":"/agent-eval","ts":"2026-09-08T19:57:11Z","project":"agent-eval-composable-search","target":"configurable scoped and sequential optimization methods","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/agent-eval","ts":"2026-09-10T01:01:49Z","project":"agent-eval-search-projector","target":"search ledger projector extraction","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/simplify","ts":"2026-09-10T01:01:49Z","project":"agent-eval-search-projector","target":"search ledger module ownership; 89 preserved declarations and unchanged public API","operatorPrompt":"","durationMin":null,"verdict":"KEEP","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/reconcile","ts":"2026-09-13T01:32:41Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"ML benchmark book against agent-eval: source-backed adoption decisions and offline boundary probes","operatorPrompt":"","durationMin":null,"verdict":"ADAPT","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/report","ts":"2026-09-13T01:32:42Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Complete linked ML benchmark book review and agent-eval capability map at fe1cc511","operatorPrompt":"","durationMin":null,"verdict":"COMPLETE","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/review-to-green","ts":"2026-09-13T01:47:58Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"PR #748: source identity reproduces from the reviewed archive and detects local source changes; 11 focused checks passed","operatorPrompt":"","durationMin":null,"verdict":"FIXED","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/report","ts":"2026-09-14T01:27:02Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Book-inspired methodology and historical selfImprove evidence: 10 registry entries, 20 notebook records, 4 raw analyst results; efficacy limits retained in docs/design/self-improvement-evidence-audit.md","operatorPrompt":"","durationMin":null,"verdict":"COMPLETE","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/dont-collapse-the-architecture","ts":"2026-09-14T01:27:02Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"SelfImprove usefulness: fixed-roster development remains reusable; candidates survive inconclusive or negative final results; historical nulls do not isolate optimizer mechanisms or prove cross-domain failure","operatorPrompt":"","durationMin":null,"verdict":"OPEN","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/deep-clean","ts":"2026-09-14T02:09:17Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Evaluation APIs: removed retired digest readers and canonical-JSON waiver paths; root types align with contract; breaking upgrades documented; caller coverage 139/201 refs with 62 exclusions","operatorPrompt":"","durationMin":null,"verdict":"IMPLEMENTED","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/polish","ts":"2026-09-14T02:09:17Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Evaluation integrity: shared units, outcome validity, evaluator admission, reusable claims, README and public exports; final package checks and 107 affected tests passed; full dedicated CI pending","operatorPrompt":"","durationMin":null,"verdict":"OPEN","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/verify","ts":"2026-09-14T02:09:17Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Final judge snapshot: 107/107 affected tests; typecheck, examples, scripts, lint, build, package verification and public offline example passed; full local suite has timing failures; dedicated CI pending","operatorPrompt":"","durationMin":null,"verdict":"OPEN","dispatchedTo":"/review-to-green","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/review-to-green","ts":"2026-09-14T02:09:18Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Local Standards/Spec review: fixed mutable-evaluator false promotion and matrix docs; six adversarial regressions and independent fix review passed; hosted review quota exhausted, formal approval remains required","operatorPrompt":"","durationMin":null,"verdict":"LOCAL_FIXED","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/converge","ts":"2026-09-14T02:16:07Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"PR #749: clean CI checkout cannot resolve public example imports before build; moved example typecheck after package build without altering check coverage","operatorPrompt":"","durationMin":null,"verdict":"LOCAL_FIXED","dispatchedTo":"/review-to-green","operatorOverride":null,"transcriptPath":null,"traceDir":null} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7f71b31..478d41ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] jobs: ci: @@ -43,9 +42,6 @@ jobs: - name: Typecheck run: pnpm typecheck - - name: Typecheck examples - run: pnpm typecheck:examples - - name: Typecheck scripts run: pnpm typecheck:scripts @@ -57,6 +53,9 @@ jobs: NODE_OPTIONS: --max-old-space-size=8192 run: pnpm build + - name: Typecheck examples + run: pnpm typecheck:examples + - name: Verify packed package exports run: pnpm run verify:package diff --git a/CHANGELOG.md b/CHANGELOG.md index f988ac9d..7d73ccb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,47 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ## Unreleased +### Changed + +- **Breaking:** Root `Scenario`, `JudgeScore`, and `GateDecision` now match `/contract`. + Product workflows use `ProductScenario`, `DimensionJudgeScore`, and `HeldOutGateDecision`. +- **Breaking:** Current canonical envelopes and algorithm identifiers are required for seals, attestations, and profile identities. + Retired digest readers and canonical-JSON waiver paths are removed. + Historical evidence retains its original identity. +- **Breaking:** Cluster interval registrations bind their measured `value` field. + Row execution evidence supplies rows; it cannot select another metric under the same seal. +- **Breaking:** Power calculations and power-floor gates require `minimumEffect` and assess adequacy at that effect. +- **Breaking:** Predictive validity requires a declared outcome direction and uses descriptive `aligned`, `inverse`, and `weak` associations. + Research proposals retain hypotheses instead of invented expected gains. +- **Breaking:** Adaptation comparisons require matched identified scenario cohorts and report paired uncertainty and inconclusive results. + Contamination diagnostics use `alpha`; heuristic per-item `qValue` values are removed. +- Method comparisons expose unit-level scores, raw paired-cell counts, and the deciding statistical evidence. + `favored: null` replaces the ambiguous `tie` sentinel for inconclusive comparisons. + Continuous mean decisions cannot use a small-sample sign test as evidence about the mean. + Binary and explicit median decisions retain their appropriate confidence-dependent observation requirements. + +### Added + +- Optional top-level `claim` metadata declares populations and independent source units without consuming reusable regression evidence. + New-unit claims keep source families together across automatic partitions. + Self-improvement retains the selected candidate when release evidence is negative or inconclusive. +- Optional `finalEvidence` reserves fresh final units before search and records exposure before final measurement. + The shared journal detects conflicting use, concurrent ownership, corrupted history, and deleted trusted heads. +- `/meta-eval` exports evaluator admission from actual controls, simultaneous error bounds, and explicit unknown and excluded evidence. + Existing position and self-preference audits are public alongside calibration and verbosity diagnostics. +- `calibrationFromPairs()` accepts direct measured rows without requiring trace and outcome stores. +- The [evaluation-integrity guide](docs/evaluation-integrity.md) explains methodology and limits. + Its offline example composes public imports and exports actual fixture results. + ### Fixed +- Outcome queries select the latest finite requested metric instead of an unrelated latest observation. + Outcome-store corruption and unavailable evidence remain visible failures. +- Calibration preserves clipped observations, measures constant predictors, and honors the requested bin count. +- Registered-unit gates pair complete cells before aggregation; repetitions and source variants cannot multiply independent evidence. +- Opened experiments and outcome research retain validated snapshots instead of mutable caller-owned rules. +- Comparisons capture judge configuration and callbacks before asynchronous work. + Replacing a caller's judge between arms cannot create artificial lift under the original evaluator identity. - Pareto promotion applies regression floors to the deciding confidence interval. Tied binary outcomes cannot bypass a safety floor through a zero-width diagnostic bootstrap. Zero-width or non-finite deciding intervals now produce an `indeterminate` axis and `not_evaluated` check. diff --git a/README.md b/README.md index 919ba455..eb12bb49 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,15 @@ # `@tangle-network/agent-eval` -Measure agent behavior, compare changes on the same cases, and improve prompts or skills without showing the final test cases to the optimizer. +Run agent evaluations, compare changes on the same cases, and decide whether a candidate has enough evidence to release. [![npm](https://img.shields.io/npm/v/@tangle-network/agent-eval.svg)](https://www.npmjs.com/package/@tangle-network/agent-eval) [![pypi](https://img.shields.io/pypi/v/agent-eval-rpc.svg)](https://pypi.org/project/agent-eval-rpc/) [![tests](https://github.com/tangle-network/agent-eval/actions/workflows/ci.yml/badge.svg)](https://github.com/tangle-network/agent-eval/actions/workflows/ci.yml) [![license: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](./LICENSE) -The evaluation path runs in your TypeScript process. -Model calls happen only through the clients and agents you configure. - -New to the package? Read [concepts](./docs/concepts.md) first — it takes five minutes and defines every word used here. - -Looking for a measured result (a lift, a null, a parity verdict)? The canonical registry is [`evidence/`](./evidence/README.md) — machine-readable records, a generated index, and a freshness gate. +Eval runs in your TypeScript process. +You supply agent execution, judges, and model transports. +It records outputs, failures, costs, and evidence for each comparison. ## Install @@ -22,8 +19,8 @@ pnpm add @tangle-network/agent-eval ## Quickstart -This example is offline and complete. -Copy it, run it, then replace the agent and the judge with your product code. +This complete example runs offline. +Replace the agent and judge with your product functions when it works. ```ts import { defineAgentEval } from '@tangle-network/agent-eval/contract' @@ -60,138 +57,103 @@ console.log( ) ``` -Each call runs every case, records what the agent produced, applies the same judge, and returns score distributions. +The baseline scores `0`; the candidate scores `1` on all three cases. +These scores describe the three examples. +They do not establish a release decision or performance on new tasks. -Three words carry this example. -A **case** is one task the agent must do. -A **surface** is the value being changed: a prompt, a skill, or a serialized configuration. -A **judge** is a function that scores one produced result. +A **case** is one task. +A **surface** is the prompt, skill, or configuration being changed. +A **judge** scores the agent's result. -`expectUsage: 'off'` is set because this agent makes no paid calls. -The default, `'assert'`, fails a run whose cells report no cost receipt. -Keep the default whenever real model calls happen. +`expectUsage: 'off'` applies because this example makes no paid calls. +Keep the default, `'assert'`, for model calls so missing cost receipts fail visibly. +The [runnable example](./examples/evaluate-a-change/) uses the same evaluation. -Runnable copy: [`examples/evaluate-a-change`](./examples/evaluate-a-change/). +## Choose a workflow -## Auditable optimization history +| Intent | Start with | Result | +|---|---|---| +| Score one change | [`defineAgentEval()`](./examples/evaluate-a-change/) from `/contract` | Cell results, failures, score distributions, and measured cost. | +| Search for a better surface | [`selfImprove()`](./examples/selfimprove-quickstart/) from `/contract` | A selected surface, final comparison, and `gateDecision`. | +| Compare search methods | [`compareOptimizationMethods()`](./examples/compare-optimization-methods/) from `/campaign` | Paired final comparisons, uncertainty, coverage, and costs under declared budgets. | +| Register evidence and decision rules | [`defineEvaluationClaim()` and `sealExperiment()`](./docs/evaluation-integrity.md) from `/experiment` | A declared population, independent unit, optional practical effect, and sealed rules. | +| Check the evaluator | [`auditEvaluator()`](./docs/evaluation-integrity.md) and [calibration tools](./docs/outcome-validity.md) from `/meta-eval` | Error rates, admission evidence, bias diagnostics, and outcome associations. | +| Analyze completed work | [`analyzeRuns()`](./examples/analyze-existing-runs/) from `/contract`; [trace analysts](./docs/trace-analysis.md) from `/analyst` | Comparisons and findings with links to recorded evidence. | -Optimization methods may return a bounded `SearchHistoryReceipt` over Eval's canonical hash-chained `SearchLedger`. Existing callers keep working and see missing-history coverage. Autonomous and publication-grade runs set `searchHistoryPolicy: 'require-complete'` to refuse an incomplete planned denominator before the untouched final cases are opened. +`defineAgentEval()` also exposes `improve()` when the same agent, cases, judge, and baseline should share configuration. +Use direct [campaign controls](./docs/eval-surface-map.md) for scheduling, durable caches, model matrices, or custom release rules. +The [example index](./examples/README.md) covers fixtures, trace intake, code verification, replay, and training-data exports. -The receipt is a small proof envelope, not another event log. Exact candidates, attempts, failures, decisions, and missing ids remain in the ledger. See [complete optimization search history](./docs/search-history-receipts.md). +## Make automated improvement accountable -## Which Front Door +An optimizer can write candidates and evaluations faster than a person can review them. +Use reusable evaluations for development feedback. +Add stronger controls when a result must support performance on new tasks or an adaptive release decision. -Every row is a function you call. Each links to a runnable example. +1. Pass a `claim` describing the population, sampling frame, and independent unit to the comparison. + Declare `minimumEffect` when the decision concerns a useful improvement. +2. When introducing an evaluator, check known good and known bad controls with `auditEvaluator()`. +3. Give search separate training and selection cases. +4. For fresh confirmation, supply `finalEvidence` with a shared ledger, request ID, and evaluator digest. + This reserves final units before search and records exposure before measurement. +5. Inspect the final comparison, gate contributions, exclusions, uncertainty, cost, and search history before releasing. -| When to call it | What you give it | What you get back | -|---|---|---| -| [`defineAgentEval()`](./examples/evaluate-a-change/) — you changed a surface and must know whether it helped | cases, an agent, a judge, a starting surface | `evaluate()` for scores, `improve()` for a search plus a release decision | -| [`selfImprove()`](./examples/selfimprove-quickstart/) — you want candidate generation, scoring, and a release decision in one call | cases, an agent, a judge, a starting surface | a report, a winner surface, and a `gateDecision` | -| [`analyzeRuns()`](./examples/analyze-existing-runs/) — the runs already happened and no agent needs to run again | `RunRecord[]` | an `InsightReport`: distributions, paired lift, judge agreement, cost, failure clusters | -| `fromFeedbackTable()` ([example](./examples/customer-feedback-loop/)) / `fromOtelSpans()` ([example](./examples/customer-otel-traces/)) — your data is in a table or an OTel collector, not in `RunRecord` shape | source rows or spans | `RunRecord[]` ready for `analyzeRuns()` | -| [`planCampaignRun()` / `runCampaign()`](./examples/plan-before-you-spend/) — you need direct control of the case grid, or you must see it before paying for it | cases, a dispatch function, judges, a run directory | a per-cell schedule, then a campaign result with cached cells | -| [`loadEvalFixtureScenarios()`](./examples/eval-fixtures-quickstart/) — agents should add cases as folders on disk | `evals//PROMPT.md` plus checks | `Scenario[]` for `runCampaign()` | -| [`compareOptimizationMethods()`](./examples/compare-optimization-methods/) — two search methods must be compared at equal budget | methods, a starting surface, train, selection, and final cases | per-method final lift, intervals, pairwise contrasts, and cost | -| [`gepaOptimizationMethod()` / `skillOptOptimizationMethod()`](./examples/compare-optimization-methods/) — official GEPA or Microsoft SkillOpt should own the search | an objective, a recipe or trainer, an optimizer budget | an optimization method for the comparison above | -| [`externalTextOptimizationMethod()`](./examples/adapt-a-text-optimizer/) — another package owns text search and you keep the scoring | the package identity, limits, and a `run` callback | the same, with the final cases never exposed | -| [`SurfaceProposer`](./examples/selfimprove-quickstart/) — candidate generation belongs to your product | a `propose()` function | candidates the campaign executes, scores, and gates | -| [`runProfileMatrix()`](./examples/profile-matrix/) — the same cases must run across models or profiles | axes of models and profiles, cases | one row per cell, with an explicit `unknown` model rather than an invented one | -| [`ExperimentTracker`](./examples/experiment-evidence/) — a candidate must beat its parent across N repetitions | reps with scores, run ids, and evidence references | a KEEP / ITERATE / NOISE / REGRESSION verdict with git provenance | -| [`sealExperiment()` / `openSealedExperiment()`](./examples/sealed-experiment/) — the result must convince someone who does not trust you | arms, an admission funnel, an estimand, an interval, a decision table | a hashed rule tree, and executors that can run no other rule | -| [`runEquivalenceCheck()` / `VERIFICATION_STRATEGIES`](./examples/verify-without-an-answer-key/) — the work has no held-out test suite | a claim, two blind arms, an injected checker | a certification that names who vouched and how it can fail | -| [`AnalystRegistry.runExact()`](./examples/custom-trace-analyst/) — a batch of runs failed and you need cited findings | recorded evidence, a declared analyst list | findings with evidence references, an execution plan, and a receipt | -| [`analyzeTraces()`](./docs/trace-analysis.md#answer-one-question) — you have one question about a recorded run ("what first caused this failure?") | stored traces, the question, a DSPy RLM engine with a cost cap | an answer, findings with evidence references, and the investigation trajectory | -| [`runAnalystBenchmark()`](./docs/trace-analysis.md) — an analyst's accuracy must be measured, not assumed | labeled issues and exact span locations | scored findings, trace reads, model calls, tokens, cost, and runtime | -| [`deltaRepair()`](./docs/trace-repair-grader.md) — a finding must be graded by executing the repair it proposes | a trajectory, an analyst finding, a sandbox | the repair's measured effect against a no-fix control | -| [`replayVerify()`](./docs/trajectory-replay.md) — you must know whether a recorded failure still reproduces | a recorded shell trajectory and its pinned image | a re-execution verdict and the divergences found | -| [`analyzeSupervisorRun()`](./docs/adapters-observability.md) — a recursive or supervised run directory must be read | a run directory | counts that stay missing when a measurement is missing, never zero | -| [`plantByPerturbation()` / `seedPlants()` / `catchRate()`](./docs/plants.md) — you must know whether the grader catches a wrong answer, not only how the work scored | a grading set, and claims the grader verified | items authored wrong by one value, a sealed manifest, then a catch rate that refuses rather than guessing | -| [`buildRlDataset()`](./examples/publish-rl-dataset/) — scored runs should become training data | run records and preferences | reward, preference, and supervised rows | - -## Configure Model Calls - -Benchmarks, user drivers, executors, built-in judges, completion checkers, and judge adapters all take the same `ChatClient`. -You own model execution, and Agent Eval never goes looking for a credential: it reads no environment variable to find one, and every transport is bound at the call site. - -Bind a transport one of two ways. Pass a `chat` function you wrote: +Repeated attempts on one task do not create new independent tasks. +The top-level `claim` controls unit aggregation for reusable comparisons. +Power checks assess the declared minimum effect. +Optional `finalEvidence` binds fresh confirmation to that claim and refuses reused final units across campaigns sharing the ledger. -```ts -import { createChatClient } from '@tangle-network/agent-eval' +The host must enforce access isolation and author/auditor separation. +A digest records identity; it cannot prove secrecy or that a benchmark represents future users. +Custom gates remain responsible for their decision rules. +See [evaluation integrity](./docs/evaluation-integrity.md) for the complete API and its boundaries. -const chat = createChatClient({ - transport: 'custom', - defaultModel: 'openai/gpt-4.1', - maximumAttempts: 3, - chat: async (request, opts) => myProviderClient(request, opts), -}) -``` +Set `searchHistoryPolicy: 'require-complete'` when every attempted search slot must be accounted for before final evidence is exposed. +The [search-history receipt](./docs/search-history-receipts.md) binds the planned denominator to Eval's existing search ledger. + +A `gateDecision` is `ship`, `hold`, `need_more_work`, `model_ceiling`, or `arch_ceiling`. +Gate contributions distinguish missing evidence from measured failures and successful checks. +[Concepts](./docs/concepts.md) explains these decisions and how gates compose. + +## Configure model calls -Or name an OpenAI-compatible endpoint and hand over a credential as values, and Agent Eval drives `POST {baseUrl}/chat/completions` for you: +Pass a `ChatClient` to model judges, analysts, and adapters. +Eval obtains credentials from the values you supply; it does not search your environment. ```ts +import { createChatClient } from '@tangle-network/agent-eval/contract' + const chat = createChatClient({ transport: 'openai-compatible', - baseUrl: 'https://router.example/v1', // ends at /v1; the path is ours to append - apiKey: process.env.MY_ROUTER_KEY, // or `bearer`, or `authHeader` - defaultModel: 'claude-sonnet-4-6', + baseUrl: 'https://router.example/v1', + apiKey: process.env.MY_ROUTER_KEY, + defaultModel: process.env.EVAL_MODEL_ID, }) ``` -Prefer the second over hand-rolling a fetch loop. It carries the retry, degrade, and — load-bearing — the `servedModel` echo that `assertServedModel` and `assertCrossFamilyServed` read; a transport that omits that field makes both checks report `unreported`, so the cross-vendor rules they enforce measure nothing. `baseUrl` and one credential form are required arguments with no default and no fallback: a half-configured client is refused at construction rather than reaching an endpoint you did not name. - -On Agent Runtime, `profileChatClient({ profile, executor, context })` from `@tangle-network/agent-runtime/kernel` is that transport: every call runs one exact `AgentProfile` and reports its measured usage, retries, and served model identity. -Use `sandbox-sdk` for Sandbox and `mock` in tests. -A custom adapter must return a `ChatResponse` and declare `maximumAttempts` before a capped cost account can dispatch it. - -`ChatResponse` carries the whole execution record across that boundary: the served model id, measured input/output/reasoning/cached tokens, billed USD or an explicit unknown, the finish reason, and the per-token log probabilities the expectation judge scores on. - -The official GEPA and SkillOpt optimizers run through a Python bridge. -Install commands, version pins, and the reason for each pin: -[GEPA](./docs/campaign-proposers.md#install-official-gepa), -[SkillOpt](./docs/campaign-proposers.md#install-official-skillopt), -and [DSPy](./docs/campaign-proposers.md#use-official-dspy-optimizers). - -## Entry Points - -| Import | Use | -|---|---| -| `@tangle-network/agent-eval/contract` | Define an evaluation, run it, improve it, and analyze existing runs. | -| `@tangle-network/agent-eval/campaign` | Campaigns, optimization methods, comparisons, storage, and release rules. | -| `@tangle-network/agent-eval/experiment` | Experiments as sealed objects: registered rules, funnels, estimands, refusals. | -| `@tangle-network/agent-eval/analyst` | Built-in and custom trace analysts, labeled comparison, costs, and reports. | -| `@tangle-network/agent-eval/trace-repair` | Grade one analyst finding by executing the repair it proposes. | -| `@tangle-network/agent-eval/trajectory-replay` | Re-execute a recorded shell trajectory and check whether its failure reproduces. | -| `@tangle-network/agent-eval/traces` | Store, replay, and inspect structured traces. | -| `@tangle-network/agent-eval/reporting` | Statistical comparisons and report rendering. | -| `@tangle-network/agent-eval/supervisor-run` | Read recursive run directories without collapsing missing measurements to zero; `agent-eval supervisor-run report ` prints one. | -| `@tangle-network/agent-eval/meta-eval` | Measure the grader itself: judge calibration, sentinels, and seeded known-wrong plants. | -| `@tangle-network/agent-eval/profile-cell` | Create and validate portable agent-profile identities. | -| `@tangle-network/agent-eval/ledger-core` | Append-only hash-chained journal with idempotent append and chain verification. | -| `@tangle-network/agent-eval/benchmarks` | Benchmark adapters and retrieval metrics. | -| `@tangle-network/agent-eval/rl` | Export rewards, preferences, and training rows. | -| `@tangle-network/agent-eval/wire` | HTTP and RPC schemas for other languages. | -| `@tangle-network/agent-eval/adapters/http` | Run campaign cells on remote workers over HTTP. | - -Use the root import for common primitives. -Use a subpath when you want an explicit capability boundary. - -## Documentation - -| Question | Read | -|---|---| -| What do these words mean? | [`docs/concepts.md`](./docs/concepts.md) | -| Why does this package exist, and where is it going? | [`docs/charter.md`](./docs/charter.md) | -| Which `run*` function do I want? | [`docs/eval-surface-map.md`](./docs/eval-surface-map.md) | -| How do I choose a candidate-generation method? | [`docs/campaign-proposers.md`](./docs/campaign-proposers.md) | -| What is in an `InsightReport`? | [`docs/insight-report.md`](./docs/insight-report.md) | -| How do I register an experiment as a sealed object? | [`docs/experiment.md`](./docs/experiment.md) | -| How is something certified without an answer key? | [`docs/verification-strategies.md`](./docs/verification-strategies.md) | -| Where does every verifier land its result? | [`docs/verdicts.md`](./docs/verdicts.md) | -| Does the grader catch a claim that is known to be wrong? | [`docs/plants.md`](./docs/plants.md) | -| How do I turn a coding-agent session log into runs? | [`docs/code-agent-intake.md`](./docs/code-agent-intake.md) | -| How do I score a string from another language? | [`docs/wire-protocol.md`](./docs/wire-protocol.md) | - -The [example index](./examples/README.md) lists every runnable example. +Use your deployed model identifier and preserve the returned `servedModel` identity and cost receipt. +For an existing SDK, use `transport: 'custom'` with your `chat` callback and an explicit `maximumAttempts`. +Agent Runtime callers can bind `profileChatClient()` from `@tangle-network/agent-runtime/kernel`. +Eval has no dependency on Runtime. + +Official GEPA, SkillOpt, and DSPy integrations use a Python bridge. +Their maintained installation instructions and execution contracts are in [campaign proposers](./docs/campaign-proposers.md). +The [Python client](./clients/python/README.md) and [wire protocol](./docs/wire-protocol.md) support other-language consumers. + +## Public imports and evidence + +Use `/contract` for a product integration, `/campaign` for execution controls, `/experiment` for registered decisions, and `/meta-eval` for evaluator checks. +Root `Scenario`, `JudgeScore`, and `GateDecision` are the same types as `/contract`. +Product judging retains the explicit root names `ProductScenario` and `DimensionJudgeScore` beside its functions. +`HeldOutGate.evaluate()` returns `HeldOutGateDecision`. + +Specialist subpaths and their examples are listed in the [surface map](./docs/eval-surface-map.md). +Current canonical envelopes are required for seals, attestations, and profile identities. +Retired or incomplete formats fail verification; historical reports retain their recorded identities. + +Published measurements live in the [evidence registry](./evidence/README.md). +The [benchmark-book review](./docs/design/mlbenchmarks-book-review.md) records the source analysis and reproduced defects behind these integrity changes. +[The charter](./docs/charter.md) defines package ownership and the remaining research boundaries. ## Development @@ -199,8 +161,10 @@ The [example index](./examples/README.md) lists every runnable example. pnpm install pnpm typecheck pnpm typecheck:examples +pnpm typecheck:scripts pnpm test pnpm build +pnpm verify:package ``` Python compatibility tests use the locked dependencies: @@ -222,17 +186,3 @@ uv run --frozen pytest tests/test_dspy_metric.py ## License MIT. - - -## Supervisor-run resource receipts - -The `/supervisor-run` reader preserves named-resource measurements in `economics.resourceRecords`. -Each record identifies its node and source within the normalized journal or terminal result. -Journal row indices refer to parsed rows after reader normalization, not original file line numbers. -The Markdown report renders each resource name, unit, amount, and completeness flag. -Comparison cells retain those same records without combining them. - -A false `known` flag means the amount is a recorded subtotal, not complete usage. -Missing maps, explicit empty maps, and invalid fields remain distinct from measured zero. -Parent settlements and terminal results can include child usage, so these records are not additive totals. -The reporter reads evidence; it does not enforce budgets or infer missing measurements. diff --git a/docs/adapters-observability.md b/docs/adapters-observability.md index bdf74785..f31330b1 100644 --- a/docs/adapters-observability.md +++ b/docs/adapters-observability.md @@ -134,3 +134,17 @@ No new dependencies. No new peer deps. No `@traceai/*`, no `@langfuse/*`, no `@opentelemetry/*` in our manifest. You bring the observability stack you want; agent-eval's exporter emits the same OTLP wire format independently, keyed on the endpoint you point it at. + + +## Supervisor-run resource receipts + +The `/supervisor-run` reader preserves named-resource measurements in `economics.resourceRecords`. +Each record identifies its node and source within the normalized journal or terminal result. +Journal row indices refer to parsed rows after reader normalization, not original file line numbers. +The Markdown report renders each resource name, unit, amount, and completeness flag. +Comparison cells retain those same records without combining them. + +A false `known` flag means the amount is a recorded subtotal, not complete usage. +Missing maps, explicit empty maps, and invalid fields remain distinct from measured zero. +Parent settlements and terminal results can include child usage, so these records are not additive totals. +The reporter reads evidence; it does not enforce budgets or infer missing measurements. diff --git a/docs/charter.md b/docs/charter.md index fb49b3c8..6a559910 100644 --- a/docs/charter.md +++ b/docs/charter.md @@ -1,112 +1,108 @@ -# Charter: what agent-eval is for - -This document states what this package is, derived from the four end-states the stack must reach. -It was written on 2026-08-10 from a measured inventory of this repo, agent-runtime, discovery, discovery-lab, braid, traces, and supervisor-lab. -Every claim about current code cites the module that carries it. -When behavior moves, move this document in the same change. - -## One sentence - -agent-eval is the honesty layer of the agent stack: instruments that make it structurally hard to fool ourselves at machine speed, and that never choose a research method. - -The anchor is discovery's covenant (discovery `docs/01-vision.md`): shared code enforces evidence integrity and immutable observations; it never chooses roles, methods, or winners. -Everything above this package — runtime, discovery, braid, verticals — gets its freedom because honesty is enforced below. - -## The four end-states, and what each demands from this package - -1. **Build complex software rapidly, end to end, without issues.** - The gate is not generation. - The gate is knowing what is true about the work while it runs. - Measured on our own corpus: 87% of failed long runs end on a clean exit with the agent claiming success. - Demand: executable verification wired into the runtime loop, not beside it. - -2. **Do novel research: physics, quantum computing, math, unsolved problems.** - An unsolved problem has no held-out test suite by definition. - Every grader this package shipped before 2026-08 assumes an answer key. - Demand: verification strategies that certify without one — proof kernels, invariant checks, independent derivation agreement, replication — plus the statistics of a careful experimentalist. - This demand is measured, not speculative: discovery-lab holds 99 pursuit directories and 71 blind-graded oracle files on frontier problems, all verified today by hand-rolled run tools outside this package. - -3. **Build self-improving agents easily, and explain them easily.** - Improvement requires an ungameable signal; the grader, not the edit, decides improve-versus-game. - Explanation requires a portable artifact: the receipt that proves B beat A, verifiable by a third party who does not trust us. - -4. **Build new interfaces (braid-class): detached sessions, forking, analysts on tap.** - An interface can only expose what the layer below makes addressable. - Demand: sessions, traces, experiments, and verdicts as durable, forkable, queryable objects. - -## What already exists (the 2026-08-10 inventory, corrected) - -The fragmentation story is smaller than it feels. -The audit refuted "built four times": traces imports this package's whole analyst suite; agent-runtime's live detectors import the detection kernel verbatim; supervisor-lab's judges are AgentProfiles dispatched through this package's judge primitives. - -- **Ask any question over any trace: exists.** - `TraceAnalysisEngine` (`src/analyst/engine.ts`) takes a free-text question. - A new custom question costs zero library files: `defineTraceAnalyst` + `runTraceAnalyst` on the `./analyst` subpath, with the DSPy RLM engine, seven byte-budgeted trace tools, and a metered model proxy behind it. -- **Default failure analysts: exist.** - `buildDefaultAnalystRegistry` ships failure-mode, intent-divergence, knowledge-gap, knowledge-poisoning, improvement, control-integrity, and skill-usage kinds, engine-agnostic and versionable. -- **Statistics: most of an A-plus toolkit, publicly exported.** - Paired bootstrap, clustered paired binary, exact and score risk differences, McNemar with power and required-n, MDE, multiplicity (Holm, Benjamini-Hochberg), e-process sequential gates, corpus inter-rater agreement, pre-registration manifests with content hashes. -- **Executable process verification: proven this week.** - The trace-repair grader scores a proposed fix by executing it and running the task's own held-out suite from outside the container. - Oracle-fix separates from inert-probe (+0.353 vs 0.000 on milestone 1) with the floor pinned at zero. -- **Integrity instruments hardened by this week's burns:** - served-model assertions (a gateway can answer one id with another model), oracle determinism certification (a wall-clock grader flipped 8 of 16 units on identical bytes), control-policy declaration (a zero-step control screened two milestones and could never fire), and equal-terms refusal between comparison arms (`repairArmAsymmetries`). - -## What is missing (the honest, short list) - -1. **Cluster-aware power with design-time refusal.** - "Four task clusters cannot certify any effect size, including 1.0" was learned by running the experiment. - A `clusteredPower` simulator must refuse the design before a dollar is spent. -2. **Pre-registration as code, bound everywhere.** - The manifest binds exactly one statistic family today. - The registered decision rule must be the object the runner executes; drift between registered and ran must be unrepresentable. -3. **A general prime query surface.** - The prime engine is benchmark-bound; there is no `runPrimeAnalyst` symmetric to `runTraceAnalyst`. -4. **The funnel as a first-class object.** - Denominator chains are assembled by hand every run. -5. **The unified analyst definition.** - One declarative unit — AgentProfile + evidence projection + reply contract + budget declaration — compiled to any engine, guarded by a byte-identity kill test against the bespoke arms. -6. **Verification without an answer key.** - A strategy family where the held-out suite is one member: proof kernels, invariant and metamorphic checks, independent derivation agreement, replication. - The type layer exists (`docs/verification-strategies.md`) and the certifications are now produced, not just typed: every in-package verifier — the layer pipeline, the completion oracle, trace contracts, declarative oracles, trajectory replay, the repair grader — lands in `DefaultVerdict` with a certification naming its checker, strategy member, and unverified assumptions (`docs/verdicts.md`). - What remains is execution without an answer key: a real kernel checker bound through the port. -7. **Session forking as one primitive.** - Both halves exist unjoined: agent-runtime's `SandboxLineage.fork` (live checkpoint) and this package's trajectory replay (recorded prefix). - Braid branches are metadata pointers; the provider session is always new. -8. **The improvement receipt.** - Digests, attestations, and sealed manifests exist; the single portable, third-party-verifiable file does not. - Five gaming attacks on the receipt are named; each refusal must live inside the receipt. - -## Build order - -Wave 1 — in flight now. -The analyst definition contract with its byte-identity CI kill test; prime as a HarnessType in agent-interface; the three-arm review fixes. - -Wave 2 — the experiment subpath. **Shipped: [docs/experiment.md](./experiment.md).** -`./experiment`: compose the exported statistics into `defineExperiment` / `sealExperiment` with cluster-aware power refusal, pre-registration as executable decision rules, the funnel object, and matched-budget verification. -Kill test first: re-derive this week's three hand-written PREREG.md files as decision-rule objects; any rule that needs an opaque escape hatch kills or extends the design. -The kill test extended the design — ten node families beyond the seed AST, no opaque node — and the three preregistrations are the subpath's acceptance suite. - -Wave 3 — runtime wiring (blocked until agent-runtime's supervision merge resolves). -The executable checker bound as a validator at the live-sandbox seam; a stop policy that consumes executable verdicts; then the supervisor budget-allocation experiment at equal compute. -The published negative result to beat: StateSeal, −3.0pp, CI [−8.5, 1.1], n=540. -Our measured headroom: blind continuation rescues 4.7% of rollouts; the done-signal has 62.5% precision. - -Wave 4 — the joins. -`fork(session, step, modification)` joining lineage-fork and trajectory replay, consumed by braid; the improvement receipt v1 serializing evidence vector, pre-registration hash, grader calibration, and refusal outcomes into one attested file. - -Wave 5 — science. -The verification-strategy family, proof kernels first. -Its pilot is live: two independent Lean formalizations of the BCWW (4.6) inequality — one from the paper, one from the campaign's artifacts — with a kernel-checked equivalence verdict and the campaign's counterexample checked against both. -A statement mismatch is a successful outcome; it is the formalization gap made visible. -The contract layer is shipped: the family, checker port, and blind two-arm equivalence protocol (`src/verification-strategy.ts`) and verdict certifications (`src/verdict.ts`) — see `docs/verification-strategies.md` for the family, each member's failure mode, and the pilot as the worked example. - -## Standing principles (each earned by a measured burn) - -- **Instruments, never methods.** This package refuses to choose roles, prompts, models, or winners; it makes whatever runs honest. -- **No upward dependencies.** Consumers import this package; never the reverse. -- **A check that cannot run must never render as green.** (505 of 5,459 automated reviews published verdicts no evidence supported.) -- **Certification is task-scoped.** A prompt certified on one task carries nothing onto another; re-authoring voids it. -- **Access to the world beats loop sophistication.** Measured twice in one week: the analyst that could execute beat the one that could only read; framing carried more than the agent loop. -- **Controls must be able to fire.** A screening control that cannot in principle produce the outcome it screens for is uncalibrated, not conservative. -- **The refusal lives inside the artifact.** An adequacy check, an equal-terms check, a determinism check that runs beside the result can be skipped; one that the artifact carries cannot. +# Charter: what agent-eval owns + +`agent-eval` owns evaluation data, scoring, experiment decisions, and release evidence. +It lets a host automate candidate generation while preserving the evidence needed to challenge the result. + +## Package boundary + +| Concern | Owner | +|---|---| +| Portable agent contracts and canonical encodings | `agent-interface` | +| Cases, judge scores, run records, statistical comparisons, evidence admission, and release rules | `agent-eval` | +| Agent sessions, workers, tool access, model execution, and research orchestration | The host, including `agent-runtime` | +| Product activation, business outcomes, storage authorization, and access to final data | The consuming application | + +`agent-runtime` and `agent-knowledge` can depend on Eval. +Eval must not depend on either consumer, including through development or type-only imports. +Execution enters through caller-supplied functions. + +## Decisions the package supports + +**Did the agent perform the required task?** +A clean process exit or a fluent answer cannot establish that the required artifact works. +Campaigns retain results, failures, deterministic checks, semantic judgments, traces, and measured usage. +[Completion verification](../src/completion-verifier.ts), [layered verification](../src/multi-layer-verifier.ts), and [trace replay](./trajectory-replay.md) support checks on produced work. + +**Did a change improve the agent?** +A comparison needs paired evidence, explicit exclusions, and an appropriate independent observation unit. +An improvement decision also needs a declared meaningful effect. +[Campaign gates](./eval-surface-map.md) and [registered experiments](./experiment.md) make those decisions inspectable. +Train and selection data can guide search; final evidence supports the resulting comparison. + +**Does the evaluator measure the intended outcome?** +Known good and known bad controls test different errors. +[Evaluator admission](./evaluation-integrity.md), [judge calibration](./concepts.md#judge-calibration), and [outcome validity](./outcome-validity.md) describe what the measurements establish. +Outcome association can motivate an experiment; it cannot establish that changing a rubric causes improvement. + +**Can another reader verify the evidence?** +[Evidence receipts](./experiment.md) bind reports to declared identities and provenance. +[Search-history receipts](./search-history-receipts.md) account for planned and attempted search slots. +[Verdict certifications](./verdicts.md) name the checker and its unverified assumptions. +The [evidence registry](../evidence/README.md) retains published measurements and their freshness state. + +## Implemented foundations + +The current implementation includes: + +- Registered decision rules, sealed experiments, admission funnels, matched-budget checks, and cluster-aware power refusal in [`/experiment`](../src/experiment/index.ts). +- Paired comparisons, exact binary inference, multiplicity corrections, and sequential gates in [statistics](../src/statistics/index.ts) and [campaign gates](../src/campaign/gates/). +- Declarative analyst definitions and caller-owned engine binding in [`/analyst`](../src/analyst/index.ts). +- Executed repair grading and replay in [`/trace-repair`](../src/trace-repair/index.ts) and [`/trajectory-replay`](../src/trajectory-replay/index.ts). +- Verification strategies and blind equivalence checks through a [caller-supplied checker](./verification-strategies.md). +- Claim metadata, durable final-evidence reservations, and evaluator admission through the [evaluation integrity API](./evaluation-integrity.md). + +The [benchmark-book review](./design/mlbenchmarks-book-review.md) separates observed defects, existing capabilities, and proposed research. +Its archived measurements describe the reviewed revision. +Current source and regression tests define present behavior. + +## Automating evaluation engineering + +The host can generate candidate cases, checks, rubrics, and agent changes. +Eval checks whether their evidence supports the declared decision. +The same authoring loop must not silently turn its own generated labels into independent certification. + +A host can compose this loop: + +```mermaid +flowchart LR + A[Production failures and task requirements] --> B[Candidate cases and evaluators] + B --> C[Independent evaluator audit] + C --> D[Search on train and selection cases] + D --> E[Reserved final evidence] + E --> F[Paired comparison and release decision] + F --> G[Observed deployment outcomes] + G --> A +``` + +Every revision to an evaluator or candidate changes the object being tested. +Once final evidence influences that revision, the next confirmation needs fresh evidence. +Reusable comparisons can declare their population and unit without consuming final evidence. +Opting into a shared final-evidence ledger records fresh-confirmation exposure across campaigns. +It cannot enforce secrecy outside the host that uses it. + +## Remaining boundaries + +A package cannot establish population coverage from a dataset name. +Sampling plans still need production context, source lineage, and checks for missing groups. +More repetitions improve measurements on existing units; they do not add independent tasks. + +Generated evaluators need an independent source of expected behavior. +The host must enforce author/auditor separation and prevent access to final evidence. +Declared identities and digests make these assumptions inspectable without proving them. + +A checker for an open research problem must run through its actual verification backend. +The checker port supports proof kernels, invariants, replication, and agreement checks. +A strategy name alone supplies no evidence that any of those checks executed. + +Product activation and continuous monitoring stay with the host. +Eval returns evidence and decisions; it does not grant deployment authority or choose a research agenda. + +## Standing rules + +- Keep missing evidence distinct from measured zero, failed execution, and a successful empty result. +- Preserve every attempted slot and its cost, including rejected candidates and service failures. +- Check practical effect, independence, power, and capture completeness before interpreting a positive score. +- Keep refusals and exclusions inside the result artifact. +- Require current canonical envelopes for seals and attestations. +- Preserve historical evidence as recorded, even when current APIs reject its retired format. +- Treat a negative result as evidence about the measured conditions and mechanism. diff --git a/docs/concepts.md b/docs/concepts.md index 35bcae03..230fe78a 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -2,52 +2,51 @@ `agent-eval` records agent runs, scores their outputs, compares variants, and applies caller-defined release rules. -A model can say a task is complete while the build fails, a browser flow is broken, an integration is disconnected, or required sources are missing. +An agent can claim success while a build, browser flow, or integration fails. +Required source evidence can also be missing. This package lets code, model judges, and human feedback check those outcomes through the same run format. ## The top-level functions -Start with `/contract` and `defineAgentEval()` for a new integration. -Use the lower-level functions when you need direct control over execution, storage, or statistics. +Start with `defineAgentEval()` from `/contract` for one agent, judge, case set, and baseline surface. +Its `evaluate()` method returns campaign measurements. +Its `improve()` method searches and returns a final comparison with a release decision. +Use `selfImprove()` directly when you do not need shared configuration. -| Function | When to call it | What you give it | What you get back | -|---|---|---|---| -| **`defineAgentEval()`** | You have scenarios, an agent, a judge, and a baseline surface, and you want one object you can score or improve. | scenarios, agent, judge, baseline surface | `{ evaluate(), improve() }` where `evaluate()` returns a campaign result and `improve()` returns a report | -| **`selfImprove()`** | You want candidate generation, scoring, and a release decision in one call. | scenarios, agent, judge, baseline surface | report, winner surface, and a `gateDecision` (see below) | -| **`loadEvalFixtureScenarios()`** | You want agents to add evals as folders with `PROMPT.md`, checks, and starter files. | `evals//PROMPT.md + EVAL.ts + package.json` | `Scenario[]` that runs through `runCampaign`; pair with `planEvalFixtureRun()` before spending tokens | -| **`analyzeRuns()`** | You have existing runs and do not need to invoke an agent. | `RunRecord[]` and options | `InsightReport` | -| **Intake adapters** (`fromFeedbackTable`, `fromOtelSpans`) | Your data isn't already in `RunRecord` shape: it's in Obsidian, Sheets, an OTel collector, etc. | source-specific input | `RunRecord[]` ready to pipe into `analyzeRuns()` | -| **`sealExperiment()` / `openSealedExperiment()`** | The result must convince a reader who does not trust you, so the rules must be fixed before the data arrives. | arms, admission funnel, estimand, interval, decision table | a hashed rule tree plus executors that can run no other rule ([`experiment.md`](./experiment.md)) | -| **`runEquivalenceCheck()`** | The work has no held-out test suite, so no answer key exists to grade against. | a claim, two blind arms, an injected checker | a certification naming who vouched and how it can fail ([`verification-strategies.md`](./verification-strategies.md)) | -| **`AnalystRegistry.runExact()`** | A batch of runs failed and you need cited findings, with the caller owning every execution choice. | recorded evidence, a declared analyst list | findings with evidence references, an execution plan, and a receipt ([`trace-analysis.md`](./trace-analysis.md)) | +Use `analyzeRuns()` from `/contract` for existing `RunRecord[]` evidence. +[The README workflows](../README.md#choose-a-workflow) link to runnable examples. +[The surface map](./eval-surface-map.md) lists lower-level execution and analysis APIs. -See [`customer-journeys.md`](./customer-journeys.md) for runnable paths from existing logs, human ratings, and a callable agent. -The [README front-door table](../README.md#which-front-door) lists every callable entry point with a runnable example. +Root `Scenario`, `JudgeScore`, and `GateDecision` use the same definitions as `/contract` and `/campaign`. +The product-judging shapes have explicit root names: `ProductScenario` and `DimensionJudgeScore`. +The separate `HeldOutGate` class returns `HeldOutGateDecision` over `RunRecord` comparisons. ### The five release decisions -`selfImprove()` and every gate return a `GateDecision`, not a two-way ship/hold flag. -Folding the last three into `hold` throws away the action each one names. +`selfImprove()` returns a `gateDecision` from the campaign `GateDecision` union. +Keep its five values distinct because they require different actions. | Decision | What it means | What to do next | |---|---|---| -| `ship` | Every gate passed on sufficient evidence. | Release the candidate. | -| `hold` | A gate failed on sufficient evidence. | Reject this candidate. | -| `need_more_work` | A gate could not decide: the evidence was missing, or the paired sample was too small to claim significance. | Gather more runs, then gate again. | +| `ship` | All required configured checks support release. | Review the evidence and release the candidate. | +| `hold` | The gate does not justify release. A required check can fail or lack sufficient evidence. | Inspect the contributions to distinguish regression from an unresolved comparison. | +| `need_more_work` | The gate reports that more work or evidence is required. | Address the reported gap before another decision. | | `model_ceiling` | Reserved for a caller-supplied gate that attributes the limit to the model. | Handle it; no gate in this package emits it. | | `arch_ceiling` | Reserved for a caller-supplied gate that attributes the limit to the architecture. | Handle it; no gate in this package emits it. | The last two are part of the taxonomy and of the composition order, but no built-in gate returns them today. Handle all five anyway: a caller's own gate may return either, and the type will not let you ignore them. -`need_more_work` is not a quiet `hold`. -"Gather more evidence" and "reject this candidate" are different actions, and folding the first into the second abandons a real gain that was only underpowered. +Read the contributing checks before interpreting a refused release. +An unresolved comparison does not establish that the candidate is worse. +Absent optional checks remain `not_evaluated`, including when the required checks support `ship`. When gates are composed, `ship` requires every gate to ship. Otherwise the strongest hold wins, in this order: `arch_ceiling`, `model_ceiling`, `hold`, `need_more_work`. -`analyzeRuns()` and the high-level contract return the same `InsightReport` shape. -It contains score distributions, paired lift intervals, judge agreement, cost, failure clusters, contamination checks, outcome correlation, and recommendations. +`analyzeRuns()` returns an `InsightReport`; `selfImprove()` includes one in its result. +The report includes score distributions, cost, and recommendations. +Paired lift, failure clusters, contamination checks, and outcome associations require their corresponding inputs. [`insight-report.md`](./insight-report.md) defines every field. ## Package Boundary @@ -68,7 +67,8 @@ Use the profile improvement functions from `/contract` when a host owns immutabl This API never activates a candidate or runs an agent itself. The host owns authorization, billing, task isolation, profile materialization, execution, and durable evidence. -The first portable profile contract accepts prompt and skill changes only; a host must add its own exact-state adapter before measuring tools, MCP servers, hooks, subagents, or external knowledge. +The portable profile contract accepts prompt and skill changes. +A host needs an adapter for exact state before measuring tools, MCP servers, hooks, subagents, or external knowledge. ## Main Objects @@ -126,14 +126,14 @@ that can seed memory, replay scenarios, and optimization. | **Layer** | One stage of a verifier pipeline (install, typecheck, build, semantic, …). | | **Finding** | A specific issue a judge found: file, line, severity, message. | | **Trace store** | The append-only log of every span/event during a run. Replay = read this back. | -| **Composite score** | A 0..1 number combining all dimensions. The single number you gate on. | +| **Composite score** | An aggregate on the judge's declared scale. Gates must use thresholds on that scale. | | **Rubric version** | A stable hash of the rubric. Scores from different rubric versions are not comparable. | ### Running an evaluation | Term | Plain English | |---|---| -| **Case** (`Scenario`) | One task the agent must do. The unit every score is per. | +| **Case** (`Scenario`) | One task the agent must do. Variants can share an independent source unit. | | **Surface** | The value being changed: a prompt, a skill, or a serialized configuration. | | **Dispatch** | The function that runs your agent on one case and returns the artifact. | | **Campaign** | One complete pass of every case, executed, scored, and cached under a run directory. | @@ -156,20 +156,52 @@ that can seed memory, replay scenarios, and optimization. | **Selection cases** | Evidence the optimizer reads to choose among its candidates. | | **Final cases** | Held back from the optimizer entirely. They produce the reported lift. | -The three-way split is the reason a reported lift means anything. -An optimizer that saw the final cases can score well on them without the agent getting better. +Keep scenario identifiers disjoint across the three partitions. +For new-unit claims or fresh final evidence, also keep source units separate between development and final cases. +Fixed-roster development can share sources while retaining independent-unit counts in its reports. +Renamed variants from one incident can leak information across splits. +A final comparison supports only the declared population and measured conditions. ### Proving a result -| Term | Plain English | +| Term | Meaning | |---|---| -| **Experiment** | The rules — arms, funnel, estimand, interval, decision — written as data before the data arrives. | -| **Seal** | A hash of that whole rule tree. The execution surface accepts no rule outside it. | -| **Estimand** | The exact quantity being measured, for example the paired difference in pass rate. | -| **Funnel** | The denominator chain: how many rows entered, what each stage removed, and how many remain. | -| **Verification strategy** | One of ten ways to certify a result, each with a documented way it can certify a wrong one. | -| **Certification** | Who vouched for a verdict, with what checker version, and what the checker did not check. | -| **Analyst** | A function that reads recorded evidence and returns findings that cite it. | +| **Claim** | The intended use, population, sampling frame, independent unit, generalization target, and optional minimum useful effect. | +| **Independent unit** | The source task, incident, or family that contributes one independent observation to an inference. | +| **Experiment** | Arms, admission, estimand, interval, and decision rules declared before results are inspected. | +| **Seal** | A digest binding the experiment's rules and claim to the executed specification. | +| **Estimand** | The quantity being estimated, such as the mean difference across independent task families. | +| **Funnel** | Counts of input rows, exclusions at each stage, and retained evidence. | +| **Final-evidence reservation** | A durable claim on source units before search; exposure records the evaluated candidates before dispatch. | +| **Verification strategy** | A method of checking a result, with documented assumptions and failure modes. | +| **Certification** | The checker identity, strategy, unverified assumptions, and evidence associated with a verdict. | +| **Analyst** | A function that reads recorded evidence and returns cited findings. | + +Use `defineEvaluationClaim()` from `/experiment` to declare what a result can describe. +Pass it as the top-level `claim` when improving a surface or comparing optimization methods. +`fixed-roster` concerns the listed units; `new-units` attempts to generalize to further units from the declared population. +A declared sampling frame does not itself establish representative sampling. + +Count repetitions separately from independent units. +For example, 100 retries of one incident produce 100 observations and one independent incident. +Campaign aggregate `n` describes its observed scores. +Registered gates report their independent-unit count and paired-cell count separately. + +Set `minimumEffect` when the decision concerns a practically useful change. +Development and absolute-rate claims can omit it. +Sealed power checks assess the declared effect. +A design that detects only much larger effects cannot pass that adequacy check. +Inference also needs the interval and clustering rule to match the claim. + +Unit-aware comparison does not require a final-evidence ledger. +For fresh confirmation, add `finalEvidence: { ledger, requestId, evaluatorDigest }` alongside the top-level `claim`. +This reserves final units before search. +Use one durable ledger across related campaigns and stable source identities across renamed variants. +Exposure remains recorded if measurement fails or the process stops. +The host enforces access isolation; the ledger cannot inspect reads outside this workflow. + +See [evaluation integrity](./evaluation-integrity.md) for claims, final evidence, and evaluator admission. +[Registered experiments](./experiment.md) describes seals, decision rules, and refusal artifacts. ## The feedback trajectory loop @@ -189,7 +221,8 @@ rows, optimizer rows, and held-out examples for overfit checks. ## Code Generator Eval -When the artifact is generated code, agent-eval scores it at three independent layers. Each layer fails differently, and you want to know which one broke: +Generated-code evaluations can score the agent session, the build, and the running application. +Each layer detects different failures: ``` L0 builder Did the agent's session itself work? @@ -204,18 +237,23 @@ L2 app-runtime Does the artifact actually run end-to-end? (Dynamic signal: only worth checking if L1 passed.) ``` -`BuilderSession` orchestrates this. It opens at `startChat`, runs the build at `ship`, runs the runtime check at `runAppScenario`. Each layer emits a trace span. Composite score aggregates them with `scoreProject`. +`BuilderSession` coordinates these checks. +It opens at `startChat`, runs the build at `ship`, and runs the application check at `runAppScenario`. +Each layer emits a trace span. +`scoreProject` combines their measured scores. + +These layers detect different failures: -Why three? Because each catches a different failure mode: -- L0 misses: agent crashed mid-generation, you have a half-written file. -- L1 misses: files exist but typecheck fails. LLM judges can't reliably catch this. -- L2 misses: code compiles but does the wrong thing at runtime. +- L0: The agent crashed during generation and left an incomplete artifact. +- L1: Files exist but do not typecheck or build. +- L2: Code compiles but behaves incorrectly when executed. If you only check one layer, you ship the bugs that the other two layers would have caught. ## How rubrics work A rubric describes: + 1. **Dimensions**: the axes you score on (e.g. `buyer_quality`, `voice`, `signal`). 2. **Weights**: how to combine dimensions into a composite (`0.5 * buyer_quality + 0.3 * voice + 0.2 * signal`). 3. **Failure modes**: named patterns the judge looks for ("ai-cadence", "vague-claim"). @@ -225,7 +263,10 @@ A rubric describes: Built-in rubrics ship in `src/wire/rubrics.ts`, including `anti-slop` for technical-buyer voice. You can also pass the same rubric shape inline at the call site. -A rubric is plain data. The digest of that data, tagged with the scheme that produced it, is the `rubricVersion`. Two scores are only comparable if they used the same `rubricVersion`: change the rubric and you start a new comparison series. +A rubric is plain data. +Its digest and encoding scheme identify the `rubricVersion`. +Changing the rubric starts a new comparison series. +Evaluate rubric revisions against independent labels before combining their scores. ## How verifiers work @@ -254,16 +295,18 @@ Use `blendedScore` only to inspect the measurements that did complete. Two rules that will save you bugs: -1. **Run both gates.** Build gates catch code that doesn't compile; structural assertions catch missing files. Run both unconditionally: they catch orthogonal failures. - -2. **Pair LLM judges with build outcomes.** An LLM judge will rate non-compiling code as "looks right" (0.8). Always short-circuit on `buildOutcome.passed === false` before any LLM judging. +1. Run build checks and structural assertions. + They detect different failures. +2. Preserve a failed build as a deterministic release failure. + A semantic score cannot override it. ## Judge calibration Two questions to answer before trusting any LLM judge: 1. **Does it agree with humans?** `calibrateJudge(golden, candidate)` reports Pearson, MAE, integer-rounded κ, and worst-N miscalibrations vs a human golden set. -2. **Does it agree with itself / other judges?** `continuousAgreement(scores)` and `calibrateJudgeContinuous(golden, candidate)` report κ_w + ICC(2,1) + Pearson + Spearman with bootstrap 95% CIs on the raw [0,1] scores. +2. **Does it agree with other judges?** + `continuousAgreement()` and `calibrateJudgeContinuous()` report agreement and bootstrap intervals on continuous scores. Each statistic answers a different question: @@ -284,15 +327,36 @@ ICC(2,1) catches a bias Pearson cannot see. If judge B always scores twice judge A, the two move together perfectly and Pearson stays near 1, while ICC drops. That drop is the signal. -Every reported interval is a bootstrap 95 % interval: the statistic is recomputed on many resamples of the data, and the middle 95 % of those values is the interval. +These agreement intervals use bootstrap resampling. +The middle 95% of the recomputed statistics forms each reported interval. + +Import calibration and bias functions from `/meta-eval`. + +| Probe | Input | Observation | +|---|---|---| +| `positionalBias()` | The same items judged with their presentation order swapped. | Mean paired score difference by position. | +| `verbosityBias()` | Output lengths and judge scores. | Correlation between length and score. | +| `selfPreference()` | Scores grouped by whether judge and output share a model family. | Difference between the group means. | + +These probes are descriptive diagnostics. +Length and family groups can also differ in task quality; an observed association alone does not isolate bias. +Inspect sample counts before interpreting a diagnostic, especially `n: 0`. + +Use `auditEvaluator()` for admission against predeclared false-acceptance and false-rejection limits. +Its observation records distinguish fresh controls, development exposure, and unknown judgments. +It counts source families rather than repeated variants and reports simultaneous exact bounds for both error rates. +The host must enforce independent authorship and control access. -`verbosityBias` is the one exported bias probe: it finds a judge that rewards length regardless of quality. -The `JudgeInsight` report shape also carries optional `positionalBias` and `selfPreference` fields for caller-computed probes. -No built-in computes those two fields. +Use `rubricPredictiveValidity()` to compare rubric scores with declared deployment outcomes. +Specify whether each outcome should increase or decrease. +The report preserves signed associations, direction-aligned associations, and exclusions. +An `inverse` association is a reason to investigate; it does not prove that reversing a rubric will improve behavior. +See [outcome validity](./outcome-validity.md). ## Trace Model -Every operation emits structured spans into a `TraceStore`. A run is a tree: +Instrumented execution writes structured spans into a `TraceStore`. +A builder run can have this tree: ``` builder-session [span] @@ -305,7 +369,9 @@ builder-session [span] └── scenario.run [span] ``` -Spans are append-only and have stable ids: replay is reading the same store back. OTLP export ships them out for distributed tracing. +Recorded spans preserve their identifiers and relationships. +Trace inspection reads this evidence; executable replay separately reruns recorded operations. +OTLP export sends spans to distributed tracing systems. You usually should not build this tree by hand. Product runtimes, `runAgentControlLoop`, harnesses, and verifiers should emit it while they run. @@ -328,4 +394,4 @@ release decision. - **Certifying a result with no answer key?** Read [verification-strategies.md](./verification-strategies.md) for the ten-member family and the blind two-arm protocol. - **Reading a verdict someone else produced?** Read [verdicts.md](./verdicts.md) for what `certification` carries and what an absent one means. - **Grading a finding by executing its repair?** Read [trace-repair-grader.md](./trace-repair-grader.md), and [trajectory-replay.md](./trajectory-replay.md) for re-executing a recorded failure. -- **Wondering why this package exists at all?** Read [charter.md](./charter.md) for the four end-states it is built against. +- **Checking package ownership?** Read [charter.md](./charter.md) for the implemented foundations and host responsibilities. diff --git a/docs/design.md b/docs/design.md index e912b0f4..e8669f95 100644 --- a/docs/design.md +++ b/docs/design.md @@ -66,5 +66,6 @@ They are not adoption reference: - [`building-doctrine.md`](./building-doctrine.md): conventions our agents follow when consuming this package (reachable model defaults, probe-before-debug, experiment integrity checklist) - [`design/loop-taxonomy.md`](./design/loop-taxonomy.md): the internal vocabulary for execution drivers, workers, measurements, and proposers - [`design/statistics-decisions.md`](./design/statistics-decisions.md): per-statistic trust status, the no-runtime-dependency verdict, and the exact-versus-asymptotic policy at 3–10 repetitions +- [`design/mlbenchmarks-book-review.md`](./design/mlbenchmarks-book-review.md): book review, source-backed gaps, and proposals for automated evaluation engineering at a pinned repository revision - [`research-report-methodology.md`](./research-report-methodology.md): the evidence standard our own research reports are held to - [`.claude/skills/agent-eval/SKILL.md`](../.claude/skills/agent-eval/SKILL.md): directives for LLM agents writing integration code, encoding bug classes we have already shipped and fixed once diff --git a/docs/design/mlbenchmarks-book-review.md b/docs/design/mlbenchmarks-book-review.md new file mode 100644 index 00000000..efff66e3 --- /dev/null +++ b/docs/design/mlbenchmarks-book-review.md @@ -0,0 +1,440 @@ +# What the benchmark book changes for agent-eval + +Agent-eval should make the intended claim, evaluation population, and evidence lifecycle explicit before adding more autonomous search. +Its existing statistics and execution records provide a strong base. +The largest opportunity is to connect those instruments into a system that can propose, test, and revise evaluations independently of the agent being improved. + +This assessment is for agent-eval maintainers deciding what to build next. +It reviews Moritz Hardt’s [The Emerging Science of Machine Learning Benchmarks](https://mlbenchmarks.org/) against repository revision `fe1cc5111aab5d588bf7db3a3785325635937a91`. +The exact inspected revision is also recorded in the [source manifest](./mlbenchmarks-review/sources.json). +The review date is September 12, 2026, in America/Los_Angeles. +Recommendations below are design proposals, not measured improvements or implemented runtime changes. + +## Decision + +Retain the current package boundaries and measurement primitives. +Prioritize three additions: + +1. Bind each certification to its target population, independent observation unit, and intended use. +2. Track final-data exposure across campaigns, including what feedback reached candidate authors. +3. Admit generated evaluations through independent calibration and challenge before they can authorize agent improvements. + +First repair the narrow measurement defects documented below. +An automated researcher that consumes the wrong outcome metric can optimize in the wrong direction faster than a human reviewer notices. + +Do not start with another optimizer, runner, statistics package, or general benchmark leaderboard. +Do not treat a high judge agreement score as proof that a model ranking is correct. +Do not make an evaluator’s own approval rate its optimization objective. + +## Reading scope and method + +The live index exposes **16 reading pages**, totaling **109,771 whitespace-delimited body words**. +That count includes notes, references, tables, and equation text. +We read all 16: preface, prologue, chapters 1–8, and chapters 10–15. +Three parallel reviews covered the full text; the repository mapping checked implementations, exports, relevant callers, tests, and offline behavior. +HTML text omitted by paragraph extraction was separately audited and read. +Figures and tables supporting the conclusions were checked against their HTML or PDF context. + +**Chapter 9 remains unavailable.** +The book references an annotation chapter, but the live index jumps from 8 to 10. +The checked `/09-annotation.html`, `/09-data-annotation.html`, and `/09-annotations.html` paths returned HTTP 404. +This review does not claim coverage of an unavailable chapter or the forthcoming print edition. +We read chapter references as part of the book; we did not independently reproduce every cited study or read every cited paper. + +The [manifest](./mlbenchmarks-review/sources.json) records every page URL, title, body count, and downloaded HTML SHA-256. +It preserves source identity without vendoring the book. +The [offline probes](./mlbenchmarks-review/probes.mts) and [observations](./mlbenchmarks-review/observations.json) preserve the local checks behind concrete findings. +The probes use deterministic or synthetic data and make zero paid model calls. +They establish behavior and assumption boundaries, not production defect rates or expected improvement sizes. + +The search covered `src`, `tests`, `docs`, `examples`, and package exports at the inspected revision. +It did not audit deployed consumers, production traffic, or the implementation of agent-runtime and agent-knowledge. +An absent connection here may already exist in a downstream application. + +## The book, chapter by chapter + +The book supplies several kinds of support. +Mathematical results depend on their assumptions; benchmark studies describe particular settings; historical interpretations suggest mechanisms worth testing. +The architecture proposals are our applications of those sources. + +| Reading | Main contribution and qualification | Consequence for agent-eval | +| --- | --- | --- | +| [Preface](https://mlbenchmarks.org/00-preface.html) | Frames benchmarks as both development instruments and scientific institutions. The book is a synthesis, not an agent evaluation implementation guide. | Evaluate the measurement process and its incentives, alongside individual agents. | +| [Prologue](https://mlbenchmarks.org/00-prologue.html) | Uses a learning-rate anecdote to question the search for a universal modeling trick. Its empirical lesson unfolds in later chapters. | Preserve empirical comparison and let the researcher choose methods. | +| [1. Introduction](https://mlbenchmarks.org/01-introduction.html#the-iron-rule) | Explains benchmark-driven competition through the ImageNet and language-model eras. Benchmark success and explanatory scientific progress are different achievements. | Keep mechanism tests and replication alongside winner selection. | +| [2. Populations and predictions](https://mlbenchmarks.org/02-populations-predictions.html#errors-and-metrics) | Defines prediction risk relative to a population, loss, and decision problem. Calibration, accuracy, precision, and recall answer different questions. | Record the target distribution and error consequences; a scenario digest does not establish population validity. | +| [3. Detecting differences](https://mlbenchmarks.org/03-detecting-differences.html#comparing-similar-models) | Develops sample requirements, iid assumptions, multiplicity, and discordance in paired correctness outcomes. Small differences can require substantial independent evidence. | Reuse paired and discordance-aware power checks; distinguish task count from repeated execution count. | +| [4. Holdout method](https://mlbenchmarks.org/04-holdout-method.html#whats-the-holdout-method-for) | Separates development feedback, model ranking, and capability measurement. Valid intervals within a dataset do not establish external validity. | Give exploratory reports, fixed-roster comparisons, and deployment claims different evidence requirements. | +| [5. Test set reuse](https://mlbenchmarks.org/05-test-set-reuse.html#guarantees-of-the-holdout-method-under-adaptivity) | Shows why adaptive feedback changes holdout guarantees. Worst-case attacks demonstrate possibility, not the prevalence of practical overfitting. | Track exposure and adaptive claim history across calls; preserve useful development feedback. | +| [6. Scientific crisis](https://mlbenchmarks.org/06-scientific-crisis.html#researcher-degrees-of-freedom) | Explains selection, low power, publication incentives, and researcher flexibility. A p-value is not the probability that a claim is true. | Retain failed attempts and amended rules, practical effect sizes, controls, and unresolved outcomes. | +| [7. Replication in machine learning](https://mlbenchmarks.org/07-replication-machine-learning.html#measurement-versus-ranking) | Studies cases where new test sets shift absolute accuracy while preserving much of the ranking. Evidence from ImageNet does not guarantee language-agent stability. | Distinguish rerunning identical artifacts from sampling new tasks and reproducing the conclusion independently. | +| [8. Forces against crisis](https://mlbenchmarks.org/08-forces-against-crisis.html#biases-and-heuristics) | Examines leaderboard mechanisms, human information filtering, and shared code. These partly explain empirical robustness; they are not universal protections. | Autonomous search needs explicit feedback policies because it may exploit details that humans ignored. | +| [10. Generative models](https://mlbenchmarks.org/10-generative-models.html#the-limits-of-scaling-laws) | Connects language modeling, scaling, training distributions, and downstream benchmarks. Better likelihood or fitted scaling laws need not establish product capability. | Measure complete executable profiles and user outcomes; keep model-level proxies in their stated role. | +| [11. Evaluating language models](https://mlbenchmarks.org/11-evaluating-language-models.html#confounded-evaluations) | Covers post-training, generative judges, shortcuts, contamination, and tune-before-test. Unequal task preparation can confound claims about base-model capability. | Declare whether the comparison concerns deployed products or adaptation potential; account for preparation when that claim requires it. | +| [12. The problem of aggregation](https://mlbenchmarks.org/12-problem-aggregation.html#problems-of-aggregation-and-voting-systems) | Uses social choice and empirical comparisons to expose ranking tradeoffs. No theorem says every task-specific aggregate is useless. | Preserve dimensions, target weights, subgroup denominators, and sensitivity to defensible alternative aggregation policies. | +| [13. When the model moves the data](https://mlbenchmarks.org/13-model-moves-data.html#what-performativity-means-for-model-evaluation) | Models deployments that change future data. Stability, optimality, and welfare differ; feedback-loop stories require evidence. | Record assignment, time, exposure, and affected populations; distinguish monitoring correlation from causal deployment effects. | +| [14. Evaluation at the frontier](https://mlbenchmarks.org/14-evaluation-frontier.html#agreement-alone-is-not-enough) | Shows why judge agreement can coexist with wrong rankings. Discusses debiasing, verification, simulation, and live experiments, each with limits. | Calibrate ranking errors against independent labels, challenge evaluators, and connect offline decisions to later outcomes. | +| [15. Epilogue](https://mlbenchmarks.org/15-epilogue.html) | Returns to the social and scientific choices behind measurement. More automation does not remove judgment about desirable outcomes. | Let domain owners define value and acceptable failures; automate evidence collection and scrutiny. | + +### What transfers, and what does not + +The most useful distinction is **development signal versus ranking versus capability certification**. +These uses require progressively stronger evidence. +A regression suite can be useful after repeated exposure without supporting a fresh claim about unseen tasks. +A ranking can reproduce across populations while every absolute success rate changes. +A perfectly reproducible computation can measure the wrong construct. + +Chapter 8 suggests a specific automation risk. +Human researchers often discarded most benchmark feedback through heuristics and limited attention. +An autonomous optimizer can retain every score, failed attempt, trace, and explanation. +The book motivates testing whether that additional feedback increases overfitting; it does not establish that our optimizers currently do so. +Its [Ladder mechanism](https://mlbenchmarks.org/08-forces-against-crisis.html#leaderboard-error) releases score updates only after sufficiently large improvements. +The guarantees depend on the specified mechanism and observation assumptions. +A minimum-effect gate alone does not reproduce them. +Keep complete private audit evidence even when an optimizer receives restricted feedback. + +Chapter 11 also requires a careful distinction. +Comparing two products with their actual prompts and tools is appropriate when those products are the alternatives being deployed. +Comparing underlying models’ learning potential may require equal task preparation, adaptation curves, and total preparation cost. +Automatically tuning every model would change the first question into the second. + +At the frontier, independent agreement and formal verification remain conditional evidence. +Two agents can share a blind spot. +A proof kernel checks a formal statement, leaving the connection to the intended claim as a separate obligation. +The package’s [verification strategy model](../verification-strategies.md) already captures this distinction. + +## What agent-eval already has + +These are implementation findings, not an assessment of adoption in every consumer. +“Partial” means the named behavior exists but leaves a specific contract or integration gap. + +| Concern | Checked implementation | Assessment and remaining gap | +| --- | --- | --- | +| Statistical comparisons | [statistics](../../src/statistics/index.ts), [paired decisions](../../src/paired-promotion-decision.ts), [heldout pairing](../../src/campaign/gates/statistical-heldout.ts) | Present. Paired tests, uncertainty, exact binary methods, multiplicity, and power do not need replacement. Claim scope and independent sampling units need stronger binding. | +| Cluster-aware design | [power](../../src/experiment/power.ts), [registered rule AST](../../src/experiment/ast.ts) | Present. The high-level campaign path does not automatically select these methods from a declared generalization target. | +| Executable preregistration | [define/seal/open](../../src/experiment/define.ts), [acceptance tests](../../tests/experiment/preregistration-acceptance.test.ts) | Present. Extend this rule representation instead of creating a second experiment language. | +| Sequential testing | [sequential gate](../../src/campaign/gates/sequential.ts), [e-process](../../src/statistics/sequential-eprocess.ts) | Present. Optional stopping within a valid stream differs from adapting hypotheses across streams or counting dependent replicas as independent evidence. | +| Search/final separation | [selfImprove](../../src/contract/self-improve.ts), [method comparison](../../src/campaign/presets/compare-optimization-methods.ts) | Present within calls. Final cases are withheld from method inputs. A persistent cross-campaign exposure policy is missing from these paths. | +| Dataset identity and access | [Dataset](../../src/dataset.ts), [contamination helpers](../../src/contamination-guard.ts), [labeled store](../../src/campaign/labeled-store/fs-adapter.ts) | Partial. Hashes, split labels, mutation locks, temporal sampling, and access logs exist. They do not establish secrecy, lineage independence, or one-time certification use. | +| Complete search history | [SearchLedger](../../src/campaign/search-ledger.ts), [history receipt documentation](../search-history-receipts.md) | Present. Reuse the canonical ledger; do not create another optimizer event log. Requiring complete history remains a caller policy. | +| Evidence identity and authority | [EvidenceReceipt](../../src/experiment/evidence-receipt.ts), [campaign receipts](../../src/experiment/campaign-evidence.ts), [registry](../../src/experiment/evidence-record.ts) | Present. These bind identities and preserve declared authority. Hashes and authority labels alone do not prove independent execution or valid sampling. | +| Judge quality and drift | [calibration](../../src/judge-calibration.ts), [sentinel](../../src/meta-eval/sentinel.ts), [plants](../../src/meta-eval/plants.ts) | Present. Per-candidate residual bias, ranking validity, and independent evaluator admission need composition. Position and self-preference helpers exist internally; only verbosity has a root public export. | +| Multiple objectives | [promotion policy](../../src/campaign/gates/promotion-policy.ts), [production gate](../../src/campaign/gates/default-production-gate.ts) | Present. Per-dimension regression guards and evidence vectors exist. Explicit target-population weights and aggregation sensitivity remain useful additions. | +| Outcome validity | [correlation study](../../src/meta-eval/correlation-study.ts), [rubric validity](../../src/meta-eval/rubric-predictive-validity.ts), [outcome store](../../src/meta-eval/outcome-store.ts) | Present. Repair the metric-selection defect below; define direction and observational limits before using correlations for automated steering. | +| Adaptation and causal primitives | [adaptation evaluation](../../src/rl/adaptation-eval.ts), [off-policy estimators](../../src/rl/off-policy.ts) | Present. The adaptation comparison needs pairing repair. IPS, SNIPS, and doubly robust estimation still depend on supplied propensities, overlap, and identification assumptions. | +| Generating reusable cases | [fixtures](../../src/campaign/fixtures.ts), [feedback trajectories](../../src/feedback-trajectory.ts), [analysts](../../src/analyst/index.ts) | Partial. Cases can be authored, replayed, and scored. The inspected package lacks a complete admission protocol for an autonomously authored evaluation. | +| Active and adversarial case selection | [curriculum](../../src/rl/active-curriculum.ts), [adversarial scenarios](../../src/rl/adversarial.ts), [fuzzing](../../src/fuzz/fuzz-agent.ts), [discrimination](../../src/campaign/scenario-selection.ts) | Present. Extend population and exposure accounting around these primitives; do not propose a first automatic case-generation loop. | +| Automated improvement | [contract](../../src/contract/index.ts), [Researcher](../../src/researcher.ts), [predictive-validity researcher](../../src/rl/predictive-validity-researcher.ts) | Partial. Optimizer adapters and inspect/propose/apply/evaluate contracts exist. The predictive-validity researcher recommends changes but does not execute plans. | +| Verification without answer keys | [strategy/checker port](../../src/verification-strategy.ts), [verdicts](../../src/verdict.ts), [repair grading](../../src/trace-repair/index.ts) | Present as contracts and applicable execution paths. Domain checkers remain injected; a universal verifier is neither provided nor justified. | + +The [charter](../charter.md) is a dated inventory with some later additions described beneath its original missing list. +Current source already supplies cluster-aware power, sealed rules, funnels, and evidence receipts. +Treat those as foundations to connect, not unbuilt modules. + +## Concrete findings from offline checks + +The observations below are intentionally narrower than production reliability claims. +They use the actual library functions at the inspected revision. +The [probe source](./mlbenchmarks-review/probes.mts) contains the complete inputs and invocation paths. +The archived probes target review snapshot `dda9941437190c9c541b3f54946bfeeb153366fe`, whose implementation matches the inspected source. +Run `pnpm exec tsx docs/design/mlbenchmarks-review/probes.mts` there after installing the locked dependencies. +Use an isolated checkout without concurrent source edits. +Current APIs have breaking changes, so these historical probes cannot run unchanged against current implementation code. +Current regressions and the [integrity example](../../examples/evaluation-integrity/) verify the replacement behavior. +It prints current observations without asserting that the recorded defects must persist. +The source identity hashes actual files under `src`, plus `package.json`, `pnpm-lock.yaml`, and `tsconfig.json`. +A separate hash identifies the diagnostic itself. +These identities survive documentation commits and change with local source edits, including untracked files. +They assume dependencies were installed from the lockfile; they do not fingerprint installed packages or the host environment. + +| Finding | Observed result | Consequence and bounded correction | +| --- | --- | --- | +| Final evidence can be reused across independent calls | Two `selfImprove()` calls each dispatched baseline and candidate on the same six final cases: 12 final dispatches per call. Both returned `ship`, with the same final-set digest. | Confirms no cross-call consumption guard on this path. Add exposure-aware certification policy; this probe does not demonstrate empirical overfitting. | +| Outcome correlation can select the wrong metric | Ten synthetic runs had `csat = score` and an earlier object key with the opposite trend. Default `latest` returned correlation −1 for `csat`; `mean` returned +1. | [The reducer](../../src/meta-eval/correlation-study.ts) does not receive the requested metric name. Preserve metric identity when selecting the latest eligible outcome. | +| Adaptation comparison accepts unrelated tasks | Curves with disjoint scenario IDs can return `a_better`. The implementation computes separate marginal intervals and compares point summaries. | [The comparison](../../src/rl/adaptation-eval.ts) claims pairing without joining IDs. Use the existing paired machinery, report missing pairs, and distinguish descriptive curves from release evidence. | +| Exchangeability alone cannot validate the sequential gate | Choose one fair sign per experiment and repeat it for 100 cells. The positive state promotes after 15 observations; the negative state does not. | [The gate commentary](../../src/campaign/gates/sequential.ts) overstates exchangeability and shuffling. Under this marginal-zero construction, false promotion is 50%; the required conditional-mean assumption fails. | + +The sequential example is an exact two-state counterexample, not a Monte Carlo estimate. +It challenges the stated assumption boundary, not the conditional-mean theorem underlying an e-process. +Shuffling correlated replicas does not manufacture independent tasks. + +Two additional source findings matter before automating interpretation. +In [rubric predictive validity](../../src/meta-eval/rubric-predictive-validity.ts), the `load_bearing` classification uses absolute correlation. +Strong negative association can therefore earn that label; tests make magnitude-based behavior intentional despite contradictory interface prose. +Require an explicit outcome direction before treating this label as a recommendation to increase rubric weight. +This already affects [PredictiveValidityResearcher](../../src/rl/predictive-validity-researcher.ts). +Given nonempty failures, it can recommend up-weighting the top `load_bearing` rubric without checking the correlation’s sign. +The correction must reach that consumer as well as the report vocabulary. + +In the [contamination probe](../../src/rl/contamination.ts), per-item `qValue` is derived from `1 - abs(delta)` before adjustment. +That quantity has no demonstrated p-value calibration. +The global paired Wilcoxon calculation is separate and should not be conflated with these display values. +Remove inferential naming from the heuristic or introduce a justified repeated-sample model. +Perturbation sensitivity can reflect changed difficulty as well as contamination. + +The open safety-floor change was a separate worktree and PR during this review. +This assessment does not duplicate ownership of that gate or assume its changes were present in the inspected base. + +## Prioritized improvements + +### 1. Bind the claim to the design + +**Recommendation: adapt; highest architectural priority.** +Extend the existing experiment definition with claim metadata and mechanically checked obligations. +Do not create a new runner or parallel estimator family. + +The record should name the intended use, target population, sampling frame, observation unit, clustering, timeframe, outcome direction, and practical effect threshold. +It should identify whether the population is fixed, sampled independently, or affected by deployment. +It should retain exclusions, selection probabilities when known, and reasons when they are unknown. + +For a frozen task roster, repeated executions can estimate execution variability conditional on that roster. +For unseen-task claims, resample independent tasks or task families through the existing cluster-aware methods. +Variants derived from one incident should retain their common source identity. +Twenty rewrites of one failure are not twenty independent examples of customer demand. +The method-comparison path reduces repetitions to scenario means; the primary heldout gate counts `scenario:rep` cells. +Make that choice follow the claim instead of treating either convention as universally correct. + +Preflight power at the minimum worthwhile effect using the actual registered decision procedure. +The existing `power-floor` gate asks whether maximum power anywhere on its supplied effect grid reaches the target. +A fixture with power 0.1 at effect 0.01 and power 1 at effect 1 passes its target-0.8 check. +Those values are supplied curve points, not measured operating characteristics. +The result matches the gate’s structural-feasibility semantics; it does not establish adequacy for detecting an effect of 0.01. + +**Smallest decisive check:** compare one task repeated 100 times with 100 independent tasks. +Both designs should expose the same execution count and different independent-unit counts. +A claim about unseen tasks must not acquire precision merely by duplicating the first task. +Reject this addition if the same safety and clarity can be obtained by composing existing typed fields without a new contract. + +### 2. Account for evidence exposure across campaigns + +**Recommendation: adapt; prerequisite for autonomous certification.** +Bind final-set commitments, task-family lineage, claim identity, and released feedback to the canonical ledger. +The host should reserve and consume final evidence through a durable operation with retry identity. +The record must distinguish replaying one completed measurement from opening evidence for a new adaptive decision. + +Keep ordinary regression and exploratory reuse available and labeled. +For certification, support a frozen comparison, registered sequential collection of new valid observations, or refreshed final data. +A seed change or new run directory does not make an exposed task fresh. +Concurrent hosts must not each mint an apparently unused reservation for the same claim and evidence. + +The runtime owns file permissions, model context, storage credentials, and separation between authors and final evaluators. +Eval owns the portable access/consumption record and the certification refusal. +An append-only record cannot prove secrecy if the host lets the author read the answer files. + +**Smallest decisive check:** reproduce the two-call probe with one final-set reservation shared across restarts and two competing workers. +Require an explicit reuse policy before another adaptive certification can consume it. +Then compare full-trace, score-only, and thresholded development feedback on a null benchmark and an untouched replica at equal total spend. +That experiment tests whether formal information restrictions are worth their operational cost. +Defer a differential-privacy or reusable-holdout implementation until this measurement supports it. + +### 3. Treat evaluation authoring as a measured task + +**Recommendation: adapt the existing composition; largest automation opportunity.** +An evaluation candidate should be a versioned bundle of fixture references, checker identities, rubric, population description, and sealed decision rules. +The bundle should reuse `Scenario`, `JudgeConfig`, `VerificationStrategy`, `SealedExperiment`, and `EvidenceReceipt`. +Introduce only the missing admission and lineage fields. + +The evaluator’s objective is detecting consequential defects while accepting independently verified good behavior. +High agent scores, high judge agreement, and large test counts are insufficient objectives. +A checker that rejects every output has excellent defect recall and no useful decision quality. + +**Smallest decisive check:** give an eval author one real failure trace and a bounded budget. +Require a reproducible fixture, a correct reference, realistic negative controls, and an independently held audit set. +Compare its selected checker with a maintained human checker and a simple deterministic baseline. +Measure false acceptance and rejection, coverage by defect family, unknowns, flakiness, cost, and decision changes. +The winning author must improve audited decisions without weakening the accepted behavior. + +### 4. Measure ranking validity and aggregation sensitivity + +**Recommendation: adapt the calibration and reporting modules.** +Keep the evidence vector and per-dimension gates. +Add reports for per-candidate judge residuals against independent labels and uncertainty in pairwise ranking differences. +Good global agreement can hide a small directional bias that reverses a close comparison. + +Retain subgroup counts and report the result under a few domain-approved population weights and normalization choices. +Record rank reversals and subgroup regressions rather than automatically choosing favorable weights. +Any data-driven choice of aggregation belongs to development and needs a later independent assessment. + +**Smallest decisive check:** construct a panel with high aggregate agreement and known candidate-specific bias. +The ranking audit should catch the reversal while the existing agreement summary remains high. +For aggregation, change irrelevant alternatives and approved weights while keeping the focal models’ raw scores unchanged. +Report sensitivity without declaring that every aggregate is invalid. + +[Prediction-powered inference](https://mlbenchmarks.org/14-evaluation-frontier.html#prediction-powered-inference) is a promising later experiment. +It combines many inexpensive predictions with a smaller independent labeled sample to correct measurement bias. +Compare its interval coverage and cost against an equally funded human-only estimator under candidate-specific and shifting bias. +The reference/proxy pairs and proxy-only sample must represent the same target population. +A bias correction does not resolve an undefined target construct. +The chapter derives a factor-two effective-sample-size ceiling for its considered unbiased estimators under a specified binary-score regime. +That regime requires agreement between 0.5 and the candidate’s reference score. +It does not bound every form of judge assistance or tool-based verification. +Measure coverage and total cost instead of assuming inexpensive proxy labels produce large savings. + +### 5. Distinguish reproducibility from replication and transfer + +**Recommendation: adapt existing receipts and experiment comparisons.** +Record whether a result reuses exact data, samples new tasks, changes the implementation, or tests another environment or population. +Use those distinctions in evidence records and reports. +Replicas should state which conclusion must reproduce: absolute performance, pairwise lift, ranking, or a proposed mechanism. + +For claims about adaptation potential, compose repaired adaptation curves with method comparison and complete preparation costs. +Pin what each arm may change and which demonstrations it sees. +For product selection, preserve the actual deployed profile as the treatment being compared. + +**Smallest decisive check:** rerun a fixed comparison on a newly sampled task cohort with preserved inclusion rules. +Check absolute score movement and paired/ranking movement separately. +Do not call a cache replay an independent replication. +Measure selection regret: the target loss incurred by choosing a candidate from the source benchmark. +A mostly preserved ranking can still select the wrong winner or leave every candidate below an operational reliability threshold. + +### 6. Make deployment feedback interpretable + +**Recommendation: adapt; execution remains downstream.** +After fixing outcome selection, extend outcome provenance with assignment, eligibility, exposure, observation window, and censoring information where the host can supply it. +Distinguish missing outcomes from users who experienced no event. +Record when the agent changes which tasks arrive, which users remain, or which feedback is observed. + +Use outcome correlation as a diagnostic hypothesis. +Use randomized deployment comparisons when feasible, or existing off-policy estimators when their assumptions and propensities are defensible. +Neither correlation nor a stable feedback loop establishes that a change caused an improvement. + +**Smallest decisive check:** build a two-cohort example where aggregate satisfaction rises because difficult users disappear while both cohorts worsen. +The report should expose changed denominators and within-cohort effects. +A production experiment should then measure whether the suspected selection mechanism actually occurs. + +## A design for automated evaluation engineering + +Here, “Software 3.0” means agents authoring executable evaluation assets and learning how to improve them. +This is a proposed system design derived from the review, not terminology or an architecture prescribed by the book. + +Use three connected loops with separately versioned objectives and evidence. +Keep the evaluator fixed during each agent comparison. +Keep the independent evaluator audit fixed during each evaluation-design comparison. +Use deployment observations to challenge whether either comparison still represents the intended outcome. + +```mermaid +flowchart TD + O[Production traces and independent outcomes] --> D[Diagnose missing behaviors] + D --> A[Host authors evaluation candidate] + A --> C[Calibrate on good and bad controls] + C --> H[Development challenge] + H --> S[Seal evaluation candidate and audit rules] + S --> U[Independent final audit] + U --> V[Admit evaluation version] + V --> P[Optimize agent against development cases] + P --> F[Measure selected agent on reserved final cases] + F --> R[Evidence receipt and release decision] + R --> O + H --> E[Reject or revise evaluation candidate] + E --> A + U --> X[Do not admit] + X -->|Fresh audit evidence or valid registered reuse| A +``` + +The host chooses methods and coordinates workers. +The diagram does not move agent execution into this package. +Final-audit feedback consumes its independence, whether the candidate passes or fails. +Another revision requires fresh audit evidence or a registered protocol that justifies the proposed reuse. + +| Stage | Required artifact | Existing building block | New obligation | +| --- | --- | --- | --- | +| Diagnose | Cited failure hypothesis and affected population | Trace analysts, feedback trajectories, replay | Explain why the case matters beyond being easy to generate. | +| Author | Fixture/checker/rubric bundle with source identities | Eval fixtures, datasets, checker ports | Separate author-visible material from sealed labels and final cases. | +| Calibrate | Known-good and known-bad outcomes, with execution evidence | Plants, golden calibration, verifiers | Reject inert checks, always-reject checks, and checks that reward the author’s own wording. | +| Challenge | Independently constructed development counterexamples and adjudicated disagreements | Blind equivalence protocol, repair execution, judge calibration | Measure defect-family coverage, false decisions, and source independence. | +| Seal and audit | Frozen candidate, registered rules, and independent final audit | Sealed experiments, power checks, funnels | Bind population, observation unit, exposure policy, and audit authority; consume final-audit evidence on disclosure. | +| Improve agent | Complete candidate history and detached selected artifact | `selfImprove`, optimizer adapters, SearchLedger | Prevent evaluator or final-case changes inside the comparison. | +| Certify | Paired final results and exact evidence bindings | Campaigns, gates, EvidenceReceipt | Verify final-evidence reservation and required evaluator health. | +| Revalidate | Later outcomes and an explicit replication decision | Outcome stores, sentinel, evidence registry | Distinguish drift, changed populations, and candidate-caused feedback. | + +### How to evaluate the eval author + +The unit of observation should be an independent task or incident family with independently adjudicated outcomes. +Split entire families and generator sources across authoring, selection, and final audit. +Do not let the author choose which of its failures disappear from the denominator. + +Give every author the same source evidence, tool access, labeling allowance, and total measured budget. +Account for generation, calibration, challenge, failed executions, review, and final scoring. +Equal numbers of generated tests are not equal resources. + +Compare three approaches initially: a maintained human procedure, a simple fixture-and-mutation procedure, and one autonomous author. +Use existing method comparison and sealed experiment machinery wherever their contracts fit. +Prefer one bounded pilot domain with executable ground truth before semantic or frontier research tasks. +For optimizing the target agent, include simple sample-and-verify selection as a baseline when outcomes are executable. +Charge its sampling and verification against the same total resource allowance. + +The primary decision should concern false acceptance of consequential defects under a predeclared acceptable false-rejection limit. +Report both error rates with denominators and uncertainty, along with coverage, inconclusive outcomes, run failures, latency, and complete cost. +Report the distribution across task families; an average can hide a completely untested failure class. +Use an independently chosen practical effect threshold and cluster-aware power calculation to set the final sample size. +Calibrate the complete workflow on known nulls and known improvements across repeated independent experiments. +Include case authoring, candidate selection, stopping, and final confirmation in that measurement. +An isolated estimator’s error rate does not certify the larger adaptive procedure. + +Test transfer to new repositories, task families, and later failure incidents before claiming a general eval engineer. +Measure whether the admitted evaluator changes actual release decisions correctly. +Generating plausible tests is only an intermediate artifact. + +### Stop the recursive trust problem at explicit authority + +The same agent may propose changes to the system and to its evaluator in separate experiments. +It must not certify its own change by altering the standard during the comparison. +When a rubric or checker changes, version it and remeasure both arms under that version. +A prompt edit to the evaluator cannot retroactively improve the candidate’s recorded result. +Before combining both changes, score old and new agent outputs under both old and new evaluators. +Independent references in this crossed comparison distinguish an agent improvement from a changed measuring instrument. + +The final audit should use a separately controlled evidence source: executable ground truth, independent labels, replication, or an appropriate checker. +Different model families can reduce one source of shared bias; they do not prove independence or correctness. +The host must enforce separation and retain actual access evidence. +An authority field records the declaration; it does not authenticate the declaration by itself. + +For tasks without answer keys, retain the checker’s stated assumptions and unresolved obligations. +Request a better observation or narrower claim when no available check can discriminate success from failure. +Do not resolve that limit by adding more mutually agreeing agents. + +### Allocate work by the uncertainty that changes the decision + +A downstream controller can choose between collecting new tasks, adding repetitions, buying independent labels, challenging a checker, or testing another mechanism. +Eval should provide the evidence and cost estimates for that choice. +It should not hard-code a research strategy into the substrate. +Compose existing curriculum, discriminative selection, and adversarial exploration where they fit. +The proposed addition concerns valid allocation and comparison of those choices, rather than their initial implementation. + +If uncertainty comes from task diversity, collect new task families. +If it comes from execution noise on a fixed roster, add repetitions. +If the judge is biased, buy independent labels or execute a stronger check. +If every candidate ties, inspect case discrimination and metric resolution before proposing another architecture. +Retain a representative final sample when development uses deliberately difficult or discriminative cases. + +This controller is worth implementing only after a pilot compares it with a fixed allocation at equal actual cost. +Measure correct decisions per budget, not the amount of activity it schedules. + +## Build order and decisions to defer + +| Order | Deliverable | Evidence required before continuing | +| --- | --- | --- | +| First | Repair outcome metric selection and adaptation pairing; correct sequential assumption guidance and ambiguous metric labels. | Focused regressions using the reproduced counterexamples and valid controls. | +| Next | One claim contract and final-exposure integration through existing seals, ledgers, and gates. | Independent-unit, restart, duplicate-use, concurrency, and information-access boundary checks. | +| Pilot | One runtime-owned eval author that emits ordinary fixtures and registered evaluations. | Independent final audit against human and simple baselines at equal measured resources. | +| Expand | Ranking-bias audits, aggregation sensitivity, and fresh-cohort replication. | Show that each addition changes a previously wrong or unresolved decision. | +| Conditional | Prediction-powered inference and adaptive allocation of labels or compute. | Demonstrated coverage, reduced decision error, and measured cost benefit under relevant bias and drift. | + +Retain official optimizers and the single campaign execution path. +Reject duplicating the ledger, experiment language, receipt format, or generic researcher loop merely to package this proposal. +Defer a universal scalar capability score, an automatically learned definition of user value, and unrestricted evaluator self-modification. +The book supplies reasons to distrust those shortcuts, not evidence that a larger autonomous loop will overcome them. + +## Verification and limits + +Local typechecking, build, and package verification passed at the inspected revision. +The diagnostic passed a separate strict TypeScript check, and all seven outputs reproduced exactly on a second execution. +An isolated archive of the reviewed source reproduced the complete diagnostic output, including source identity. +Eleven focused provenance checks covered that reproduction, local changes, documentation stability, and refusal of symlinks and special files. +The Vitest run completed with **399 passed files, 2 skipped files; 5,876 passed tests, 3 skipped tests**. +The recorded test invocation expanded to the full suite; the exact command is preserved with the observations. +No production evaluation campaign, paid optimization experiment, or deployment was performed. + +Source inspection and the offline probes support the gaps and defects identified here. +They do not establish downstream prevalence, adoption cost, or expected performance gains. +The proposed experiments state what evidence would justify implementation or reject the proposal. diff --git a/docs/design/mlbenchmarks-review/observations.json b/docs/design/mlbenchmarks-review/observations.json new file mode 100644 index 00000000..085f7892 --- /dev/null +++ b/docs/design/mlbenchmarks-review/observations.json @@ -0,0 +1,713 @@ +{ + "reviewedBaseRevision": "fe1cc5111aab5d588bf7db3a3785325635937a91", + "sourceIdentity": { + "algorithm": "sha256-canonical-file-manifest", + "paths": [ + "src", + "package.json", + "pnpm-lock.yaml", + "tsconfig.json" + ], + "fileCount": 765, + "digest": "sha256:d743fe180fb7f89c2add26f3f66344a8586b2798f1edcaf8b881b914c10bc009", + "dependencyScope": "Records the manifest and lockfile; assumes dependencies were installed from that lockfile." + }, + "diagnosticIdentity": { + "path": "docs/design/mlbenchmarks-review/probes.mts", + "sha256": "29d0bba079d8b76247bd3d92a24ea8b256080e22ee37db9ca28f0ac3330c793a" + }, + "command": "pnpm exec tsx docs/design/mlbenchmarks-review/probes.mts", + "paidModelCalls": 0, + "execution": { + "kind": "offline deterministic diagnostic", + "modelCalls": 0, + "callbackImplementation": "Local arithmetic and string checks only; no provider clients are supplied.", + "temporaryRunStorage": "Allocated under the OS temporary directory and removed in finally.", + "outputPolicy": "Preserves current returned measurements; excludes temporary paths, run IDs, and wallclock fields.", + "assertionPolicy": "No assertions require the observed defects or policy boundaries to persist." + }, + "probes": { + "holdoutReuse": { + "inputs": { + "calls": 2, + "trainCases": 6, + "finalCases": 6, + "generationsPerCall": 1, + "populationPerGeneration": 1, + "replicatesPerCase": 1, + "sameFinalPayloadsOnBothCalls": true, + "firstResultFedToSecondProposer": false, + "baselineSurface": "baseline", + "proposedSurface": "marker", + "scoring": "1 if artifact contains marker, otherwise 0", + "expectUsage": "off" + }, + "results": { + "rounds": [ + { + "round": 1, + "gateDecision": "ship", + "finalDispatches": 12, + "distinctFinalIds": 6, + "finalSplitDigest": "sha256:fe055aa9390b6483701c8686564c1d4d96426d6a545f59e405fa5350efae84fa", + "agentCallbacks": 24, + "judgeCallbacks": 24, + "proposerCallbacks": 1 + }, + { + "round": 2, + "gateDecision": "ship", + "finalDispatches": 12, + "distinctFinalIds": 6, + "finalSplitDigest": "sha256:fe055aa9390b6483701c8686564c1d4d96426d6a545f59e405fa5350efae84fa", + "agentCallbacks": 24, + "judgeCallbacks": 24, + "proposerCallbacks": 1 + } + ], + "accessPurposes": [ + "debugging", + "evaluation" + ], + "temporaryRunDirectoriesRemoved": true + }, + "limitations": [ + "Measures repeated final-set access and debugging access, not empirical overfitting.", + "The calls use deterministic local callbacks and independent temporary run directories.", + "Does not estimate false-promotion frequency or test a downstream access-control service." + ] + }, + "outcomeKeyOrder": { + "inputs": { + "n": 10, + "rows": [ + { + "score": 0.1, + "retention": 0.9, + "csat": 0.1 + }, + { + "score": 0.2, + "retention": 0.8, + "csat": 0.2 + }, + { + "score": 0.3, + "retention": 0.7, + "csat": 0.3 + }, + { + "score": 0.4, + "retention": 0.6, + "csat": 0.4 + }, + { + "score": 0.5, + "retention": 0.5, + "csat": 0.5 + }, + { + "score": 0.6, + "retention": 0.4, + "csat": 0.6 + }, + { + "score": 0.7, + "retention": 0.30000000000000004, + "csat": 0.7 + }, + { + "score": 0.8, + "retention": 0.19999999999999996, + "csat": 0.8 + }, + { + "score": 0.9, + "retention": 0.09999999999999998, + "csat": 0.9 + }, + { + "score": 1, + "retention": 0, + "csat": 1 + } + ], + "outcomeRowsPerRun": 1, + "requestedMetric": "csat", + "expectedPearsonForRequestedMetric": 1, + "expectedSpearmanForRequestedMetric": 1, + "seed": 1, + "bootstrapIterations": 500 + }, + "results": [ + { + "keyOrder": "retention-first", + "latest": { + "pairs": [ + { + "evalMetric": "score", + "outcomeMetric": "csat", + "n": 10, + "pearson": -1, + "spearman": -1, + "pearsonCi95": { + "lower": -1.0000000000000002, + "upper": -0.9999999999999998 + }, + "verdict": "strong" + } + ], + "joinedSamples": 10, + "skippedRuns": 0 + }, + "mean": { + "pairs": [ + { + "evalMetric": "score", + "outcomeMetric": "csat", + "n": 10, + "pearson": 1, + "spearman": 1, + "pearsonCi95": { + "lower": 1, + "upper": 1 + }, + "verdict": "strong" + } + ], + "joinedSamples": 10, + "skippedRuns": 0 + } + }, + { + "keyOrder": "csat-first", + "latest": { + "pairs": [ + { + "evalMetric": "score", + "outcomeMetric": "csat", + "n": 10, + "pearson": 1, + "spearman": 1, + "pearsonCi95": { + "lower": 1, + "upper": 1 + }, + "verdict": "strong" + } + ], + "joinedSamples": 10, + "skippedRuns": 0 + }, + "mean": { + "pairs": [ + { + "evalMetric": "score", + "outcomeMetric": "csat", + "n": 10, + "pearson": 1, + "spearman": 1, + "pearsonCi95": { + "lower": 1, + "upper": 1 + }, + "verdict": "strong" + } + ], + "joinedSamples": 10, + "skippedRuns": 0 + } + } + ], + "limitations": [ + "Tests metric selection and JSON key order with constructed data, not deployment validity.", + "Each run has one outcome row, so latest and mean refer to the same requested observation." + ] + }, + "adaptationPairing": { + "inputs": { + "scenariosA": [ + { + "scenarioId": "easy-only", + "score": 0.9 + } + ], + "scenariosB": [ + { + "scenarioId": "hard-only", + "score": 0.1 + } + ], + "ks": [ + 0, + 1 + ], + "reps": 1, + "observationsPerArm": 2, + "commonScenarios": 0, + "sameRunnerForBothArms": true, + "bootstrapSeed": 1 + }, + "results": { + "perK": [ + { + "k": 0, + "deltaMean": 0.8, + "aLow": 0.9, + "aHigh": 0.9, + "bLow": 0.1, + "bHigh": 0.1 + }, + { + "k": 1, + "deltaMean": 0.8, + "aLow": 0.9, + "aHigh": 0.9, + "bLow": 0.1, + "bHigh": 0.1 + } + ], + "areaDelta": 0.8, + "firstPassKDelta": null, + "verdict": "a_better", + "rationale": "mean per-k delta=0.800, area delta=0.800" + }, + "limitations": [ + "The two arms differ in task difficulty; zero task identities overlap.", + "This probes the adaptation helper, not the separately implemented campaign paired comparison." + ] + }, + "contaminationDisplay": { + "inputs": { + "n": 12, + "originalScorePerCase": 1, + "perturbedScorePerCase": 0.4, + "observationsPerCasePerCondition": 1, + "modelTrainingExposure": "No model is used; scores are constructed fixture values." + }, + "results": { + "perScenario": [ + { + "scenarioId": "case-0", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-1", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-2", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-3", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-4", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-5", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-6", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-7", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-8", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-9", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-10", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + }, + { + "scenarioId": "case-11", + "originalScore": 1, + "perturbedScore": 0.4, + "delta": -0.6, + "qValue": 0.4 + } + ], + "pairedTest": { + "w": 0, + "p": 0.00048828125, + "method": "exact", + "pFloor": 0.00048828125, + "nNonZero": 12 + }, + "medianDelta": -0.6, + "meanDelta": -0.5999999999999999, + "contaminationSuspected": true, + "reason": "paired p=0.0005 < 0.05 and median drop -0.6000 ≥ 0.05", + "n": 12 + }, + "limitations": [ + "The global Wilcoxon test measures the constructed paired difference; it does not identify contamination as its cause.", + "Per-item qValue uses BH on 1 - abs(delta), without a per-item sampling null; it is a display aid in the inspected source.", + "The per-item qValues do not drive the global contaminationSuspected result." + ] + }, + "negativeOutcomeDirection": { + "inputs": { + "n": 8, + "rows": [ + { + "quality": 0, + "successRate": 1 + }, + { + "quality": 0.14285714285714285, + "successRate": 0.8571428571428572 + }, + { + "quality": 0.2857142857142857, + "successRate": 0.7142857142857143 + }, + { + "quality": 0.42857142857142855, + "successRate": 0.5714285714285714 + }, + { + "quality": 0.5714285714285714, + "successRate": 0.4285714285714286 + }, + { + "quality": 0.7142857142857143, + "successRate": 0.2857142857142857 + }, + { + "quality": 0.8571428571428571, + "successRate": 0.1428571428571429 + }, + { + "quality": 1, + "successRate": 0 + } + ], + "rubric": "quality", + "outcome": "success_rate", + "desiredOutcomeDirection": "increase", + "seed": 1, + "bootstrapResamples": 100, + "researcherBootstrapResamples": 500, + "researcherSeed": "Derived deterministically by the validity helper", + "researcherFailureThreshold": 0.5 + }, + "results": { + "report": { + "pairs": [ + { + "rubric": "quality", + "outcome": "success_rate", + "n": 8, + "pearson": -1, + "spearman": -1, + "ci95": { + "low": -1.0000000000000002, + "high": -0.9999999999999998 + }, + "verdict": "load_bearing" + } + ], + "ranked": [ + { + "rubric": "quality", + "bestOutcome": "success_rate", + "spearman": -1, + "pearson": -1, + "n": 8, + "verdict": "load_bearing" + } + ], + "joinedSamples": 8, + "skippedRuns": 0, + "rubricsWithoutData": [] + }, + "researcher": { + "report": { + "pairs": [ + { + "rubric": "quality", + "outcome": "success_rate", + "n": 8, + "pearson": -1, + "spearman": -1, + "ci95": { + "low": -1.0000000000000002, + "high": -0.9999999999999998 + }, + "verdict": "load_bearing" + } + ], + "ranked": [ + { + "rubric": "quality", + "bestOutcome": "success_rate", + "spearman": -1, + "pearson": -1, + "n": 8, + "verdict": "load_bearing" + } + ], + "joinedSamples": 8, + "skippedRuns": 0, + "rubricsWithoutData": [] + }, + "failureGroups": 1, + "failures": [ + { + "code": "low-score-same-candidate", + "description": "same-candidate scored < 0.5 on 4 run(s) (mean 0.214)", + "samples": 4 + } + ], + "proposedChanges": [ + { + "kind": "reviewer_prompt", + "payload": { + "rubric": "quality", + "action": "up-weight", + "spearman": -1, + "bestOutcome": "success_rate" + }, + "rationale": "predictive-validity Spearman=-1.000 vs success_rate (load-bearing); recommend up-weighting", + "expectedDelta": 0.05 + } + ] + } + }, + "limitations": [ + "Magnitude-based bucketing is intentional in existing tests, despite contradictory interface prose.", + "A negative association can be desirable for an outcome such as failure rate; direction needs explicit interpretation.", + "The researcher recommends increasing rubric weight despite its negative association with desired success rate; it does not execute or deploy that recommendation.", + "Constructed perfect correlation establishes neither causal validity nor held-out predictive performance." + ] + }, + "sequentialDependence": { + "inputs": { + "alpha": 0.05, + "minN": 5, + "maxN": 100, + "shuffleSeed": 1337, + "branchesEnumerated": 2, + "cellsPerBranch": 100, + "independentRandomSignsPerExperiment": 1, + "dataGeneratingProcess": "Draw one fair sign Z; set all 100 paired deltas equal to Z.", + "exchangeable": true, + "marginalMeanDelta": 0, + "conditionalMeanAfterFirstObservation": "Z, not necessarily <= 0" + }, + "results": { + "branches": [ + { + "commonDelta": -1, + "probability": 0.5, + "result": { + "decision": "hold", + "reasons": [ + "sequentialPairedGate: undecided at pre-registered maxN=100 (e-value 1.00 < 1/α=20.00). This is NOT evidence of no effect — the effect may be real but smaller than this budget can detect; re-register with a larger N to test that" + ], + "contributingGates": [ + { + "name": "sequentialPairedGate", + "status": "fail", + "detail": { + "wealth": 1, + "n": 100, + "decided": false, + "alpha": 0.05, + "maxBet": 0.5, + "nullMean": 0.5, + "threshold": 20, + "sumX": 0, + "varSum": 0.15877048244745845, + "decision": "undecided-at-maxN", + "minN": 5, + "maxN": 100, + "scale": 1, + "shuffleSeed": 1337, + "direction": "increase", + "minEffect": 0, + "pairedN": 100 + } + } + ], + "delta": -1 + } + }, + { + "commonDelta": 1, + "probability": 0.5, + "result": { + "decision": "ship", + "reasons": [ + "sequentialPairedGate: e-value 22.74 ≥ 1/α=20.00 at n=15 (minN=5): the paired improvement exceeds 0 at anytime-valid level α=0.05" + ], + "contributingGates": [ + { + "name": "sequentialPairedGate", + "status": "pass", + "detail": { + "wealth": 22.737367544323206, + "n": 15, + "decided": true, + "alpha": 0.05, + "maxBet": 0.5, + "nullMean": 0.5, + "threshold": 20, + "decidedAtN": 15, + "sumX": 15, + "varSum": 0.14608663336124675, + "decision": "promote", + "minN": 5, + "maxN": 100, + "scale": 1, + "shuffleSeed": 1337, + "direction": "increase", + "minEffect": 0, + "pairedN": 100 + } + } + ], + "delta": 1 + } + } + ], + "promotionProbabilityUnderMarginalZeroProcess": 0.5 + }, + "limitations": [ + "Enumerates both equiprobable branches exactly; this is not a Monte Carlo estimate.", + "The process violates the conditional-mean null required by the e-process core.", + "This refutes sufficiency of exchangeability and shuffling, not the valid e-process theorem or any measured production dataset." + ] + }, + "powerFloor": { + "inputs": { + "gate": { + "kind": "power-floor", + "target": 0.8, + "effectGrid": [ + 0.01, + 1 + ], + "sim": { + "trials": 1, + "resamples": 1, + "seed": 1 + } + }, + "curve": [ + { + "effect": 0.01, + "power": 0.1 + }, + { + "effect": 1, + "power": 1 + } + ], + "practicalEffectForInterpretation": 0.01, + "curveSource": "Supplied deterministic fixture; no power simulation is run." + }, + "results": { + "id": "floor", + "passed": true, + "evidence": { + "target": 0.8, + "maxPower": 1, + "curve": [ + { + "effect": 0.01, + "power": 0.1 + }, + { + "effect": 1, + "power": 1 + } + ] + } + }, + "limitations": [ + "The inspected gate documents a maximum-over-grid structural feasibility check; this output matches that contract.", + "Passing this gate does not establish target power at the practical effect of 0.01.", + "The fixture powers are inputs, not measured or simulated power estimates." + ] + } + }, + "separateVerificationAtReviewedBase": { + "provenance": "Historical checks at reviewedBaseRevision; this diagnostic does not rerun them.", + "sourceRevision": "fe1cc5111aab5d588bf7db3a3785325635937a91", + "checks": [ + { + "command": "pnpm typecheck", + "result": "passed" + }, + { + "command": "pnpm build", + "result": "passed" + }, + { + "command": "pnpm verify:package", + "result": "passed" + } + ], + "tests": { + "command": "pnpm test -- tests/experiment/preregistration-acceptance.test.ts tests/experiment/power.test.ts tests/contamination-guard.test.ts tests/rl-predictive-validity-researcher.test.ts tests/rubric-predictive-validity.test.ts tests/meta-eval.test.ts", + "observedScope": "The package command expanded to the full Vitest suite.", + "files": { + "passed": 399, + "skipped": 2 + }, + "tests": { + "passed": 5876, + "skipped": 3 + }, + "result": "passed" + }, + "limitation": "Passing repository checks do not establish correctness of the counterexample behaviors recorded above." + } +} diff --git a/docs/design/mlbenchmarks-review/probes.mts b/docs/design/mlbenchmarks-review/probes.mts new file mode 100644 index 00000000..06669215 --- /dev/null +++ b/docs/design/mlbenchmarks-review/probes.mts @@ -0,0 +1,476 @@ +import { createHash } from 'node:crypto' +import { lstat, mkdtemp, readFile, readdir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { campaignSplitDigest } from '../../../src/campaign/coverage.ts' +import { sequentialPairedGate } from '../../../src/campaign/gates/sequential.ts' +import type { JudgeScore } from '../../../src/campaign/types.ts' +import { HoldoutAuditor } from '../../../src/contamination-guard.ts' +import { selfImprove } from '../../../src/contract/self-improve.ts' +import { evaluatePowerFloorGate } from '../../../src/experiment/ast.ts' +import { compareCodeUnits, hashCanonical } from '../../../src/ledger-core/canonical.ts' +import { correlationStudy } from '../../../src/meta-eval/correlation-study.ts' +import { InMemoryOutcomeStore } from '../../../src/meta-eval/outcome-store.ts' +import { rubricPredictiveValidity } from '../../../src/meta-eval/rubric-predictive-validity.ts' +import { compareAdaptationCurves, runAdaptationCurve } from '../../../src/rl/adaptation-eval.ts' +import { runContaminationProbe } from '../../../src/rl/contamination.ts' +import { PredictiveValidityResearcher } from '../../../src/rl/predictive-validity-researcher.ts' +import type { RunRecord } from '../../../src/run-record.ts' +import { TraceEmitter } from '../../../src/trace/emitter.ts' +import { InMemoryTraceStore } from '../../../src/trace/store.ts' + +// These diagnostics record behavior without asserting that a defect must remain. +const reviewedBaseRevision = 'fe1cc5111aab5d588bf7db3a3785325635937a91' +const command = 'pnpm exec tsx docs/design/mlbenchmarks-review/probes.mts' +const repositoryRoot = fileURLToPath(new URL('../../../', import.meta.url)) +const sourcePaths = ['src', 'package.json', 'pnpm-lock.yaml', 'tsconfig.json'] + +async function fileEntries(path: string): Promise> { + const absolutePath = join(repositoryRoot, path) + const metadata = await lstat(absolutePath) + if (metadata.isDirectory()) { + const children = await readdir(absolutePath) + return (await Promise.all(children.map(child => fileEntries(`${path}/${child}`)))).flat() + } + if (!metadata.isFile()) throw new Error(`Source identity requires a regular file: ${path}`) + return [{ path, sha256: createHash('sha256').update(await readFile(absolutePath)).digest('hex') }] +} + +// Hash working files, including untracked files, so Git index state cannot hide changes. +const sourceFiles = (await Promise.all(sourcePaths.map(fileEntries))) + .flat() + .sort((left, right) => compareCodeUnits(left.path, right.path)) +const sourceIdentity = { + algorithm: 'sha256-canonical-file-manifest', + paths: sourcePaths, + fileCount: sourceFiles.length, + digest: hashCanonical({ domain: 'agent-eval-mlbenchmarks-review-source-v1', files: sourceFiles }), + dependencyScope: 'Records the manifest and lockfile; assumes dependencies were installed from that lockfile.', +} +const diagnosticPath = 'docs/design/mlbenchmarks-review/probes.mts' +const diagnosticIdentity = { + path: diagnosticPath, + sha256: createHash('sha256').update(await readFile(join(repositoryRoot, diagnosticPath))).digest('hex'), +} + +async function holdoutReuse() { + const runRoot = await mkdtemp(join(tmpdir(), 'agent-eval-mlbenchmarks-review-')) + const train = Array.from({ length: 6 }, (_, i) => ({ + id: `train-${i}`, + kind: 'offline-probe', + })) + const final = Array.from({ length: 6 }, (_, i) => ({ + id: `final-${i}`, + kind: 'offline-probe', + })) + const rounds = [] + try { + for (const round of [1, 2]) { + const finalDispatches: string[] = [] + let agentCallbacks = 0 + let judgeCallbacks = 0 + let proposerCallbacks = 0 + const result = await selfImprove({ + scenarios: train, + baselineSurface: 'baseline', + model: 'deterministic-probe@2026-09-12', + agent: async (surface, scenario) => { + agentCallbacks++ + if (scenario.id.startsWith('final-')) finalDispatches.push(scenario.id) + return String(surface) + }, + judge: { + name: 'marker', + dimensions: [{ key: 'pass', description: 'Output has the marker' }], + score: ({ artifact }) => { + judgeCallbacks++ + const pass = artifact.includes('marker') ? 1 : 0 + return { dimensions: { pass }, composite: pass, notes: '' } + }, + }, + proposer: { + kind: 'offline-probe', + propose: async () => { + proposerCallbacks++ + return [{ + surface: 'marker', + label: 'marker', + rationale: 'Synthetic repeat-access probe', + }] + }, + }, + budget: { generations: 1, populationSize: 1, holdoutScenarios: final }, + runDir: join(runRoot, `round-${round}`), + expectUsage: 'off', + }) + rounds.push({ + round, + gateDecision: result.gateDecision, + finalDispatches: finalDispatches.length, + distinctFinalIds: new Set(finalDispatches).size, + finalSplitDigest: campaignSplitDigest(final, 1), + agentCallbacks, + judgeCallbacks, + proposerCallbacks, + }) + } + } finally { + // Remove only the directory allocated for this invocation. + await rm(runRoot, { recursive: true, force: true }) + } + const auditor = new HoldoutAuditor([ + { id: 'heldout', payload: 'hidden', split: 'holdout' }, + ]) + auditor.get('heldout', 'debugging') + auditor.get('heldout', 'evaluation') + return { + inputs: { + calls: 2, + trainCases: train.length, + finalCases: final.length, + generationsPerCall: 1, + populationPerGeneration: 1, + replicatesPerCase: 1, + sameFinalPayloadsOnBothCalls: true, + firstResultFedToSecondProposer: false, + baselineSurface: 'baseline', + proposedSurface: 'marker', + scoring: '1 if artifact contains marker, otherwise 0', + expectUsage: 'off', + }, + results: { + rounds, + accessPurposes: auditor.getAccessLog().map(entry => entry.purpose), + temporaryRunDirectoriesRemoved: true, + }, + limitations: [ + 'Measures repeated final-set access and debugging access, not empirical overfitting.', + 'The calls use deterministic local callbacks and independent temporary run directories.', + 'Does not estimate false-promotion frequency or test a downstream access-control service.', + ], + } +} + +async function outcomeKeyOrder() { + const rows = Array.from({ length: 10 }, (_, i) => { + const score = (i + 1) / 10 + return { score, retention: 1 - score, csat: score } + }) + const results = [] + for (const keyOrder of ['retention-first', 'csat-first'] as const) { + const traces = new InMemoryTraceStore() + const outcomes = new InMemoryOutcomeStore() + let tick = 0 + for (const [i, row] of rows.entries()) { + const emitter = new TraceEmitter(traces, { + runId: `outcome-fixture-${i}`, + now: () => ++tick, + }) + await emitter.startRun({ scenarioId: `scenario-${i}` }) + await emitter.endRun({ pass: true, score: row.score }) + await outcomes.append({ + runId: emitter.runId, + capturedAt: ++tick, + metrics: keyOrder === 'retention-first' + ? { retention: row.retention, csat: row.csat } + : { csat: row.csat, retention: row.retention }, + }) + } + results.push({ + keyOrder, + latest: await correlationStudy(traces, outcomes, [{ id: 'score' }], ['csat'], { + seed: 1, + bootstrapIterations: 500, + }), + mean: await correlationStudy(traces, outcomes, [{ id: 'score' }], ['csat'], { + seed: 1, + bootstrapIterations: 500, + reduction: 'mean', + }), + }) + } + return { + inputs: { + n: rows.length, + rows, + outcomeRowsPerRun: 1, + requestedMetric: 'csat', + expectedPearsonForRequestedMetric: 1, + expectedSpearmanForRequestedMetric: 1, + seed: 1, + bootstrapIterations: 500, + }, + results, + limitations: [ + 'Tests metric selection and JSON key order with constructed data, not deployment validity.', + 'Each run has one outcome row, so latest and mean refer to the same requested observation.', + ], + } +} + +async function adaptationPairing() { + const scenariosA = [{ scenarioId: 'easy-only', score: 0.9 }] + const scenariosB = [{ scenarioId: 'hard-only', score: 0.1 }] + const runner = { + run: async ({ scenario }: { scenario: { score: number } }) => scenario.score, + } + const ks = [0, 1] + const reps = 1 + const a = await runAdaptationCurve<{ scenarioId: string; score: number }>({ + scenarios: scenariosA, ks, reps, runner, + }) + const b = await runAdaptationCurve<{ scenarioId: string; score: number }>({ + scenarios: scenariosB, ks, reps, runner, + }) + return { + inputs: { + scenariosA, + scenariosB, + ks, + reps, + observationsPerArm: ks.length * reps, + commonScenarios: scenariosA.filter(a => scenariosB.some(b => a.scenarioId === b.scenarioId)).length, + sameRunnerForBothArms: true, + bootstrapSeed: 1, + }, + results: compareAdaptationCurves(a, b, { seed: 1 }), + limitations: [ + 'The two arms differ in task difficulty; zero task identities overlap.', + 'This probes the adaptation helper, not the separately implemented campaign paired comparison.', + ], + } +} + +async function contaminationDisplay() { + const originals = Array.from({ length: 12 }, (_, i) => ({ + id: `case-${i}`, + score: 1, + })) + const perturbed = originals.map(scenario => ({ ...scenario, score: 0.4 })) + const result = await runContaminationProbe({ + scenarioId: scenario => scenario.id, + originals, + perturbed, + scoreFn: async scenario => scenario.score, + }) + return { + inputs: { + n: originals.length, + originalScorePerCase: 1, + perturbedScorePerCase: 0.4, + observationsPerCasePerCondition: 1, + modelTrainingExposure: 'No model is used; scores are constructed fixture values.', + }, + results: result, + limitations: [ + 'The global Wilcoxon test measures the constructed paired difference; it does not identify contamination as its cause.', + 'Per-item qValue uses BH on 1 - abs(delta), without a per-item sampling null; it is a display aid in the inspected source.', + 'The per-item qValues do not drive the global contaminationSuspected result.', + ], + } +} + +async function negativeOutcomeDirection() { + const rows = Array.from({ length: 8 }, (_, i) => ({ + quality: i / 7, + successRate: 1 - i / 7, + })) + const runs: RunRecord[] = [] + const outcomes = new InMemoryOutcomeStore() + for (const [i, row] of rows.entries()) { + const runId = `direction-fixture-${i}` + runs.push({ + runId, + experimentId: 'book-review-fixture', + candidateId: 'same-candidate', + scenarioId: `direction-scenario-${i}`, + seed: 0, + model: 'fixture@1', + promptHash: '0'.repeat(64), + configHash: '1'.repeat(64), + commitSha: reviewedBaseRevision, + wallMs: 0, + costUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + tokenUsage: { input: 0, output: 0 }, + terminalOutcome: 'succeeded', + splitTag: 'holdout', + outcome: { holdoutScore: row.quality, raw: { quality: row.quality } }, + }) + await outcomes.append({ + runId, + capturedAt: i, + metrics: { success_rate: row.successRate }, + }) + } + const report = await rubricPredictiveValidity({ + runs, + outcomes, + outcomeMetrics: ['success_rate'], + rubrics: ['quality'], + seed: 1, + bootstrapResamples: 100, + }) + const researcher = new PredictiveValidityResearcher({ + outcomes, + outcomeMetrics: ['success_rate'], + rubrics: ['quality'], + }) + const researcherReport = await researcher.runValidityCheck(runs) + const failures = await researcher.inspectFailures(runs) + const changes = await researcher.proposeChange(failures) + return { + inputs: { + n: rows.length, + rows, + rubric: 'quality', + outcome: 'success_rate', + desiredOutcomeDirection: 'increase', + seed: 1, + bootstrapResamples: 100, + researcherBootstrapResamples: 500, + researcherSeed: 'Derived deterministically by the validity helper', + researcherFailureThreshold: 0.5, + }, + results: { + report, + researcher: { + report: researcherReport, + failureGroups: failures.length, + failures: failures.map(failure => ({ + code: failure.code, + description: failure.description, + samples: failure.evidence.samples, + })), + proposedChanges: changes, + }, + }, + limitations: [ + 'Magnitude-based bucketing is intentional in existing tests, despite contradictory interface prose.', + 'A negative association can be desirable for an outcome such as failure rate; direction needs explicit interpretation.', + 'The researcher recommends increasing rubric weight despite its negative association with desired success rate; it does not execute or deploy that recommendation.', + 'Constructed perfect correlation establishes neither causal validity nor held-out predictive performance.', + ], + } +} + +async function sequentialDependence() { + const options = { alpha: 0.05, minN: 5, maxN: 100, shuffleSeed: 1337 } + const branches = [] + for (const delta of [-1, 1]) { + const scores = (composite: number): Record => ({ + judge: { composite, dimensions: {}, notes: '' }, + }) + const baseline = new Map(Array.from({ length: 100 }, (_, i) => [ + `task:${i}`, + scores(delta > 0 ? 0 : 1), + ])) + const candidate = new Map(Array.from({ length: 100 }, (_, i) => [ + `task:${i}`, + scores(delta > 0 ? 1 : 0), + ])) + const result = await sequentialPairedGate(options).decide({ + scenarios: [{ id: 'task', kind: 'synthetic-common-sign' }], + judgeScores: candidate, + baselineJudgeScores: baseline, + candidateArtifacts: new Map(), + baselineArtifacts: new Map(), + cost: { candidate: 0, baseline: 0 }, + signal: new AbortController().signal, + }) + branches.push({ commonDelta: delta, probability: 0.5, result }) + } + return { + inputs: { + ...options, + branchesEnumerated: branches.length, + cellsPerBranch: 100, + independentRandomSignsPerExperiment: 1, + dataGeneratingProcess: 'Draw one fair sign Z; set all 100 paired deltas equal to Z.', + exchangeable: true, + marginalMeanDelta: 0, + conditionalMeanAfterFirstObservation: 'Z, not necessarily <= 0', + }, + results: { + branches, + promotionProbabilityUnderMarginalZeroProcess: branches.reduce( + (sum, branch) => sum + (branch.result.decision === 'ship' ? branch.probability : 0), + 0, + ), + }, + limitations: [ + 'Enumerates both equiprobable branches exactly; this is not a Monte Carlo estimate.', + 'The process violates the conditional-mean null required by the e-process core.', + 'This refutes sufficiency of exchangeability and shuffling, not the valid e-process theorem or any measured production dataset.', + ], + } +} + +function powerFloor() { + const gate = { + kind: 'power-floor' as const, + target: 0.8, + effectGrid: [0.01, 1], + sim: { trials: 1, resamples: 1, seed: 1 }, + } + const curve = [{ effect: 0.01, power: 0.1 }, { effect: 1, power: 1 }] + return { + inputs: { + gate, + curve, + practicalEffectForInterpretation: 0.01, + curveSource: 'Supplied deterministic fixture; no power simulation is run.', + }, + results: evaluatePowerFloorGate('floor', gate, curve), + limitations: [ + 'The inspected gate documents a maximum-over-grid structural feasibility check; this output matches that contract.', + 'Passing this gate does not establish target power at the practical effect of 0.01.', + 'The fixture powers are inputs, not measured or simulated power estimates.', + ], + } +} + +const observations = { + reviewedBaseRevision, + sourceIdentity, + diagnosticIdentity, + command, + paidModelCalls: 0, + execution: { + kind: 'offline deterministic diagnostic', + modelCalls: 0, + callbackImplementation: 'Local arithmetic and string checks only; no provider clients are supplied.', + temporaryRunStorage: 'Allocated under the OS temporary directory and removed in finally.', + outputPolicy: 'Preserves current returned measurements; excludes temporary paths, run IDs, and wallclock fields.', + assertionPolicy: 'No assertions require the observed defects or policy boundaries to persist.', + }, + probes: { + holdoutReuse: await holdoutReuse(), + outcomeKeyOrder: await outcomeKeyOrder(), + adaptationPairing: await adaptationPairing(), + contaminationDisplay: await contaminationDisplay(), + negativeOutcomeDirection: await negativeOutcomeDirection(), + sequentialDependence: await sequentialDependence(), + powerFloor: powerFloor(), + }, + separateVerificationAtReviewedBase: { + provenance: 'Historical checks at reviewedBaseRevision; this diagnostic does not rerun them.', + sourceRevision: reviewedBaseRevision, + checks: [ + { command: 'pnpm typecheck', result: 'passed' }, + { command: 'pnpm build', result: 'passed' }, + { command: 'pnpm verify:package', result: 'passed' }, + ], + tests: { + command: 'pnpm test -- tests/experiment/preregistration-acceptance.test.ts tests/experiment/power.test.ts tests/contamination-guard.test.ts tests/rl-predictive-validity-researcher.test.ts tests/rubric-predictive-validity.test.ts tests/meta-eval.test.ts', + observedScope: 'The package command expanded to the full Vitest suite.', + files: { passed: 399, skipped: 2 }, + tests: { passed: 5876, skipped: 3 }, + result: 'passed', + }, + limitation: 'Passing repository checks do not establish correctness of the counterexample behaviors recorded above.', + }, +} + +console.log(JSON.stringify(observations, null, 2)) diff --git a/docs/design/mlbenchmarks-review/sources.json b/docs/design/mlbenchmarks-review/sources.json new file mode 100644 index 00000000..ae6f19b0 --- /dev/null +++ b/docs/design/mlbenchmarks-review/sources.json @@ -0,0 +1,200 @@ +{ + "accessed_at_utc": "2026-09-13T01:01:32.062338+00:00", + "index_url": "https://mlbenchmarks.org/", + "index_sha256": "edd63754d75578741714d1fcbf8766e97782c865a4e65345f00967a26b8b4589", + "pages": [ + { + "file": "00-preface.html", + "url": "https://mlbenchmarks.org/00-preface.html", + "title": "Preface - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "df83e9d6dd7ee845e05a4086db47f53af0d85f807be0f512f22f2323f7c00391", + "body_words": 2689, + "review_status": "complete-text" + }, + { + "file": "00-prologue.html", + "url": "https://mlbenchmarks.org/00-prologue.html", + "title": "Prologue - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "1885c0e15e19f55c2f267d96a0ebd06ff694582f0ad8905ccddd6e85b8033553", + "body_words": 625, + "review_status": "complete-text" + }, + { + "file": "01-introduction.html", + "url": "https://mlbenchmarks.org/01-introduction.html", + "title": "1 – Introduction - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "507797a0b6c72a0036b8367602ee41a0d0c339a7269a3ea7934128bb3a9185a0", + "body_words": 3377, + "review_status": "complete-text" + }, + { + "file": "02-populations-predictions.html", + "url": "https://mlbenchmarks.org/02-populations-predictions.html", + "title": "2 – Populations and predictions - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "e6ff7c7ddd73154d199f04d3a43f92b3a16195456f4f23cc0296e257bbb225bc", + "body_words": 5563, + "review_status": "complete-text" + }, + { + "file": "03-detecting-differences.html", + "url": "https://mlbenchmarks.org/03-detecting-differences.html", + "title": "3 – Detecting differences - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "b9b1df7d9339a71a70d825f0d51c27d17293a0d3dd55a6272f84591c78285cde", + "body_words": 5542, + "review_status": "complete-text" + }, + { + "file": "04-holdout-method.html", + "url": "https://mlbenchmarks.org/04-holdout-method.html", + "title": "4 – Holdout method - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "c3dd581b0fbc8227d989f0c5863b2060d902cf866c1167dc1fb840a224a04907", + "body_words": 8441, + "review_status": "complete-text" + }, + { + "file": "05-test-set-reuse.html", + "url": "https://mlbenchmarks.org/05-test-set-reuse.html", + "title": "5 – Test set reuse - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "09261cb5ddeea95fcb84b042eaa2b721393e1a52c40377d13247085d394d6138", + "body_words": 5585, + "review_status": "complete-text" + }, + { + "file": "06-scientific-crisis.html", + "url": "https://mlbenchmarks.org/06-scientific-crisis.html", + "title": "6 – Scientific crisis - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "500d42c5bce9fbd67b4908e5a5a3fd956d954969ab73c0828e22211be952ef15", + "body_words": 7256, + "review_status": "complete-text" + }, + { + "file": "07-replication-machine-learning.html", + "url": "https://mlbenchmarks.org/07-replication-machine-learning.html", + "title": "7 – Replication in machine learning - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "c5103477aa1a72b2bbd75fe58e8f1ab2254b247e6529f06c0bebb12950335bf4", + "body_words": 8899, + "review_status": "complete-text" + }, + { + "file": "08-forces-against-crisis.html", + "url": "https://mlbenchmarks.org/08-forces-against-crisis.html", + "title": "8 – Forces against crisis - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "874764d22ba2c712877fc86fa2e465297670071ca8c58fd011769c1dfb1be4f8", + "body_words": 8128, + "review_status": "complete-text" + }, + { + "file": "10-generative-models.html", + "url": "https://mlbenchmarks.org/10-generative-models.html", + "title": "10 – Generative models - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "2e89354b87cc473e1190f687a4286f80754efc86257ec06b3569ea13d6f23729", + "body_words": 11490, + "review_status": "complete-text" + }, + { + "file": "11-evaluating-language-models.html", + "url": "https://mlbenchmarks.org/11-evaluating-language-models.html", + "title": "11 – Evaluating language models - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "01a08570e33f9446bc33f0b0ac6fa60c18ec9d356672944da51b4ff802059bad", + "body_words": 13399, + "review_status": "complete-text" + }, + { + "file": "12-problem-aggregation.html", + "url": "https://mlbenchmarks.org/12-problem-aggregation.html", + "title": "12 – The problem of aggregation - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "f09917ed330f01aa573d9609469e82745ebc50a218484c88685ff21065217104", + "body_words": 8857, + "review_status": "complete-text" + }, + { + "file": "13-model-moves-data.html", + "url": "https://mlbenchmarks.org/13-model-moves-data.html", + "title": "13 – When the model moves the data - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "673f0e4f300cf4d27b4bdab92593a0e2b72fcd63092a51afbebf614ed4173ae8", + "body_words": 8832, + "review_status": "complete-text" + }, + { + "file": "14-evaluation-frontier.html", + "url": "https://mlbenchmarks.org/14-evaluation-frontier.html", + "title": "14 – Evaluation at the frontier - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "71a5d336dc99e9135651218344564ffaea2e9140e67295860735c3b76df15154", + "body_words": 9464, + "review_status": "complete-text" + }, + { + "file": "15-epilogue.html", + "url": "https://mlbenchmarks.org/15-epilogue.html", + "title": "15 – Epilogue - The Emerging Science of Machine Learning Benchmarks", + "html_sha256": "a0b57ccdb46107e3a82d2a9f3bd16601e950fd3fb13006030062edc61379c651", + "body_words": 1624, + "review_status": "complete-text" + } + ], + "review_date_local": "2026-09-12", + "review_timezone": "America/Los_Angeles", + "repository": { + "url": "https://github.com/tangle-network/agent-eval", + "revision": "fe1cc5111aab5d588bf7db3a3785325635937a91", + "package_version": "0.180.0", + "base": "origin/main" + }, + "coverage": { + "linked_reading_pages": 16, + "reviewed_pages": 16, + "body_words": 109771, + "word_count_method": "Whitespace split of BeautifulSoup chapter-body get_text with spaces; includes headings, tables, equations, notes, and references.", + "extraction_audit": "Paragraph extraction plus every omitted non-whitespace text node were read; figures/tables supporting conclusions were checked in HTML or PDF context.", + "missing_chapter": { + "number": 9, + "status": "unavailable from inspected live index", + "checked_404_paths": [ + "https://mlbenchmarks.org/09-annotation.html", + "https://mlbenchmarks.org/09-data-annotation.html", + "https://mlbenchmarks.org/09-annotations.html" + ] + }, + "independent_reproduction_of_cited_studies": false, + "print_edition_reviewed": false + }, + "review_partitions": [ + { + "pages": [ + "00-preface.html", + "00-prologue.html", + "01-introduction.html", + "02-populations-predictions.html", + "03-detecting-differences.html", + "04-holdout-method.html", + "05-test-set-reuse.html", + "06-scientific-crisis.html" + ], + "body_words": 39078 + }, + { + "pages": [ + "07-replication-machine-learning.html", + "08-forces-against-crisis.html", + "10-generative-models.html", + "11-evaluating-language-models.html" + ], + "body_words": 41916 + }, + { + "pages": [ + "12-problem-aggregation.html", + "13-model-moves-data.html", + "14-evaluation-frontier.html", + "15-epilogue.html" + ], + "body_words": 28777 + } + ], + "visual_inspection": { + "chapter_7": "Figures 7.2–7.4 checked using linked chapter PDF; HTML used for references.", + "chapter_11": "Figures 11.3–11.6 checked using linked chapter PDF; HTML used for references.", + "chapters_12_to_14": "All eight SVG figures checked; all three chapter-12 tables checked in HTML.", + "chapters_1_to_6": "Captions and surrounding arguments read; chapter-2 confusion matrix and chapter-5 rate table checked in HTML." + } +} diff --git a/docs/design/self-improvement-evidence-audit.md b/docs/design/self-improvement-evidence-audit.md new file mode 100644 index 00000000..abaecb11 --- /dev/null +++ b/docs/design/self-improvement-evidence-audit.md @@ -0,0 +1,263 @@ +# Evidence audit: selfImprove and optimizer benefit + +Historical runs contain gains, nulls, and regressions. +They do not establish optimizer superiority over a direct edit or current-branch improvement across tasks. +A historical GEPA analyst comparison reported 0.4285 to 0.4809 pooled micro F1, with important validity limits described below. +A later GEPA challenger lost 0.0561 on fresh agent families. +These results justify preserving candidates, checking transfer, and reporting negative or inconclusive outcomes. + +This was a read-only audit on 2026-09-13 of `feat/evaluation-integrity` over `dda9941437190c9c541b3f54946bfeeb153366fe`. +The implementation changes were uncommitted during inspection. +No paid calls or new model experiments ran for this audit. +Current-branch fixture tests are distinct from all historical model results below. + +## Inspected evidence + +Paths below are relative to the repository root unless stated otherwise. + +- All 10 [registry records](../../evidence/INDEX.md): 2 `CERTIFIED`, 6 `MEASURED-ONCE`, 1 `RESOLVED-NULL`, and 1 `UNVERIFIED`. + These are registry labels; this audit did not recertify those records. +- The archived extraction comparison, its original implementation, and 20 optimization-related notebook records. +- Four committed CodeTraceBench result files, both certification reports, and both preregistrations. +- Current selfImprove, final-comparison, method-integrity, and final-evidence fixtures. + +Searches covered `evidence/`, `examples/`, `benchmarks/`, `docs/`, and `.evolve/`. +There was no root `results/` directory and only one archived method-comparison JSON in examples. +Queries used `rg`, `git show` at the producing revision, and Python JSON parsing. +For each committed analyst result, model observation counts, positive-label micro F1, call totals, and cost sums were independently recomputed. + +The following referenced raw artifacts were unavailable locally: + +- `~/bench-cache/ctb-20260801/certification/`: incumbent and manual-width-adaptive arms. +- `~/bench-cache/ctb-20260801/cert2/`: rejected G2 arms. +- `.evolve/substrate-proof/appworld/d3-scaled-comparison.json`. +- `.evolve/compare-optimization-methods/` and `.evolve/compare-drivers/`. +- `examples/findings-ablation/index.ts` and the referenced session scratch artifacts for bridge and CAD proofs. + +The registry's `.evolve/certification-2026-08-02-preregistration.md` reference is stale. +The committed replacement is [benchmarks/trace-analysis/codetracebench-crossfamily-cert-20260802/preregistration.md](../../benchmarks/trace-analysis/codetracebench-crossfamily-cert-20260802/preregistration.md). + +## Model-backed evidence + +| Date and task | Optimizer and comparison | Observed result | Units and costs | Evidence limit | +| --- | --- | --- | --- | --- | +| June 1, transaction extraction | Package-local GEPA-style reflection, GEPA-style Pareto, and SkillOpt-style patching versus an intentionally underspecified prompt | Baseline 0.625; reflection and SkillOpt 1.0; Pareto 0.958 | 8 search cases, 6 reported final cases; 182 captured worker calls; $0.013237 captured worker cost | SkillOpt selected on the reported final set; optimizer model costs omitted; no official Python optimizers | +| June 1, findings ablation | Local GEPA with versus without analyst findings | Both 1.0 from baseline 0.625; difference 0, archived CI [0, 0] | 6 reported cases, 130 calls, $0.009, 131 seconds | Zero findings were generated; the proposed mechanism never activated; notebook only | +| May 30, legal agents | Local gepaDriver candidates versus incumbent | Candidate fee scores 100 to 83/92; hallucination-free score 100 to 85; selected baseline | 4 scorable personas; 2 named final personas; gen1/pop2/reps2 | Model, cost, and complete paired rows missing; notebook only | +| June 1, AppWorld difficulty 3 | Local drivers versus a competent baseline, deepseek-v4-pro worker | Small baseline 0.794, lift 0; scaled baseline 0.885, both GEPA lifts 0; memory -0.047 | Small n=6; scaled notebook n=8 with unclear repeated-cell meaning; scaled cost $2.58 | Raw comparison absent; null does not isolate whether remaining errors are prompt-fixable | +| August 1, CodeTraceBench analyst | Real Python GEPA instructions versus stock and manual width-adaptive arm, glm-5.2 | Pooled micro F1: GEPA 0.4809, stock 0.4285, manual 0.4047 | 69 cases, 2 reps, 138 model observations per arm; search $4.46; six comparison runs documented $50.63 | Paired intervals cross zero; point-estimate promotion; split3 later retired; baseline raw files unavailable | +| August 2, CodeTraceBench transfer | Real Python GEPA G2 versus shipping G1, same analyst engine/model | Fresh pooled micro F1 0.1928 versus 0.2489; G2 rejected | 64 cases, 128 observations per arm; search $7.05; four comparison runs documented $31.07 | G2 raw files unavailable; intervals cross zero; Terminus2 has 30 source clusters for 32 cases | +| August 3, cross-family selection | Real GEPA prompt search with a macro objective | Macro 0.163 to 0.187; micro 0.3399 to 0.1897; TP 26 to 11; recall 0.325 to 0.138 | 24 train / 16 selection cases, 56 evaluations, $6.42 | Selection-only rejection; no fresh final experiment; notebook only | +| August 19–20, AIME | Official Python GEPA and live bridge; glm-5.3 worker, deepseek-v4-flash optimizer | Search completed in attempts 11–14; final comparison never completed | Train8/selection8/final10; 14 launches; about $4 reported | Machinery evidence, no lift verdict; substantial shared-host contention and task timeouts | +| August 20, CLI bridge toy | Official GEPA autoresearch engine and unmodified Claude CLI through a loopback route | Deterministic length objective about 0.286 to 1.0 | 12 optimizer evaluations, 2 submitted candidates, 8/8 wire calls, $0.029778336, 39.7 seconds | Real candidate-authoring calls; deterministic toy score; registry-only artifacts | +| June 8, OpenSCAD directive | GEPA versus handwritten directive | Reported +9.5 percentage points on compiled-CAD quality | One final split; task count and cost missing | Weak external-repository pointer; no pinned command or run directory | + +The AppWorld notebook also reports GSM8K baseline 1.0 with both deepseek-v4-pro and deepseek-v4-flash. +Its sample counts and costs are absent. +Its aggregate phrase “five configs” does not reconcile with its enumerated task variants; no total run count is inferred here. +The findings and extraction records are related demonstrations and must not be counted as independent replication. + +### June extraction: exact limitations + +Artifact: [examples/compare-optimization-methods/results/deepseek-chat-20260601.json](../../examples/compare-optimization-methods/results/deepseek-chat-20260601.json). +It was created at `a648fae334c740d8e0f368e81f68ae933cdb1135` under `examples/compare-drivers-canonical/results/`. +The current directory name does not identify the historical optimizer implementation. + +At that revision, inspect these producing sources with `git show`: + +- `examples/compare-drivers-canonical/index.ts`. +- `examples/_shared/extraction-task.ts`. +- `src/campaign/presets/compare-drivers.ts`. +- `src/campaign/presets/run-skill-opt.ts`. +- `src/campaign/presets/run-improvement-loop.ts`. +- `src/campaign/drivers/gepa.ts` and `src/campaign/drivers/skill-opt.ts`. + +The baseline was `Extract the transaction info from the message as JSON.` +The optimizer received the omitted schema, formatting rules, and suggested mutation primitives. +Winners mainly supplied merchant, amount, date, category, and formatting requirements. +The deterministic composite averages four normalized field matches; six transactions are the sample units, not 24 independent fields. +No direct edit control tested whether copying the supplied requirements achieved the same benefit. + +The same `HOLDOUT` entered the inner optimizers and the outer comparison. +SkillOpt accepted patches using those six cases and fed rejection scores plus accepted-delta notes into subsequent proposals. +Its reported final score is therefore selection-set performance. +The GEPA-style entries selected on train, then exposed those cases to an inner gate before outer rescoring. +Their returned surface did not depend on that gate. +This establishes unequal final-data access; it does not prove that every numerical GEPA gain was false. + +| Historical driver | Baseline | Candidate | Lift | Archived lift CI | Captured driver scoring cost | +| --- | ---: | ---: | ---: | --- | ---: | +| gepa-reflection | 0.625 | 1.000 | 0.375 | [0.167, 0.542] | $0.002921 | +| skill-opt | 0.625 | 1.000 | 0.375 | [0.167, 0.542] | $0.004005 | +| gepa-pareto | 0.625 | 0.958 | 0.333 | [0.167, 0.542] | $0.002929 | + +Reflection versus SkillOpt had delta 0, CI [0, 0]. +Reflection versus Pareto had delta 0.042, CI [0, 0.125]. +Both archived comparisons used `favored: 'tie'`; that does not establish equivalence. + +Captured worker usage was 18,226 input tokens and 7,560 output tokens across 182 records over 126 seconds. +The rate calculation `18226 * 0.27 / 1e6 + 7560 * 1.10 / 1e6 = 0.01323702` matches the rounded artifact cost. +The listed driver costs sum to $0.009855; $0.003382 of captured worker cost has no named phase breakdown in this artifact. +More importantly, the captured `records` array was populated only by the extraction worker. +The optimizer drivers called `callLlm` directly and discarded model usage and cost. +Therefore, end-to-end optimization cost and total model-call count are unknown. +The archived `honestVerdict: 'lift-proven'` is not valid current certification or proof of optimizer superiority. + +### CodeTraceBench: useful gains and failed transfer + +Full source reports and all measured fields: + +- [benchmarks/trace-analysis/codetracebench-glm52-certified-20260801/README.md](../../benchmarks/trace-analysis/codetracebench-glm52-certified-20260801/README.md) and `preregistration.md`. +- [benchmarks/trace-analysis/codetracebench-crossfamily-cert-20260802/README.md](../../benchmarks/trace-analysis/codetracebench-crossfamily-cert-20260802/README.md) and `preregistration.md`. +- `.evolve/experiments.jsonl`, lines 26, 29, 33, 34, and 41. + +Round 1 used real Python GEPA at historical script revision `0eb2e32`, with 40 evaluations on 10 train and 6 selection cases. +The output contract stayed fixed. +The selected prompt hash was `d3829fb855690a3a385f498049801c14bb990c6e49858a6739bd331c0ab324e1`. +Search selection composite rose from 0.281 to 0.331; this is separate from final micro F1. + +The final experiment used glm-5.2 through z.ai, stock DSPy RLM execution, seed0, and two repetitions. +Arms ran serially; each run used concurrency6, maxOutput8192, timeout1,200,000ms, maxCost30, and maxArtifact8MiB. +The environment was Node24.16.0 on Linux x64. +The dataset revision was `aa213b84ffb6690fc37ca15766d6ca174ec36d4d`. +Model names were recorded; a provider-served immutable model snapshot was not demonstrated by this audit. + +| Arm and final split | Micro F1 | Macro F1 | Recall | Precision | Failed model observations | Documented cost | +| --- | ---: | ---: | ---: | ---: | --- | ---: | +| Incumbent / holdout2 | 0.5641 | 0.5596 | 0.6436 | 0.5021 | 0/64 | $7.97 | +| Manual W / holdout2 | 0.5224 | 0.5125 | 0.5426 | 0.5037 | 1/64 | $7.67 | +| GEPA G / holdout2 | 0.6288 | 0.5789 | 0.6622 | 0.5986 | 0/64 | $7.96 | +| Incumbent / split3 | 0.1693 | 0.1830 | 0.3276 | 0.1141 | 1/74 | $9.24 | +| Manual W / split3 | 0.1805 | 0.1791 | 0.3190 | 0.1259 | 1/74 | $8.72 | +| GEPA G / split3 | 0.1799 | 0.1844 | 0.3017 | 0.1282 | 0/74 | $9.07 | + +There were 32 holdout2 cases and 37 split3 cases: 69 cases and 138 model observations per arm. +Only 30 holdout2 cases had positive labels; the paired F1 table consequently used 30 + 37 = 67 positive cases. +The notebook's “67 fresh cases” must not replace the execution denominator. +Pooled macro F1 was G0.3611, incumbent0.3516, and W0.3284. + +G versus incumbent paired F1 intervals were [-0.027, +0.067] on holdout2 and [-0.042, +0.046] on split3. +W intervals were [-0.096, +0.001] and [-0.064, +0.053], respectively. +Reported median paired delta was zero for all four comparisons. +Promotion followed a preregistered pooled point-estimate rule; it did not require statistical exclusion of zero. +The wide-cascade holdout2 gain of 0.0647 is a useful historical signal. + +After certification, split3 was retired: 27/37 cases label the final submit step. +A constant last-step prediction scored micro F1 0.568 there, versus about 0.180 for the analyst. +Consequently, the pooled result cannot support a general analysis-quality claim. +The reports' stronger language about unbiased instruments and certification must be read with this disclosed correction. + +G2 search later improved a weighted selection objective by 0.064 on 12 cases, with micro F1 0.400 to 0.509 and macro -0.032. +On fresh OpenHands and Terminus2, G2 pooled micro was 0.1928 versus stock0.2489, with 3/128 versus 0/128 failures. +Per-family G2 micro was OH0.2086/T20.1822, versus stock OH0.2896/T20.2162. +Paired intervals were OH[-0.126, +0.058] and T2[-0.143, +0.023]. +The fixed promotion rule rejected G2; its selection gain did not transfer. +The later August3 macro-versus-micro divergence was found on selection data and is not another fresh final result. + +### Raw recomputation and accounting + +These four files contain the surviving model arm and an empty control; they do not contain the unavailable incumbent/W/G2 arms. +Historical comparison deltas and intervals therefore remain documented evidence rather than independently reconstructed comparisons. + +| Label | Committed raw result | Run identity SHA256 | +| --- | --- | --- | +| G/h2 | [benchmarks/trace-analysis/codetracebench-glm52-certified-20260801/result-holdout2.json](../../benchmarks/trace-analysis/codetracebench-glm52-certified-20260801/result-holdout2.json) | `24883695f29e0b928f3a55d000e985682d18f810eb63c2c70b00e95418a34fac` | +| G/s3 | [benchmarks/trace-analysis/codetracebench-glm52-certified-20260801/result-split3.json](../../benchmarks/trace-analysis/codetracebench-glm52-certified-20260801/result-split3.json) | `417161b661138fb03977e475ae76be22e54d2153b32bb1842c0a6dde49ecc200` | +| Stock/OH | [benchmarks/trace-analysis/codetracebench-crossfamily-cert-20260802/result-stock-openhands.json](../../benchmarks/trace-analysis/codetracebench-crossfamily-cert-20260802/result-stock-openhands.json) | `30d80958be3d71f56a780850c7791cb4d078a8d98bc696140181611f94f7be95` | +| Stock/T2 | [benchmarks/trace-analysis/codetracebench-crossfamily-cert-20260802/result-stock-terminus2.json](../../benchmarks/trace-analysis/codetracebench-crossfamily-cert-20260802/result-stock-terminus2.json) | `abda3035ded04b3625981b2073d6b24a152fb4223836471481d4858f8a9589a3` | + +| Label | Observations / cases / clusters | Positive / negative / unlabeled observations | Positive-label TP / FP / FN | Recomputed micro F1 | Model calls | Estimated model cost | +| --- | --- | --- | --- | ---: | ---: | ---: | +| G/h2 | 64 / 32 / 32 | 60 / 0 / 4 | 249 / 167 / 127 | 0.628787879 | 912 | $7.9586602 | +| G/s3 | 74 / 37 / 37 | 74 / 0 / 0 | 35 / 238 / 81 | 0.179948586 | 1045 | $9.0744110 | +| Stock/OH | 64 / 32 / 32 | 32 / 28 / 4 | 43 / 80 / 131 | 0.289562290 | 855 | $7.3335628 | +| Stock/T2 | 64 / 32 / 30 | 32 / 20 / 12 | 40 / 130 / 160 | 0.216216216 | 883 | $7.5553616 | + +Counts, call totals, and estimated cost sums reconcile against each result summary. +All four model arms have zero failed observations, zero unknown-cost observations, and zero unknown token-usage observations. +Every model observation explicitly labels its cost `estimated`; the reports' wording “measured cost” must not imply provider-billed receipts. +The table does not include GEPA search cost or missing comparison arms. + +| Label | Input tokens | Output tokens | Cached tokens | Reasoning tokens | Unknown cache-write usage observations | +| --- | ---: | ---: | ---: | ---: | ---: | +| G/h2 | 2,055,464 | 578,167 | 9,089,024 | 0 | 64/64 | +| G/s3 | 2,553,890 | 659,891 | 10,150,528 | 0 | 74/74 | +| Stock/OH | 2,336,470 | 445,036 | 8,254,336 | 0 | 64/64 | +| Stock/T2 | 2,257,624 | 457,760 | 8,656,192 | 0 | 64/64 | + +Cache-write usage is unknown for every observation; summary zero totals do not establish measured zeros. +Trusted-negative false-positive rates are OH0.5 on 28 observations and T20.6 on 20 observations. +They are null on h2/s3 because those model arms have no trusted-negative observations. +The normalized calibration F1 includes negative predictions, giving OH0.243626062 and T20.189573460. +Those are different quantities from the historical positive-label micro F1 above. + +| Label | Latency minimum ms | Median ms | Mean ms | p95 ms | Maximum ms | +| --- | ---: | ---: | ---: | ---: | ---: | +| G/h2 | 58,387.888 | 134,080.197 | 149,395.929 | 280,446.514 | 388,678.939 | +| G/s3 | 66,049.095 | 131,120.154 | 150,738.482 | 281,274.392 | 348,210.556 | +| Stock/OH | 50,270.252 | 101,125.396 | 112,446.304 | 203,914.120 | 233,030.922 | +| Stock/T2 | 40,764.354 | 108,713.527 | 118,436.742 | 206,919.628 | 223,641.819 | + +Latency is per completed analyst observation under concurrency6; it is not serial campaign duration. +The exact protocol, implementation, dependency-lock, labels, trace, and candidate hashes remain in the linked raw results. + +## Current selected-candidate behavior + +The implementation preserves the search-selected candidate independently of its final gate decision. +`runFinalComparison()` compares that selected surface without selecting a replacement. +`src/contract/self-improve-method.ts` returns `selected.winnerSurface`. +`src/contract/self-improve.ts` returns `result.winnerSurface`. + +Inspected fixtures establish these intended behaviors: + +- `tests/campaign/final-evidence-integration.test.ts`: eight independent binary pairs produce lift1 and default-gate `ship`. + The declared-unit comparison runs twice without consuming evidence when no ledger policy is supplied. +- The same file: four fractional pairs preserve `WIN` and lift0.215 while the default gate returns `hold`. +- `tests/contract-self-improve-method-integrity.test.ts`: method-selected `WIN` remains selected despite losing on train. + Four final dispatches produce lift0.4; this test injects an always-ship gate. +- Deferred-final fixtures preserve the selected candidate, execute zero final calls, and leave final score/lift absent. +- No-op fixtures return the baseline and hold; these reflect unchanged selection rather than gate-driven candidate replacement. +- Ledger lifecycle fixtures cover method and proposer paths with an injected hold gate. + +These are fixed, marker, or echo fixtures. +Some controlled receipt fixtures set backend `real`; that literal does not turn them into model-backed efficacy evidence. +This historical audit inspected fixture source independently of the implementation's test runs. +The implementation also adds a default-gate regression for a selected candidate that loses on final tasks. +It checks negative lift, a refusing gate, and preservation of the selected candidate. + +The branch's `claim` option declares units separately from optional fresh-evidence accounting. +This separation supports repeated development comparisons without misrepresenting them as new confirmation. +The interval API also seals the cluster-bootstrap `value` selector in `IntervalSpec`; submitted row evidence no longer chooses it after sealing. +Neither API correction is itself evidence of improved model behavior. + +## Smallest meaningful benefit experiment + +Use the current public `selfImprove({ method, claim })` path with the maintained official optimizer and the production agent entrypoint. +Choose a real task panel with observed prompt-fixable errors under a reasonable current baseline. +Do not manufacture benefit solely by withholding known output requirements from that baseline. + +1. Run a small execution-and-capture smoke before a full search. + Verify candidate identity, worker and optimizer receipts, missingness, and retained raw scores. +2. Use three arms: unchanged baseline, a direct edit from the same development evidence, and the official optimizer. + Fix authoring/search resource limits and preserve actual spending separately for each arm. +3. Partition by source task or incident into development, selection, and fresh final units. + Pin the population, practical effect, primary metric, failure policy, and stopping rule before final exposure. +4. Estimate final sample size from development variation and the desired practical effect using maintained power helpers. + Count independent source units and account for clustering; do not substitute repeated calls or a universal 20-unit floor. +5. Freeze the selected candidates and compare all arms on paired final units under a balanced execution schedule. + Report baseline-to-candidate lift and optimizer-to-direct-edit lift, including uncertainty, failures, costs, and latency. +6. Use the final-evidence ledger when making a fresh-confirmation claim. + Retain the candidate, its diff, and all observed scores even when the gate holds or evidence is inconclusive. + +No paid experiment was launched by this audit. +A cost forecast should use the current configured model rates and observed smoke usage before the search starts. +A single successful search demonstrates that run's benefit; repeat searches are required to estimate optimizer reliability across seeds or tasks. + +## Supported communication + +The implementation improves how users declare claims, preserve evidence, inspect uncertainty, and compare selected candidates. +Historical useful-task optimization has produced gains, nulls, and regressions. +The evidence supports testing automated evaluation engineering through explicit outcome checks and independent confirmation. +It does not support claiming universal self-improvement, optimizer superiority over a direct edit, or current-branch model-quality lift. + +See [evaluation integrity](../evaluation-integrity.md) for the public API and the book chapters motivating its methodology. diff --git a/docs/eval-surface-map.md b/docs/eval-surface-map.md index a68ead9f..76d9344c 100644 --- a/docs/eval-surface-map.md +++ b/docs/eval-surface-map.md @@ -1,20 +1,21 @@ # Eval surface map: which primitive, when -The eval surface is a small set of orthogonal primitives. They compose; they do -not overlap. If two seem interchangeable, read the "use when": the distinction -is real and load-bearing. **Do not add a new wrapper to bridge two of these; the -composition point already exists (see Produced-state grading below).** - -## The run\* primitives - -| Primitive | Use when | Returns | -|---|---|---| -| `runCampaign` | The measurement primitive. Run a dispatch over scenarios × seeds × reps, score each with judges, aggregate. Caller owns the dispatch. | `CampaignResult` | -| `runEval` | The simplest preset over `runCampaign`: just score, no loop, no gate. The 80% "I want a scorecard" case. | `CampaignResult` | -| `runProfileMatrix` | Factor the SAME scenarios across N agent **profiles** (models / prompt variants), with RunRecord stamping + a real-backend integrity guard. | `RunRecord[]` | -| `runOptimization` | GENERATE: measured or validated premeasured baseline → N generations of propose → measure → rank → promote. No release gate. | generations + winner | -| `runImprovementLoop` | The release-gate shell around `runOptimization`: adds a held-out re-score + a promotion gate (+ optional auto-PR). | gate decision + winner | -| `runEvalCampaign` | Inversion-of-control variant of `runCampaign`: the runner is handed a pre-wired trace/sink/emitter and integrity gating as a precondition. Use when you need full capture by construction. | `CampaignResult` + records | +Choose the public entry point by the decision you need to make. +The [README](../README.md#choose-a-workflow) starts with the common product workflows. +This reference covers direct execution controls and specialist modules. + +## The run primitives + +| Primitive | Import | Use | Returns | +|---|---|---|---| +| `runCampaign()` | `/campaign` | Execute and judge a scenarios × repetitions grid through caller-owned dispatch. | `CampaignResult` | +| `runEval()` | `/contract` or `/campaign` | Score one surface with campaign defaults. | `CampaignResult` | +| `runProfileMatrix()` | `/campaign` | Run the same cases across named agent profiles with provenance and backend checks. | `RunProfileMatrixResult`, including `.records`. | +| `runOptimization()` | `/campaign` | Generate, measure, and select candidates on development cases. | Generations and a winner surface. | +| `runImprovementLoop()` | `/contract` or `/campaign` | Search, compare on final cases, and apply a release gate. | Final comparison, winner, and gate decision. | +| `compareOptimizationMethods()` | `/campaign` | Compare selected surfaces from several methods under declared budgets. | Final paired contrasts, uncertainty, and costs. | +| `runEvalCampaign()` | Root | Run with a caller-supplied trace sink and emitter. | Campaign result and records. | +| `runAgentMatrix()` | `/matrix` | Schedule a general Cartesian grid without campaign scoring semantics. | Cell results. | When variants of the same task run inside one `runCampaign`, give those scenarios the same `seedGroup` so each repetition uses common randomness. Use `runProfileMatrix` instead when profiles are separate campaign axes. @@ -34,14 +35,61 @@ to retry failed cells from Eval's durable campaign cache. Finalization refuses missing, overlapping, stale, corrupt, or duplicate rows, then returns the ordinary `runProfileMatrix` result and its distributions. Coverage reports missing, failed, and zero-score rows separately. -| `runAgentMatrix` | The bare N-axis cartesian scheduler with concurrency control. The layer beneath the eval surface: reach for it only when you need raw scheduling, not eval semantics. | cell results | -Mental model: **measure** (`runCampaign`/`runEval`) → **factor** (`runProfileMatrix`) → -**generate** (`runOptimization`) → **gate** (`runImprovementLoop`). `runEvalCampaign` -is `runCampaign` with capture inverted; `runAgentMatrix` is the scheduler underneath. +## Claims, evaluator checks, and final evidence -Merging any two of these conflates distinct mental models (measure ≠ search ≠ -release-gate). Keep them separate; pick by the table. +| Concern | Import | Public API | +|---|---|---| +| Declare population and independent units | `/experiment` | `defineEvaluationClaim()`, `summarizeEvaluationUnits()` | +| Register an executable decision | `/experiment` | `defineExperiment()`, `sealExperiment()`, `openSealedExperiment()` | +| Check design adequacy at a practical effect | `/experiment` | `clusteredPower()`, `assertDesignAdequate()` | +| Track final-data reservation and exposure | `/experiment` | `openFinalEvidenceLedger()` | +| Group reusable comparisons by source unit | `/contract` or `/campaign` | The top-level `claim` option. | +| Reserve fresh evidence for confirmation | `/contract` or `/campaign` | Optional `finalEvidence: { ledger, requestId, evaluatorDigest }`. | +| Admit an evaluator against both error limits | `/meta-eval` | `auditEvaluator()` | +| Measure agreement and known bias patterns | `/meta-eval` | `calibrateJudgeContinuous()`, `continuousAgreement()`, `positionalBias()`, `verbosityBias()`, `selfPreference()` | +| Relate scores to declared deployment outcomes | `/meta-eval` | `rubricPredictiveValidity()`, `correlationStudy()`, `calibrationFromPairs()`, `calibrationCurve()` | + +The top-level `claim` declares the independent unit for ordinary reusable comparisons. +Include `minimumEffect` when the decision concerns a practical improvement. +Optional `finalEvidence` requires a comparison or certification claim. +It reserves units before search and records exposure before final dispatch. +Its ledger must be shared across related campaigns. +Keep source unit identifiers stable when scenarios or populations are renamed. +The host controls access to private evidence and must preserve author/auditor separation. +[Evaluation integrity](./evaluation-integrity.md) describes these boundaries and the public result shapes. + +Outcome associations require an explicit desired direction for predictive validity. +They produce descriptive evidence and experiment hypotheses. +They do not establish a causal benefit from changing a rubric. +See [outcome validity](./outcome-validity.md). + +Method comparisons retain `scenarioScores`, `unitScores`, the `units` summary, and `pairedCellN` separately. +`favored: null` means the paired decision did not establish a preferred method. +It does not establish equivalence. +Use the decision diagnostics and intervals to distinguish insufficient evidence from a supported improvement. + +## Specialist imports + +| Subpath | Use | +|---|---| +| `/traces`, `/trace-attributes` | Store and inspect trace evidence; use canonical measurement attribute names. | +| `/analyst` | Execute declared analysts against recorded evidence. | +| `/reporting`, `/pipelines` | Compare runs, render reports, and extract recorded failure patterns. | +| `/supervisor-run` | Read recursive run directories and their evidence coverage. | +| `/trace-repair`, `/trajectory-replay` | Execute proposed repairs or replay recorded shell trajectories. | +| `/benchmarks`, `/fuzz` | Adapt benchmark data and explore a declared behavior space. | +| `/builder-eval`, `/multishot`, `/multishot/golden` | Evaluate generated applications and multi-turn conversations. | +| `/matrix` | Schedule Cartesian experiment grids. | +| `/rl` | Build reward, preference, and supervised datasets from eligible evidence. | +| `/profile-cell` | Create and validate portable agent-profile identities. | +| `/authenticity`, `/ledger-core` | Check evidence authenticity and maintain canonical hash-chained journals. | +| `/rollout`, `/storyboard` | Represent rollout trees and render recorded work. | +| `/hosted`, `/wire`, `/adapters/http` | Connect hosted storage or expose evaluation through HTTP and RPC. | + +Root `Scenario`, `JudgeScore`, and `GateDecision` match `/contract`. +Use root `ProductScenario` and `DimensionJudgeScore` for the product-judging functions. +`HeldOutGate.evaluate()` returns the separate root type `HeldOutGateDecision`. ## What a campaign result reports: the mean and the spread @@ -50,6 +98,8 @@ release-gate). Keep them separate; pick by the table. `byScenario` holds one `ScenarioAggregate` per scenario that produced at least one composite. Each aggregate reports a mean, a seeded bootstrap `ci95` band, `n`, and a `distribution`. +Here, `n` counts observed scores. +Use registered gates for inference across independent source units; their `pairedCellN` retains the raw paired denominator. `distribution` is the `SeriesDistribution` value `summarizeNumberSeries` returns: `n`, `min`, `p50`, `p90`, `max`, and `sum` over the exact scores the mean was taken over. Quantiles use the nearest-rank definition, so every reported quantile is a score the campaign measured. @@ -66,17 +116,22 @@ It is not the `ScalarDistribution` the insight report uses; see `insight-report. ## Planning the cell grid without a run directory `buildCellSchedule(scenarios, seed, reps)` returns the `(scenario × rep)` fan-out: one `CellScheduleSlot` per cell, with its `cellId` and its per-cell seed. -It touches no filesystem, so a caller can size a design, or assert a design's cell count and seeds in a test, before a run directory exists. +This function does not access the filesystem. +It can size a design and check cell counts and seeds before a run directory exists. Scenarios that share a `seedGroup` receive the same per-replicate seeds, which is what makes a paired comparison see common randomness. -Use `planCampaignRun` instead when you also need the cached, to-run, and blocked classification; that call needs a real run directory because it reads the durable cache. +Use `planCampaignRun()` to classify cached, pending, and blocked cells. +That call reads the durable cache in a real run directory. `cellDirectory` and `cellCachePath` name a cell's location once a run directory is chosen. ## Evidence receipts: `attest` -`attest(report, provenance)` content-addresses any serializable report and binds that address to the provenance needed to reproduce it: model versions, seeds, price-table hash, code SHA, and inputs hash. +`attest(report, provenance)` binds a serializable report to its provenance through content hashes. +Provenance records model versions, seeds, the price-table hash, code revision, and input digest. `verifyAttestation(report, attested)` returns a typed outcome rather than throwing, so a pipeline records why a report failed to verify instead of dying. `ATTESTATION_ALGORITHM` is the hash-scheme tag every attestation carries, and a verifier rejects an unknown algorithm instead of guessing. +Verification requires an `envelopeHash` that binds the report hash to its provenance. +An absent, malformed, or mismatched envelope makes the attestation invalid. Signing stays with the consumer: an `AttestedReport` is a stable byte-identical payload to sign, and this package never holds keys. ## Failed cells: receipts and bounded retry @@ -93,24 +148,18 @@ A retried attempt keeps its receipt at `/failure-receipt.attempt-.json` With `abortOnCellError`, the abort fires only when a cell's final attempt fails. Without `cellRetry`, a failed cell is final: one transient 503 leaves campaign coverage incomplete, and `runImprovementLoop` then refuses the holdout comparison. -## Produced-state grading: there is NO persona-dispatch wrapper +## Grade produced state through a judge -To grade what an agent actually **produced** (filed the proposal, wrote the -artifact) rather than what it said, the composition point is a **judge that wraps -`verifyCompletion`**: not a dedicated runner. The pipeline: +Use `extractProducedState()` and `verifyCompletion()` inside a `JudgeConfig` to grade the work an agent produced. +The same judge can run through `runCampaign()` or `runProfileMatrix()`. -``` -runtime/app-tool events ──► extractProducedState(events) ──► ProducedState - │ - verifyCompletion(taskGold, state, correctnessChecker) - │ - inject as a JudgeConfig into runProfileMatrix / runCampaign +```text +runtime events -> extractProducedState(events) -> ProducedState + -> verifyCompletion(taskGold, state, checker) -> JudgeConfig score ``` -`extractProducedState` is a pure function over the produced-event stream; the -judge calls it inline. This is why **`runProducedStatePersonaDispatch` does not -exist and should not be built**: it would be a fourth layer over a composition -that is already one judge. (Archetype: `playback.ts` `scoreUserStory`.) +The host supplies the correctness checker and the events. +This composition shares campaign execution, capture, and reporting without another runner. ### The in-band body contract @@ -123,6 +172,5 @@ product database to recover it: proposal is graded presence-only (and, by the completion oracle's rule, does not count as a completed deliverable). -A consumer that finds itself re-fetching a deliverable's body from its own DB to -grade it is working around a thin event: fix the event (carry `content`), don't -add an enrichment band-aid. +Carry deliverable content in the event when the host persists it. +This lets the grader inspect the recorded artifact without querying mutable product storage. diff --git a/docs/evaluation-integrity.md b/docs/evaluation-integrity.md new file mode 100644 index 00000000..4be3c255 --- /dev/null +++ b/docs/evaluation-integrity.md @@ -0,0 +1,214 @@ +# Evaluation claims and automated improvement + +Use reusable evaluations to guide development and select candidates. +Declare stronger evidence requirements when a result must support a broader claim. +`selfImprove()` returns its selected surface and measured lift even when its release gate remains inconclusive. + +The package separates three questions: + +| Question | Evidence | Public entry | +| --- | --- | --- | +| Did this change help on these cases? | Paired scores, failures, cost, and case coverage. | `defineAgentEval()` or `selfImprove()` from `/contract`. | +| Does the improvement extend to new tasks? | Representative independent tasks, a declared effect, and an appropriate comparison. | Optional `claim` on campaign comparisons; registered rules from `/experiment`. | +| Can fresh final evidence support this adaptive decision? | A frozen comparison, retained access boundaries, and a durable exposure record. | Optional `finalEvidence` on the same comparison. | + +These controls reuse the existing execution path, paired estimators, sealed experiments, and locked journal. +They do not add another optimizer or agent runner. + +## Declare the comparison without consuming data + +```ts +import { defineEvaluationClaim } from '@tangle-network/agent-eval/experiment' + +const claim = defineEvaluationClaim({ + use: 'comparison', + population: { + id: 'support-incidents', + description: 'Support incidents from the deployed product', + }, + samplingFrame: 'A random sample of incidents from the declared collection window', + independentUnit: 'source.incidentId', + generalization: 'new-units', + minimumEffect: 0.05, +}) +``` + +Pass `claim` to `selfImprove()`, `runImprovementLoop()`, or `compareOptimizationMethods()`. +It remains independent of final-evidence storage. +`minimumEffect` is optional because development reports and absolute-rate measurements need not test an improvement threshold. +When omitted, each existing comparison keeps its documented decision threshold. + +`independentUnit` names a field path in each scenario or evidence row. +Variants from one incident must carry the same source identity. +For new-unit claims, automatic `selfImprove()` partitions keep those variants together. +Fixed-roster claims retain source units in reports without requiring disjoint sources between development partitions. +Explicit partitions for new-unit claims must not share source units between development and final evaluation. + +The default self-improvement gate averages paired cells within registered units. +Method comparison first averages repetitions within each scenario, then averages scenarios within each source unit. +It weights source units equally. +Results retain `scenarioScores`, `unitScores`, `units`, and `pairedCellN` so callers can inspect each denominator. +Custom gates receive the same measured evidence and remain responsible for their own decision rules. + +`fixed-roster` describes the specified cases. +Repeated executions can measure execution variability on that roster. +They do not establish task diversity or performance on unseen users. +Claim metadata records the intended scope; it does not authenticate sampling or turn an exploratory result into certification. + +## Interpret small and inconclusive results + +There is no universal task count that proves an improvement. +The effect, outcome type, dependence, confidence level, and decision procedure determine what the evidence supports. + +Paired binary decisions use the shared score interval and exact discordance check. +A sufficiently large binary gain can pass with fewer than 20 independent pairs. +Continuous mean decisions require the existing bootstrap path's 20-pair eligibility threshold. +That implementation threshold does not establish adequate power or guarantee interval coverage for every distribution. +The smaller-sample sign test answers a different question about directional or median change. + +Method rankings describe observed lift. +`favored: null` means the evidence does not establish a favored method; it does not establish equivalence. +Each score and pairwise contrast retains its full `decision`, including the estimator, threshold, minimum, and sufficiency. +An inconclusive gate leaves the selected candidate available for further development or a narrower evaluation. + +Use `clusteredPower()` or a registered `power-floor` gate before an expensive population-level comparison. +Both assess power at the declared `minimumEffect`. +High power at a much larger effect cannot substitute for power at the improvement that matters. + +## Opt into fresh final evidence + +```ts +import { openFinalEvidenceLedger } from '@tangle-network/agent-eval/experiment' + +const finalEvidence = { + ledger: openFinalEvidenceLedger({ path: '.agent-eval/final-evidence.jsonl' }), + requestId: 'support-comparison-2026-09-13', + evaluatorDigest, // Content identity of the actual evaluator and its configuration. +} + +// Pass both claim and finalEvidence to the existing comparison entrypoint. +``` + +Ordinary regression and development evaluations remain reusable. +Supply this policy when freshness is part of the evidence supporting a particular final comparison. +It requires a comparison or certification claim and measured final execution. + +Comparison entrypoints capture judge configuration and callbacks before search or final exposure. +The host must keep callback receiver and closed-over state stable throughout the comparison. + +The campaign reserves source units before candidate search. +It records exposure before dispatching the baseline, selected candidates, or an optional neutralized control. +Failed or interrupted final execution still consumes that evidence. +An already exposed request cannot start another final measurement. + +The ledger permits exact retries of reservation and exposure writes. +Campaign entrypoints refuse a replayed exposure so competing workers cannot each start a new measurement. +Read retained campaign artifacts after exposure; a fresh request ID does not restore freshness. + +Source unit identities are unique across the shared ledger, including across population labels. +The ledger also rejects the same dataset digest under another request. +Use one persistent ledger for related decisions and preserve stable source identities. +Opening another empty ledger or inventing new lineage identities cannot establish independent evidence. + +`FinalEvidenceLedger` returns typed outcomes. +Inspect `succeeded` before reading `value`. +Failures distinguish conflicting use, invalid input, and unavailable or damaged storage. +Campaign errors preserve these categories through `FinalEvidenceError.kind`. + +The filesystem implementation uses the existing hash-chained journal, process locks, durable writes, and required trusted head. +Preserve both the journal and its `.head` file. +The head detects truncation while it remains trusted. +An actor who can replace both files can replace the recorded history. + +The host owns answer-file permissions, model context, credentials, and author/evaluator separation. +The ledger records exposure; it cannot prove that earlier undisclosed access never occurred. + +## Seal the rule and measured field + +Attach the same `claim` to `defineExperiment()` before calling `sealExperiment()`. +Cluster intervals register both the source unit and measured field: + +```ts +const interval = { + kind: 'cluster-bootstrap' as const, + clusterBy: 'source.incidentId', + value: 'pairedDelta', + resamples: 2000, + seed: 7, + level: 0.95, + method: 'percentile' as const, +} + +// Register interval in the experiment's intervals map before sealing. +// Then execute registered.interval('lift', { kind: 'rows', rows }). +``` + +New-unit claims reject intervals that resample a different field. +Registered binomial intervals require one unique `unitId` per trial for new-unit claims. +Opened seals capture validated rules before asynchronous execution. +Caller mutation cannot change the opened experiment's rules. + +Only current canonical digest schemes can execute. +Retain historical artifacts with their original identities; re-register current work under the supported format. +See [registered experiments](./experiment.md) for the complete rule language. + +## Audit an evaluator's errors + +Use `auditEvaluator()` from `/meta-eval` when admitting a new checker or model judge. +Provide actual judgments of independently verified good and bad controls. +Each observation names its source unit, evidence reference, expected decision, observed decision, and development exposure. + +The audit measures false acceptance and false rejection separately. +A source unit fails a class when any variant in that class is misjudged. +Repeated variants increase case coverage without increasing the independent-unit count. +If a source appeared during evaluator development, every supplied variant from that source is excluded from fresh audit evidence. + +The audit uses exact binomial bounds and adjusts the two intervals for simultaneous confidence. +Unknown judgments remain visible and contribute their most adverse possible outcomes to each upper bound. +Admission is possible when both worst-case upper bounds meet policy. +If unresolved outcomes affect the measured error rate, that rate is `null`. +An always-accept checker fails false acceptance; an always-reject checker fails false rejection. + +Reports retain inputs, source coverage, exclusions, unknowns, limits, and content digests. +The declared audit authority must differ from the evaluator author. +Different identifiers alone do not prove independence; the host must enforce and record separation. +Audit cases must represent the stated population and a consistent control-generation procedure. +Changing the number or kind of variants changes the meaning of an any-variant error rate. + +`auditEvaluator()` measures supplied judgments. +It does not execute models or automatically approve a deployment. +For outcome associations and direct score calibration, use [the outcome-validity tools](./outcome-validity.md). + +## Test whether self-improvement is useful + +These integrity checks establish execution and measurement behavior. +They do not establish that a particular optimizer improves agents across domains. +The [historical evidence audit](./design/self-improvement-evidence-audit.md) records prior gains, nulls, regressions, and their limits. + +For a benefit experiment, define the user behavior and useful effect before search. +Compare the starting agent, a direct edit or simple search baseline, and the proposed improvement method at equal actual resources. +Give every method the same allowed preparation, feedback, tools, and candidate surface. +Retain all attempts, costs, failures, selected candidates, and final comparisons. + +Before interpreting a null result, verify that candidate generation executed and the evaluator distinguishes plausible improvements from regressions. +Measure improvement under the conditions where the method claims an advantage. +Use fresh tasks when the conclusion concerns unseen tasks. +Treat a result on a fixed product workflow as evidence for that workflow. +Repeat across distinct domains before making a broad claim. + +The [offline example](../examples/evaluation-integrity/) exercises these public APIs and exports its report without paid calls. +It verifies the integration with deterministic fixtures; it is not an optimizer-benefit study. + +## Source and design rationale + +The book motivates the distinctions; the API and policies are project design choices. + +| Source | Applied idea | +| --- | --- | +| [Chapter 4: purposes of holdout](https://mlbenchmarks.org/04-holdout-method.html#whats-the-holdout-method-for) | Development feedback, selection, and capability measurement require different evidence. | +| [Chapter 3: detecting differences](https://mlbenchmarks.org/03-detecting-differences.html#comparing-similar-models) | Pair comparisons and evaluate precision against the actual effect and independent observations. | +| [Chapter 5: test-set reuse](https://mlbenchmarks.org/05-test-set-reuse.html) | Preserve development feedback while tracking adaptive final-data exposure. | +| [Chapter 11: confounded evaluations](https://mlbenchmarks.org/11-evaluating-language-models.html#confounded-evaluations) | Give methods comparable preparation before judging their adaptation potential. | +| [Chapter 14: judge agreement](https://mlbenchmarks.org/14-evaluation-frontier.html#agreement-alone-is-not-enough) | Measure consequential evaluator errors; agreement alone does not establish correct rankings. | + +The [complete review](./design/mlbenchmarks-book-review.md) records all available chapters, repository evidence, and remaining research questions. diff --git a/docs/experiment.md b/docs/experiment.md index be7da5d7..d3e02926 100644 --- a/docs/experiment.md +++ b/docs/experiment.md @@ -44,22 +44,37 @@ const halt = registered.halt([gate]) // refuse-spend fires before any const outcome = registered.decide(quantities) // the sealed table; non-total tables throw ``` -#### Digest schemes and retention +Cluster intervals register both `clusterBy` and `value` inside the sealed `IntervalSpec`. +Call `registered.interval('gain95', { kind: 'rows', rows })` to apply those fields. +The row evidence cannot override the registered value field. +Changing the measured field requires a new seal. -A seal and a signed `HypothesisManifest` are durable records: each is written -once and verified later, possibly by a different release. Both carry an `algo` -field that names the digest scheme, and verification selects the encoder from -that field. +For a paired contrast, prepare one difference per pair and register that difference field as `value`. +A pooled pass rate from both arms measures a different quantity. +The [runnable sealed experiment](../examples/sealed-experiment/index.ts) demonstrates the paired path. +Confidence levels, field paths, seeds, and resample counts are validated before sealing and direct computation. -| `algo` | Serialization | Status | -|---|---|---| -| `sha256-rfc8785` | RFC 8785 canonical JSON, from `ledger-core/canonical` | What `sealExperiment` and `signManifest` write | -| `sha256-content` | key-sorted `JSON.stringify` | Read-only. Records written before the RFC 8785 scheme carry it, or carry no `algo` at all, and still verify | +Older cluster interval registrations omitted `value` and require their original package version for execution. +Retain their original bytes and evidence; create a new registration for subsequent measurements. + +#### Canonical identities and migration -The `sha256-content` encoder is private to the module that verifies with it and -is unreachable from any path that writes a digest. Retire it once no record -carrying that tag needs to verify; until then, deleting it would make those -records unverifiable rather than invalid. +Readers and writers use RFC 8785 canonical JSON from `ledger-core/canonical`. +Verification refuses missing or unsupported digest schemes. + +| Artifact | Required identity | Refusal | +|---|---|---| +| Sealed experiment | `algo: 'sha256-rfc8785'` | `verifySealedExperiment()` returns `false`; `openSealedExperiment()` refuses execution | +| Signed hypothesis | `algo: 'sha256-rfc8785'` | `verifyManifest()` returns `false`; synchronous digest checks and hypothesis evaluation refuse the record | +| Agent profile cell | `agent-profile-cell:sha256-rfc8785:` | Cell validation refuses any other scheme | +| Report attestation | Report hash and required `envelopeHash` over its provenance | `verifyAttestation()` returns an invalid result with a reason | + +The package no longer verifies `sha256-content` records, untagged manifests, bare `agent-profile-cell:sha256:` identifiers, or attestations without provenance envelopes. +Keep those records unchanged as historical artifacts with their original package version. +Create new registrations with `sealExperiment()` or `signManifest()` before collecting new decision evidence. +Use `buildAgentProfileCell()` and `attest()` to produce current identities from independently verified source material. +Never relabel an existing digest or reconstruct a provenance envelope from unverified metadata. +A new digest cannot establish that a registration existed before its evidence was observed. `openSealedExperiment` is the only execution surface. A rule that is not in the sealed spec cannot run; a rule that is cannot run differently. @@ -74,7 +89,28 @@ Two floors, one simulation: - **Seeded simulation.** Per-row paired contrasts are drawn under a registered effect model (base win/loss rates, optional noisy clusters), each trial takes a whole-cluster percentile bootstrap, and power is the fraction of trials whose interval excludes zero. The refusal is a verdict inside the returned artifact (`result.refusal`), with `assertDesignAdequate` as the throwing form. -The `power-floor` validity gate consumes a power curve as evidence and fails when the curve tops out under the registered target. +Both `clusteredPower` and the registered `power-floor` gate require `minimumEffect`. +The effect must appear exactly in the supplied grid; the API does not interpolate. +Adequacy requires target power at that effect. +`maxPower` describes the grid and cannot establish adequacy at a smaller effect. + +```ts +const power = clusteredPower({ + clusterSizes: Array.from({ length: 24 }, () => 3), + effects: [0.05, 0.1, 0.2], + minimumEffect: 0.1, + targetPower: 0.8, + seed: 17, +}) + +assertDesignAdequate(power) +``` + +Simulation effects are expected paired contrasts within signal clusters. +The zero-effect model requires equal `baseWinRate` and `baseLossRate`. +Configured noisy clusters retain zero expected contrast, so they dilute the pooled population effect. +Power remains conditional on this outcome model, the registered sampling structure, and the simulated test. +The [statistical evidence guide](./statistical-evidence.md) explains unit counts, adaptation comparisons, and sequential assumptions. ### The funnel diff --git a/docs/outcome-validity.md b/docs/outcome-validity.md new file mode 100644 index 00000000..9a62bde2 --- /dev/null +++ b/docs/outcome-validity.md @@ -0,0 +1,165 @@ +# Connect rubric scores to deployment outcomes + +`rubricPredictiveValidity()` measures associations between rubric scores and observations from deployment. +Declare the desired direction for each outcome before reading the results. +Higher rubric scores always mean better evaluated behavior. + +```ts +import { + InMemoryOutcomeStore, + rubricPredictiveValidity, + type DeploymentOutcome, + type OutcomeMetricSpec, +} from '@tangle-network/agent-eval/meta-eval' +import type { RunRecord } from '@tangle-network/agent-eval' + +async function assessOutcomes(runs: RunRecord[], observations: DeploymentOutcome[]) { + const outcomes = new InMemoryOutcomeStore() + for (const observation of observations) await outcomes.append(observation) + const metrics: OutcomeMetricSpec[] = [ + { id: 'success_rate', direction: 'higher-is-better' }, + { id: 'failure_rate', direction: 'lower-is-better' }, + ] + + return rubricPredictiveValidity({ + runs, + outcomes, + outcomeMetrics: metrics, + rubrics: ['task_quality'], + }) +} +``` + +Populate `RunRecord.outcome.raw.task_quality` with the captured rubric score. +Append deployment observations with the same `runId` and exact outcome metric keys. +The store accepts finite numbers, including zero. +Omit an unmeasured metric instead of replacing it with zero. + +The default reduction selects the latest finite observation of each requested metric. +A newer row containing another metric cannot supply its value or erase an older observation. +The `mean` and `max` reductions operate on that metric alone. + +## Read the report + +| Field | Interpretation | +|---|---| +| `pearson`, `spearman` | Association with the recorded outcome, retaining its original sign. | +| `alignedPearson`, `alignedSpearman` | Positive means higher rubric scores associate with better outcomes. | +| `pearsonCi95`, `spearmanCi95` | Bootstrap intervals for the raw associations. | +| `alignedSpearmanCi95` | Spearman interval after applying the declared outcome direction. | +| `verdict` | `aligned` at aligned Spearman ≥ 0.4; `inverse` at ≤ −0.4; otherwise `weak`. | +| `n` | Finite joined run observations for this rubric and outcome. | +| `excludedPairs` | Unestimated pairs, their observation counts, and the reason for exclusion. | +| `rubricsWithoutData` | Declared rubrics with no finite score in the supplied runs. | + +A rubric can correlate negatively with failure rate and still receive `aligned`. +The same negative correlation with success rate receives `inverse`. +These labels describe association strength and direction. +They do not grant release authority or establish that changing a rubric weight will improve outcomes. + +Pairs require eight observations by default. +`minSamples` can set another integer of at least three. +Insufficient observations and constant scores or outcomes remain explicit exclusions. +Constant observations cannot establish a perfect predictor. +Intervals remain `null` when no bootstrap resample has an estimable correlation. + +`joinedSamples + skippedRuns` equals the supplied run count. +A run is joined when at least one finite score and outcome pair exists. +It remains joined even if that pair has too few observations for estimation. +Duplicate run IDs are rejected. + +Each run is an independent bootstrap observation. +Repeated observations from the same person or task can violate that assumption. +Aggregate at the independent unit or use a study with an appropriate grouped estimator. + +`ranked` selects each rubric's highest direction-aligned association across the declared outcomes. +This ordering is exploratory and can conceal conflicts among outcomes. +Inspect all `pairs` and use a target chosen before analysis for an automated recommendation. +Confirm any proposed change with fresh evidence and fixed scoring rules. + +## Propose an experiment against one target + +```ts +import { PredictiveValidityResearcher } from '@tangle-network/agent-eval/rl' +import { InMemoryOutcomeStore } from '@tangle-network/agent-eval/meta-eval' + +const outcomes = new InMemoryOutcomeStore() +const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'success_rate', direction: 'higher-is-better' }, + rubrics: ['task_quality'], +}) +``` + +Supply observed outcomes before calling `runValidityCheck(runs)`. +Pass the resulting failure groups to `proposeChange(failures)`. +The researcher uses its declared target even if another outcome has a stronger association. +It proposes increased-weight experiments for aligned associations and reversal or replacement experiments for inverse associations. +Both require an aligned Spearman interval that excludes zero. +Weak or uncertain associations produce requests for calibration evidence. +Missing estimates produce requests for more outcome observations. + +The proposals contain their association, interval, sample count, and target direction. +They contain no predicted improvement because a correlation does not identify a causal treatment effect. +`applyChange()` appends proposals to a plan. +`evaluateChange()` declines promotion because the caller owns experiment execution. + +`runRLCampaign()` accepts the same outcome specifications through `outcomeMetrics` when `outcomeStore` is supplied. +Supply both options together; incomplete or empty declarations fail before campaign execution. +Its summary reports direction-aligned association and preserves missing estimates. +The target declaration remains fixed while the campaign runs. + +The researcher copies its target declaration, rubric list, and cached reports. +Changing callback arguments, returned reports, or proposal payloads cannot rewrite its cached evidence. + +## Store observations without losing failures + +`InMemoryOutcomeStore` and `FileSystemOutcomeStore` copy observations at ingestion and retrieval. +Changing a caller's metric or label object cannot rewrite stored evidence. +The filesystem store rereads observations so another instance's later writes remain visible. +Use one writer per directory; operations on that writer are serialized. + +A nonexistent directory is an empty store. +An unreadable file, malformed JSON, or invalid outcome record throws `OutcomeStoreError`. +The error carries its operation, path, source line when available, and original cause. +Read errors never become empty studies or partial successful results. +Repair the source and retry the read. + +For trace data, `correlationStudy()` accepts outcome names and reports descriptive associations without a desired direction. +It shares the metric reduction, bootstrap, and exclusion behavior. +Its optional capture window includes only observations captured after the run started. + +## Inspect calibration by score range + +Use `calibrationFromPairs()` when scores and outcomes are already joined. +It accepts readonly observations directly. + +```ts +import { calibrationFromPairs } from '@tangle-network/agent-eval/meta-eval' + +const calibration = calibrationFromPairs([ + { evalScore: 1, outcome: 0 }, + { evalScore: 1, outcome: 0 }, +], 'predicted-success', 'observed-success') + +console.log(calibration?.ece) // 1: confident predictions, observed failures. +``` + +Direct input rejects nonfinite pairs with the offending index. +It does not silently discard them from the denominator. +`calibrationCurve()` performs the join for trace and outcome stores using the latest finite observation of the named outcome metric. +It reports each bin's count, mean score, mean outcome, and absolute gap. +`ece` weights each bin's gap by its share of the finite joined observations. +Both quantities must use comparable numerical scales for this difference to measure calibration. + +The `range` option clips scores before binning and retains every joined observation. +Constant scores form one bin, so a consistently wrong predictor still receives a measured calibration error. +Equal-frequency binning produces the requested number of bins, capped by the observation count. +Bin counts differ by at most one, except when all scores share one value. +Equal-width binning omits empty bins. +On either path, bin counts sum to the reported `n`. + +Fewer than two finite pairs returns `null`. +Both entry points validate metric identities, binning options, and range bounds. +Invalid requests fail even when no evidence is available. +The store entry point validates before reading evidence. diff --git a/docs/statistical-evidence.md b/docs/statistical-evidence.md new file mode 100644 index 00000000..40e775d1 --- /dev/null +++ b/docs/statistical-evidence.md @@ -0,0 +1,129 @@ +# Statistical evidence + +Repeated runs measure execution variation on the cases you supplied. +A claim about new cases also needs independent observations from its target population. +Repeating one incident 100 times supplies one incident, even when every execution has a distinct identifier. + +## Declare the independent unit + +Pass `independentUnitByScenarioId` to `defaultProductionGate`, `heldoutSignificance`, or `dimensionRegressions` when scenarios share a source. +The map assigns each scenario to the incident, document, task family, or other unit sampled independently. +Choose this assignment before examining scores. + +```ts +import { defaultProductionGate } from '@tangle-network/agent-eval/campaign' + +const holdoutScenarios = [ + { id: 'incident:17:original', kind: 'support' }, + { id: 'incident:17:paraphrase', kind: 'support' }, + { id: 'incident:28:original', kind: 'support' }, +] + +const gate = defaultProductionGate({ + holdoutScenarios, + independentUnitByScenarioId: new Map([ + ['incident:17:original', 'incident-17'], + ['incident:17:paraphrase', 'incident-17'], + ['incident:28:original', 'incident-28'], + ]), + deltaThreshold: 0.05, + criticalDimensions: ['factualAccuracy'], +}) +``` + +This example has two independent units and cannot meet its observation minimum. +More repetitions of these scenarios will not change that count. +The default gate copies the map when it is constructed. +Later mutations cannot change the grouping used by that gate. + +Pairing precedes aggregation. +Candidate and baseline must contain the same full cell identifiers and the same selected judges within each cell. +The last numeric suffix identifies a repetition, so colons within scenario identifiers remain valid. +The implementation averages matched cells within each unit, then gives every unit equal weight. +It refuses asymmetric cells, asymmetric selected judges, duplicate cells, non-finite scores, and missing unit assignments. + +Reports distinguish the following counts: + +| Field | Meaning | +|---|---| +| `n` | Paired observation units used for inference | +| `pairedCellN` | Matched execution cells before grouping | +| `observationUnit` | `registered` when a supplied map defines units; `cell` on an ungrouped fixed-roster path | +| `unitIds` | The units represented in a held-out significance result | + +Without a map, fixed-roster significance uses execution cells as observations. +Its uncertainty concerns independently sampled execution outcomes conditional on that roster. +It does not establish generalization to new incidents or task families. +Identifiers and aggregation do not establish independence; the sampling design must justify it. + +## Read the interval that decided + +`heldoutSignificance().decision` contains the test, interval, observation minimum, and promotion decision. +Pass/fail scores use the shared paired risk-difference rule. +Continuous scores use the shared paired bootstrap or its small-sample test. +`bootstrap` and `medianBootstrap` remain diagnostics when another test decides. + +Continuous mean targets require 20 observations for bootstrap eligibility. +Below that count, a reported exact sign-test diagnostic cannot establish a mean effect. +Binary outcomes and explicitly requested median targets use their actual confidence-dependent observation minimum. +The observation minimum establishes estimator eligibility; it does not establish statistical power or representative sampling. +A zero-width bootstrap interval cannot establish improvement under the shared decision rule. + +Required dimensions also need sufficient observations and complete coverage. +The default gate reports `not_evaluated` when a required dimension lacks enough units or omits matched cells or scenarios. +`fewRuns`, `missingCellIds`, and `missingScenarioIds` preserve the missing evidence. +An observed regression can still hold the gate while evidence remains incomplete. +Passing the regression guard does not certify every safety property of the candidate. + +## Sequential decisions require a conditional-mean assumption + +`sequentialPairedGate` consumes one observation per scenario by default in `decide()`. +Its `independentUnitByScenarioId` option groups related scenarios before testing and copies the mapping at construction. +`maxN` counts these independent units. +The report preserves both the consumed count `n` and the available counts `pairedN` and `pairedCellN`. + +The statistical guarantee requires each next delta's conditional expectation to remain below the registered null boundary. +Independent sampling with that bound is sufficient. +Shuffling an exchangeable sequence does not establish the condition. +One random sign repeated 100 times has only one independent draw. +After its first observation, later signs reveal no new evidence. + +Direct `observe(delta)` callers must perform the required aggregation themselves. +They must also justify the sampling assumption. +`sequentialDecide` stops candidate exploration heuristically; candidate selection and reused incumbent scores prevent a general type-I error guarantee. +The selected candidate still requires fresh held-out evidence. + +## Adaptation comparisons pair whole scenarios + +`runAdaptationCurve` and `compareAdaptationCurves` are exported from `@tangle-network/agent-eval/rl`. +Scenarios require explicit, unique `scenarioId` values. +The runner validates repetitions, the demonstration grid, and finite scores in `[0,1]`. + +Comparisons require identical scenario cohorts and identical demonstration grids. +Missing pairs, duplicate identities, and changing cohorts are errors. +The comparison computes one area per scenario before applying the existing paired estimators. +This preserves dependence across demonstration counts. +Repeated executions improve each scenario mean without increasing the number of independent scenarios. + +`perK` intervals describe the curve. +The paired area decisions determine `a_better`, `b_better`, `inconclusive`, or `insufficient_evidence`. +Continuous area outcomes require the bootstrap minimum of 20 paired scenarios. +Binary area outcomes use the shared score-interval rule and can support a decision with fewer observations. +The deciding interval must remain nondegenerate. +`inconclusive` does not establish equivalence. +`firstPassK` describes the first observed crossing and carries no reliability guarantee. + +## Perturbation sensitivity is a diagnostic + +`runContaminationProbe` reports observed score differences and one global Wilcoxon paired test. +Use `alpha` for that test's significance threshold. +Per-item differences carry no p-values or q-values because the probe defines no calibrated item-level sampling null. + +The report retains every observed pair and lists `excludedScenarioIds` when a score floor excludes evidence. +Summaries describe the included population. +Fewer than four included pairs produce `pairedTest: null` while preserving measured means and medians. +Zero included pairs produce null summaries. + +A significant drop can reflect changed task difficulty, broken perturbations, or contamination. +`contaminationSuspected` therefore requests investigation; it does not identify the cause. +The global test also relies on independent pairs and the Wilcoxon assumptions for paired differences. diff --git a/examples/README.md b/examples/README.md index 234c1162..d6086425 100644 --- a/examples/README.md +++ b/examples/README.md @@ -30,6 +30,7 @@ pnpm tsx examples/evaluate-a-change/index.ts | Goal | Example | Requirements | |---|---|---| | Improve with your own candidate generator | [`selfimprove-quickstart`](./selfimprove-quickstart/) | Offline | +| Declare source units, audit a checker, and retain fresh final evidence | [`evaluation-integrity`](./evaluation-integrity/) | Offline; build the package first | | Improve one prompt with official GEPA in one call | [`self-improve-optimizer`](./self-improve-optimizer/) | Python GEPA package and an LLM endpoint | | Let a metered coding agent drive the optimization | [`agent-engine-optimizer`](./agent-engine-optimizer/) | Python GEPA package, the `claude` CLI, and an LLM endpoint | | Let another package own the text search | [`adapt-a-text-optimizer`](./adapt-a-text-optimizer/) | Offline | diff --git a/examples/evaluation-integrity/README.md b/examples/evaluation-integrity/README.md new file mode 100644 index 00000000..2215d2b7 --- /dev/null +++ b/examples/evaluation-integrity/README.md @@ -0,0 +1,30 @@ +# Evaluation integrity example + +Run an offline candidate search, audit its deterministic checker, and export the comparison report. +The example imports only public package entrypoints and makes zero paid calls. + +From the repository root: + +```sh +pnpm build +pnpm exec tsx examples/evaluation-integrity/index.ts +``` + +The search evaluates two arithmetic behaviors and selects the one that adds correctly. +Forty source tasks have two variants each. +The final comparison keeps source families separate from candidate selection. +The script checks the selected surface, observed lift, gate decision, cost, and durable exposure record. + +Outputs appear under `.agent-eval/evaluation-integrity-example/`: + +- `audit-controls.json` contains the executed good and bad controls. +- `report.json` contains evaluator error bounds and the actual final comparison. +- `final-evidence.jsonl` and its `.head` file retain reservation and exposure. + +The final-evidence guard refuses a second measurement with the same reserved source units. +For disposable integration checks, pass a scratch output directory as the first argument. +Production decisions need a persistent shared ledger and fresh source evidence. + +The fixture verifies public API composition. +It does not measure model-backed improvement or establish independent audit authority. +See [evaluation integrity](../../docs/evaluation-integrity.md) for methodology and the controls appropriate to each claim. diff --git a/examples/evaluation-integrity/index.ts b/examples/evaluation-integrity/index.ts new file mode 100644 index 00000000..198c8488 --- /dev/null +++ b/examples/evaluation-integrity/index.ts @@ -0,0 +1,153 @@ +import assert from 'node:assert/strict' +import { mkdirSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { + costFromLedgerSummary, + type OptimizationMethod, + runCampaign, +} from '@tangle-network/agent-eval/campaign' +import { type JudgeConfig, type Scenario, selfImprove } from '@tangle-network/agent-eval/contract' +import { + defineEvaluationClaim, + openFinalEvidenceLedger, +} from '@tangle-network/agent-eval/experiment' +import { canonicalString, hashCanonical } from '@tangle-network/agent-eval/ledger-core' +import { auditEvaluator } from '@tangle-network/agent-eval/meta-eval' + +interface SumCase extends Scenario { + sourceId: string + left: number + right: number +} +interface Answer { + total: number +} +const runDir = resolve(process.argv[2] ?? '.agent-eval/evaluation-integrity-example') +mkdirSync(runDir, { recursive: true }) + +function correct(artifact: Answer, scenario: SumCase): boolean { + return artifact.total === scenario.left + scenario.right +} +const judge: JudgeConfig = { + name: 'sum', + dimensions: [{ key: 'correct', description: 'The sum equals the reference' }], + score: ({ artifact, scenario }) => { + const value = Number(correct(artifact, scenario)) + return { composite: value, dimensions: { correct: value }, notes: '' } + }, +} +const evaluatorDigest = hashCanonical({ rule: correct.toString(), version: 'offline-sum-v1' }) +const scenarios: SumCase[] = Array.from({ length: 40 }, (_, source) => + [0, 1].map((variant) => ({ + id: `task-${source}:${variant}`, + kind: 'sum', + sourceId: `source-${source}`, + left: source, + right: variant + 1, + })), +).flat() + +// Both candidates execute through the public evaluation path before selection. +const method: OptimizationMethod = { + name: 'two-candidate-search', + optimize: async (input) => { + let selected = input.baselineSurface + let best = -Infinity + for (const surface of ['subtract', 'add']) { + const result = await runCampaign({ + ...input.runOptions, + scenarios: [...input.selectionScenarios], + judges: [...input.judges], + dispatch: (scenario, ctx) => input.dispatchWithSurface(surface, scenario, ctx), + costLedger: input.costLedger, + runDir: `${input.runDir}/${surface}`, + }) + const mean = + result.cells.reduce((sum, cell) => sum + cell.judgeScores.sum!.composite, 0) / + result.cells.length + if (mean > best) { + best = mean + selected = surface + } + } + return { winnerSurface: selected, cost: costFromLedgerSummary(input.costLedger.summary()) } + }, +} + +const controls = Array.from({ length: 100 }, (_, i) => + (['accept', 'reject'] as const).map((expected) => { + const scenario: SumCase = { + id: `audit-${i}`, + kind: 'sum', + sourceId: `audit-source-${i}`, + left: i, + right: 2, + } + return { scenario, artifact: { total: i + 2 + Number(expected === 'reject') }, expected } + }), +).flat() +writeFileSync(`${runDir}/audit-controls.json`, `${canonicalString(controls)}\n`) +const evaluatorAudit = auditEvaluator({ + evaluatorDigest, + population: 'Deterministic arithmetic controls', + samplingFrame: 'One correct and one incorrect answer for each of 100 fixture sources', + authority: { + evaluatorAuthorId: 'fixture-author', + auditorId: 'fixture-auditor', + independenceEvidenceRef: 'offline-example:declared-roles-only', + }, + policy: { confidence: 0.95, maxFalseAcceptanceRate: 0.05, maxFalseRejectionRate: 0.05 }, + observations: controls.map((control, i) => ({ + id: `control-${i}`, + independentUnitId: control.scenario.sourceId, + evidenceRef: hashCanonical(control), + expected: control.expected, + observed: correct(control.artifact, control.scenario) ? 'accept' : 'reject', + exposure: 'fresh', + })), +}) +assert.equal(evaluatorAudit.verdict, 'admit') + +const result = await selfImprove({ + scenarios, + judge, + method, + baselineSurface: 'subtract', + runDir, + model: 'deterministic-arithmetic@2026-09-13', + expectUsage: 'off', + agent: async (surface, scenario) => ({ + total: surface === 'add' ? scenario.left + scenario.right : scenario.left - scenario.right, + }), + claim: defineEvaluationClaim({ + use: 'comparison', + population: { id: 'arithmetic-fixtures', description: 'The deterministic fixture roster' }, + samplingFrame: 'Forty source tasks with two variants; no deployment population is sampled', + independentUnit: 'sourceId', + generalization: 'fixed-roster', + minimumEffect: 0.05, + }), + finalEvidence: { + ledger: openFinalEvidenceLedger({ path: `${runDir}/final-evidence.jsonl` }), + requestId: 'arithmetic-comparison', + evaluatorDigest, + }, +}) +assert.equal(result.winner.surface, 'add') +assert.equal(result.lift, 1) +assert.equal(result.gateDecision, 'ship') +assert.equal(result.totalCostUsd, 0) +assert.ok(result.finalEvidence?.record.exposure) + +const report = { + purpose: 'Offline integration fixture; not evidence of general optimizer effectiveness', + evaluatorAudit, + claim: result.claim, + finalEvidence: result.finalEvidence, + winner: result.winner, + lift: result.lift, + gateDecision: result.gateDecision, + totalCostUsd: result.totalCostUsd, +} +writeFileSync(`${runDir}/report.json`, `${canonicalString(report)}\n`) +console.log(`${runDir}/report.json`) diff --git a/examples/sealed-experiment/index.ts b/examples/sealed-experiment/index.ts index a692020b..53791f68 100644 --- a/examples/sealed-experiment/index.ts +++ b/examples/sealed-experiment/index.ts @@ -57,6 +57,7 @@ const spec: ExperimentSpec = { pairedContrast95: { kind: 'cluster-bootstrap', clusterBy: 'queue', + value: 'diff', resamples: 2_000, seed: 20260815, level: 0.95, @@ -129,7 +130,6 @@ const differenceRows: EvidenceRecord[] = admission.survivors.map((row) => ({ const interval = registered.interval('pairedContrast95', { kind: 'rows', rows: differenceRows, - value: 'diff', }) console.log('95% interval: ', [interval.lower, interval.upper]) diff --git a/scripts/check-canonical-json.mjs b/scripts/check-canonical-json.mjs index dbbef248..839c470c 100644 --- a/scripts/check-canonical-json.mjs +++ b/scripts/check-canonical-json.mjs @@ -18,11 +18,8 @@ * A reviewer is the only other check, and a reviewer already missed it eleven * times. * - * ALLOWLIST below names the encoders that are deliberately not the home: the - * private legacy verifiers each durable-record module keeps so a record signed - * under the retired scheme still verifies. Each is unreachable from any path - * that writes a digest, and each is retired with its retention window (see - * docs/experiment.md). + * Readers and writers use the same encoder. Retired digest schemes are refused, + * so verification introduces no additional encoders or exceptions. */ import { readdirSync, readFileSync, statSync } from 'node:fs' @@ -35,30 +32,6 @@ const REPOSITORY_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..') /** The one canonical-JSON home. */ const HOME = 'src/ledger-core/canonical.ts' -/** - * Read-only legacy encoders, kept private inside the module that verifies with - * them so a durable record written under the retired scheme still verifies. - * `file` and `fn` must both match; a new function in the same file is not - * covered. - */ -const ALLOWLIST = [ - { - file: 'src/pre-registration.ts', - fn: 'legacyContentDigest', - reason: 'verifies a manifest signed under sha256-content; never writes a digest', - }, - { - file: 'src/agent-profile-cell.ts', - fn: 'legacyCellDigest', - reason: 'verifies an agent-profile-cell:sha256: id; never mints one', - }, - { - file: 'src/experiment/define.ts', - fn: 'specDigest', - reason: 'selects the encoder by the seal algo; the legacy branch only verifies', - }, -] - /** Tests and type declarations ship no encoder. */ const SKIPPED = [/\.test\.ts$/, /\.test-support\.ts$/, /\.d\.ts$/] @@ -78,22 +51,18 @@ const SERIALIZERS = new Set(['stringify', 'digest', 'update']) * Throws on an unparseable source file — a gate that cannot read its own input * must not report a pass. */ -export function checkCanonicalJson({ root = REPOSITORY_ROOT, allowlist = ALLOWLIST } = {}) { +export function checkCanonicalJson({ root = REPOSITORY_ROOT } = {}) { const offences = [] - const usedWaivers = new Set() const sourceRoot = resolve(root, 'src') if (statSync(sourceRoot, { throwIfNoEntry: false })?.isDirectory()) { for (const file of sourceFiles(sourceRoot)) { const relativePath = relative(root, file).replaceAll('\\', '/') if (relativePath === HOME) continue if (SKIPPED.some((pattern) => pattern.test(relativePath))) continue - collect({ file, relativePath, allowlist, usedWaivers, offences }) + collect({ file, relativePath, offences }) } } - return { - offences, - unusedWaivers: allowlist.filter((entry) => !usedWaivers.has(`${entry.file} ${entry.fn}`)), - } + return { offences } } function* sourceFiles(directory) { @@ -104,7 +73,7 @@ function* sourceFiles(directory) { } } -function collect({ file, relativePath, allowlist, usedWaivers, offences }) { +function collect({ file, relativePath, offences }) { const source = readFileSync(file, 'utf8') const { program, errors } = parseSync(file, source) if (errors.length > 0) throw new Error(`${relativePath}: parse failed — ${errors[0].message}`) @@ -143,9 +112,7 @@ function collect({ file, relativePath, allowlist, usedWaivers, offences }) { const { sortsKeys, serializes } = bodyHas(fn, { sorters }) if (!(sortsKeys && serializes)) return const name = functionName(fn, source) - const waiver = allowlist.find((e) => e.file === relativePath && e.fn === name) - if (waiver !== undefined) usedWaivers.add(`${waiver.file} ${waiver.fn}`) - else offences.push({ file: relativePath, line: lineOf(source, fn.start), fn: name }) + offences.push({ file: relativePath, line: lineOf(source, fn.start), fn: name }) }) } @@ -258,7 +225,7 @@ function lineOf(source, offset) { if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { try { - const { offences, unusedWaivers } = checkCanonicalJson() + const { offences } = checkCanonicalJson() for (const offence of offences) { console.error( `${offence.file}:${offence.line}: ${offence.fn}() sorts object keys and serializes in one ` + @@ -266,25 +233,15 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur `${HOME} instead.`, ) } - for (const entry of unusedWaivers) { - console.error( - `scripts/check-canonical-json.mjs: the allowlist entry for ${entry.fn}() in ${entry.file} ` + - 'matches nothing — delete it.', - ) - } if (offences.length > 0) { console.error( `\n${offences.length} hand-rolled canonical-JSON encoder(s). Eleven copies of this code ` + 'disagreed on undefined-valued keys, Date, and integer-like key order, so one value ' + - `hashed differently depending on the caller. ${HOME} is the one encoder; a legacy ` + - 'verifier that must keep the retired bytes is added to ALLOWLIST with its reason.', + `hashed differently depending on the caller. ${HOME} is the one encoder.`, ) } - if (offences.length > 0 || unusedWaivers.length > 0) process.exit(1) - console.log( - `canonical json gate valid: ${HOME} is the only encoder under src/ ` + - `(${ALLOWLIST.length} legacy verifiers waived, all matched)`, - ) + if (offences.length > 0) process.exit(1) + console.log(`canonical json gate valid: ${HOME} is the only encoder under src/`) } catch (error) { console.error(error instanceof Error ? error.message : String(error)) process.exit(1) diff --git a/scripts/check-canonical-json.test.mjs b/scripts/check-canonical-json.test.mjs index 86f97d83..6706d551 100644 --- a/scripts/check-canonical-json.test.mjs +++ b/scripts/check-canonical-json.test.mjs @@ -11,7 +11,7 @@ afterEach(() => { }) /** Build a throwaway repo containing only `src` files. */ -function run(files, allowlist = []) { +function run(files) { const root = mkdtempSync(join(tmpdir(), 'canonical-json-gate-')) tempRoots.push(root) for (const [name, content] of Object.entries(files)) { @@ -19,7 +19,7 @@ function run(files, allowlist = []) { mkdirSync(join(path, '..'), { recursive: true }) writeFileSync(path, content) } - return checkCanonicalJson({ root, allowlist }) + return checkCanonicalJson({ root }) } /** @@ -180,7 +180,7 @@ describe('code that sorts keys without encoding', () => { }) }) -describe('the allowlist', () => { +describe('verification encoders', () => { const legacy = { 'verify.ts': [ 'export function verifyLegacy(value: Record): string {', @@ -190,28 +190,16 @@ describe('the allowlist', () => { ].join('\n'), } - test('waives exactly the named function in the named file', () => { - const { offences, unusedWaivers } = run(legacy, [ - { file: 'src/verify.ts', fn: 'verifyLegacy', reason: 'verifies retired bytes; never writes' }, - ]) - - expect(offences).toEqual([]) - expect(unusedWaivers).toEqual([]) - }) - - test('reports a waiver that matches nothing, so a stale entry cannot hide a new copy', () => { - const { offences, unusedWaivers } = run(legacy, [ - { file: 'src/verify.ts', fn: 'renamedAway', reason: 'stale entry' }, - ]) + test('rejects a second encoder even when it only verifies retired records', () => { + const { offences } = run(legacy) expect(offences).toHaveLength(1) - expect(unusedWaivers).toHaveLength(1) + expect(offences[0]).toMatchObject({ file: 'src/verify.ts', fn: 'verifyLegacy' }) }) }) describe('the shipped repository', () => { test('passes its own gate', () => { - const { offences, unusedWaivers } = checkCanonicalJson() - expect({ offences, unusedWaivers }).toEqual({ offences: [], unusedWaivers: [] }) + expect(checkCanonicalJson()).toEqual({ offences: [] }) }) }) diff --git a/scripts/tb-gated-stop-ab.ts b/scripts/tb-gated-stop-ab.ts index db85dab6..293dfb7c 100644 --- a/scripts/tb-gated-stop-ab.ts +++ b/scripts/tb-gated-stop-ab.ts @@ -300,6 +300,7 @@ function powerAt(sizes: readonly number[], effect: number, trials: number): numb return clusteredPower({ clusterSizes: [...sizes], effects: [effect], + minimumEffect: effect, seed: POWER_SIM.seed, trials, resamples: POWER_SIM.resamples, @@ -519,6 +520,7 @@ function buildSpec(certifiedTasks: string[], sealedRowIds: string[], take: numbe pairedContrast95: { kind: 'cluster-bootstrap', clusterBy: 'taskName', + value: 'diff', resamples: 10_000, seed: 20260814, level: 0.95, @@ -529,6 +531,7 @@ function buildSpec(certifiedTasks: string[], sealedRowIds: string[], take: numbe powerFloor: { kind: 'power-floor', target: POWER_TARGET, + minimumEffect: SETTLING_EFFECT, effectGrid: [...REGISTERED_EFFECT_GRID], sim: { ...POWER_SIM }, }, @@ -1057,6 +1060,7 @@ async function main(): Promise { const power = clusteredPower({ clusterSizes: clusters.map((cluster) => cluster.rowIds.length), effects: [...REGISTERED_EFFECT_GRID], + minimumEffect: SETTLING_EFFECT, seed: POWER_SIM.seed, trials: POWER_SIM.trials, resamples: POWER_SIM.resamples, @@ -1382,9 +1386,21 @@ async function main(): Promise { ['C-vs-B-final', finalPassed], ] as const) { const evidence = [...armRows('blind-continue', controlPassed), ...treatment] + const byPair = new Map() + for (const row of evidence) { + const rowId = String(row.rowId) + const pair = byPair.get(rowId) ?? [] + pair.push(row) + byPair.set(rowId, pair) + } + const differences = [...byPair].map(([rowId, pair]) => ({ + rowId, + taskName: pair[0]!.taskName, + diff: registered.estimate('pairedContrast', pair).value, + })) contrasts[label] = { estimand: registered.estimate('pairedContrast', evidence), - interval: registered.interval('pairedContrast95', { kind: 'rows', rows: evidence, value: 'passed' }), + interval: registered.interval('pairedContrast95', { kind: 'rows', rows: differences }), } } diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 6265ffbf..91016301 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -686,6 +686,111 @@ try { ] `, ) + writeFileSync( + join(appDir, 'integrity-imports.ts'), + ` + import type { + Scenario as RootScenario, + JudgeScore as RootJudgeScore, + GateDecision as RootGateDecision, + ProductScenario, + DimensionJudgeScore, + HeldOutGate, + HeldOutGateDecision, + JudgeFn, + JudgeInput, + } from '@tangle-network/agent-eval' + import type { + Scenario, + JudgeScore, + GateDecision, + FinalEvidencePolicy, + } from '@tangle-network/agent-eval/contract' + import { + defineEvaluationClaim, + summarizeEvaluationUnits, + openFinalEvidenceLedger, + FinalEvidenceError, + FinalEvidenceConflictError, + } from '@tangle-network/agent-eval/experiment' + import { + auditEvaluator, + calibrationFromPairs, + calibrateJudgeContinuous, + positionalBias, + selfPreference, + OutcomeStoreError, + type EvaluatorAuditInput, + type OutcomeMetricSpec, + type RubricOutcomeExclusion, + } from '@tangle-network/agent-eval/meta-eval' + + type Equal = + (() => T extends A ? 1 : 2) extends (() => T extends B ? 1 : 2) ? true : false + const scenarioMatches: Equal = true + const scoreMatches: Equal = true + const decisionMatches: Equal = true + const productMatches: Equal = true + const dimensionMatches: Equal>, DimensionJudgeScore[]> = true + const heldoutMatches: Equal, HeldOutGateDecision> = true + + const claim = defineEvaluationClaim({ + use: 'comparison', + population: { id: 'support', description: 'Support incidents.' }, + samplingFrame: 'A queue sample before optimization.', + independentUnit: 'incidentId', + generalization: 'new-units', + minimumEffect: 0.05, + }) + const units = summarizeEvaluationUnits(claim, [ + { incidentId: 'same-incident' }, { incidentId: 'same-incident' }, + ]) + const finalEvidence: FinalEvidencePolicy = { + ledger: openFinalEvidenceLedger({ path: 'final-evidence.jsonl' }), + requestId: 'comparison-1', + evaluatorDigest: 'sha256:1111111111111111111111111111111111111111111111111111111111111111', + } + const outcome: OutcomeMetricSpec = { id: 'failure-rate', direction: 'lower-is-better' } + const exclusion: RubricOutcomeExclusion = { + rubric: 'correctness', outcome: outcome.id, outcomeDirection: outcome.direction, + n: 0, reason: 'insufficient_samples', + } + const audit: EvaluatorAuditInput = { + evaluatorDigest: finalEvidence.evaluatorDigest, + population: 'Support outputs.', + samplingFrame: 'Independent held-back incidents.', + authority: { + evaluatorAuthorId: 'author', auditorId: 'auditor', + independenceEvidenceRef: 'artifact://audit-policy', + }, + policy: { + confidence: 0.95, maxFalseAcceptanceRate: 0.1, maxFalseRejectionRate: 0.1, + }, + observations: [{ + id: 'unknown', independentUnitId: 'incident-1', evidenceRef: 'artifact://unknown', + expected: 'reject', observed: 'unknown', exposure: 'fresh', + }], + } + const calibration = calibrationFromPairs([ + { evalScore: 1, outcome: 0 }, { evalScore: 1, outcome: 0 }, + ], 'confidence', 'success') + if (calibration?.n !== 2 || calibration.ece !== 1) { + throw new Error('packed calibration lost direct observations') + } + const report = auditEvaluator(audit) + if (units.independentUnits !== 1 || units.observations !== 2) { + throw new Error('packed claim conflated observations and independent units') + } + if (report.verdict !== 'inconclusive' || report.coverage.unknownCases !== 1) { + throw new Error('packed evaluator audit lost unknown evidence') + } + void [ + scenarioMatches, scoreMatches, decisionMatches, productMatches, dimensionMatches, + heldoutMatches, exclusion, calibrateJudgeContinuous, positionalBias, selfPreference, + OutcomeStoreError, FinalEvidenceError, FinalEvidenceConflictError, + ] + `, + ) writeFileSync( join(appDir, 'tsconfig.json'), JSON.stringify({ @@ -697,10 +802,11 @@ try { skipLibCheck: true, outDir: 'dist', }, - include: ['index.ts', 'quickstart.ts'], + include: ['index.ts', 'quickstart.ts', 'integrity-imports.ts'], }), ) run(join(repoRoot, 'node_modules', '.bin', 'tsc'), ['-p', 'tsconfig.json'], appDir) + run(process.execPath, [join(appDir, 'dist', 'integrity-imports.js')], appDir) const quickstartOutput = run(process.execPath, [join(appDir, 'dist', 'quickstart.js')], appDir) const plainQuickstartOutput = quickstartOutput.replace(/\x1b\[[0-9;]*m/g, '') // Whitespace-tolerant: Node's inspector wraps the aggregate across lines once @@ -847,7 +953,13 @@ try { throw new Error('obsolete rl export isTrainingRunEligible') } const metaEval = await import('@tangle-network/agent-eval/meta-eval') - if (!('InMemoryOutcomeStore' in metaEval)) throw new Error('missing meta-eval export InMemoryOutcomeStore') + for (const name of [ + 'InMemoryOutcomeStore', 'OutcomeStoreError', 'auditEvaluator', 'calibrateJudge', + 'calibrationFromPairs', 'calibrateJudgeContinuous', 'continuousAgreement', 'positionalBias', + 'selfPreference', 'verbosityBias', 'rubricPredictiveValidity', + ]) { + if (!(name in metaEval)) throw new Error('missing meta-eval export ' + name) + } const wire = await import('@tangle-network/agent-eval/wire') if (!('dispatchRpc' in wire)) throw new Error('missing wire export dispatchRpc') const hosted = await import('@tangle-network/agent-eval/hosted') diff --git a/src/agent-profile-cell.ts b/src/agent-profile-cell.ts index 6cdd9af8..d0f7174e 100644 --- a/src/agent-profile-cell.ts +++ b/src/agent-profile-cell.ts @@ -1,4 +1,3 @@ -import { createHash } from 'node:crypto' import type { AgentProfile } from '@tangle-network/agent-interface' import { ValidationError } from './errors' import { hashJson } from './pre-registration' @@ -67,14 +66,9 @@ export class AgentProfileCellValidationError extends ValidationError { } const SHA256_HEX = /^[0-9a-f]{64}$/ -/** - * A cell id names the digest scheme that produced it. `sha256-rfc8785` is what - * {@link buildAgentProfileCell} mints; the bare `sha256` form is read-only, - * carried by cells built under an earlier release, and still verifies. - */ -const CELL_ID = /^agent-profile-cell:sha256(?:-rfc8785)?:[0-9a-f]{64}$/ +/** A cell id carries the canonical digest scheme required for verification. */ +const CELL_ID = /^agent-profile-cell:sha256-rfc8785:[0-9a-f]{64}$/ const CELL_ID_PREFIX = 'agent-profile-cell:sha256-rfc8785:' -const LEGACY_CELL_ID_PREFIX = 'agent-profile-cell:sha256:' export async function buildAgentProfileCell( input: AgentProfileCellInput, @@ -94,36 +88,13 @@ export function agentProfileCellHashMaterial( /** * Verify an `AgentProfileCell`'s `cellId` matches the sha256 of its hash-material - * fields, confirming the record has not been tampered with. The id names its own - * digest scheme, so a cell minted by an earlier release verifies under that scheme. + * fields, confirming the record has not been tampered with. Unsupported digest + * schemes are refused before comparing the material. */ export async function verifyAgentProfileCell(cell: AgentProfileCell): Promise { validateAgentProfileCell(cell) const material = agentProfileCellHashMaterial(cell) - if (cell.cellId.startsWith(CELL_ID_PREFIX)) { - return cell.cellId === `${CELL_ID_PREFIX}${await hashJson(material)}` - } - return cell.cellId === `${LEGACY_CELL_ID_PREFIX}${legacyCellDigest(material)}` -} - -/** - * Key-sorted `JSON.stringify` digest. Private and read-only: it verifies a cell - * id minted before the RFC 8785 scheme, and no path that MINTS an id calls it. - */ -function legacyCellDigest(value: unknown): string { - return createHash('sha256') - .update(JSON.stringify(sortKeysDeep(value)), 'utf8') - .digest('hex') -} - -function sortKeysDeep(value: unknown): unknown { - if (value === null || typeof value !== 'object') return value - if (Array.isArray(value)) return value.map(sortKeysDeep) - const out: Record = {} - for (const key of Object.keys(value as Record).sort()) { - out[key] = sortKeysDeep((value as Record)[key]) - } - return out + return cell.cellId === `${CELL_ID_PREFIX}${await hashJson(material)}` } export function validateAgentProfileCell(input: unknown): AgentProfileCell { @@ -134,7 +105,7 @@ export function validateAgentProfileCell(input: unknown): AgentProfileCell { expectLiteral(obj.schemaVersion, 'agent-profile-cell/v1', 'schemaVersion') if (typeof obj.cellId !== 'string' || !CELL_ID.test(obj.cellId)) { throw new AgentProfileCellValidationError( - 'cellId must match agent-profile-cell:sha256:<64 lowercase hex chars>', + 'cellId must match agent-profile-cell:sha256-rfc8785:<64 lowercase hex chars>', 'cellId', ) } diff --git a/src/attestation.test.ts b/src/attestation.test.ts index bc9da43c..b995c5f7 100644 --- a/src/attestation.test.ts +++ b/src/attestation.test.ts @@ -92,13 +92,18 @@ describe('verifyAttestation', () => { expect(result.reason).toMatch(/envelope hash mismatch/) }) - it('marks legacy attestations as valid report hashes with unbound provenance', () => { - const { envelopeHash: _envelopeHash, ...legacy } = attest(report, provenance) - expect(verifyAttestation(report, legacy)).toEqual({ - valid: true, - legacyUnboundProvenance: true, - }) - }) + it.each([undefined, null, '', 'not-a-hash', 123])( + 'refuses an absent or invalid provenance envelope (%s)', + (envelopeHash) => { + const input: AttestedReport = JSON.parse( + JSON.stringify({ ...attest(report, provenance), envelopeHash }), + ) + expect(verifyAttestation(report, input)).toEqual({ + valid: false, + reason: 'attestation envelope hash is missing or invalid', + }) + }, + ) it('rejects an unknown algorithm instead of guessing', () => { const attested = attest(report, provenance) diff --git a/src/attestation.ts b/src/attestation.ts index b805f514..f03bb40f 100644 --- a/src/attestation.ts +++ b/src/attestation.ts @@ -45,20 +45,14 @@ export interface AttestedReport { reportHash: string provenance: AttestationProvenance algorithm: typeof ATTESTATION_ALGORITHM - /** - * Hex sha-256 over `{ reportHash, provenance, algorithm }`. New attestations - * always carry it. Optional only so persisted pre-envelope attestations can - * still be read and explicitly recognized as legacy by callers. - */ - envelopeHash?: string + /** Hex sha-256 over `{ reportHash, provenance, algorithm }`. */ + envelopeHash: string } export interface AttestationVerification { valid: boolean /** Populated iff `valid` is false — names the exact mismatch. */ reason?: string - /** True only for a valid pre-envelope attestation whose provenance is not cryptographically bound. */ - legacyUnboundProvenance?: true } function envelopeMaterial( @@ -92,9 +86,8 @@ export function attest(report: unknown, provenance: AttestationProvenance): Atte * canonicalizes) is a verification failure with the cause in `reason`, not a * crash — verifiers run in pipelines that must record WHY, not die. * - * Legacy attestations without `envelopeHash` remain readable, but verification - * explicitly marks their provenance as unbound so a promotion path can refuse - * them instead of accidentally treating old metadata as cryptographic proof. + * Both the report hash and the provenance envelope must verify. + * Missing envelope hashes cannot establish provenance and fail verification. */ export function verifyAttestation( report: unknown, @@ -106,6 +99,9 @@ export function verifyAttestation( reason: `unknown algorithm '${attested.algorithm}' — this verifier only checks '${ATTESTATION_ALGORITHM}'`, } } + if (typeof attested.envelopeHash !== 'string' || !/^[0-9a-f]{64}$/.test(attested.envelopeHash)) { + return { valid: false, reason: 'attestation envelope hash is missing or invalid' } + } let recomputed: string try { recomputed = contentHash(report) @@ -122,10 +118,6 @@ export function verifyAttestation( } } - if (attested.envelopeHash === undefined) { - return { valid: true, legacyUnboundProvenance: true } - } - let envelopeHash: string try { envelopeHash = contentHash( diff --git a/src/campaign/campaign-manifest.ts b/src/campaign/campaign-manifest.ts index d060edb2..cf1e5a70 100644 --- a/src/campaign/campaign-manifest.ts +++ b/src/campaign/campaign-manifest.ts @@ -26,7 +26,7 @@ export function computeManifestHash(input }) } -function judgeVersionFor( +export function judgeVersionFor( judge: JudgeConfig, ): string { if (judge.judgeVersion !== undefined) { diff --git a/src/campaign/final-evidence.ts b/src/campaign/final-evidence.ts new file mode 100644 index 00000000..cdd3b3e1 --- /dev/null +++ b/src/campaign/final-evidence.ts @@ -0,0 +1,131 @@ +import { ValidationError } from '../errors' +import { + defineEvaluationClaim, + type EvaluationClaim, + summarizeEvaluationUnits, +} from '../experiment/claim' +import { + FinalEvidenceConflictError, + FinalEvidenceError, + type FinalEvidenceLedger, + type FinalEvidenceRecord, +} from '../experiment/final-evidence' +import { + compareCodeUnits, + hashCanonical, + LEDGER_HASH_PATTERN, + type LedgerHash, +} from '../ledger-core/canonical' +import type { MutableSurface, Scenario } from './types' + +/** Durable final-data policy. Keep its ledger shared across related campaigns. */ +export interface FinalEvidencePolicy { + ledger: FinalEvidenceLedger + requestId: string + evaluatorDigest: LedgerHash +} + +export interface FinalEvidenceUse { + claim: EvaluationClaim + record: FinalEvidenceRecord +} + +/** Capture identities before an asynchronous search can mutate its caller's options. */ +export function captureFinalEvidencePolicy(policy: FinalEvidencePolicy): FinalEvidencePolicy { + return Object.freeze({ + ledger: policy.ledger, + requestId: policy.requestId, + evaluatorDigest: policy.evaluatorDigest, + }) +} + +export function evaluationUnitMap( + claim: EvaluationClaim, + scenarios: readonly TScenario[], +): ReadonlyMap { + const map = new Map() + for (const scenario of scenarios) { + const summary = summarizeEvaluationUnits(claim, [scenario]) + const unit = summary.units[0] + if (!unit) + throw new ValidationError('final evidence has a scenario without an independent unit') + if (map.has(scenario.id)) throw new ValidationError(`duplicate final scenario '${scenario.id}'`) + map.set(scenario.id, unit.id) + } + return map +} + +/** Shared source variants cannot serve as unseen-unit evidence after development. */ +export function assertIndependentEvaluationSplit( + claim: EvaluationClaim, + finalScenarios: readonly TScenario[], + developmentScenarios: readonly TScenario[], +): void { + const finalUnitIds = new Set(evaluationUnitMap(claim, finalScenarios).values()) + const developmentUnits = summarizeEvaluationUnits(claim, developmentScenarios) + const overlap = developmentUnits.units.filter((unit) => finalUnitIds.has(unit.id)) + if (overlap.length) { + throw new ValidationError( + `development and final evidence share independent units: ${overlap.map((unit) => unit.id).join(', ')}`, + ) + } +} + +/** Reserve before search. An identical retry may resume until final data is exposed. */ +export async function reserveFinalEvidence( + policy: FinalEvidencePolicy, + claimInput: EvaluationClaim | undefined, + scenarios: readonly TScenario[], + developmentScenarios: readonly TScenario[] = [], +): Promise { + policy = captureFinalEvidencePolicy(policy) + if (claimInput === undefined) + throw new ValidationError('fresh final evidence requires an evaluation claim') + const claim = defineEvaluationClaim(claimInput) + if (claim.use === 'development') { + throw new ValidationError('final evidence requires a comparison or certification claim') + } + if (!LEDGER_HASH_PATTERN.test(policy.evaluatorDigest)) { + throw new ValidationError('final evidence requires the evaluator content digest') + } + const units = evaluationUnitMap(claim, scenarios) + const finalUnitIds = new Set(units.values()) + assertIndependentEvaluationSplit(claim, scenarios, developmentScenarios) + const result = await policy.ledger.reserve({ + requestId: policy.requestId, + claimDigest: hashCanonical({ claim, evaluatorDigest: policy.evaluatorDigest }), + populationId: claim.population.id, + inputDigest: hashCanonical([...scenarios].sort((a, b) => compareCodeUnits(a.id, b.id))), + unitIds: [...finalUnitIds].sort(compareCodeUnits), + }) + if (!result.succeeded) throw new FinalEvidenceError(result.error.kind, result.error.message) + if (result.value.record.exposure !== null) { + throw new FinalEvidenceConflictError( + `final evidence for '${policy.requestId}' was already exposed; read its recorded result or use fresh evidence`, + ) + } + return { claim, record: result.value.record } +} + +/** Append exposure before dispatch, so a failed or interrupted measurement still consumes evidence. */ +export async function exposeFinalEvidence( + policy: FinalEvidencePolicy, + claim: EvaluationClaim | undefined, + scenarios: readonly TScenario[], + surfaces: readonly MutableSurface[], +): Promise { + policy = captureFinalEvidencePolicy(policy) + const measurement = { + evaluatorDigest: policy.evaluatorDigest, + candidateDigests: surfaces.map((surface) => hashCanonical(surface)), + } + const reserved = await reserveFinalEvidence(policy, claim, scenarios) + const result = await policy.ledger.expose(policy.requestId, measurement) + if (!result.succeeded) throw new FinalEvidenceError(result.error.kind, result.error.message) + if (result.value.replayed) { + throw new FinalEvidenceConflictError( + `final evidence for '${policy.requestId}' is already being measured`, + ) + } + return { claim: reserved.claim, record: result.value.record } +} diff --git a/src/campaign/gates/default-production-gate.test.ts b/src/campaign/gates/default-production-gate.test.ts index 608e2c0e..2b1c6630 100644 --- a/src/campaign/gates/default-production-gate.test.ts +++ b/src/campaign/gates/default-production-gate.test.ts @@ -38,8 +38,8 @@ function context(): GateContext<{ text: string }, Scenario> { scenarios.map((scenario) => [`${scenario.id}:0`, { text: 'ordinary output' }]), ), baselineArtifacts: new Map(), - judgeScores: scores([0.8, 0.9, 0.7, 0.8, 0.9, 0.7]), - baselineJudgeScores: scores([0.5, 0.6, 0.4, 0.5, 0.6, 0.4]), + judgeScores: scores([1, 1, 1, 1, 1, 1]), + baselineJudgeScores: scores([0, 0, 0, 0, 0, 0]), scenarios, cost: { candidate: 1, baseline: 1 }, signal: new AbortController().signal, @@ -108,6 +108,138 @@ function statuses(result: GateResult): Record { } describe('defaultProductionGate input status', () => { + it('routes both composite and dimension evidence through the registered independent units', async () => { + const input = context() + for (const scenario of scenarios) { + const candidate = input.judgeScores.get(`${scenario.id}:0`)! + const baseline = input.baselineJudgeScores!.get(`${scenario.id}:0`)! + candidate.judge!.dimensions.safety = 0.9 + baseline.judge!.dimensions.safety = 0.9 + for (let rep = 1; rep < 20; rep++) { + input.judgeScores.set(`${scenario.id}:${rep}`, candidate) + input.baselineJudgeScores!.set(`${scenario.id}:${rep}`, baseline) + } + } + const result = await defaultProductionGate({ + holdoutScenarios: scenarios, + independentUnitByScenarioId: new Map( + scenarios.map((scenario, i) => [scenario.id, `family-${i % 2}`]), + ), + criticalDimensions: ['safety'], + }).decide(input) + expect(result.decision).toBe('hold') + expect( + result.contributingGates.find((gate) => gate.name === 'heldout-significance')?.detail, + ).toMatchObject({ + n: 2, + pairedCellN: 120, + observationUnit: 'registered', + fewRuns: true, + }) + expect( + result.contributingGates.find((gate) => gate.name === 'dimension-regression')?.detail, + ).toMatchObject({ + regressions: [{ dimension: 'safety', n: 2, pairedCellN: 120, observationUnit: 'registered' }], + }) + expect(statuses(result)['dimension-regression']).toBe('not_evaluated') + }) + + it('holds when composite lift is supported but a required dimension has only one measured unit', async () => { + const input = context() + input.judgeScores.get('one:0')!.judge!.dimensions.safety = 0.9 + input.baselineJudgeScores!.get('one:0')!.judge!.dimensions.safety = 0.9 + const result = await defaultProductionGate({ + holdoutScenarios: scenarios, + independentUnitByScenarioId: new Map(scenarios.map((scenario) => [scenario.id, scenario.id])), + criticalDimensions: ['safety'], + }).decide(input) + expect(statuses(result)['heldout-significance']).toBe('pass') + expect(statuses(result)['dimension-regression']).toBe('not_evaluated') + expect(result.decision).toBe('hold') + expect(result.reasons.join(' ')).toContain('safety has 1 paired observation units') + }) + + it('refuses partial dimension coverage even when enough independent units remain', async () => { + const input = context() + for (const scenario of scenarios) { + input.judgeScores.get(`${scenario.id}:0`)!.judge!.dimensions.safety = 0.9 + input.baselineJudgeScores!.get(`${scenario.id}:0`)!.judge!.dimensions.safety = 0.9 + } + input.judgeScores.set('one:1', { + judge: { ...input.judgeScores.get('one:0')!.judge!, dimensions: {} }, + }) + input.baselineJudgeScores!.set('one:1', { + judge: { ...input.baselineJudgeScores!.get('one:0')!.judge!, dimensions: {} }, + }) + const result = await defaultProductionGate({ + holdoutScenarios: scenarios, + independentUnitByScenarioId: new Map(scenarios.map((scenario) => [scenario.id, scenario.id])), + criticalDimensions: ['safety'], + }).decide(input) + expect(statuses(result)['heldout-significance']).toBe('pass') + expect(statuses(result)['dimension-regression']).toBe('not_evaluated') + expect(result.decision).toBe('hold') + expect( + result.contributingGates.find((gate) => gate.name === 'dimension-regression')?.detail, + ).toMatchObject({ + incompleteDimensions: ['safety'], + regressions: [{ n: 6, fewRuns: false, missingCellIds: ['one:1'], missingScenarioIds: [] }], + }) + }) + + it('requires a unit for every configured holdout scenario and snapshots the mapping', async () => { + expect(() => + defaultProductionGate({ + holdoutScenarios: scenarios, + independentUnitByScenarioId: new Map([['one', 'family']]), + }), + ).toThrow(/missing independent unit.*two/) + const mapping = new Map(scenarios.map((scenario) => [scenario.id, scenario.id])) + const gate = defaultProductionGate({ + holdoutScenarios: scenarios, + independentUnitByScenarioId: mapping, + }) + mapping.clear() + const result = await gate.decide(context()) + expect( + result.contributingGates.find((check) => check.name === 'heldout-significance')?.detail, + ).toMatchObject({ + n: 6, + pairedCellN: 6, + observationUnit: 'registered', + }) + }) + + it('refuses an absent replica before averaging the remaining cells', async () => { + const input = context() + input.judgeScores.set('one:1', input.judgeScores.get('one:0')!) + const gate = defaultProductionGate({ + holdoutScenarios: scenarios, + independentUnitByScenarioId: new Map(scenarios.map((scenario) => [scenario.id, 'family'])), + }) + await expect(gate.decide(input)).rejects.toThrow(/do not align/) + }) + + it('names the exact test when an explicit median target remains undecided', async () => { + const input = context() + input.baselineJudgeScores = scores([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) + input.judgeScores = scores([0.9, 0.85, 0.8, 0.75, 0.7, 0.1]) + const result = await defaultProductionGate({ + holdoutScenarios: scenarios, + heldoutStatistic: 'median', + }).decide(input) + expect(result.decision).toBe('hold') + expect( + result.contributingGates.find((check) => check.name === 'heldout-significance')?.detail, + ).toMatchObject({ + decisionMethod: 'exact-sign', + fewRuns: false, + }) + expect(result.reasons.join(' ')).toContain('exact one-sided sign test') + expect(result.reasons.join(' ')).toContain('p=0.03125 does not reject at α=0.0250') + expect(result.reasons.join(' ')).not.toContain('≤ threshold') + }) + it('reports absent optional inputs as not evaluated without a passed alias', async () => { const result = await defaultProductionGate({ holdoutScenarios: scenarios, diff --git a/src/campaign/gates/default-production-gate.ts b/src/campaign/gates/default-production-gate.ts index 894a80c2..cae76764 100644 --- a/src/campaign/gates/default-production-gate.ts +++ b/src/campaign/gates/default-production-gate.ts @@ -43,10 +43,15 @@ export interface DefaultProductionGateOptions { /** Required: scenarios held out from training; substrate compares * candidate-on-holdout vs baseline-on-holdout. */ holdoutScenarios: Scenario[] - /** Minimum held-out lift the **paired-bootstrap CI lower bound** must clear - * to ship — NOT a point estimate. Default 0 ⇒ "confidently positive at the - * confidence level". Interpreted in the judge's native composite scale (set - * e.g. 2 for a 0-100 rubric to require a ≥2-point significant gain). */ + /** + * Independent sampling unit for every held-out scenario. Pair cells first, + * then average scores within each unit and give units equal weight. + * Omit only when execution cells on a fixed roster are the observation units. + */ + independentUnitByScenarioId?: ReadonlyMap + /** Minimum held-out improvement under the shared paired decision rule. + * Default 0. The outcome shape and sample count determine the statistical + * test. Use native composite units, such as 2 for a 0-100 rubric. */ deltaThreshold?: number /** Confidence level for the held-out + dimension bootstraps. Default 0.95. */ confidence?: number @@ -54,7 +59,7 @@ export interface DefaultProductionGateOptions { bootstrapResamples?: number /** Fixed bootstrap seed for a deterministic verdict. Default 1337. */ bootstrapSeed?: number - /** Minimum paired holdout observations (scenarios × reps) before a + /** Minimum paired independent units (or cells when unconfigured) before a * significance claim is allowed. The exact small-sample test may require * more observations at the selected confidence. Default 3. */ minProductiveRuns?: number @@ -62,10 +67,9 @@ export interface DefaultProductionGateOptions { * (tie-robust — see `heldoutSignificance`). Pass `'median'` for * outlier-robustness at the cost of tie-blindness. */ heldoutStatistic?: 'mean' | 'median' - /** Critical judge dimensions that must NOT significantly regress even when - * the net composite rises (anti-Goodhart). The gate HOLDS if any listed - * dimension's paired-delta CI lower bound < −`regressionTolerance`. E.g. - * `['hallucination_free']` for a legal agent. */ + /** Required judge dimensions checked for regression and complete evidence. + * A bootstrap lower bound below negative tolerance or a paired test proving + * a drop holds the gate. Thin or incomplete coverage is not_evaluated. */ criticalDimensions?: string[] /** Tolerance for the per-dimension regression guard, in the dimension's * native scale. When omitted it auto-scales off observed magnitudes: @@ -105,6 +109,21 @@ export function defaultProductionGate( const minProductiveRuns = options.minProductiveRuns ?? 3 const heldoutStatistic = options.heldoutStatistic ?? 'mean' const explicitlyRequired = new Set(options.requiredChecks ?? []) + const scenarioIds = new Set(options.holdoutScenarios.map((scenario) => scenario.id)) + const independentUnitByScenarioId = + options.independentUnitByScenarioId === undefined + ? undefined + : new Map(options.independentUnitByScenarioId) + if (independentUnitByScenarioId !== undefined) { + for (const id of scenarioIds) { + const unit = independentUnitByScenarioId.get(id) + if (typeof unit !== 'string' || unit.length === 0 || unit.trim() !== unit) { + throw new Error( + `defaultProductionGate: missing independent unit for holdout scenario '${id}'`, + ) + } + } + } return { name: 'defaultProductionGate', @@ -129,14 +148,8 @@ export function defaultProductionGate( } } - // ── (1) heldout composite lift — paired-bootstrap CI, NOT a point estimate - // The shipped false positive: the baseline re-scored against itself read - // run-to-run model noise (91 vs 95) as a "+4 lift" and shipped, because a - // point estimate carries no confidence interval. Pair candidate vs - // baseline holdout cells by FULL cellId (never averaging reps away) and - // ship only when the bootstrap CI lower bound clears the threshold — - // i.e. the gain is real at the confidence level, not noise. - const scenarioIds = new Set(options.holdoutScenarios.map((s) => s.id)) + // Full cell pairing precedes aggregation, so a missing replica cannot + // disappear into two independently computed unit means. let delta: number | undefined if (!ctx.baselineJudgeScores) { unavailable( @@ -154,6 +167,7 @@ export function defaultProductionGate( resamples, seed, statistic: heldoutStatistic, + independentUnitByScenarioId, }, ) // The DECIDING interval, not the diagnostic bootstrap: on a pass/fail @@ -167,6 +181,9 @@ export function defaultProductionGate( status: sig.fewRuns ? 'not_evaluated' : heldoutPass ? 'pass' : 'fail', detail: { n: sig.n, + pairedCellN: sig.pairedCellN, + observationUnit: sig.observationUnit, + unitIds: sig.unitIds, delta, decisionStatistic: sig.decisionStatistic, decisionMethod: sig.decisionMethod, @@ -192,17 +209,19 @@ export function defaultProductionGate( if (!heldoutPass) { const tieNote = sig.tieFraction >= TIE_WARN_FRACTION - ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied scenarios` + ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied observation units` : '' const ci = `${(dec.confidence * 100).toFixed(0)}% CI [${dec.low.toFixed(3)}, ${dec.high.toFixed(3)}]` reasons.push( sig.fewRuns - ? `held-out: only ${sig.n} paired runs (< ${sig.minimumRequired}) — too few to claim significance` + ? `held-out: only ${sig.n} paired observation units (< ${sig.minimumRequired}) — too few to claim significance` : dec.indeterminate ? `held-out: ${dec.indeterminateCause}, so the paired CI is ${ci} and carries no direction — it cannot clear threshold ${deltaThreshold} on evidence${tieNote}` : dec.exactTestVetoes ? `held-out: McNemar exact p=${dec.mcnemar?.pValue.toExponential(2)} does not reject at α=${(1 - dec.confidence).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})` - : `held-out CI.low ${dec.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})`, + : dec.method === 'exact-sign' + ? `held-out: exact one-sided sign test p=${dec.pValue} does not reject at α=${((1 - dec.confidence) / 2).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, diagnostic ${ci}${tieNote})` + : `held-out CI.low ${dec.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})`, ) } } @@ -242,15 +261,28 @@ export function defaultProductionGate( ctx.baselineJudgeScores, scenarioIds, criticalDimensions, - { tolerance: options.regressionTolerance, confidence, resamples, seed }, + { + tolerance: options.regressionTolerance, + confidence, + resamples, + seed, + independentUnitByScenarioId, + minProductiveRuns, + }, ) const measured = new Set(dimRegs.map((result) => result.dimension)) const missingDimensions = criticalDimensions.filter((dimension) => !measured.has(dimension)) const regressed = dimRegs.filter((result) => result.regressed) + const incompleteDimensions = dimRegs.filter( + (result) => + result.fewRuns || + result.missingCellIds.length > 0 || + result.missingScenarioIds.length > 0, + ) const dimensionStatus = regressed.length > 0 ? ('fail' as const) - : missingDimensions.length > 0 + : missingDimensions.length > 0 || incompleteDimensions.length > 0 ? ('not_evaluated' as const) : ('pass' as const) contributing.push({ @@ -259,6 +291,7 @@ export function defaultProductionGate( detail: { guarded: criticalDimensions, missingDimensions, + incompleteDimensions: incompleteDimensions.map((result) => result.dimension), regressions: dimRegs.map((result) => ({ dimension: result.dimension, ciLow: result.ci.low, @@ -269,6 +302,12 @@ export function defaultProductionGate( median: result.bootstrap.median, tolerance: result.tolerance, n: result.n, + pairedCellN: result.pairedCellN, + observationUnit: result.observationUnit, + minimumRequired: result.minimumRequired, + fewRuns: result.fewRuns, + missingCellIds: result.missingCellIds, + missingScenarioIds: result.missingScenarioIds, regressed: result.regressed, })), }, @@ -277,9 +316,27 @@ export function defaultProductionGate( requiredUnavailable.add('dimension-regression') reasons.push(`critical dimension(s) were not scored: ${missingDimensions.join(', ')}`) } + if (incompleteDimensions.length > 0) { + requiredUnavailable.add('dimension-regression') + reasons.push( + `critical dimension evidence is incomplete: ${incompleteDimensions + .map( + (result) => + `${result.dimension} has ${result.n} paired observation units (minimum ${result.minimumRequired}), ` + + `${result.missingCellIds.length} unscored cells, ${result.missingScenarioIds.length} unscored scenarios`, + ) + .join('; ')}`, + ) + } if (regressed.length > 0) { reasons.push( - `critical dimension(s) regressed: ${regressed.map((result) => `${result.dimension} CI.low ${result.ci.low.toFixed(3)} < -${result.tolerance}`).join('; ')}`, + `critical dimension(s) regressed: ${regressed + .map((result) => + result.bootstrap.low < -result.tolerance + ? `${result.dimension} bootstrap CI.low ${result.bootstrap.low.toFixed(3)} < -${result.tolerance}` + : `${result.dimension} paired test supports a drop greater than ${result.tolerance}`, + ) + .join('; ')}`, ) } } diff --git a/src/campaign/gates/heldout-gate.test.ts b/src/campaign/gates/heldout-gate.test.ts index 3c4b4f9e..0a164dc3 100644 --- a/src/campaign/gates/heldout-gate.test.ts +++ b/src/campaign/gates/heldout-gate.test.ts @@ -9,17 +9,17 @@ const score = (composite: number): Record => ({ function cells(values: number[]): Map> { const map = new Map>() for (const [i, v] of values.entries()) { - map.set(`s${Math.floor(i / 2)}:${i % 2}`, score(v)) + map.set(`s${i}:0`, score(v)) } return map } -const scenarios = Array.from({ length: 6 }, (_, i) => ({ id: `s${i}`, kind: 'fixture' })) +const scenarios = Array.from({ length: 24 }, (_, i) => ({ id: `s${i}`, kind: 'fixture' })) describe('heldOutGate', () => { it('HOLDS on same-distribution noise even when the candidate mean is higher', async () => { - const baseline = cells([0.9, 0.5, 0.7, 0.95, 0.6, 0.8, 0.85, 0.55, 0.75, 0.9, 0.65, 0.7]) - const candidate = cells([0.95, 0.6, 0.65, 0.9, 0.75, 0.85, 0.8, 0.7, 0.85, 0.8, 0.75, 0.8]) + const baseline = cells(Array(24).fill(0.5)) + const candidate = cells(Array.from({ length: 24 }, (_, i) => (i % 2 === 0 ? 0.7 : 0.4))) const gate = heldOutGate({ scenarios, deltaThreshold: 0.02 }) const result = await gate.decide({ judgeScores: candidate, @@ -30,8 +30,8 @@ describe('heldOutGate', () => { }) it('SHIPS a real, consistent lift whose CI clears the threshold', async () => { - const baseline = cells([0.5, 0.52, 0.48, 0.51, 0.5, 0.49, 0.5, 0.52, 0.51, 0.5, 0.49, 0.5]) - const candidate = cells([0.8, 0.82, 0.78, 0.81, 0.8, 0.79, 0.8, 0.82, 0.81, 0.8, 0.79, 0.8]) + const baseline = cells(Array(24).fill(0.5)) + const candidate = cells(Array.from({ length: 24 }, (_, i) => 0.78 + (i % 5) * 0.01)) const gate = heldOutGate({ scenarios, deltaThreshold: 0.1 }) const result = await gate.decide({ judgeScores: candidate, @@ -42,11 +42,13 @@ describe('heldOutGate', () => { }) it('uses the default threshold and exposes the configured bootstrap seed', async () => { - const baseline = cells([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) + const baseline = cells(Array(24).fill(0.1)) // Not a uniform delta: identical deltas give a zero-width interval, which // the gate refuses regardless of how large the gain is. - const candidate = cells([0.92, 0.88, 0.95, 0.89, 0.93, 0.9]) - const gate = heldOutGate({ scenarios: scenarios.slice(0, 3), bootstrapSeed: 99 }) + const candidate = cells( + Array.from({ length: 24 }, (_, i) => [0.92, 0.88, 0.95, 0.89, 0.93, 0.9][i % 6]!), + ) + const gate = heldOutGate({ scenarios, bootstrapSeed: 99 }) const result = await gate.decide({ judgeScores: candidate, baselineJudgeScores: baseline, @@ -71,7 +73,7 @@ describe('heldOutGate', () => { const baseline = cells([0.1, 0.1]) const candidate = cells([0.9, 0.9]) const gate = heldOutGate({ - scenarios: [{ id: 's0', kind: 'fixture' }], + scenarios: scenarios.slice(0, 2), deltaThreshold: 0.1, minProductiveRuns: 3, }) diff --git a/src/campaign/gates/power-preflight.test.ts b/src/campaign/gates/power-preflight.test.ts index 9818f498..2e629b39 100644 --- a/src/campaign/gates/power-preflight.test.ts +++ b/src/campaign/gates/power-preflight.test.ts @@ -22,7 +22,7 @@ describe('powerPreflight (minimum-detectable-lift calculator)', () => { const p = powerPreflight({ baselineComposites: composites }) expect(p.underpowered).toBe(false) expect(p.mde).toBeGreaterThan(0.25) - expect(p.recommendation).toMatch(/cannot clear the gate/) + expect(p.recommendation).toMatch(/recheck using measured paired deltas/) }) it('flags a structurally underpowered run: high baseline leaves less headroom than the MDE', () => { @@ -43,7 +43,7 @@ describe('powerPreflight (minimum-detectable-lift calculator)', () => { expect(p.mde).toBeLessThan(0.07) }) - it('more reps (pairedN) shrink the MDE toward the gate threshold', () => { + it('more independent observations (pairedN) shrink the MDE toward the gate threshold', () => { const composites = [1, 0, 1, 0, 1, 0, 1, 0, 1, 0] const at10 = powerPreflight({ baselineComposites: composites, pairedN: 10 }) const at40 = powerPreflight({ baselineComposites: composites, pairedN: 40 }) @@ -59,7 +59,7 @@ describe('powerPreflight (minimum-detectable-lift calculator)', () => { expect(p.mde).toBeGreaterThan(0) }) - it('flags the shared scoring channel: MDE reads as a lower bound, reps cannot debias', () => { + it('flags systematic bias outside the shared-channel estimate', () => { const composites = [1, 0, 1, 0, 1, 0, 1, 0] const shared = powerPreflight({ baselineComposites: composites, sharedScorerChannel: true }) expect(shared.sharedChannelCaveat).toMatch(/independent second scoring channel/) diff --git a/src/campaign/gates/power-preflight.ts b/src/campaign/gates/power-preflight.ts index 2c97d9ec..809b606a 100644 --- a/src/campaign/gates/power-preflight.ts +++ b/src/campaign/gates/power-preflight.ts @@ -1,68 +1,54 @@ /** - * Power preflight — "can this budget detect the effect you are hunting?" - * - * The failure it prevents (measured, twice): a live prompt-improvement campaign ran - * 333 sandbox cells over 5.6 hours and produced a +0.08 holdout lift the ship gate - * (paired bootstrap, CI.low > 0.05) could not distinguish from zero — because at - * that holdout size and worker variance the MINIMUM DETECTABLE lift was larger than - * any effect a prompt change plausibly produces. The budget was spent learning what - * a 30-second calculation on the baseline cells already knew. No eval framework we - * know of surfaces this; every underpowered improvement run everywhere ends in an - * uninformative "hold". + * Approximate detectable lift from baseline variance and the intended sample size. * * Model: the ship rule is `CI.low(paired Δ) > deltaThreshold`. Approximating the * bootstrap CI as normal, `CI.low ≈ effect − z·sd_Δ/√n`, so the smallest shippable - * true effect is `MDE = deltaThreshold + z·sd_Δ/√n`. The paired-delta SD is unknown - * before the candidate exists; we bound it by the zero-correlation case - * `sd_Δ ≤ √2·sd_baseline` — a CONSERVATIVE (upper) MDE, which is the correct - * direction for a warning. Pairing is per cell (`scenario:rep`), so reps multiply n. + * effect is `MDE = deltaThreshold + z·sd_Δ/√n`. This estimates `sd_Δ` as + * `√2·sd_baseline`, assuming equal arm variances and zero paired correlation. + * Actual candidate variance and paired correlation can move the threshold in either direction. + * This diagnostic does not replace the gate or guarantee a detection probability. * - * Standalone by design: feed it any baseline composites (a `gate:'none'` run, a - * live-proof table) BEFORE budgeting the real search; `selfImprove` also attaches - * it to every result and warns when the run was structurally unable to ship. + * Supply one baseline mean per independent observation used by the comparison. + * With declared source units, repetitions and variants refine those means without increasing n. + * `selfImprove` attaches this diagnostic after its final measurement. */ export interface PowerPreflightOptions { - /** Per-cell baseline composites on the HOLDOUT scenarios (one per scenario:rep cell). */ + /** Baseline composites at the comparison's independent observation unit. */ baselineComposites: number[] - /** Paired observations the budgeted comparison will produce - * (holdout scenarios × reps). Defaults to `baselineComposites.length`. */ + /** Independent paired observations planned for the comparison. + * Defaults to `baselineComposites.length`. */ pairedN?: number /** The ship gate's effect-size threshold. Default 0.05 (defaultProductionGate). */ deltaThreshold?: number /** CI confidence the gate uses. Default 0.95. */ confidence?: number - /** True when the holdout is scored by the SAME judge/scorer family as the gate - * (selfImprove's default composition — one judge scores everything). Under a - * shared channel, raising paired n reduces only the IDIOSYNCRATIC noise share; - * systematic judge bias is untouched, so the MDE here is a lower bound and the - * only full debiaser is an independent second scoring channel - * (recursive-self-improvement S1c, closed form in EXP-023 P0). Default false. */ + /** Whether the holdout uses the gate's judge family. + * More observations cannot establish freedom from systematic scoring bias. + * Default false. */ sharedScorerChannel?: boolean } export interface PowerPreflight { /** Paired observations the comparison will have. */ n: number - /** Baseline per-cell composite standard deviation (the variance the effect must beat). */ + /** Sample standard deviation of baseline observation means. */ sd: number - /** Minimum detectable lift: the smallest TRUE effect the gate could ship at this budget. */ + /** Approximate lift needed to put a normal interval above the gate threshold. */ mde: number /** Baseline holdout composite mean. */ baselineMean: number /** Headroom to a perfect 1.0 composite (the largest achievable lift on a [0,1] judge). */ headroom: number - /** True when even the largest achievable effect (headroom) is below the MDE — - * the run is structurally unable to ship regardless of proposal quality. - * Only asserted for [0,1]-scaled judges (see `scaleAssumed`). */ + /** Whether this approximation exceeds the estimated [0,1] score headroom. + * This is a planning warning, not a proof that promotion is impossible. */ underpowered: boolean /** True when composites look [0,1]-scaled; headroom/underpowered are only * meaningful under that convention (0-100 judges get mde/sd/n but no verdict). */ scaleAssumed: boolean deltaThreshold: number confidence: number - /** Set when the holdout shares the gate's scoring channel: more cells cannot - * buy back systematic judge bias — treat the MDE as a lower bound. */ + /** Notes unmeasured systematic bias when the gate shares its scoring channel. */ sharedChannelCaveat?: string /** One actionable sentence for humans and logs. */ recommendation: string @@ -76,10 +62,7 @@ function zFor(confidence: number): number { return 1.282 } -/** Estimate the minimum detectable lift a paired-holdout improvement run can - * ship at a given budget, from the baseline holdout composites — call it BEFORE - * spending a search to learn whether the effect you are hunting is even - * observable at this holdout size and worker variance. */ +/** Estimate detectable lift from baseline independent observations before budgeting a comparison. */ export function powerPreflight(opts: PowerPreflightOptions): PowerPreflight { const composites = opts.baselineComposites.filter((v) => Number.isFinite(v)) if (composites.length < 3) { @@ -104,12 +87,12 @@ export function powerPreflight(opts: PowerPreflightOptions): PowerPreflight { const underpowered = scaleAssumed && mde > headroom const sharedChannelCaveat = opts.sharedScorerChannel - ? 'Holdout and gate share one scoring channel: raising n/reps reduces only idiosyncratic noise — systematic judge bias remains and this MDE is a lower bound. Full debiasing needs an independent second scoring channel (different judge/benchmark family).' + ? 'Holdout and gate share one scoring channel: systematic judge bias remains outside this estimate. An independent second scoring channel can help test that bias.' : undefined const recommendation = underpowered - ? `UNDERPOWERED: minimum detectable lift ${mde.toFixed(3)} exceeds the ${headroom.toFixed(3)} headroom above the baseline (${mean.toFixed(3)}) — no achievable effect can ship at this budget. Raise paired n (scenarios x reps) to ~${Math.ceil(((z * Math.SQRT2 * sd) / Math.max(headroom - deltaThreshold, 0.01)) ** 2)} or reduce worker variance before searching.` - : `Minimum detectable lift at n=${n}: ${mde.toFixed(3)} (baseline sd ${sd.toFixed(3)}). Effects smaller than this cannot clear the gate; budget the search for effects you believe exceed it.` + ? `UNDERPOWERED under this approximation: detectable lift ${mde.toFixed(3)} exceeds the ${headroom.toFixed(3)} headroom above the baseline (${mean.toFixed(3)}). Raise paired n using independent observations to ~${Math.ceil(((z * Math.SQRT2 * sd) / Math.max(headroom - deltaThreshold, 0.01)) ** 2)} or reduce observation variance. Recheck with measured paired deltas.` + : `Approximate detectable lift at n=${n}: ${mde.toFixed(3)} (baseline sd ${sd.toFixed(3)}). Compare this estimate with the effect you expect, then recheck using measured paired deltas.` return { n, diff --git a/src/campaign/gates/sequential.test.ts b/src/campaign/gates/sequential.test.ts index 5a33260d..e917a2f4 100644 --- a/src/campaign/gates/sequential.test.ts +++ b/src/campaign/gates/sequential.test.ts @@ -267,7 +267,7 @@ describe('sequentialPairedGate.observe — anytime validity', () => { describe('sequentialPairedGate.decide — gate contract', () => { const better = ctxFrom( - Array.from({ length: 10 }, (_, i) => ({ + Array.from({ length: 30 }, (_, i) => ({ scenarioId: `s${i}`, reps: 3, candidate: 0.9, @@ -304,7 +304,7 @@ describe('sequentialPairedGate.decide — gate contract', () => { expect(result.reasons[0]).toContain('NOT evidence of no effect') }) - it('maps a stream that ends undecided before maxN to need_more_work (more reps could decide)', async () => { + it('maps an unfinished independent-scenario stream to need_more_work', async () => { const gate = sequentialPairedGate({ alpha: 0.05, minN: 5, maxN: 100 }) const result = await gate.decide(flat) expect(result.decision).toBe('need_more_work') @@ -333,6 +333,76 @@ describe('sequentialPairedGate.decide — gate contract', () => { expect(gate.state().n).toBe(0) expect(gate.state().decision).toBe('continue') }) + + it('does not count a repeated random scenario effect as independent evidence', async () => { + // Choose one fair sign per experiment, then repeat it within one task. + // Both equiprobable states have one independent unit, despite 100 cells. + for (const sign of [-1, 1]) { + const gate = sequentialPairedGate({ alpha: 0.05, minN: 5, maxN: 100 }) + const result = await gate.decide( + ctxFrom([ + { + scenarioId: 'family:one-random-task', + reps: 100, + candidate: sign > 0 ? 1 : 0, + baseline: sign > 0 ? 0 : 1, + }, + ]), + ) + expect(result.decision).toBe('need_more_work') + expect(result.contributingGates[0]!.detail).toMatchObject({ + n: 1, + pairedN: 1, + pairedCellN: 100, + observationUnit: 'scenario', + }) + } + }) + + it('uses equal scenario weights when repetition counts differ', async () => { + const gate = sequentialPairedGate({ maxN: 100 }) + const result = await gate.decide( + ctxFrom([ + { scenarioId: 'a', reps: 100, candidate: 1, baseline: 0 }, + { scenarioId: 'b', reps: 1, candidate: 0, baseline: 1 }, + ]), + ) + expect(result.delta).toBe(0) + expect(result.decision).toBe('need_more_work') + expect(result.contributingGates[0]!.detail).toMatchObject({ pairedN: 2, pairedCellN: 101 }) + }) + + it('groups distinct variants from one source and snapshots the registered mapping', async () => { + const variants = Array.from({ length: 20 }, (_, i) => ({ + scenarioId: `family:variant-${i}`, + reps: 5, + candidate: 1, + baseline: 0, + })) + const independentUnitByScenarioId = new Map( + variants.map((variant) => [variant.scenarioId, 'shared-source']), + ) + const gate = sequentialPairedGate({ maxN: 20, independentUnitByScenarioId }) + independentUnitByScenarioId.clear() + const result = await gate.decide(ctxFrom(variants)) + expect(result.decision).toBe('need_more_work') + expect(result.contributingGates[0]!.detail).toMatchObject({ + n: 1, + pairedN: 1, + pairedCellN: 100, + observationUnit: 'registered', + unitIds: ['shared-source'], + }) + }) + + it('refuses an incomplete registered map and asymmetric cell evidence', async () => { + const input = ctxFrom([{ scenarioId: 'family:task', reps: 3, candidate: 1, baseline: 0 }]) + await expect( + sequentialPairedGate({ maxN: 20, independentUnitByScenarioId: new Map() }).decide(input), + ).rejects.toThrow(/missing independent unit.*family:task/) + input.baselineJudgeScores!.delete('family:task:1') + await expect(sequentialPairedGate({ maxN: 20 }).decide(input)).rejects.toThrow(/do not align/) + }) }) // ── sequentialPairedGate — restart reconstruction ───────────────────── diff --git a/src/campaign/gates/sequential.ts b/src/campaign/gates/sequential.ts index c313d33b..86b5ed8b 100644 --- a/src/campaign/gates/sequential.ts +++ b/src/campaign/gates/sequential.ts @@ -20,21 +20,23 @@ * shifts the null boundary by `manifest.minEffect` — re-deciding the same * stream under different parameters after seeing data would reopen optional * stopping under a fancier name. The manifest's content hash is verified at - * construction (sync, same `sha256-content` scheme as `signManifest`). + * construction with the canonical RFC 8785 digest used by signManifest. * - * Non-iid caveat (stated honestly): the supermartingale guarantee needs each - * delta's conditional mean under H0 to stay ≤ the null boundary given the - * past — exchangeable scenario deltas suffice. Scenario streams ordered by - * difficulty or by scenario family violate this; `decide(ctx)` therefore - * shuffles the paired deltas with a SEEDED permutation by default (the - * permutation is data-independent, so bet predictability is preserved). - * Stratified betting (per-stratum λ) is future work, not implemented here. + * The supermartingale guarantee requires E[delta_t | past] ≤ the null + * boundary under H0. Independent sampling with that mean bound suffices. + * Exchangeability or shuffling alone does not: one fair sign repeated many + * times is exchangeable and has marginal mean zero, but its conditional mean + * becomes the revealed sign. A seeded shuffle cannot remove that dependence. + * `decide(ctx)` pairs cells, then averages repetitions within each scenario + * before testing. Pass independentUnitByScenarioId for scenarios sharing a + * latent source or incident. Units must still satisfy the conditional-mean null. + * Direct `observe()` callers own this sampling and aggregation contract. */ import { manifestContentDigest, type SignedManifest } from '../../pre-registration' import { type EProcessState, eProcess, mulberry32 } from '../../statistics' import type { Gate, GateContext, GateResult, GenerationRecord, Scenario } from '../types' -import { pairHoldout } from './statistical-heldout' +import { aggregatePairedHoldout, pairHoldout } from './statistical-heldout' export type SequentialDecision = 'promote' | 'continue' | 'undecided-at-maxN' @@ -53,11 +55,11 @@ export interface SequentialPairedGateOptions { /** Type-I budget. With `preRegistration` bound this MUST match * `manifest.alpha` (conflict throws). Default 0.05. */ alpha?: number - /** Minimum paired deltas before a promote may fire. The stopping rule is + /** Minimum independent paired units before a promote may fire. The stopping rule is * "first n ≥ minN with e-value ≥ 1/alpha" — still a valid stopping time. * Default 5. */ minN?: number - /** Pre-registered observation budget. Required unless `preRegistration` + /** Pre-registered independent-unit budget (scenarios by default in decide()). Required unless `preRegistration` * supplies it via `preRegisteredN` (conflict throws). */ maxN?: number /** Bet truncation forwarded to `eProcess`. Default 0.5. */ @@ -66,9 +68,13 @@ export interface SequentialPairedGateOptions { * x = (d/scale + 1)/2 ∈ [0,1]. A delta outside ±scale throws (use * `detectScale` to pick 1 vs 100 BEFORE streaming). Default 1. */ scale?: number - /** Seed for the data-independent shuffle of paired deltas in `decide(ctx)` - * (exchangeability guard). Default 1337. */ + /** Seed for reproducible ordering of scenario deltas in decide(). + * Shuffling does not establish the conditional-mean null. Default 1337. */ shuffleSeed?: number + /** Independent unit for every scenario passed to decide(). Full cell pairs + * are averaged within each unit before testing. Defaults to scenario IDs. + * The mapping is copied at construction. Direct observe() is unchanged. */ + independentUnitByScenarioId?: ReadonlyMap /** Bind the pre-registered hypothesis. Verified (content hash) at * construction; alpha/maxN/direction/minEffect come FROM the manifest. */ preRegistration?: SignedManifest @@ -91,8 +97,10 @@ export type SequentialStreamState = EProcessState & { decision: SequentialDecisi export interface SequentialPairedGate extends Gate { - /** Streaming entry point: feed one paired per-scenario delta - * (candidate − baseline, native scale). Each gate instance carries ONE + /** Streaming entry point: feed one paired independent-unit delta + * (candidate − baseline, native scale). Aggregate correlated repetitions + * before calling; the caller must justify E[delta_t | past] under H0. + * Each gate instance carries ONE * observe-stream; `decide(ctx)` runs on its own fresh stream and never * consumes or advances this one. 'promote' is sticky; observing past the * pre-registered maxN throws (extending a finished stream after seeing @@ -335,15 +343,16 @@ function seededShuffle(items: T[], seed: number): T[] { /** * Anytime-valid sequential paired gate. Conforms to the existing `Gate` - * contract (`decide(ctx)` consumes candidate vs baseline judge scores via - * `pairHoldout` — same pairing granularity as the fixed-n gates: full cellId, - * never scenarioId) and adds a streaming `observe(delta)` entry for campaigns - * that score cells incrementally and want to stop mid-stream. + * contract: decide(ctx) pairs candidate and baseline by full cellId, then + * averages deltas within each configured unit (scenario by default). The + * e-process consumes one observation per unit, with equal unit weights. + * The streaming observe(delta) entry consumes caller-aggregated independent + * units. Repetitions improve a unit's precision and do not increase n. * * Decision mapping onto the substrate's five-valued `GateDecision`: * - 'promote' → 'ship' * - 'continue' → 'need_more_work' (stream ended before maxN with - * the e-value undecided — more reps could decide) + * the e-value undecided — more independent units needed) * - 'undecided-at-maxN' → 'hold', with the reason stating it is NOT * evidence of no effect (never a silent default) */ @@ -354,6 +363,10 @@ export function sequentialPairedGate s.id)) + const unitMap = independentUnitByScenarioId ?? new Map([...scenarioIds].map((id) => [id, id])) + for (const id of scenarioIds) { + const unit = unitMap.get(id) + if (typeof unit !== 'string' || unit.length === 0 || unit.trim() !== unit) { + throw new Error(`${name}: missing independent unit for scenario '${id}'`) + } + } const paired = pairHoldout( ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, (s) => s.composite, ) - const deltas = paired.after.map((a, i) => a - paired.before[i]!) + for (let i = 0; i < paired.cellIds.length; i++) { + const delta = paired.after[i]! - paired.before[i]! + if (Math.abs(delta) > cfg.scale) { + throw new Error( + `${name}: cell '${paired.cellIds[i]}' delta ${delta} outside ±scale=${cfg.scale}`, + ) + } + } + const observations = aggregatePairedHoldout(paired, unitMap) + const deltas = observations.after.map((after, i) => after - observations.before[i]!) seededShuffle(deltas, cfg.shuffleSeed) const stream = makeStream(cfg) @@ -396,6 +425,9 @@ export function sequentialPairedGate => const composite = (s: JudgeScore) => s.composite describe('pairHoldout + heldoutSignificance — promotion gate decision core', () => { + it('pairs namespaced scenario IDs using the final repetition suffix', () => { + const paired = pairHoldout( + new Map([ + ['family:task:0', { quality: score(0.8) }], + ['family:other:0', { quality: score(0.9) }], + ]), + new Map([ + ['family:task:0', { quality: score(0.5) }], + ['family:other:0', { quality: score(0.4) }], + ]), + new Set(['family:task']), + composite, + ) + expect(paired).toEqual({ before: [0.5], after: [0.8], cellIds: ['family:task:0'] }) + }) + + it('counts registered independent units and preserves original paired-cell counts', () => { + const paired = { + before: Array.from({ length: 100 }, () => 0.5), + after: Array.from({ length: 100 }, (_, i) => (i < 50 ? 0.9 : 0.8)), + cellIds: Array.from({ length: 100 }, (_, i) => `task:${i}`), + } + const ungrouped = heldoutSignificance(paired) + expect(ungrouped.significant).toBe(true) + expect(ungrouped.n).toBe(100) + const grouped = heldoutSignificance(paired, { + independentUnitByScenarioId: new Map([['task', 'task-population-unit']]), + }) + expect(grouped.significant).toBe(false) + expect(grouped.fewRuns).toBe(true) + expect(grouped.n).toBe(1) + expect(grouped.pairedCellN).toBe(100) + expect(grouped.unitIds).toEqual(['task-population-unit']) + expect(grouped.bootstrap.mean).toBeCloseTo(0.35, 10) + }) + + it('rejects missing unit assignments and keeps scenario families equally weighted', () => { + const paired = { + before: [0.5, 0.5, 0.5, 0.5], + after: [0.9, 0.9, 0.9, 0.1], + cellIds: ['large:first:0', 'large:first:1', 'large:second:0', 'small:task:0'], + } + expect(() => heldoutSignificance(paired, { independentUnitByScenarioId: new Map() })).toThrow( + /missing independent unit.*large:first/, + ) + const result = heldoutSignificance(paired, { + independentUnitByScenarioId: new Map([ + ['large:first', 'large-family'], + ['large:second', 'large-family'], + ['small:task', 'small-family'], + ]), + }) + expect(result.n).toBe(2) + expect(result.bootstrap.mean).toBeCloseTo(0, 10) + }) + + it('retains the same inference when every unit has more identical execution repeats', () => { + const pairedAt = (reps: number) => ({ + before: Array.from({ length: 30 * reps }, () => 0.5), + after: Array.from({ length: 30 * reps }, (_, i) => 0.625 + (Math.floor(i / reps) % 4) / 32), + cellIds: Array.from( + { length: 30 * reps }, + (_, i) => `family:task-${Math.floor(i / reps)}:${i % reps}`, + ), + }) + const independentUnitByScenarioId = new Map( + Array.from({ length: 30 }, (_, i) => [`family:task-${i}`, `unit-${i}`]), + ) + const single = heldoutSignificance(pairedAt(1), { independentUnitByScenarioId }) + const repeated = heldoutSignificance(pairedAt(20), { independentUnitByScenarioId }) + expect(repeated.pairedCellN).toBe(600) + expect(repeated.n).toBe(single.n) + expect(repeated.decision).toEqual(single.decision) + expect(repeated.bootstrap).toEqual(single.bootstrap) + }) + + it('refuses duplicated cells and non-finite scores before inference', () => { + expect(() => + heldoutSignificance({ + before: [0, 0], + after: [1, 1], + cellIds: ['same:0', 'same:0'], + }), + ).toThrow(/duplicate cellIds/) + expect(() => + heldoutSignificance({ + before: [0], + after: [NaN], + cellIds: ['same:0'], + }), + ).toThrow(/scores must be finite/) + expect(() => + heldoutSignificance({ + before: [0], + after: [1], + cellIds: [], + }), + ).toThrow(/same length/) + }) + + it('refuses asymmetric judge selection before computing a paired mean', () => { + const baseline = new Map([ + [ + 'sc0:0', + { + strict: score(0.1, { safety: 0.1 }), + lenient: score(0.9, { safety: 0.9 }), + }, + ], + ]) + const candidate = new Map([['sc0:0', { lenient: score(0.9, { safety: 0.9 }) }]]) + expect(() => pairHoldout(candidate, baseline, scenarioIds(1), composite)).toThrow( + /selected judge IDs do not align/, + ) + expect(() => dimensionRegressions(candidate, baseline, scenarioIds(1), ['safety'])).toThrow( + /selected judge IDs do not align/, + ) + const noSafety = new Map([ + ['sc0:0', { strict: score(0.9), lenient: score(0.9, { safety: 0.9 }) }], + ]) + expect(() => dimensionRegressions(noSafety, baseline, scenarioIds(1), ['safety'])).toThrow( + /selected judge IDs do not align/, + ) + }) + it('a clear held-out gain is SIGNIFICANT (gate ships)', () => { // Deltas must not be IDENTICAL: n identical deltas give a zero-width // interval, which carries no information about how far the estimate could // be wrong and is refused whatever the sign test says (pinned below). const paired = pairHoldout( - cells([0.82, 0.78, 0.85, 0.79, 0.83, 0.8]), - cells([0.5, 0.5, 0.5, 0.5, 0.5, 0.5]), - scenarioIds(6), + cells(Array.from({ length: 24 }, (_, i) => [0.82, 0.78, 0.85, 0.79, 0.83, 0.8][i % 6]!)), + cells(Array(24).fill(0.5)), + scenarioIds(24), composite, ) - expect(paired.before).toEqual([0.5, 0.5, 0.5, 0.5, 0.5, 0.5]) + expect(paired.before).toEqual(Array(24).fill(0.5)) const sig = heldoutSignificance(paired) - expect(sig.n).toBe(6) + expect(sig.n).toBe(24) expect(sig.fewRuns).toBe(false) - expect(sig.decisionMethod).toBe('exact-sign') - expect(sig.pValue).toBeCloseTo(1 / 64, 12) + expect(sig.decisionMethod).toBe('bootstrap-ci') + expect(sig.pValue).toBeNull() expect(sig.bootstrap.low).toBeGreaterThan(0) expect(sig.significant).toBe(true) }) diff --git a/src/campaign/gates/statistical-heldout.ts b/src/campaign/gates/statistical-heldout.ts index 372a25d1..0915b8e2 100644 --- a/src/campaign/gates/statistical-heldout.ts +++ b/src/campaign/gates/statistical-heldout.ts @@ -1,27 +1,12 @@ /** - * Statistical held-out promotion machinery — the trustworthy core the - * point-estimate `heldout-delta` gate lacked. + * Held-out inference pairs execution cells and judge identities before taking + * means within registered independent units. Repetitions can improve a unit's + * precision without increasing n. Ungrouped inference concerns independently + * sampled execution cells conditional on a fixed scenario roster. * - * The shipped false positive it prevents: a winner re-scored against the - * baseline on the holdout read run-to-run model NOISE (e.g. 91 vs 95) as a - * "+4 lift" and shipped, because the gate compared point estimates with no - * confidence interval. Here we pair candidate vs baseline holdout observations - * and bootstrap a CI on the paired delta — a candidate ships only when the CI - * lower bound clears the effect-size threshold (the gain is real at the - * confidence level, not noise), and is blocked when a critical dimension - * (e.g. `hallucination_free` for a legal agent) significantly regresses even if - * the net composite rose (anti-Goodhart). - * - * Two traps this module is built around (both produce a NEW false positive if - * gotten wrong): - * 1. PAIRING GRANULARITY — pairs by FULL `cellId` (`scenario:rep`), never by - * `scenarioId` (which averages reps away and destroys the within-pair - * variance reduction that makes a paired bootstrap tighter than unpaired). - * One paired observation per cell ⇒ reps multiply n. - * 2. SCALE — a judge may emit composites/dimensions on [0,1] or 0-100. The - * threshold + tolerance are interpreted in the judge's NATIVE scale; the - * per-dimension tolerance auto-scales off the observed baseline magnitudes - * so `-0.10` on [0,1] doesn't silently become a no-op on a 0-100 dimension. + * The shared paired decision rule selects the estimator and statistical test. + * Dimension reports retain missing coverage so required safety checks cannot + * pass through absent evidence. Thresholds use the judge's native score scale. */ import { @@ -53,13 +38,75 @@ export interface PairedHoldout { cellIds: string[] } +/** Campaign cell IDs append a numeric repetition after the scenario's full ID. */ +export function scenarioIdFromCellId(cellId: string): string { + const separator = cellId.lastIndexOf(':') + const repetition = cellId.slice(separator + 1) + if ( + separator < 1 || + cellId.trim() !== cellId || + !/^(0|[1-9]\d*)$/.test(repetition) || + !Number.isSafeInteger(Number(repetition)) + ) { + throw new Error(`pairHoldout: malformed cellId '${cellId}'; expected scenarioId:rep`) + } + return cellId.slice(0, separator) +} + +/** Preserve cell pairing before taking equal-weight independent-unit means. */ +export function aggregatePairedHoldout( + paired: PairedHoldout, + independentUnitByScenarioId?: ReadonlyMap, +): { before: number[]; after: number[]; unitIds: string[] } { + if ( + paired.before.length !== paired.after.length || + paired.before.length !== paired.cellIds.length + ) { + throw new Error('aggregatePairedHoldout: scores and cellIds must have the same length') + } + if (new Set(paired.cellIds).size !== paired.cellIds.length) { + throw new Error('aggregatePairedHoldout: duplicate cellIds cannot count as new observations') + } + if ( + paired.before.some((value) => !Number.isFinite(value)) || + paired.after.some((value) => !Number.isFinite(value)) + ) { + throw new Error('aggregatePairedHoldout: paired scores must be finite') + } + if (independentUnitByScenarioId === undefined) { + return { before: [...paired.before], after: [...paired.after], unitIds: [...paired.cellIds] } + } + const scenarioIds = paired.cellIds.map(scenarioIdFromCellId) + const groups = new Map() + for (let i = 0; i < paired.cellIds.length; i++) { + const scenarioId = scenarioIds[i]! + const unitId = independentUnitByScenarioId.get(scenarioId) + if (typeof unitId !== 'string' || unitId.length === 0 || unitId.trim() !== unitId) { + throw new Error( + `aggregatePairedHoldout: missing independent unit for scenario '${scenarioId}'`, + ) + } + const group = groups.get(unitId) ?? { before: 0, after: 0, n: 0 } + group.before += paired.before[i]! + group.after += paired.after[i]! + group.n += 1 + groups.set(unitId, group) + } + const unitIds = [...groups.keys()].sort() + return { + before: unitIds.map((id) => groups.get(id)!.before / groups.get(id)!.n), + after: unitIds.map((id) => groups.get(id)!.after / groups.get(id)!.n), + unitIds, + } +} + /** * Pair candidate vs baseline holdout observations by FULL cellId. `select` * pulls the scalar from a cell's judge reports (composite, or a named * dimension); a cell contributes the mean of `select` across its judges. Cells * whose scenario is not in `scenarioIds`, or where `select` is undefined for - * every judge on either side, are skipped on BOTH sides so the arrays stay - * paired. Throws when the two maps disagree on which holdout cells exist — a + * every judge on both sides, are skipped. The selected judge IDs must agree + * within each pair. Throws when the two maps disagree on holdout cell IDs — a * load-bearing invariant: the baseline + winner holdout campaigns run the same * scenarios with the same seed base, so their cellIds MUST align; a mismatch * means a silent pairing bug, not a soft fallback. @@ -70,14 +117,14 @@ export function pairHoldout( scenarioIds: Set, select: (s: JudgeScore) => number | undefined, ): PairedHoldout { - const cellValue = ( + const cellValues = ( byCell: Map>, cellId: string, - ): number | undefined => { + ): Map => { const scores = byCell.get(cellId) - if (!scores) return undefined - const vals: number[] = [] - for (const s of Object.values(scores)) { + const values = new Map() + if (!scores) return values + for (const [judgeId, s] of Object.entries(scores)) { if (s.failed === true) { throw new Error(`pairHoldout: cell '${cellId}' contains a failed judge score`) } @@ -85,13 +132,12 @@ export function pairHoldout( if (typeof v === 'number' && !Number.isFinite(v)) { throw new Error(`pairHoldout: cell '${cellId}' contains a non-finite selected score`) } - if (typeof v === 'number') vals.push(v) + if (typeof v === 'number') values.set(judgeId, v) } - if (vals.length === 0) return undefined - return vals.reduce((a, b) => a + b, 0) / vals.length + return values } - const inScope = (cellId: string) => scenarioIds.has(cellId.split(':')[0] ?? '') + const inScope = (cellId: string) => scenarioIds.has(scenarioIdFromCellId(cellId)) const candCells = [...candidate.keys()].filter(inScope).sort() const baseCells = [...baseline.keys()].filter(inScope).sort() // Alignment invariant — the holdout campaigns share scenarios + seed, so the @@ -108,16 +154,20 @@ export function pairHoldout( const after: number[] = [] const cellIds: string[] = [] for (const cellId of candCells) { - const b = cellValue(baseline, cellId) - const a = cellValue(candidate, cellId) + const b = cellValues(baseline, cellId) + const a = cellValues(candidate, cellId) // A scalar absent on both sides means that dimension was not scored. A // one-sided absence is asymmetric evidence loss, never a row to discard. - if (b === undefined && a === undefined) continue - if (b === undefined || a === undefined) { + if (b.size === 0 && a.size === 0) continue + if (b.size === 0 || a.size === 0) { throw new Error(`pairHoldout: cell '${cellId}' has a selected score on only one arm`) } - before.push(b) - after.push(a) + if (b.size !== a.size || [...b.keys()].some((id) => !a.has(id))) { + throw new Error(`pairHoldout: cell '${cellId}' selected judge IDs do not align`) + } + const judgeIds = [...b.keys()].sort() + before.push(judgeIds.reduce((sum, id) => sum + b.get(id)!, 0) / judgeIds.length) + after.push(judgeIds.reduce((sum, id) => sum + a.get(id)!, 0) / judgeIds.length) cellIds.push(cellId) } return { before, after, cellIds } @@ -156,9 +206,14 @@ export interface HeldoutSignificance { * high tie fraction is WHY a median-based gate would have missed a real lift; * it is the observability the tie fix adds. */ tieFraction: number - /** n paired observations. */ + /** Number of paired observation units, after configured aggregation. */ n: number - /** Effective minimum after applying the bootstrap's hard statistical floor. */ + /** Original matched execution cells, before aggregation. */ + pairedCellN: number + observationUnit: 'registered' | 'cell' + /** Registered unit IDs, or cell IDs on the ungrouped path. */ + unitIds: string[] + /** Effective minimum for the requested target and chosen estimator. */ minimumRequired: number /** Statistical method that carried the decision. */ decisionMethod: PairedDecisionMethod @@ -180,6 +235,8 @@ export interface HeldoutSignificanceOptions { /** Fixed by default for a deterministic, reproducible gate verdict. */ seed?: number statistic?: 'mean' | 'median' + /** Group full cell pairs into equal-weight independent units before inference. */ + independentUnitByScenarioId?: ReadonlyMap } /** @@ -205,8 +262,8 @@ export interface HeldoutSignificanceOptions { * and 88.50 % at n = 6 under a bounded asymmetric null whose true mean paired * delta is exactly 0. * - * At small n, where the percentile bootstrap is descriptive only, a - * pre-registered exact sign test still carries the bootstrap path. + * Continuous mean targets require bootstrap eligibility. Explicit median + * targets can use the exact sign test at its confidence-dependent minimum. */ export function heldoutSignificance( paired: PairedHoldout, @@ -229,7 +286,8 @@ export function heldoutSignificance( // median is kept as a reported diagnostic. Callers wanting outlier-robustness at // the cost of tie-blindness can still pass `statistic: 'median'`. const statistic = opts.statistic ?? 'mean' - const decision = decidePairedPromotion(paired.before, paired.after, { + const observations = aggregatePairedHoldout(paired, opts.independentUnitByScenarioId) + const decision = decidePairedPromotion(observations.before, observations.after, { confidence, resamples, statistic, @@ -242,21 +300,26 @@ export function heldoutSignificance( // field. Same two bootstraps as before on every path. const bootstrap = decision.bootstrap ?? - pairedBootstrap(paired.before, paired.after, { confidence, resamples, statistic, seed }) + pairedBootstrap(observations.before, observations.after, { + confidence, + resamples, + statistic, + seed, + }) const medianBootstrap = statistic === 'median' ? bootstrap - : pairedBootstrap(paired.before, paired.after, { + : pairedBootstrap(observations.before, observations.after, { confidence, resamples, statistic: 'median', seed, }) - const n = paired.before.length + const n = observations.before.length let ties = 0 for (let i = 0; i < n; i += 1) { - const after = paired.after[i] ?? 0 - const before = paired.before[i] ?? 0 + const after = observations.after[i]! + const before = observations.before[i]! if (Math.abs(after - before) < 1e-9) ties += 1 } const tieFraction = n === 0 ? 0 : ties / n @@ -269,6 +332,9 @@ export function heldoutSignificance( mcnemar: decision.mcnemar, tieFraction, n, + pairedCellN: paired.cellIds.length, + observationUnit: opts.independentUnitByScenarioId === undefined ? 'cell' : 'registered', + unitIds: observations.unitIds, minimumRequired: decision.minimumPairs, decisionMethod: decision.method, pValue: decision.pValue, @@ -294,13 +360,21 @@ export interface DimensionRegression { mcnemar: PairedMcNemarEvidence | null /** `ci` has zero width — no evidence in either direction. */ indeterminate: boolean - /** True iff the candidate may have regressed this dimension by more than - * tolerance: the lower bound of the DECIDING interval on (candidate − - * baseline) is below −tolerance, OR the exact small-sample test proves a drop - * past tolerance. */ + /** The bootstrap lower bound is below negative tolerance, or the shared + * paired test supports a drop exceeding tolerance. Missing coverage and + * insufficient observations are reported separately. */ regressed: boolean tolerance: number n: number + pairedCellN: number + observationUnit: 'registered' | 'cell' + /** Statistical minimum for the configured independent observation unit. */ + minimumRequired: number + fewRuns: boolean + /** Both arms lack this dimension on these otherwise matched execution cells. */ + missingCellIds: string[] + /** Configured scenarios with no paired dimension measurement at all. */ + missingScenarioIds: string[] } /** Detect the native scale of a set of scores: 0-100 when any magnitude clears @@ -310,24 +384,14 @@ export function detectScale(values: number[]): 1 | 100 { return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1 } -/** Per-critical-dimension regression guard. For each dimension, pair the - * candidate vs baseline values by full cellId and bootstrap the paired delta; - * a dimension is "regressed" when the CI lower bound < −tolerance (conservative - * — blocks if the credible worst case exceeds tolerance, which is the right - * posture for safety dimensions like `hallucination_free`). When `tolerance` - * is omitted it auto-scales: 0.05 on [0,1], 5 on 0-100. - * - * The interval comes from {@link decidePairedPromotion}, so a pass/fail - * dimension is judged on Tango's score interval rather than a percentile - * bootstrap of the mean — `tolerance` is a NONZERO margin, and the bootstrap - * is not a valid interval at one. That matters most here because this guard - * fails OPEN by construction: `tolerance` is positive, so an interval pinned at - * [0,0] never satisfies `low < −tolerance` and a real regression on a safety - * dimension would be reported as `regressed: false`. On the median it fails the - * same way for the same reason — when most pairs tie, which is automatic for a - * pass/fail dimension on {0,1} and on the 0-100 encoding `detectScale` exists - * to support, the median CI collapses to [0,0]. Pass `statistic: 'median'` to - * restore the pre-0.134 behaviour. */ +/** + * Report required-dimension evidence after full pairing and optional unit means. + * A bootstrap floor breach or a shared paired test supporting a drop marks + * regression. These two criteria are distinct; `ci` records the shared + * estimator and `bootstrap` records the floor interval. Missing observations + * and insufficient n remain explicit for the caller's evidence policy. + * The default tolerance is 0.05 on [0,1] and 5 on a detected 0-100 scale. + */ export function dimensionRegressions( candidate: Map>, baseline: Map>, @@ -341,12 +405,20 @@ export function dimensionRegressions( /** Paired statistic the CI is computed on. Default `'mean'` — see * {@link DECISION_PAIRED_DELTA_STATISTIC} for why the median is not. */ statistic?: 'mean' | 'median' + independentUnitByScenarioId?: ReadonlyMap + minProductiveRuns?: number } = {}, ): DimensionRegression[] { const out: DimensionRegression[] = [] + const expectedCellIds = [...baseline.keys()] + .filter((cellId) => scenarioIds.has(scenarioIdFromCellId(cellId))) + .sort() for (const dim of criticalDimensions) { const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim]) if (paired.before.length === 0) continue // dimension not scored on this judge + const observations = aggregatePairedHoldout(paired, opts.independentUnitByScenarioId) + const measuredCells = new Set(paired.cellIds) + const measuredScenarios = new Set(paired.cellIds.map(scenarioIdFromCellId)) const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]) const bootstrapStatistic = opts.statistic ?? DECISION_PAIRED_DELTA_STATISTIC const shared = { @@ -354,13 +426,15 @@ export function dimensionRegressions( resamples: opts.resamples ?? 2000, statistic: bootstrapStatistic, seed: opts.seed ?? 1337, + minPairs: opts.minProductiveRuns, } - const guard = decidePairedPromotion(paired.before, paired.after, shared) - const regression = decidePairedPromotion(paired.after, paired.before, { + const guard = decidePairedPromotion(observations.before, observations.after, shared) + const regression = decidePairedPromotion(observations.after, observations.before, { ...shared, threshold: tolerance, }) - const bootstrap = guard.bootstrap ?? pairedBootstrap(paired.before, paired.after, shared) + const bootstrap = + guard.bootstrap ?? pairedBootstrap(observations.before, observations.after, shared) out.push({ dimension: dim, bootstrap, @@ -386,7 +460,13 @@ export function dimensionRegressions( // ties dominate. regressed: bootstrap.low < -tolerance || regression.promote, tolerance, - n: paired.before.length, + n: observations.before.length, + pairedCellN: paired.cellIds.length, + observationUnit: opts.independentUnitByScenarioId === undefined ? 'cell' : 'registered', + minimumRequired: guard.minimumPairs, + fewRuns: !guard.sufficient, + missingCellIds: expectedCellIds.filter((cellId) => !measuredCells.has(cellId)), + missingScenarioIds: [...scenarioIds].filter((id) => !measuredScenarios.has(id)).sort(), }) } return out diff --git a/src/campaign/index.ts b/src/campaign/index.ts index 2a0fab6c..c47c9ccd 100644 --- a/src/campaign/index.ts +++ b/src/campaign/index.ts @@ -125,6 +125,7 @@ export { type ExternalTextOptimizerResult, externalTextOptimizationMethod, } from './external-text-optimization' +export type { FinalEvidencePolicy, FinalEvidenceUse } from './final-evidence' // ── Fixture UX / dry-run planning ──────────────────────────────────── export { discoverEvalFixtures, diff --git a/src/campaign/judge-snapshot.ts b/src/campaign/judge-snapshot.ts new file mode 100644 index 00000000..2d4e8a60 --- /dev/null +++ b/src/campaign/judge-snapshot.ts @@ -0,0 +1,26 @@ +import { judgeVersionFor } from './campaign-manifest' +import type { JudgeConfig, Scenario } from './types' + +/** Capture configuration and callbacks; receiver and closed-over state remain caller-owned. */ +export function captureJudge( + judge: JudgeConfig, +): JudgeConfig { + const dimensions = judge.dimensions.map((dimension) => + Object.freeze({ key: dimension.key, description: dimension.description }), + ) + Object.freeze(dimensions) + const captured = { + name: judge.name, + judgeVersion: judge.judgeVersion, + dimensions, + score: judge.score, + appliesTo: judge.appliesTo, + } + return Object.freeze({ + ...captured, + // Derive identity before binding; bound callbacks hide their original source text. + judgeVersion: judgeVersionFor(captured), + score: captured.score.bind(judge), + appliesTo: captured.appliesTo?.bind(judge), + }) +} diff --git a/src/campaign/optimization-method.ts b/src/campaign/optimization-method.ts index efe88f51..165fc77e 100644 --- a/src/campaign/optimization-method.ts +++ b/src/campaign/optimization-method.ts @@ -1,4 +1,5 @@ import { assertGepaCandidatePopulationSummary } from './gepa-candidate-population' +import { captureJudge } from './judge-snapshot' import { assertComparisonCost, combineComparisonCosts, @@ -40,13 +41,7 @@ export async function executeOptimizationMethod( baselineSurface: structuredClone(input.baselineSurface), trainScenarios: cloneScenarios(input.trainScenarios), selectionScenarios: cloneScenarios(input.selectionScenarios), - judges: Object.freeze( - input.judges.map((judge) => { - const dimensions = judge.dimensions.map((dimension) => Object.freeze({ ...dimension })) - Object.freeze(dimensions) - return Object.freeze({ ...judge, dimensions }) - }), - ), + judges: Object.freeze(input.judges.map(captureJudge)), runOptions: Object.freeze({ ...input.runOptions }), costLedger: costScope.ledger, }) diff --git a/src/campaign/presets/compare-optimization-methods.ts b/src/campaign/presets/compare-optimization-methods.ts index 6a61ca7e..aa59383a 100644 --- a/src/campaign/presets/compare-optimization-methods.ts +++ b/src/campaign/presets/compare-optimization-methods.ts @@ -12,8 +12,17 @@ import { type CampaignEvidenceContext, createCampaignEvidenceReceipt, } from '../../experiment/campaign-evidence' +import { + defineEvaluationClaim, + type EvaluationClaim, + type EvaluationUnitSummary, + summarizeEvaluationUnits, +} from '../../experiment/claim' import type { EvidenceReceipt } from '../../experiment/evidence-receipt' -import { pairedBootstrap } from '../../statistics' +import { + decidePairedPromotion, + type PairedPromotionDecision, +} from '../../paired-promotion-decision' import { contentHash } from '../../verdict-cache' import { assertCampaignDesign, assertCompleteCampaign } from '../coverage' import type { ExternalOptimizerWireCounts } from '../external-optimizer-contracts' @@ -22,6 +31,7 @@ import type { ExternalOptimizerObservationSummary, } from '../external-optimizer-observations' import type { GepaCandidatePopulationSummary } from '../gepa-candidate-population' +import { captureJudge } from '../judge-snapshot' import { type ComparisonCost, combineComparisonCosts, @@ -36,6 +46,15 @@ export { } from '../optimization-cost' export { assertOptimizationResult } from '../optimization-method' +import { + assertIndependentEvaluationSplit, + captureFinalEvidencePolicy, + evaluationUnitMap, + exposeFinalEvidence, + type FinalEvidencePolicy, + type FinalEvidenceUse, + reserveFinalEvidence, +} from '../final-evidence' import { type RunCampaignOptions, runCampaign } from '../run-campaign' import { resolveRunDir } from '../run-dir' import { campaignBreakdown } from '../score-utils' @@ -209,11 +228,11 @@ export interface OptimizationMethodScore { baselineComposite: number /** Mean final-test composite of this method's selected surface. */ winnerComposite: number - /** Mean per-scenario final-test lift (winner minus baseline). */ + /** Mean final-test lift across independent units (winner minus baseline). */ lift: number - /** Simultaneous paired-bootstrap interval for per-scenario lift. - * `low > 0` excludes zero after adjustment for all reported contrasts. */ + /** Deciding interval, adjusted for every registered contrast. */ liftCi: { low: number; high: number } + decision: PairedPromotionDecision /** Search spend reconciled with recorded method calls. Excludes final test scoring. */ optimizationCost: ComparisonCost /** Optimization duration reported by the method. Excludes final test scoring. */ @@ -227,6 +246,14 @@ export interface OptimizationMethodScore { winnerComposite: number lift: number }> + /** Scenario means averaged within each registered source unit before inference. */ + unitScores: Array<{ + unitId: string + scenarios: number + baselineComposite: number + winnerComposite: number + lift: number + }> winnerSurface: MutableSurface composition?: OptimizationMethodComposition /** 1-based, by descending lift. */ @@ -237,21 +264,28 @@ export interface OptimizationMethodPairwise { /** Higher-ranked method. */ a: string b: string - /** Mean per-scenario untouched-test delta (a − b). */ + /** Mean final-test delta across independent units (a − b). */ deltaMean: number low: number high: number - /** `a` if the CI clears 0, `b` if it is entirely negative, else `'tie'`. */ - favored: string + /** Favored only when the full paired decision clears the registered effect; null is inconclusive. */ + favored: string | null + decision: PairedPromotionDecision } export interface OptimizationMethodComparison { + claim?: EvaluationClaim + finalEvidence?: FinalEvidenceUse /** Sorted by descending lift; `rank` set accordingly. */ scores: OptimizationMethodScore[] best: OptimizationMethodScore - /** Best vs each other method, using simultaneous paired-bootstrap intervals. */ + /** Best observed method versus each alternative, with simultaneous paired decisions. */ pairwise: OptimizationMethodPairwise[] testScenarioIds: string[] + units: EvaluationUnitSummary + observationUnit: 'scenario' | 'registered' + /** Complete scenario-replicate pairs per contrast, before unit aggregation. */ + pairedCellN: number /** Sum of method reports reconciled against each method's recorded calls. */ optimizationCost: ComparisonCost /** Baseline and distinct winner scoring on the final test partition. */ @@ -279,6 +313,8 @@ export interface CompareOptimizationMethodsOptions[] baselineSurface: MutableSurface /** Evidence used by every optimizer to author or fit candidates. */ @@ -314,6 +350,16 @@ export interface CompareOptimizationMethodsOptions( opts: CompareOptimizationMethodsOptions, ): Promise { + opts = { + ...opts, + methods: opts.methods.map((method) => ({ ...method })), + trainScenarios: structuredClone(opts.trainScenarios), + selectionScenarios: structuredClone(opts.selectionScenarios), + testScenarios: structuredClone(opts.testScenarios), + judges: opts.judges.map(captureJudge), + claim: opts.claim && defineEvaluationClaim(opts.claim), + finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence), + } assertOptimizationMethods(opts.methods) assertComparisonPartitions(opts) const searchHistoryPolicy = opts.searchHistoryPolicy ?? 'allow-missing' @@ -379,6 +425,32 @@ export async function compareOptimizationMethods s.id).sort() + const unitByScenario = opts.claim + ? evaluationUnitMap(opts.claim, opts.testScenarios) + : new Map(scenarioIds.map((id) => [id, id])) + const units: EvaluationUnitSummary = opts.claim + ? summarizeEvaluationUnits(opts.claim, opts.testScenarios) + : { + observations: scenarioIds.length, + independentUnits: scenarioIds.length, + units: scenarioIds.map((id) => ({ id, observations: 1 })), + } + const aggregateUnits = (arr: number[]): number[] => { + const values = new Map() + scenarioIds.forEach((id, index) => { + const unitId = unitByScenario.get(id)! + const bucket = values.get(unitId) ?? [] + bucket.push(arr[index]!) + values.set(unitId, bucket) + }) + return units.units.map((unit) => mean(values.get(unit.id)!)) + } + const decisionOptions = { + seed, + resamples, + confidence: intervalConfidence, + threshold: opts.claim?.minimumEffect ?? 0, + } const align = (byScenario: Record, label: string): number[] => { const missing = scenarioIds.filter((id) => !(id in byScenario)) if (missing.length > 0) { @@ -393,6 +465,18 @@ export async function compareOptimizationMethods { try { @@ -441,6 +525,12 @@ export async function compareOptimizationMethods method.winnerSurface), + ]) + : undefined // Reuse one final-test measurement for identical surfaces. This avoids duplicate // spend and prevents model variance from inventing a difference between equal inputs. const baselineArr = align( @@ -468,18 +558,16 @@ export async function compareOptimizationMethods { - const boot = pairedBootstrap(baselineArr, w.arr, { - seed, - resamples, - confidence: intervalConfidence, - statistic: 'mean', - }) + const baselineUnits = aggregateUnits(baselineArr) + const winnerUnits = aggregateUnits(w.arr) + const decision = decidePairedPromotion(baselineUnits, winnerUnits, decisionOptions) const score: OptimizationMethodScore = { name: w.name, - baselineComposite: mean(baselineArr), - winnerComposite: mean(w.arr), - lift: boot.mean, - liftCi: { low: boot.low, high: boot.high }, + baselineComposite: mean(baselineUnits), + winnerComposite: mean(winnerUnits), + lift: decision.delta, + liftCi: { low: decision.low, high: decision.high }, + decision, optimizationCost: w.cost, scenarioScores: scenarioIds.map((scenarioId, index) => ({ scenarioId, @@ -487,6 +575,13 @@ export async function compareOptimizationMethods ({ + unitId: unit.id, + scenarios: unit.observations, + baselineComposite: baselineUnits[index]!, + winnerComposite: winnerUnits[index]!, + lift: winnerUnits[index]! - baselineUnits[index]!, + })), winnerSurface: structuredClone(w.winnerSurface), rank: 0, } @@ -520,35 +615,21 @@ export async function compareOptimizationMethods [w.name, w])) - const bestArr = byName.get(best.name)!.arr + const bestArr = aggregateUnits(byName.get(best.name)!.arr) const pairwise: OptimizationMethodPairwise[] = scores.slice(1).map((other) => { - const otherArr = byName.get(other.name)!.arr + const otherArr = aggregateUnits(byName.get(other.name)!.arr) // before = other, after = best ⇒ delta = best − other on the test set. - const boot = pairedBootstrap(otherArr, bestArr, { - seed, - resamples, - confidence: intervalConfidence, - statistic: 'mean', - }) - // A zero-width interval names no winner. Identical per-scenario deltas make - // every resample identical, so `[g, g]` would declare `best` favored at any - // n on no spread at all; `[0, 0]` already fell through to 'tie'. - const degenerate = - !Number.isFinite(boot.low) || !Number.isFinite(boot.high) || boot.low === boot.high - const favored = degenerate - ? 'tie' - : boot.low > 0 - ? best.name - : boot.high < 0 - ? other.name - : 'tie' + const decision = decidePairedPromotion(otherArr, bestArr, decisionOptions) + const reverse = decidePairedPromotion(bestArr, otherArr, decisionOptions) + const favored = decision.promote ? best.name : reverse.promote ? other.name : null return { a: best.name, b: other.name, - deltaMean: boot.mean, - low: boot.low, - high: boot.high, + deltaMean: decision.delta, + low: decision.low, + high: decision.high, favored, + decision, } }) @@ -566,6 +647,11 @@ export async function compareOptimizationMethods ctx: Parameters['dispatch']>[1], ) => Promise gate: Gate + claim?: EvaluationClaim + finalEvidence?: FinalEvidencePolicy holdout?: 'measured' | 'deferred' label?: string neutralize?: (winner: MutableSurface, baseline: MutableSurface) => MutableSurface @@ -23,6 +32,12 @@ export interface FinalComparisonOptions export async function runFinalComparison( opts: FinalComparisonOptions, ) { + opts = { + ...opts, + judges: opts.judges?.map(captureJudge), + claim: opts.claim && defineEvaluationClaim(opts.claim), + finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence), + } const storage = opts.storage ?? fsCampaignStorage() const costLedger = opts.costLedger ?? @@ -36,6 +51,22 @@ export async function runFinalComparison( // An unchanged selection has nothing to promote, regardless of measurement noise. const winnerIsBaseline = surfaceHash(winnerSurface) === surfaceHash(baselineSurface) const holdoutDeferred = (opts.holdout ?? 'measured') === 'deferred' + if (opts.finalEvidence && holdoutDeferred) { + throw new Error('final evidence requires a measured comparison') + } + const controlSurface = + opts.neutralize && !winnerIsBaseline && !holdoutDeferred + ? structuredClone( + opts.neutralize(structuredClone(winnerSurface), structuredClone(baselineSurface)), + ) + : undefined + const finalEvidence = opts.finalEvidence + ? await exposeFinalEvidence(opts.finalEvidence, opts.claim, finalScenarios, [ + baselineSurface, + winnerSurface, + ...(controlSurface === undefined ? [] : [controlSurface]), + ]) + : undefined // An empty campaign records deferred measurement without dispatching final cases. const baselineOnHoldout = holdoutDeferred @@ -124,11 +155,8 @@ export async function runFinalComparison( let neutralizedJudgeScores: ScoreMap | undefined let neutralizedOnHoldout: CampaignResult | undefined let neutralizedSurface: MutableSurface | undefined - if (opts.neutralize && !winnerIsBaseline && !holdoutDeferred) { - const surface = opts.neutralize( - structuredClone(winnerSurface), - structuredClone(baselineSurface), - ) + if (controlSurface !== undefined) { + const surface = controlSurface neutralizedSurface = surface neutralizedOnHoldout = await runCampaign({ ...opts, @@ -198,6 +226,8 @@ export async function runFinalComparison( : renderSurfaceDiff(winnerSurface, baselineSurface) return { + ...(opts.claim ? { claim: opts.claim } : {}), + ...(finalEvidence ? { finalEvidence } : {}), baselineOnHoldout, winnerOnHoldout, ...(neutralizedOnHoldout && neutralizedSurface diff --git a/src/campaign/presets/run-improvement-loop.ts b/src/campaign/presets/run-improvement-loop.ts index dfabc00f..2695910d 100644 --- a/src/campaign/presets/run-improvement-loop.ts +++ b/src/campaign/presets/run-improvement-loop.ts @@ -1,3 +1,4 @@ +import { defineEvaluationClaim, type EvaluationClaim } from '../../experiment/claim' /** * Run a caller-owned candidate generator, compare its winner with the starting * surface on separate cases, apply a release rule, and optionally open a pull @@ -5,6 +6,14 @@ */ import { openAutoPr } from '../auto-pr' +import { + assertIndependentEvaluationSplit, + captureFinalEvidencePolicy, + type FinalEvidencePolicy, + type FinalEvidenceUse, + reserveFinalEvidence, +} from '../final-evidence' +import { captureJudge } from '../judge-snapshot' import { resolveRunDir } from '../run-dir' import { createRunCostLedger, fsCampaignStorage } from '../storage' import type { CampaignResult, Gate, MutableSurface, Scenario } from '../types' @@ -25,6 +34,8 @@ export type RunImprovementLoopOptions< /** Holdout scenarios kept OUT of the training optimization pool — used * ONLY to score baseline vs winner for the gate. */ holdoutScenarios: TScenario[] + claim?: EvaluationClaim + finalEvidence?: FinalEvidencePolicy /** Holdout policy. Default `'measured'`: baseline + winner are re-scored on * `holdoutScenarios` and the gate decides on that held-out comparison. * `'deferred'`: the improvement-set (search) campaigns run exactly as usual, @@ -63,6 +74,8 @@ export interface RunImprovementLoopResult neutralizedOnHoldout?: CampaignResult neutralizedSurface?: MutableSurface gateResult: Awaited['decide']>> + claim?: EvaluationClaim + finalEvidence?: FinalEvidenceUse /** Present iff the loop ran with `holdout: 'deferred'`. When set, * `baselineOnHoldout`/`winnerOnHoldout` are the shared EMPTY campaign (zero * cells dispatched) and the gate verdict is the forced `'hold'`. */ @@ -81,6 +94,18 @@ export interface RunImprovementLoopResult export async function runImprovementLoop( opts: RunImprovementLoopOptions, ): Promise> { + opts = { + ...opts, + judges: opts.judges?.map(captureJudge), + ...(opts.claim || opts.finalEvidence + ? { + claim: opts.claim && defineEvaluationClaim(opts.claim), + finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence), + scenarios: structuredClone(opts.scenarios), + holdoutScenarios: structuredClone(opts.holdoutScenarios), + } + : {}), + } // ── Safety pre-flight ───────────────────────────────────────────── if ((opts as { autoOnPromote?: string }).autoOnPromote === 'config') { throw new Error( @@ -134,6 +159,19 @@ export async function runImprovementLoop( const dispatchTimeoutMs = opts.dispatchTimeoutMs ?? DEFAULT_DISPATCH_TIMEOUT_MS // ── (1) optimization loop produces a winner ──────────────────────── + if (opts.claim?.generalization === 'new-units') { + assertIndependentEvaluationSplit(opts.claim, opts.holdoutScenarios, opts.scenarios) + } + if (opts.finalEvidence) { + if (opts.holdout === 'deferred') + throw new Error('final evidence requires a measured comparison') + await reserveFinalEvidence( + opts.finalEvidence, + opts.claim, + opts.holdoutScenarios, + opts.scenarios, + ) + } const optimization = await runOptimization({ ...opts, dispatchTimeoutMs, costLedger }) const comparison = await runFinalComparison({ ...opts, diff --git a/src/campaign/provenance.ts b/src/campaign/provenance.ts index a5be7110..0ca2b21c 100644 --- a/src/campaign/provenance.ts +++ b/src/campaign/provenance.ts @@ -27,6 +27,7 @@ import { createHash } from 'node:crypto' import { join } from 'node:path' import type { CostReceipt } from '../cost-ledger' +import { defineEvaluationClaim, type EvaluationClaim } from '../experiment/claim' import type { HostedClient } from '../hosted/client' import type { EvalRunCellScore, @@ -43,7 +44,12 @@ import type { } from './presets/compare-optimization-methods' import type { RunImprovementLoopResult } from './presets/run-improvement-loop' import { campaignCellExecutionEvidence, projectCampaignCellQuality } from './run-record' -import { campaignMeanComposite, campaignMeanCompositeOrNull } from './score-utils' +import { + type CampaignComparisonUnits, + campaignMeanComposite, + campaignMeanCompositeOrNull, + pairedCampaignComposites, +} from './score-utils' import type { CampaignStorage } from './storage' import { renderSurfaceDiff, @@ -116,6 +122,7 @@ export interface LoopProvenanceEvidence { splitDigest: `sha256:${string}` baselineCampaignDigest: `sha256:${string}` winnerCampaignDigest: `sha256:${string}` + observations?: CampaignComparisonUnits neutralized?: { contentHash: `sha256:${string}` campaignDigest: `sha256:${string}` @@ -161,6 +168,7 @@ export interface LoopProvenanceRecord { evidence: LoopProvenanceEvidence /** Baseline composite on the search split that generated the candidates. */ baselineSearchComposite: number + claim?: EvaluationClaim /** The gate verdict — decision + reasons + contributing gates + delta. */ gate: { decision: GateDecision @@ -216,6 +224,9 @@ export interface BuildLoopProvenanceArgs holdout?: 'measured' | 'deferred' baselineOnHoldout: CampaignResult winnerOnHoldout: CampaignResult + claim?: EvaluationClaim + /** Required with a measured claim because campaign identities redact source fields. */ + independentUnitByScenarioId?: ReadonlyMap neutralizedSurface?: MutableSurface neutralizedOnHoldout?: CampaignResult /** Settled run-wide receipts — agent calls are the source for backend provenance. */ @@ -234,6 +245,7 @@ export interface LoopProvenanceArgsFromResult totalCostUsd: number totalDurationMs: number + independentUnitByScenarioId?: ReadonlyMap } /** One translation from a completed improvement loop into durable evidence. */ @@ -264,6 +276,10 @@ export function loopProvenanceArgsFromResult( - campaign: CampaignResult, -): number { - return campaignMeanComposite(campaign) -} - /** Build the durable provenance record from a completed loop result. */ export function buildLoopProvenanceRecord( args: BuildLoopProvenanceArgs, @@ -408,6 +418,10 @@ export function buildLoopProvenanceRecord } const holdoutDeferred = args.holdout === 'deferred' + const claim = args.claim ? defineEvaluationClaim(args.claim) : undefined + if (claim && !holdoutDeferred && args.independentUnitByScenarioId === undefined) { + throw new Error('buildLoopProvenanceRecord: a measured claim requires its independent-unit map') + } if (args.baselineOnHoldout.splitDigest !== args.winnerOnHoldout.splitDigest) { throw new Error('buildLoopProvenanceRecord: baseline and winner use different holdout splits') } @@ -429,16 +443,30 @@ export function buildLoopProvenanceRecord 'buildLoopProvenanceRecord: a deferred holdout cannot include a neutralized measurement', ) } - const holdoutMeasurement = holdoutDeferred - ? { kind: 'deferred' as const } - : { - kind: 'measured' as const, - baseline: meanHoldoutComposite(args.baselineOnHoldout), - winner: meanHoldoutComposite(args.winnerOnHoldout), - ...(args.neutralizedOnHoldout - ? { neutralized: meanHoldoutComposite(args.neutralizedOnHoldout) } - : {}), - } + const holdoutScores = holdoutDeferred + ? undefined + : pairedCampaignComposites( + args.baselineOnHoldout, + args.winnerOnHoldout, + args.independentUnitByScenarioId, + ) + const holdoutMeasurement = + holdoutScores === undefined + ? { kind: 'deferred' as const } + : { + kind: 'measured' as const, + baseline: holdoutScores.beforeMean, + winner: holdoutScores.afterMean, + ...(args.neutralizedOnHoldout + ? { + neutralized: pairedCampaignComposites( + args.baselineOnHoldout, + args.neutralizedOnHoldout, + args.independentUnitByScenarioId, + ).afterMean, + } + : {}), + } const diff = surfaceContentHash(args.baselineSurface) === surfaceContentHash(args.winnerSurface) @@ -463,6 +491,7 @@ export function buildLoopProvenanceRecord splitDigest: args.baselineOnHoldout.splitDigest, baselineCampaignDigest: campaignMeasurementDigest(args.baselineOnHoldout), winnerCampaignDigest: campaignMeasurementDigest(args.winnerOnHoldout), + ...(holdoutScores ? { observations: holdoutScores.observations } : {}), ...(args.neutralizedSurface && args.neutralizedOnHoldout && holdoutMeasurement.kind === 'measured' && @@ -482,6 +511,7 @@ export function buildLoopProvenanceRecord ), }, baselineSearchComposite, + ...(claim ? { claim } : {}), gate: { decision: args.gate.decision, reasons: args.gate.reasons, diff --git a/src/campaign/score-utils.ts b/src/campaign/score-utils.ts index 8b95e1b9..1a4cba69 100644 --- a/src/campaign/score-utils.ts +++ b/src/campaign/score-utils.ts @@ -6,8 +6,80 @@ */ import { EMITTED_EVIDENCE_MAX_CHARS } from '../reflective-mutation' +import { aggregatePairedHoldout, pairHoldout } from './gates/statistical-heldout' import { projectCampaignCellQuality } from './run-record' -import type { CampaignResult, Scenario } from './types' +import type { CampaignResult, JudgeScore, Scenario } from './types' + +export interface CampaignComparisonUnits { + pairedCellN: number + unitIds: string[] + /** Cells with no complete quality score on either arm. */ + unscoredCellIds: string[] + /** Present when repetitions and source variants share an independent unit. */ + independentUnitByScenarioId?: Record +} + +/** Reduce complete paired cells using the same observation units as held-out inference. */ +export function pairedCampaignComposites( + baseline: CampaignResult, + candidate: CampaignResult, + independentUnitByScenarioId?: ReadonlyMap, +): { + before: number[] + after: number[] + beforeMean: number + afterMean: number + observations: CampaignComparisonUnits +} { + const scoresByCell = (campaign: CampaignResult) => { + const scores = new Map>() + for (const cell of campaign.cells) { + if (scores.has(cell.cellId)) { + throw new Error(`pairedCampaignComposites: duplicate cell '${cell.cellId}'`) + } + const quality = projectCampaignCellQuality(cell) + scores.set(cell.cellId, quality.score === undefined ? {} : quality.successfulJudgeScores) + } + return scores + } + const scenarioIds = new Set( + [...baseline.cells, ...candidate.cells].map((cell) => cell.scenarioId), + ) + const paired = pairHoldout( + scoresByCell(candidate), + scoresByCell(baseline), + scenarioIds, + (score) => score.composite, + ) + const observations = aggregatePairedHoldout(paired, independentUnitByScenarioId) + const scoredCellIds = new Set(paired.cellIds) + if (observations.before.length === 0) { + throw new Error('pairedCampaignComposites: campaigns have no paired quality scores') + } + return { + before: observations.before, + after: observations.after, + beforeMean: + observations.before.reduce((sum, score) => sum + score, 0) / observations.before.length, + afterMean: + observations.after.reduce((sum, score) => sum + score, 0) / observations.after.length, + observations: { + pairedCellN: paired.cellIds.length, + unitIds: observations.unitIds, + unscoredCellIds: baseline.cells + .filter((cell) => !scoredCellIds.has(cell.cellId)) + .map((cell) => cell.cellId) + .sort(), + ...(independentUnitByScenarioId + ? { + independentUnitByScenarioId: Object.fromEntries( + [...scenarioIds].sort().map((id) => [id, independentUnitByScenarioId.get(id)!]), + ), + } + : {}), + }, + } +} /** Mean composite across cells with complete task-quality evidence. * Partial judge results remain on their cells but never enter this value. diff --git a/src/contract/analyze-runs.ts b/src/contract/analyze-runs.ts index f1aa4f19..65346143 100644 --- a/src/contract/analyze-runs.ts +++ b/src/contract/analyze-runs.ts @@ -23,6 +23,7 @@ import type { AnalystRegistry } from '../analyst/registry' import type { AnalystFinding } from '../analyst/types' import { welchsTTest } from '../baseline' +import { aggregatePairedHoldout } from '../campaign/gates/statistical-heldout' import { checkCanaries } from '../contamination-guard' import type { DatasetScenario } from '../dataset' import { continuousAgreement } from '../judge-calibration' @@ -78,6 +79,9 @@ export interface AnalyzeRunsOptions { * Unmatched rows remain visible in the lift result. */ baselineCandidateId?: string candidateCandidateId?: string + /** Average matched run scores within these units before estimating lift. + * Raw run distributions and unmatched-run counts remain unchanged. */ + independentUnitByScenarioId?: ReadonlyMap /** Canary scenarios — checked against every run's raw output for * holdout contamination. */ canaryScenarios?: DatasetScenario[] @@ -192,7 +196,13 @@ export async function analyzeRuns(opts: AnalyzeRunsOptions): Promise, ): LiftInsight | undefined { let bId = baselineId let cId = candidateId @@ -891,9 +902,35 @@ function computeLift( const scoredBaseline = baseline.filter((run) => Number.isFinite(compositeOf(run, split))) const scoredCandidate = candidate.filter((run) => Number.isFinite(compositeOf(run, split))) const pairing = pairRunRecords(scoredBaseline, scoredCandidate) - const pairedBaseline = pairing.pairs.map((pair) => compositeOf(pair.baseline, split)) - const pairedCandidate = pairing.pairs.map((pair) => compositeOf(pair.treatment, split)) + let pairedBaseline = pairing.pairs.map((pair) => compositeOf(pair.baseline, split)) + let pairedCandidate = pairing.pairs.map((pair) => compositeOf(pair.treatment, split)) if (pairedBaseline.length === 0) return undefined + const grouped = independentUnitByScenarioId + ? aggregatePairedHoldout( + { + before: pairedBaseline, + after: pairedCandidate, + // Pairing is already fixed by experiment, scenario, and seed. + // The ordinal labels each matched observation for the shared reducer. + cellIds: pairing.pairs.map((pair, index) => `${pair.pairKey}:${index}`), + }, + new Map( + pairing.pairs.map((pair) => [ + pair.pairKey, + independentUnitByScenarioId.get(pair.baseline.scenarioId)!, + ]), + ), + ) + : undefined + if (grouped) { + pairedBaseline = grouped.before + pairedCandidate = grouped.after + } + const reverseInferredArms = + grouped !== undefined && + (!baselineId || !candidateId) && + mean(pairedBaseline) > mean(pairedCandidate) + if (reverseInferredArms) [pairedBaseline, pairedCandidate] = [pairedCandidate, pairedBaseline] const baselineMean = mean(pairedBaseline) const candidateMean = mean(pairedCandidate) @@ -923,10 +960,13 @@ function computeLift( ci95: [bootstrap.low, bootstrap.high], pValue: tTest.p, n: pairedBaseline.length, + ...(grouped ? { pairedRunN: pairing.pairs.length, independentUnitIds: grouped.unitIds } : {}), minimumRequired: BOOTSTRAP_GATE_MIN_N, decisionEligible: bootstrap.gateEligible, - unpairedBaseline: pairing.unpairedBaseline.length, - unpairedCandidate: pairing.unpairedTreatment.length, + unpairedBaseline: (reverseInferredArms ? pairing.unpairedTreatment : pairing.unpairedBaseline) + .length, + unpairedCandidate: (reverseInferredArms ? pairing.unpairedBaseline : pairing.unpairedTreatment) + .length, cohensD: d, mde, requiredN, diff --git a/src/contract/index.ts b/src/contract/index.ts index 6c93a200..38d09d09 100644 --- a/src/contract/index.ts +++ b/src/contract/index.ts @@ -52,6 +52,7 @@ export type { // ── Campaign primitives ────────────────────────────────────────────── export { campaignSplitDigest } from '../campaign/coverage' +export type { FinalEvidencePolicy, FinalEvidenceUse } from '../campaign/final-evidence' export { type RunEvalOptions, runEval } from '../campaign/presets/run-eval' export { type RunImprovementLoopOptions, diff --git a/src/contract/insight-report.ts b/src/contract/insight-report.ts index 387f4d84..b603c96a 100644 --- a/src/contract/insight-report.ts +++ b/src/contract/insight-report.ts @@ -287,8 +287,12 @@ export interface LiftInsight { /** Paired-t-test p-value; null when the delta is a non-zero constant, where * the t statistic is undefined. */ pValue: number | null - /** Number of paired observations. */ + /** Number of paired observations after any declared unit aggregation. */ n: number + /** Raw matched run count before declared unit aggregation. */ + pairedRunN?: number + /** Independent units used for the lift estimate; absent for ungrouped runs. */ + independentUnitIds?: string[] /** Minimum paired observations required before the interval can drive a decision. */ minimumRequired: number /** diff --git a/src/contract/measured-comparison.test.ts b/src/contract/measured-comparison.test.ts index f2eed65f..81ae8aac 100644 --- a/src/contract/measured-comparison.test.ts +++ b/src/contract/measured-comparison.test.ts @@ -379,6 +379,9 @@ function executionEvidence(input: { }, `outcomes/${executionId}.json`, ) + const startedAtMs = 1_000 + input.benchmarkCell.repetition * 100 + const durationMs = input.arm === 'baseline' ? 100 : 90 + const gradingStartedAtMs = startedAtMs + durationMs const benchmarkResult = materialEvidence( 'agent-candidate-benchmark-result', { @@ -398,8 +401,8 @@ function executionEvidence(input: { costProvenance: 'observed' as const, }, timing: { - startedAtMs: 2_000, - endedAtMs: 2_000 + (input.graderDurationMs ?? 0), + startedAtMs: gradingStartedAtMs, + endedAtMs: gradingStartedAtMs + (input.graderDurationMs ?? 0), durationMs: input.graderDurationMs ?? 0, }, }, @@ -409,8 +412,6 @@ function executionEvidence(input: { }, `results/${executionId}.json`, ) - const startedAtMs = 1_000 + input.benchmarkCell.repetition * 100 - const durationMs = input.arm === 'baseline' ? 100 : 90 const receipt = addressed({ kind: 'agent-candidate-run' as const, digestAlgorithm: 'rfc8785-sha256' as const, @@ -461,7 +462,7 @@ function profileMeasurements(): Array<{ baseline: PlatformProfileRun candidate: PlatformProfileRun }> { - return [0, 1, 2, 3, 4, 5].map((index) => { + return Array.from({ length: 24 }, (_, index) => { const baseline = 0.2 + (index % 3) * 0.05 // The per-cell gain VARIES (0.52 / 0.50 / 0.48, mean exactly 0.50). A // constant gain makes every bootstrap resample identical, and a zero-width @@ -535,13 +536,13 @@ describe('candidate experiment comparison', () => { preparationCost: { usd: 0.25, provenance: 'observed' }, }) - expect(result.overall).toMatchObject({ baseline: 0.25, candidate: 0.75, delta: 0.5, n: 6 }) + expect(result.overall).toMatchObject({ baseline: 0.25, candidate: 0.75, delta: 0.5, n: 24 }) expect(result.decision.outcome).toBe('ship') expect(result.measurementCost).toMatchObject({ provenance: 'observed' }) - expect(result.measurementCost.usd).toBeCloseTo(0.12, 12) + expect(result.measurementCost.usd).toBeCloseTo(0.48, 12) expect(result.totalCost).toMatchObject({ provenance: 'observed' }) - expect(result.totalCost.usd).toBeCloseTo(0.37, 12) - expect(result.measurementWorkDurationMs).toBe(1_200) + expect(result.totalCost.usd).toBeCloseTo(0.73, 12) + expect(result.measurementWorkDurationMs).toBe(4_800) expect(() => evaluatePairedMeasurements({ measurements: [measurements[0]!, measurements[0]!], @@ -571,7 +572,7 @@ describe('candidate experiment comparison', () => { it('uses observed paired precision instead of baseline-only variance', () => { const template = profileMeasurements()[0]!.baseline - const baselines = [0.1, 0.9, 0.1, 0.9, 0.1, 0.9] + const baselines = Array.from({ length: 24 }, (_, index) => (index % 2 === 0 ? 0.1 : 0.8)) const measurements = baselines.map((baseline, index) => { const run = (score: number): PlatformProfileRun => ({ ...template, @@ -581,7 +582,7 @@ describe('candidate experiment comparison', () => { return { cellId: `paired-precision:${index}`, baseline: run(baseline), - candidate: run(baseline + 0.1), + candidate: run(baseline + 0.1 + ((index % 3) - 1) * 0.002), } }) @@ -598,6 +599,9 @@ describe('candidate experiment comparison', () => { expect(result.overall.confidenceInterval.lower).toBeCloseTo(0.1) expect(result.overall.confidenceInterval.upper).toBeCloseTo(0.1) + expect(result.overall.confidenceInterval.upper).toBeGreaterThan( + result.overall.confidenceInterval.lower, + ) expect(result.power.minimumDetectableDelta).toBeCloseTo(0.05) expect(result.power.sufficient).toBe(true) expect(result.decision.outcome).toBe('ship') @@ -718,7 +722,7 @@ describe('candidate experiment comparison', () => { }) it('runs the exact signed matrix and derives every statistic from Runtime receipts', async () => { - const frozen = experiment(6) + const frozen = experiment(24) const observedSeeds: number[] = [] const run = await runCandidateExperiment({ experiment: frozen, @@ -761,26 +765,30 @@ describe('candidate experiment comparison', () => { }, }) - expect(comparison.overall).toMatchObject({ baseline: 0.25, candidate: 0.75, delta: 0.5, n: 6 }) + expect(comparison.overall).toMatchObject({ baseline: 0.25, candidate: 0.75, delta: 0.5, n: 24 }) expect(comparison.decision.outcome).toBe('ship') expect(comparison.diff).toContain('--- baseline/profile') expect(comparison.diff).toContain('verify every claim') - expect(comparison.measurements).toHaveLength(6) + expect(comparison.measurements).toHaveLength(24) expect(comparison.evaluation).toMatchObject({ preparation: { wallDurationMs: 50, cost: { usd: 0.25, provenance: 'observed' } }, - measurement: { workDurationMs: 1_200, cost: { usd: 0.12, provenance: 'observed' } }, + measurement: { workDurationMs: 4_800, cost: { usd: 0.48, provenance: 'observed' } }, }) - expect(comparison.evaluation.total.cost).toEqual({ usd: 0.37, provenance: 'observed' }) + expect(comparison.evaluation.total.cost).toEqual({ usd: 0.73, provenance: 'observed' }) expect(verifyCandidateExperimentComparison(comparison)).toEqual(comparison) expect(comparison.objectives).toEqual( expect.arrayContaining([ - expect.objectContaining({ kind: 'cost', baseline: 0.01, candidate: 0.01 }), + expect.objectContaining({ + kind: 'cost', + baseline: expect.closeTo(0.01, 12), + candidate: expect.closeTo(0.01, 12), + }), expect.objectContaining({ kind: 'latency', baseline: 105, candidate: 95 }), ]), ) - expect(observedSeeds.sort((left, right) => left - right)).toEqual([ - 101, 101, 102, 102, 103, 103, 104, 104, 105, 105, 106, 106, - ]) + expect(observedSeeds.sort((left, right) => left - right)).toEqual( + Array.from({ length: 24 }, (_, index) => [101 + index, 101 + index]).flat(), + ) }) it('refuses a budgeted suite before dispatch when its signed maximum cannot fit', async () => { @@ -1164,25 +1172,8 @@ describe('candidate experiment comparison', () => { }) /** - * The promotion gate's task check used to require PERFECTION: a candidate - * shipped only if `candidate.passed` held on every benchmark cell. On any - * benchmark hard enough to be worth running nothing passes everything, so the - * bar was unreachable — under the old check the CONSTRUCTED case - * `repairedNotRegressed(8)` below (baseline mean 0.3339, candidate mean 0.8685, - * paired delta +0.5346) would be held with "candidate failed 2 benchmark - * tasks", and the 2 are tasks the baseline failed too. - * - * That case is constructed, not observed: no live run of this gate is on disk. - * An earlier draft of this docblock reported it as a measurement and quoted a - * bootstrap interval that nothing produced; commit d095318 retracted the same - * claim from the source comment, and it is retracted here for the same reason. - * The argument for the change is the unreachable bar itself, visible in the - * predicate, which needs no measured candidate. - * - * The bar is now improvement WITHOUT regression. Every test below pins BOTH - * directions: the case the gate must now let through, and the neighbouring case - * it must still refuse. Each "must still refuse" case is the calibration for the - * check above it — the guard is only proven by being made to FAIL. + * Shared task failures do not block a supported improvement. + * The binary fixtures isolate pass/fail repair from regressions and other release checks. */ describe('promotion gate — improvement without regression, not perfection', () => { interface GateRun { @@ -1247,11 +1238,8 @@ describe('promotion gate — improvement without regression, not perfection', () const repairedNotRegressed = (repaired: number) => Array.from({ length: 10 }, (_, index) => ({ cellId: `task:${index}`, - baseline: gateRun(0.333 + (index % 3) * 0.001, false), - candidate: - index < repaired - ? gateRun(0.98 + (index % 3) * 0.005, true) - : gateRun(0.4 + (index % 2) * 0.01, false), + baseline: gateRun(0, false), + candidate: gateRun(index < repaired ? 1 : 0, index < repaired), })) it('SHIPS a candidate that repairs 8 tasks and still fails 2 the baseline also failed', () => { @@ -1272,8 +1260,8 @@ describe('promotion gate — improvement without regression, not perfection', () index === 9 ? { ...measurement, - baseline: gateRun(0.9, true), - candidate: gateRun(0.2, false), + baseline: gateRun(1, true), + candidate: gateRun(0, false), } : measurement, ) @@ -1295,7 +1283,7 @@ describe('promotion gate — improvement without regression, not perfection', () it('reports repaired / regressed / still-failing on a hold, so a null is separable from a gate artifact', () => { const measurements = repairedNotRegressed(8).map((measurement, index) => index === 9 - ? { ...measurement, baseline: gateRun(0.9, true), candidate: gateRun(0.2, false) } + ? { ...measurement, baseline: gateRun(1, true), candidate: gateRun(0, false) } : measurement, ) @@ -1319,7 +1307,7 @@ describe('promotion gate — improvement without regression, not perfection', () // Every task passes on both arms, so `no-task-regression` passes. The gate // must still refuse: dropping the perfection bar must not turn the gate into // a rubber stamp for a candidate that did not actually improve anything. - const measurements = Array.from({ length: 10 }, (_, index) => ({ + const measurements = Array.from({ length: 25 }, (_, index) => ({ cellId: `flat:${index}`, baseline: gateRun(0.5 + (index % 5) * 0.02, true), candidate: gateRun(0.5 + ((index + 2) % 5) * 0.02, true), @@ -1329,13 +1317,13 @@ describe('promotion gate — improvement without regression, not perfection', () expect(check(result, 'no-task-regression')).toBe(true) expect(check(result, 'paired-significance')).toBe(false) - expect(result.decision.outcome).not.toBe('ship') + expect(result.decision.outcome).toBe('hold') }) it('CALIBRATION — still HOLDS when a benchmark execution did not complete', () => { const measurements = repairedNotRegressed(8).map((measurement, index) => index === 0 - ? { ...measurement, candidate: gateRun(0.98, true, { completed: false }) } + ? { ...measurement, candidate: gateRun(1, true, { completed: false }) } : measurement, ) @@ -1373,7 +1361,7 @@ describe('promotion gate — improvement without regression, not perfection', () }) it('CALIBRATION — still HOLDS a candidate that lifted the score while regressing a critical dimension', () => { - const measurements = Array.from({ length: 10 }, (_, index) => ({ + const measurements = Array.from({ length: 24 }, (_, index) => ({ cellId: `goodhart:${index}`, baseline: { ...gateRun(0.3 + (index % 3) * 0.01, true), @@ -1462,7 +1450,7 @@ describe('promotion gate — the significance reason names the interval that dec sharedScorerChannel: true, }) - it('explains a zero-variance sample instead of claiming its own bound failed to clear', () => { + it('names the insufficient sample while retaining the historical zero-variance observation', () => { // Six cells that all move +2/3. The old reason read "lower bound // 0.6666666666666669 did not clear 0". const result = evaluate( @@ -1470,13 +1458,28 @@ describe('promotion gate — the significance reason names the interval that dec ) const reason = result.decision.reasons[0]! - expect(reason).toMatch(/interval is degenerate at \[0\.6666666666666669, 0\.6666666666666669\]/) - expect(reason).toMatch(/the mean CI collapsed to a point/) + expect(reason).toBe('only 6 paired runs; 20 required') + expect(result.overall.n).toBe(6) + expect(result.overall.delta).toBeCloseTo(2 / 3, 12) + expect(result.overall.confidenceInterval.lower).toBeCloseTo(0.6666666666666669, 12) + expect(result.overall.confidenceInterval.upper).toBeCloseTo(0.6666666666666669, 12) expect(reason).not.toMatch(/did not clear/) // The remedy is the held-out set, not a better candidate. expect(result.decision.outcome).toBe('need_more_work') }) + it('explains a degenerate interval when the mean target has enough paired observations', () => { + const result = evaluate( + Array.from({ length: 24 }, () => ({ baseline: 0.333, candidate: 0.333 + 2 / 3 })), + ) + const reason = result.decision.reasons[0]! + + expect(reason).toMatch(/interval is degenerate at/) + expect(reason).toMatch(/the mean CI collapsed to a point/) + expect(reason).not.toMatch(/did not clear/) + expect(result.decision.outcome).toBe('need_more_work') + }) + it('CALIBRATION — the zero-variance sample is still REFUSED, however large the constant gain', () => { // Same fixture. A constant +2/3 is a huge point estimate and the gate must // still not ship it: the sample carries no information about its own error, @@ -1497,16 +1500,13 @@ describe('promotion gate — the significance reason names the interval that dec it('CALIBRATION — a non-degenerate miss still reports a lower bound, and it is the DECIDING one', () => { // Varied deltas straddling zero: the interval is real, has width, and fails // honestly. This is the branch that must keep saying "did not clear". - const result = evaluate([ - { baseline: 0.5, candidate: 0.56 }, - { baseline: 0.5, candidate: 0.42 }, - { baseline: 0.5, candidate: 0.58 }, - { baseline: 0.5, candidate: 0.41 }, - { baseline: 0.5, candidate: 0.57 }, - { baseline: 0.5, candidate: 0.44 }, - { baseline: 0.5, candidate: 0.55 }, - { baseline: 0.5, candidate: 0.43 }, - ]) + const candidateScores = [0.56, 0.42, 0.58, 0.41, 0.57, 0.44, 0.55, 0.43] + const result = evaluate( + Array.from({ length: 24 }, (_, index) => ({ + baseline: 0.5, + candidate: candidateScores[index % candidateScores.length]!, + })), + ) const reason = result.decision.reasons[0]! expect(reason).toMatch(/^paired mean_bootstrap interval lower bound -?\d/) diff --git a/src/contract/profile-measured-comparison.test.ts b/src/contract/profile-measured-comparison.test.ts index 08f1e69c..908964b4 100644 --- a/src/contract/profile-measured-comparison.test.ts +++ b/src/contract/profile-measured-comparison.test.ts @@ -276,7 +276,7 @@ describe('profile improvement measured comparison', () => { }) it('runs exact profile states through one paired measurement path', async () => { - const frozen = experiment(6) + const frozen = experiment(24) const observed: Array<{ arm: string stateDigest: Sha256Digest @@ -315,7 +315,7 @@ describe('profile improvement measured comparison', () => { }, }) - expect(observed).toHaveLength(12) + expect(observed).toHaveLength(48) expect(observed.filter((entry) => entry.arm === 'baseline')).toEqual( expect.arrayContaining([ expect.objectContaining({ stateDigest: frozen.baseline.stateDigest }), @@ -326,17 +326,17 @@ describe('profile improvement measured comparison', () => { expect.objectContaining({ stateDigest: frozen.candidate.stateDigest }), ]), ) - expect(new Set(observed.map((entry) => entry.cellDigest)).size).toBe(12) - expect(comparison.overall.n).toBe(6) + expect(new Set(observed.map((entry) => entry.cellDigest)).size).toBe(48) + expect(comparison.overall.n).toBe(24) expect(comparison.overall.baseline).toBeCloseTo(0.2) expect(comparison.overall.candidate).toBeCloseTo(0.8) expect(comparison.overall.delta).toBeCloseTo(0.6) expect(comparison.decision.outcome).toBe('ship') expect(comparison.evaluation).toMatchObject({ preparation: { wallDurationMs: 50, cost: { usd: 0.25, provenance: 'observed' } }, - measurement: { workDurationMs: 1_320, cost: { usd: 0.00000132, provenance: 'observed' } }, + measurement: { workDurationMs: 5_280, cost: { usd: 0.00000528, provenance: 'observed' } }, }) - expect(comparison.evaluation.total.cost).toEqual({ usd: 0.25000132, provenance: 'observed' }) + expect(comparison.evaluation.total.cost).toEqual({ usd: 0.25000528, provenance: 'observed' }) expect(comparison.diff).toContain('add-source-and-uncertainty') expect(verifyAgentProfileImprovementExperimentComparison(comparison)).toEqual(comparison) }) diff --git a/src/contract/self-improve-fixed-roster.test.ts b/src/contract/self-improve-fixed-roster.test.ts new file mode 100644 index 00000000..39d10e7a --- /dev/null +++ b/src/contract/self-improve-fixed-roster.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from 'vitest' +import { inMemoryCampaignStorage } from '../campaign/storage' +import type { JudgeConfig, Scenario } from '../campaign/types' +import type { EvaluationClaim } from '../experiment/claim' +import { selfImprove } from './self-improve' + +interface FixedCase extends Scenario { + sourceId: string +} + +const scenarios: FixedCase[] = ['train', 'selection', 'final-a', 'final-b'].map((id) => ({ + id, + kind: 'fixture', + sourceId: 'one-fixed-source', +})) +const claim: EvaluationClaim = { + use: 'development', + population: { id: 'fixed', description: 'These fixed regression cases' }, + samplingFrame: 'Fixed regression roster', + independentUnit: 'sourceId', + generalization: 'fixed-roster', +} +const judge: JudgeConfig<{ quality: number }, FixedCase> = { + name: 'quality', + dimensions: [{ key: 'quality', description: 'Observed fixture quality' }], + score: ({ artifact }) => ({ + composite: artifact.quality, + dimensions: { quality: artifact.quality }, + notes: '', + }), +} + +describe('fixed-roster development splits', () => { + it.each(['explicit', 'automatic'] as const)( + 'keeps one-source development usable with %s partitions', + async (partition) => { + let optimized = 0 + const result = await selfImprove({ + scenarios, + claim, + judge, + baselineSurface: 'BASE', + ...(partition === 'explicit' + ? { + selectionScenarios: [scenarios[1]!], + budget: { holdoutScenarios: scenarios.slice(2) }, + } + : {}), + agent: async (surface) => ({ quality: surface === 'BASE' ? 0 : 1 }), + method: { + name: 'fixed', + optimize: async (input) => { + optimized += 1 + expect(input.trainScenarios.length).toBeGreaterThan(0) + expect(input.selectionScenarios.length).toBeGreaterThan(0) + const trainIds = new Set(input.trainScenarios.map((scenario) => scenario.id)) + expect(input.selectionScenarios.every((scenario) => !trainIds.has(scenario.id))).toBe( + true, + ) + return { + winnerSurface: 'SELECTED', + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, + } + }, + }, + model: 'fixture@2026-09-13', + expectUsage: 'off', + storage: inMemoryCampaignStorage(), + runDir: `mem://fixed-roster-${partition}`, + }) + + expect(optimized).toBe(1) + expect(result.winner.surface).toBe('SELECTED') + expect(result.lift).toBe(1) + expect(result.gateDecision).toBe('hold') + expect(result.finalEvidence).toBeUndefined() + expect( + result.raw.gateResult.contributingGates.find( + (check) => check.name === 'heldout-significance', + )?.detail, + ).toMatchObject({ n: 1, observationUnit: 'registered', fewRuns: true }) + }, + ) +}) diff --git a/src/contract/self-improve-method.ts b/src/contract/self-improve-method.ts index cdb4935f..589d883d 100644 --- a/src/contract/self-improve-method.ts +++ b/src/contract/self-improve-method.ts @@ -1,5 +1,6 @@ import { openAutoPr } from '../campaign/auto-pr' import { campaignSplitDigest } from '../campaign/coverage' +import { evaluationUnitMap } from '../campaign/final-evidence' import { defaultProductionGate } from '../campaign/gates/default-production-gate' import { executeOptimizationMethod } from '../campaign/optimization-method' import { @@ -15,12 +16,14 @@ import { campaignCellJudgeDimensions, campaignCellTaskScore, } from '../campaign/run-record' +import type { CampaignComparisonUnits } from '../campaign/score-utils' import type { SearchHistoryCoverageRow } from '../campaign/search-history-receipt' import type { CampaignStorage } from '../campaign/storage' import { surfaceContentHash, surfaceHash } from '../campaign/surface-identity' import type { Scenario } from '../campaign/types' import type { CostLedgerHandle, CostLedgerSummary } from '../cost-ledger' import { createCampaignEvidenceReceipt } from '../experiment/campaign-evidence' +import type { EvaluationClaim } from '../experiment/claim' import type { EvidenceReceipt } from '../experiment/evidence-receipt' import { createHostedClient } from '../hosted/client' import type { EvalRunGenerationSnapshot } from '../hosted/types' @@ -30,7 +33,7 @@ import type { SelfImproveOptions, SelfImproveProposerResult, } from './self-improve' -import { cellsToRunRecords, meanComposite } from './self-improve-reporting' +import { cellsToRunRecords, pairedCompositeSummary } from './self-improve-reporting' export interface SelfImproveMethodProvenance { schema: 'tangle.method-improvement' @@ -42,6 +45,7 @@ export interface SelfImproveMethodProvenance { winnerContentHash: string diff: string optimizationMethod: NonNullable['optimization']> + claim?: EvaluationClaim evidence: { trainSplitDigest: `sha256:${string}` selectionSplitDigest: `sha256:${string}` @@ -50,6 +54,7 @@ export interface SelfImproveMethodProvenance { winnerCampaignDigest: `sha256:${string}` costReceiptsDigest: `sha256:${string}` neutralizedCampaignDigest?: `sha256:${string}` + holdoutObservations?: CampaignComparisonUnits } gate: Awaited>['gateResult'] holdout?: 'deferred' @@ -121,10 +126,7 @@ export async function runSelfImproveMethod ({ ...dimension })), - } + const judge = opts.judge const { selected, cost: methodCost, @@ -176,9 +178,12 @@ export async function runSelfImproveMethod({ holdoutScenarios: holdout, - deltaThreshold: 0.05, + deltaThreshold: opts.claim?.minimumEffect ?? 0.05, + independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : undefined, }) const comparison = await runFinalComparison({ + claim: opts.claim, + finalEvidence: opts.finalEvidence, baselineSurface, winnerSurface: selected.winnerSurface, scenarios: holdout, @@ -237,10 +242,15 @@ export async function runSelfImproveMethod = { ...(evidence ? { evidence } : {}), mode: 'method', + ...(opts.claim ? { claim: opts.claim } : {}), + ...(comparison.finalEvidence ? { finalEvidence: comparison.finalEvidence } : {}), searchHistoryCoverage: history, baseline, winner: { @@ -377,6 +395,7 @@ export async function runSelfImproveMethod ({ index, surfaceHash: surfaceHash(surface), @@ -393,7 +412,7 @@ export async function runSelfImproveMethod): { - compositeMean: number - perScenario: Record -} { - const perScenario: Record = {} - const values: number[] = [] - for (const [id, agg] of Object.entries(byScenario)) { - perScenario[id] = agg.meanComposite - values.push(agg.meanComposite) - } +export function pairedCompositeSummary( + baseline: CampaignResult, + winner: CampaignResult, + independentUnitByScenarioId?: ReadonlyMap, +) { + const scores = pairedCampaignComposites(baseline, winner, independentUnitByScenarioId) + const perScenario = (campaign: CampaignResult) => + Object.fromEntries( + Object.entries(campaign.aggregates.byScenario).map(([id, aggregate]) => [ + id, + aggregate.meanComposite, + ]), + ) return { - compositeMean: values.length === 0 ? 0 : values.reduce((s, v) => s + v, 0) / values.length, - perScenario, + baseline: { compositeMean: scores.beforeMean, perScenario: perScenario(baseline) }, + winner: { compositeMean: scores.afterMean, perScenario: perScenario(winner) }, + baselineComposites: scores.before, + observations: scores.observations, } } diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index 6a88b61a..6a2fccfd 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -1,3 +1,4 @@ +import { defineEvaluationClaim, type EvaluationClaim } from '../experiment/claim' /** * Run one complete improvement job. * @@ -8,8 +9,17 @@ */ import type { ProposalFinding } from '../analyst/types' +import { + assertIndependentEvaluationSplit, + captureFinalEvidencePolicy, + evaluationUnitMap, + type FinalEvidencePolicy, + type FinalEvidenceUse, + reserveFinalEvidence, +} from '../campaign/final-evidence' import { defaultProductionGate } from '../campaign/gates/default-production-gate' import { type PowerPreflight, powerPreflight } from '../campaign/gates/power-preflight' +import { captureJudge } from '../campaign/judge-snapshot' import type { OptimizationMethod, OptimizationMethodProvenance, @@ -68,7 +78,7 @@ import { type SelfImproveMethodProvenance, type SelfImproveMethodResult, } from './self-improve-method' -import { cellsToRunRecords, meanComposite } from './self-improve-reporting' +import { cellsToRunRecords, pairedCompositeSummary } from './self-improve-reporting' export type { SelfImproveMethodProvenance, SelfImproveMethodResult } from './self-improve-method' @@ -105,7 +115,7 @@ export interface SelfImproveBudget { * static holdout scenario and recording a meaningless lift. Unless * `holdoutScenarios` reserves an explicit set, ALL scenarios train. */ holdout?: 'measured' | 'deferred' - /** Per-scenario replicates per cell — raises bootstrap-CI tightness. Default 1. */ + /** Repeated executions per scenario. A claim's independent-unit count stays unchanged. Default 1. */ reps?: number /** DEPTH dial forwarded to the proposer's `propose()` as * `ctx.maxImprovementShots` — max iterations an agentic candidate generator @@ -327,10 +337,16 @@ export interface SelfImproveOptions searchLedger?: RunOptimizationOptions['searchLedger'] /** Complete-method final measurement receipts; authority and environment remain caller-owned. */ evidence?: CampaignEvidenceContext + /** Declare the population and independent units without requiring fresh data. */ + claim?: EvaluationClaim + /** Opt into durable fresh-evidence consumption for this final comparison. */ + finalEvidence?: FinalEvidencePolicy } export interface SelfImproveProposerResult { mode: 'proposer' + claim?: EvaluationClaim + finalEvidence?: FinalEvidenceUse /** Composite mean across all scenarios, baseline run. When * `budget.holdout === 'deferred'` this is measured on the improvement * (search) split — no holdout campaign ran. */ @@ -396,10 +412,9 @@ export interface SelfImproveProposerResult( searchScenarios: TScenario[], explicitSelection: TScenario[] | undefined, fraction: number, + unitByScenario?: ReadonlyMap, ): { train: TScenario[]; selection: TScenario[] } { if (!Number.isFinite(fraction) || fraction <= 0 || fraction >= 1) { throw new Error('selfImprove: budget.selectionFraction must be in (0, 1)') @@ -534,6 +550,12 @@ function splitMethodPartitions( if (train.length === 0) { throw new Error('selfImprove: method train split is empty') } + if (unitByScenario) { + const selectedUnits = new Set([...selectionIds].map((id) => unitByScenario.get(id))) + if (train.some((scenario) => selectedUnits.has(unitByScenario.get(scenario.id)))) { + throw new Error('selfImprove: training and selection share independent units') + } + } return { train, selection: explicitSelection.map((scenario) => byId.get(scenario.id)!), @@ -542,6 +564,10 @@ function splitMethodPartitions( if (searchScenarios.length < 2) { throw new Error('selfImprove: method requires at least two non-final scenarios') } + if (unitByScenario) { + const split = splitTrainHoldout(searchScenarios, fraction, unitByScenario) + return { train: split.train, selection: split.holdout } + } const sorted = [...searchScenarios].sort( (a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id), ) @@ -568,14 +594,25 @@ function stableScenarioHash(value: string): number { return hash } -/** - * Deterministic train/holdout split by a stable hash of `scenario.id`, - * so the same scenario set always splits the same way across runs. - */ +/** Deterministic split by scenario identity, or by source identity for new-unit claims. */ function splitTrainHoldout( scenarios: TScenario[], fraction: number, + unitByScenario?: ReadonlyMap, ): { train: TScenario[]; holdout: TScenario[] } { + if (unitByScenario) { + const units = [...new Set(scenarios.map((scenario) => unitByScenario.get(scenario.id)!))].sort( + (a, b) => stableScenarioHash(a) - stableScenarioHash(b) || (a < b ? -1 : a > b ? 1 : 0), + ) + if (units.length < 2) + throw new Error('selfImprove: splitting needs at least two independent units') + const count = Math.max(1, Math.min(units.length - 1, Math.round(units.length * fraction))) + const finalUnits = new Set(units.slice(0, count)) + return { + holdout: scenarios.filter((scenario) => finalUnits.has(unitByScenario.get(scenario.id)!)), + train: scenarios.filter((scenario) => !finalUnits.has(unitByScenario.get(scenario.id)!)), + } + } const sorted = [...scenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id)) const nHoldout = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction))) return { @@ -642,6 +679,19 @@ export async function selfImprove( | SelfImproveMethodOptions | SelfImproveProposerOptions, ): Promise> { + opts = { + ...opts, + judge: captureJudge(opts.judge), + ...(opts.claim || opts.finalEvidence + ? { + claim: opts.claim && defineEvaluationClaim(opts.claim), + finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence), + scenarios: structuredClone(opts.scenarios), + selectionScenarios: opts.selectionScenarios && structuredClone(opts.selectionScenarios), + budget: opts.budget && structuredClone(opts.budget), + } + : {}), + } const startedAt = Date.now() const requestedRunDir = opts.runDir ?? @@ -681,6 +731,9 @@ async function runSelfImprove( const holdoutMode = budget.holdout ?? 'measured' const holdoutDeferred = holdoutMode === 'deferred' const expectUsage = opts.expectUsage ?? 'assert' + const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.scenarios) : undefined + const splitUnitByScenario = + opts.claim?.generalization === 'new-units' ? unitByScenario : undefined // Deferred holdout without an explicitly reserved set trains on EVERYTHING: // there is no held-out measurement in this run, so carving out a fraction @@ -694,7 +747,7 @@ async function runSelfImprove( } : holdoutDeferred ? { train: opts.scenarios, holdout: [] as TScenario[] } - : splitTrainHoldout(opts.scenarios, holdoutFraction) + : splitTrainHoldout(opts.scenarios, holdoutFraction, splitUnitByScenario) if (train.length === 0) { throw new Error( @@ -705,11 +758,20 @@ async function runSelfImprove( throw new Error('selfImprove: holdout split is empty. Pass more scenarios.') } + if (opts.claim?.generalization === 'new-units') { + assertIndependentEvaluationSplit(opts.claim, holdout, train) + } + if (opts.finalEvidence) { + if (holdoutDeferred) throw new Error('final evidence requires measured holdout') + await reserveFinalEvidence(opts.finalEvidence, opts.claim, holdout, train) + } + if (opts.method) { const partitions = splitMethodPartitions( train, opts.selectionScenarios, budget.selectionFraction ?? 0.25, + splitUnitByScenario, ) return runSelfImproveMethod({ opts: { ...opts, method: opts.method }, @@ -738,7 +800,8 @@ async function runSelfImprove( opts.gate ?? defaultProductionGate({ holdoutScenarios: holdout, - deltaThreshold: 0.05, + deltaThreshold: opts.claim?.minimumEffect ?? 0.05, + independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : undefined, }) if (opts.onProgress) { @@ -759,6 +822,8 @@ async function runSelfImprove( reps: budget.reps, maxImprovementShots: budget.maxImprovementShots, holdoutScenarios: holdout, + claim: opts.claim, + finalEvidence: opts.finalEvidence, holdout: holdoutMode, gate, neutralize: opts.neutralize, @@ -792,28 +857,21 @@ async function runSelfImprove( const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout - const baseline = meanComposite(reportBaselineCampaign.aggregates.byScenario) - const winnerStats = meanComposite(reportWinnerCampaign.aggregates.byScenario) - - // Power analysis from the baseline holdout cells — the number that says whether - // this budget could ship ANY effect. Attached to every result; loud when the - // search was structurally unable to promote (that spend should not repeat). + const reportUnitMap = opts.claim + ? evaluationUnitMap(opts.claim, holdoutDeferred ? train : holdout) + : undefined + const report = pairedCompositeSummary(reportBaselineCampaign, reportWinnerCampaign, reportUnitMap) + const baseline = report.baseline + const winnerStats = report.winner + + // Repetitions refine a declared unit's mean without increasing the sample size. let power: PowerPreflight | undefined - const baselineHoldoutComposites = result.baselineOnHoldout.cells - .filter((cell) => !cell.error) - .map((cell) => { - const scores = Object.values(cell.judgeScores) - return scores.length === 0 - ? Number.NaN - : scores.reduce((sum, s) => sum + s.composite, 0) / scores.length - }) - .filter((v) => Number.isFinite(v)) + const baselineHoldoutComposites = holdoutDeferred ? [] : report.baselineComposites if (baselineHoldoutComposites.length >= 3) { - // selfImprove's holdout is scored by the SAME judge as the gate — the - // shared-channel case by construction (S1c): flag it so the MDE reads as a - // lower bound and nobody buys reps expecting them to fix judge bias. + // The shared judge's systematic bias remains outside this variance estimate. power = powerPreflight({ baselineComposites: baselineHoldoutComposites, + deltaThreshold: opts.claim?.minimumEffect ?? 0.05, sharedScorerChannel: true, }) if (opts.onProgress) { @@ -875,6 +933,8 @@ async function runSelfImprove( )), ], baselineCandidateId: 'baseline', + independentUnitByScenarioId: reportUnitMap, + decisionThreshold: opts.claim?.minimumEffect ?? 0.05, ...(reportWinnerCampaign === reportBaselineCampaign ? {} : { candidateCandidateId: 'winner' }), }) @@ -891,6 +951,7 @@ async function runSelfImprove( costReceipts: costLedger.list(), totalCostUsd: totalCost, totalDurationMs: durationMs, + independentUnitByScenarioId: holdoutDeferred ? undefined : reportUnitMap, }), storage, hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : undefined, @@ -898,6 +959,8 @@ async function runSelfImprove( if (opts.onProvenance) opts.onProvenance(provenance) const summary: SelfImproveProposerResult = { + ...(opts.claim ? { claim: opts.claim } : {}), + ...(result.finalEvidence ? { finalEvidence: result.finalEvidence } : {}), mode: 'proposer', baseline, winner: { diff --git a/src/experiment/ast.ts b/src/experiment/ast.ts index b04ea455..6c554fac 100644 --- a/src/experiment/ast.ts +++ b/src/experiment/ast.ts @@ -396,6 +396,8 @@ export type IntervalSpec = | { kind: 'cluster-bootstrap' clusterBy: string + /** Registered numeric or boolean outcome field, including a prepared paired difference. */ + value: string resamples: number seed: number level: number @@ -409,10 +411,45 @@ export interface ComputedInterval { level: number } +/** Shared by registration and direct execution so malformed intervals cannot produce evidence. */ +export function intervalSpecProblems(spec: IntervalSpec): string[] { + if (spec === null || typeof spec !== 'object' || Array.isArray(spec)) { + return ['interval must be an object'] + } + if (spec.kind !== 'cluster-bootstrap' && spec.kind !== 'clopper-pearson') { + return ['interval kind must be cluster-bootstrap or clopper-pearson'] + } + const problems: string[] = [] + if (!Number.isFinite(spec.level) || spec.level <= 0 || spec.level >= 1) { + problems.push('level must be a finite number between zero and one') + } + if (spec.kind === 'cluster-bootstrap') { + for (const field of ['clusterBy', 'value'] as const) { + const path = spec[field] + if ( + typeof path !== 'string' || + path.split('.').some((segment) => segment.length === 0 || segment.trim() !== segment) + ) { + problems.push(`${field} must be a nonempty dot-separated field path`) + } + } + if ( + !Number.isSafeInteger(spec.resamples) || + spec.resamples < 1 || + spec.resamples > 0xffff_ffff + ) { + problems.push('resamples must be a positive integer within the supported array length') + } + if (!Number.isSafeInteger(spec.seed)) problems.push('seed must be a safe integer') + if (spec.method !== 'percentile') problems.push('method must be percentile') + } + return problems +} + /** * Execute an interval spec. * - * Cluster-bootstrap resamples whole clusters of the per-row `value` field and + * Cluster-bootstrap resamples whole clusters of its registered `value` field and * takes percentile bounds of the pooled mean. Clopper-Pearson computes the * exact binomial interval and requires `successes`/`trials` evidence instead * of rows. @@ -420,25 +457,49 @@ export interface ComputedInterval { export function computeInterval( spec: IntervalSpec, evidence: - | { kind: 'rows'; rows: readonly EvidenceRecord[]; value: string } + | { kind: 'rows'; rows: readonly EvidenceRecord[] } | { kind: 'binomial'; successes: number; trials: number }, ): ComputedInterval { + const problems = intervalSpecProblems(spec) + if (problems.length > 0) throw new ValidationError(`computeInterval: ${problems.join('; ')}`) + if (evidence === null || typeof evidence !== 'object' || Array.isArray(evidence)) { + throw new ValidationError('computeInterval: evidence must be an object') + } if (spec.kind === 'cluster-bootstrap') { if (evidence.kind !== 'rows') { throw new ValidationError('computeInterval: cluster-bootstrap requires row evidence') } + if ('value' in evidence) { + throw new ValidationError( + 'computeInterval: register value in the interval spec, not row evidence', + ) + } + if (!Array.isArray(evidence.rows)) + throw new ValidationError('computeInterval: rows must be an array') const clusters = new Map() - for (const row of evidence.rows) { - const cluster = String(readField(row, spec.clusterBy)) + for (const [index, row] of evidence.rows.entries()) { + if (row === null || typeof row !== 'object' || Array.isArray(row)) { + throw new ValidationError(`computeInterval: row ${index} must be an object`) + } + const cluster = readField(row, spec.clusterBy) + if ( + !(typeof cluster === 'string' && cluster.length > 0 && cluster.trim() === cluster) && + !(typeof cluster === 'number' && Number.isFinite(cluster)) + ) { + throw new ValidationError( + `computeInterval: cluster field '${spec.clusterBy}' must be a nonempty string or finite number on row ${index}`, + ) + } + const clusterKey = `${typeof cluster}:${cluster}` const value = readNumericOutcome( - readField(row, evidence.value), + readField(row, spec.value), 'computeInterval cluster-bootstrap', - evidence.value, + spec.value, `cluster '${cluster}'`, ) - const bucket = clusters.get(cluster) + const bucket = clusters.get(clusterKey) if (bucket) bucket.push(value) - else clusters.set(cluster, [value]) + else clusters.set(clusterKey, [value]) } const clusterValues = [...clusters.entries()] .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) @@ -459,6 +520,11 @@ export function computeInterval( count += cluster.length } means[draw] = sum / count + if (!Number.isFinite(means[draw])) { + throw new ValidationError( + 'computeInterval: cluster mean overflowed the finite numeric range', + ) + } } means.sort((a, b) => a - b) const alpha = 1 - spec.level @@ -475,7 +541,12 @@ export function computeInterval( throw new ValidationError('computeInterval: clopper-pearson requires binomial evidence') } const { successes, trials } = evidence - if (!Number.isInteger(successes) || !Number.isInteger(trials) || trials <= 0 || successes < 0) { + if ( + !Number.isSafeInteger(successes) || + !Number.isSafeInteger(trials) || + trials <= 0 || + successes < 0 + ) { throw new ValidationError( `computeInterval: clopper-pearson needs 0 <= successes <= trials, got ${successes}/${trials}`, ) @@ -677,6 +748,8 @@ export type ValidityGate = | { kind: 'power-floor' target: number + /** Minimum worthwhile effect; must be represented exactly in effectGrid. */ + minimumEffect: number effectGrid: number[] sim: { trials: number; resamples: number; seed: number } } @@ -760,8 +833,8 @@ export function evaluateIdentityGate( } /** - * The design's power curve must reach the registered target at some grid - * effect. The curve must cover the registered effect grid exactly — a curve + * The design's power at minimumEffect must reach the registered target. + * The curve must cover the registered effect grid exactly — a curve * computed on a different grid is different evidence and is refused. */ export function evaluatePowerFloorGate( @@ -769,26 +842,72 @@ export function evaluatePowerFloorGate( gate: Extract, curve: readonly { effect: number; power: number }[], ): GateResult { + const problems = powerFloorProblems(gate) + if (problems.length > 0) { + throw new ValidationError(`evaluatePowerFloorGate: ${problems.join('; ')}`) + } + if (curve.some((point) => !Number.isFinite(point.power) || point.power < 0 || point.power > 1)) { + throw new ValidationError('evaluatePowerFloorGate: powers must be finite and in [0,1]') + } const byEffect = new Map(curve.map((point) => [point.effect, point.power])) + if (byEffect.size !== curve.length) { + throw new ValidationError('evaluatePowerFloorGate: curve contains duplicate effects') + } const missing = gate.effectGrid.filter((effect) => !byEffect.has(effect)) if (missing.length > 0) { throw new ValidationError( `evaluatePowerFloorGate: curve does not cover registered effects [${missing.join(', ')}]`, ) } + const extra = curve.filter((point) => !gate.effectGrid.includes(point.effect)) + if (extra.length > 0) { + throw new ValidationError( + `evaluatePowerFloorGate: curve contains unregistered effects [${extra.map((point) => point.effect).join(', ')}]`, + ) + } const powers = gate.effectGrid.map((effect) => byEffect.get(effect)!) const maxPower = Math.max(...powers) + const powerAtMinimumEffect = byEffect.get(gate.minimumEffect)! return { id, - passed: maxPower >= gate.target, + passed: powerAtMinimumEffect >= gate.target, evidence: { target: gate.target, + minimumEffect: gate.minimumEffect, + powerAtMinimumEffect, maxPower, curve: gate.effectGrid.map((effect) => ({ effect, power: byEffect.get(effect)! })), }, } } +/** Shared by seal validation and direct execution; malformed designs never pass. */ +export function powerFloorProblems(gate: Extract): string[] { + const problems: string[] = [] + if (!Number.isFinite(gate.target) || gate.target <= 0 || gate.target > 1) { + problems.push('power target must be in (0,1]') + } + if (!Number.isFinite(gate.minimumEffect) || gate.minimumEffect <= 0) { + problems.push('minimumEffect must be positive and finite') + } + if (gate.effectGrid.length === 0 || gate.effectGrid.some((effect) => !Number.isFinite(effect))) { + problems.push('effectGrid must contain finite effects') + } + if (new Set(gate.effectGrid).size !== gate.effectGrid.length) { + problems.push('effectGrid contains duplicate effects') + } + if (!gate.effectGrid.includes(gate.minimumEffect)) { + problems.push('effectGrid must contain minimumEffect exactly; no interpolation is assumed') + } + for (const field of ['trials', 'resamples'] as const) { + if (!Number.isInteger(gate.sim[field]) || gate.sim[field] <= 0) { + problems.push(`power simulation ${field} must be a positive integer`) + } + } + if (!Number.isInteger(gate.sim.seed)) problems.push('power simulation seed must be an integer') + return problems +} + /** Checks as prerequisites: any named gate failing refuses the spend. */ export interface HaltRule { when: { kind: 'any-gate-failed'; gates: string[] } diff --git a/src/experiment/claim.ts b/src/experiment/claim.ts new file mode 100644 index 00000000..01ee0d12 --- /dev/null +++ b/src/experiment/claim.ts @@ -0,0 +1,65 @@ +import { z } from 'zod' +import { ValidationError } from '../errors' +import { compareCodeUnits } from '../ledger-core/canonical' +import { type EvidenceRecord, readField } from './ast' + +const nonEmpty = z + .string() + .min(1) + .refine((value) => value.trim() === value) + +const claimSchema = z + .object({ + use: z.enum(['development', 'comparison', 'certification']), + population: z.object({ id: nonEmpty, description: nonEmpty }).strict(), + samplingFrame: nonEmpty, + independentUnit: nonEmpty, + generalization: z.enum(['fixed-roster', 'new-units']), + minimumEffect: z.number().finite().positive().optional(), + }) + .strict() + +/** The population and independent observations a measured result can describe. */ +export type EvaluationClaim = z.infer + +/** Validate a claim before binding it to a sealed experiment or final evidence. */ +export function defineEvaluationClaim(input: EvaluationClaim): EvaluationClaim { + const parsed = claimSchema.safeParse(input) + if (!parsed.success) + throw new ValidationError(`invalid evaluation claim: ${parsed.error.message}`) + return Object.freeze({ ...parsed.data, population: Object.freeze(parsed.data.population) }) +} + +export interface EvaluationUnitSummary { + observations: number + independentUnits: number + units: Array<{ id: string; observations: number }> +} + +/** Repetitions retain their denominator without becoming additional independent units. */ +export function summarizeEvaluationUnits( + claim: EvaluationClaim, + rows: readonly object[], +): EvaluationUnitSummary { + const validated = defineEvaluationClaim(claim) + const counts = new Map() + for (const [index, row] of rows.entries()) { + if (row === null || typeof row !== 'object' || Array.isArray(row)) { + throw new ValidationError(`evaluation claim: row ${index} must be an object`) + } + const value = readField(row as EvidenceRecord, validated.independentUnit) + if (typeof value !== 'string' || !value.trim() || value.trim() !== value) { + throw new ValidationError( + `evaluation claim: row ${index} needs a nonempty string at '${validated.independentUnit}'`, + ) + } + counts.set(value, (counts.get(value) ?? 0) + 1) + } + return { + observations: rows.length, + independentUnits: counts.size, + units: [...counts] + .sort(([left], [right]) => compareCodeUnits(left, right)) + .map(([id, observations]) => ({ id, observations })), + } +} diff --git a/src/experiment/define.ts b/src/experiment/define.ts index 4bd72528..11ccad99 100644 --- a/src/experiment/define.ts +++ b/src/experiment/define.ts @@ -15,9 +15,8 @@ * registered-vs-ran drift is unrepresentable rather than checked. */ -import { createHash } from 'node:crypto' import { ValidationError } from '../errors' -import { canonicalString } from '../ledger-core/canonical' +import { hashCanonical } from '../ledger-core/canonical' import { type AdmissionRule, type BudgetRule, @@ -42,10 +41,12 @@ import { type HaltOutcome, type HaltRule, type IntervalSpec, + intervalSpecProblems, type JsonValue, type MatchedBudgetRule, type NLadderProjection, type Obligation, + powerFloorProblems, projectNLadderBudget, type ReissuePolicy, runSelectionRule, @@ -55,6 +56,12 @@ import { type ValidityGate, } from './ast' import { type ArmRealizedBudget, type MatchedBudgetVerdict, verifyMatchedBudgets } from './budget' +import { + defineEvaluationClaim, + type EvaluationClaim, + type EvaluationUnitSummary, + summarizeEvaluationUnits, +} from './claim' import { type AdmissionExecution, executeAdmissionRule } from './funnel' /** A sealed experiment whose digest no longer matches its spec. */ @@ -101,6 +108,8 @@ export interface SeedDerivation { export interface ExperimentSpec { id: string + /** Population and independent observation unit for the intended use of this result. */ + claim?: EvaluationClaim /** Human prose for the audit trail — never executable. */ hypothesis?: string arms: ArmSpec[] @@ -166,6 +175,7 @@ function conditionRefs(condition: Condition): { */ export function defineExperiment(spec: ExperimentSpec): ExperimentSpec { const problems: string[] = [] + const claim = spec.claim === undefined ? undefined : defineEvaluationClaim(spec.claim) if (!spec.id || spec.id.trim().length === 0) problems.push('id is empty') if (spec.arms.length === 0) problems.push('at least one arm is required') const armIds = new Set() @@ -184,6 +194,29 @@ export function defineExperiment(spec: ExperimentSpec): ExperimentSpec { const selectionNames = new Set(Object.keys(spec.selections ?? {})) const sealedSubsetNames = new Set(Object.keys(spec.sealedSubsets ?? {})) + for (const [name, interval] of Object.entries(spec.intervals ?? {})) { + problems.push( + ...intervalSpecProblems(interval).map((problem) => `interval '${name}': ${problem}`), + ) + } + + for (const [name, gate] of Object.entries(spec.gates ?? {})) { + if (gate.kind !== 'power-floor') continue + problems.push(...powerFloorProblems(gate).map((problem) => `gate '${name}': ${problem}`)) + if (claim?.minimumEffect !== undefined && gate.minimumEffect !== claim.minimumEffect) { + problems.push(`gate '${name}' minimumEffect differs from the evaluation claim`) + } + } + if (claim?.generalization === 'new-units') { + for (const [name, interval] of Object.entries(spec.intervals ?? {})) { + if (interval?.kind === 'cluster-bootstrap' && interval.clusterBy !== claim.independentUnit) { + problems.push( + `interval '${name}' must resample '${claim.independentUnit}' from the evaluation claim`, + ) + } + } + } + const checkCondition = (condition: Condition, where: string): void => { const refs = conditionRefs(condition) for (const name of refs.intervals) { @@ -263,7 +296,7 @@ export function defineExperiment(spec: ExperimentSpec): ExperimentSpec { if (problems.length > 0) { throw new ValidationError(`defineExperiment('${spec.id}'): ${problems.join('; ')}`) } - return deepFreeze(structuredClone(spec)) + return deepFreeze(structuredClone({ ...spec, ...(claim ? { claim } : {}) })) } function deepFreeze(value: T): T { @@ -288,22 +321,14 @@ export interface SealAmendment { digest: string } -/** - * Digest scheme of a sealed experiment. Both are sha256 hex over the - * serialized spec and differ only in the serialization: `'sha256-rfc8785'` is - * RFC 8785 canonical JSON, `'sha256-content'` is key-sorted `JSON.stringify`. - */ -export type SealAlgo = 'sha256-content' | 'sha256-rfc8785' +/** SHA-256 over the RFC 8785 canonical JSON encoding of the experiment. */ +export type SealAlgo = 'sha256-rfc8785' export interface SealedExperiment { spec: ExperimentSpec /** sha256 over the serialized spec, under the scheme `algo` names. */ digest: string - /** - * Digest scheme of `digest`. `'sha256-rfc8785'` is what {@link sealExperiment} - * emits; `'sha256-content'` is read-only, carried by seals from an earlier - * release, and still verifies. - */ + /** Required digest scheme. Unsupported or missing schemes cannot execute. */ algo: SealAlgo sealedAt: string /** Digest of the original registration, before any amendment. */ @@ -317,7 +342,7 @@ export async function sealExperiment( options: { sealedAt?: string } = {}, ): Promise { const validated = defineExperiment(spec) - const digest = specDigest(validated, 'sha256-rfc8785') + const digest = specDigest(validated) return { spec: validated, digest, @@ -337,58 +362,47 @@ export async function amendExperiment( sealed: SealedExperiment, amendment: { spec: ExperimentSpec; reason: string; blind: string[]; at?: string }, ): Promise { - await assertSealIntact(sealed) - const validated = defineExperiment(amendment.spec) - const digest = specDigest(validated, 'sha256-rfc8785') + const captured = structuredClone(sealed) + const requested = structuredClone(amendment) + await assertSealIntact(captured) + const validated = defineExperiment(requested.spec) + const digest = specDigest(validated) return { spec: validated, digest, algo: 'sha256-rfc8785', - sealedAt: sealed.sealedAt, - initialDigest: sealed.initialDigest, + sealedAt: captured.sealedAt, + initialDigest: captured.initialDigest, amendments: [ - ...sealed.amendments, + ...captured.amendments, { - at: amendment.at ?? new Date().toISOString(), - reason: amendment.reason, - blind: [...amendment.blind], + at: requested.at ?? new Date().toISOString(), + reason: requested.reason, + blind: [...requested.blind], digest, }, ], } } -/** True when the sealed digest still matches the spec it carries, under the - * scheme the seal declares. */ +/** True when the supported seal matches its canonical spec. */ export async function verifySealedExperiment(sealed: SealedExperiment): Promise { - return specDigest(sealed.spec, sealed.algo) === sealed.digest + if (sealed.algo !== 'sha256-rfc8785') return false + try { + return specDigest(sealed.spec) === sealed.digest + } catch { + return false + } } -/** - * Serialize a spec under `algo` and digest it. `'sha256-content'` is read-only - * — it exists so a seal written by an earlier release still verifies, and no - * path that WRITES a digest may pass it. - */ -function specDigest(spec: ExperimentSpec, algo: SealAlgo): string { - const serialized = - algo === 'sha256-rfc8785' ? canonicalString(spec) : JSON.stringify(sortKeysDeep(spec)) - return createHash('sha256').update(serialized, 'utf8').digest('hex') -} - -function sortKeysDeep(value: unknown): unknown { - if (value === null || typeof value !== 'object') return value - if (Array.isArray(value)) return value.map(sortKeysDeep) - const out: Record = {} - for (const key of Object.keys(value as Record).sort()) { - out[key] = sortKeysDeep((value as Record)[key]) - } - return out +function specDigest(spec: ExperimentSpec): string { + return hashCanonical(spec).slice('sha256:'.length) } async function assertSealIntact(sealed: SealedExperiment): Promise { if (!(await verifySealedExperiment(sealed))) { throw new SealIntegrityError( - `sealed experiment '${sealed.spec.id}' digest ${sealed.digest} does not match its spec — the registration was tampered with`, + `sealed experiment '${sealed.spec.id}' has an unsupported digest scheme or its digest does not match its canonical spec`, ) } } @@ -413,6 +427,8 @@ export type GateEvidence = */ export interface RegisteredExperiment { readonly sealed: SealedExperiment + /** Count independent units separately from repeated observations under the sealed claim. */ + units(records: readonly EvidenceRecord[]): EvaluationUnitSummary /** Execute the registered decision rule on derived quantities. */ decide(evidence: DerivedQuantities): DecisionOutcome /** Run the registered admission funnel over evidence rows. */ @@ -435,9 +451,9 @@ export interface RegisteredExperiment { interval( name: string, evidence: - | { kind: 'rows'; rows: readonly EvidenceRecord[]; value: string } - | { kind: 'binomial'; successes: number; trials: number }, - ): ComputedInterval + | { kind: 'rows'; rows: readonly EvidenceRecord[] } + | { kind: 'binomial'; successes: number; trials: number; unitIds?: readonly string[] }, + ): ComputedInterval & { units?: EvaluationUnitSummary } } /** @@ -448,8 +464,10 @@ export interface RegisteredExperiment { export async function openSealedExperiment( sealed: SealedExperiment, ): Promise { - await assertSealIntact(sealed) - const spec = sealed.spec + const captured = structuredClone(sealed) + await assertSealIntact(captured) + const spec = defineExperiment(captured.spec) + const frozenSeal = deepFreeze({ ...captured, spec }) const need = (value: T | undefined, what: string): T => { if (value === undefined) { throw new ValidationError(`experiment '${spec.id}' registered no ${what}`) @@ -457,7 +475,8 @@ export async function openSealedExperiment( return value } return { - sealed, + sealed: frozenSeal, + units: (records) => summarizeEvaluationUnits(need(spec.claim, 'evaluation claim'), records), decide: (evidence) => executeDecisionRule(spec.decision, evidence), admit: (records) => executeAdmissionRule(need(spec.admission, 'admission rule'), records), select: (name, records, options) => { @@ -531,7 +550,30 @@ export async function openSealedExperiment( verifyMatchedBudgets(need(spec.matchedBudget, 'matched-budget rule'), arms), estimate: (name, rows) => computeEstimand(need(spec.estimands?.[name], `estimand '${name}'`), rows), - interval: (name, evidence) => - computeInterval(need(spec.intervals?.[name], `interval '${name}'`), evidence), + interval: (name, evidence) => { + const interval = need(spec.intervals?.[name], `interval '${name}'`) + const claim = spec.claim + let units: EvaluationUnitSummary | undefined + if (claim && evidence.kind === 'rows') { + units = summarizeEvaluationUnits(claim, evidence.rows) + } else if (claim?.generalization === 'new-units' && evidence.kind === 'binomial') { + if ( + evidence.unitIds === undefined || + evidence.unitIds.length !== evidence.trials || + new Set(evidence.unitIds).size !== evidence.trials || + evidence.unitIds.some((id) => typeof id !== 'string' || !id.trim() || id.trim() !== id) + ) { + throw new ValidationError( + 'claimed binomial interval needs one unique unitId per independent trial', + ) + } + units = { + observations: evidence.trials, + independentUnits: evidence.trials, + units: evidence.unitIds.map((id) => ({ id, observations: 1 })), + } + } + return { ...computeInterval(interval, evidence), ...(units ? { units } : {}) } + }, } } diff --git a/src/experiment/evidence-receipt.test.ts b/src/experiment/evidence-receipt.test.ts index eafa9bbd..94c1b4e9 100644 --- a/src/experiment/evidence-receipt.test.ts +++ b/src/experiment/evidence-receipt.test.ts @@ -68,9 +68,10 @@ describe('evidence receipts', () => { it('refuses legacy report attestations whose provenance was never envelope-bound', () => { const receipt = createEvidenceReceipt(binding, provenance) const { envelopeHash: _envelopeHash, ...legacy } = receipt.attestation - expect(verifyEvidenceReceipt({ ...receipt, attestation: legacy })).toEqual({ + const input = JSON.parse(JSON.stringify({ ...receipt, attestation: legacy })) + expect(verifyEvidenceReceipt(input)).toEqual({ valid: false, - reason: 'evidence receipt provenance is not bound by an attestation envelope', + reason: 'attestation envelope hash is missing or invalid', }) }) diff --git a/src/experiment/evidence-receipt.ts b/src/experiment/evidence-receipt.ts index 97afa84b..82896f38 100644 --- a/src/experiment/evidence-receipt.ts +++ b/src/experiment/evidence-receipt.ts @@ -115,10 +115,8 @@ export function createEvidenceReceipt( } /** - * Verify promotion-grade evidence. Generic report attestation keeps a legacy read path, - * but an EvidenceReceipt never accepts unbound provenance: changing the evaluator code, - * model versions, input commitment provenance, or creation record must invalidate the - * evidence rather than merely annotating it as legacy. + * Verify evidence identity and its provenance envelope. Changing the evaluator, + * model versions, input commitment, or creation record invalidates the evidence. */ export function verifyEvidenceReceipt(receipt: EvidenceReceipt): EvidenceReceiptVerification { if (receipt.binding.schemaVersion !== EVIDENCE_RECEIPT_VERSION) { @@ -146,18 +144,7 @@ export function verifyEvidenceReceipt(receipt: EvidenceReceipt): EvidenceReceipt return { valid: false, reason: error instanceof Error ? error.message : String(error) } } - const verification = verifyAttestation(receipt.binding, receipt.attestation) - if (!verification.valid) return verification - if ( - verification.legacyUnboundProvenance === true || - receipt.attestation.envelopeHash === undefined - ) { - return { - valid: false, - reason: 'evidence receipt provenance is not bound by an attestation envelope', - } - } - return { valid: true } + return verifyAttestation(receipt.binding, receipt.attestation) } /** diff --git a/src/experiment/final-evidence.ts b/src/experiment/final-evidence.ts new file mode 100644 index 00000000..ede66ded --- /dev/null +++ b/src/experiment/final-evidence.ts @@ -0,0 +1,261 @@ +import { z } from 'zod' +import { compareCodeUnits, type LedgerHash } from '../ledger-core/canonical' +import { FileLedgerJournal, type LedgerJournalCodec } from '../ledger-core/journal' + +const identity = z + .string() + .min(1) + .refine((value) => value.trim() === value) +const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/) +const reservationSchema = z + .object({ + requestId: identity, + claimDigest: digest, + populationId: identity, + inputDigest: digest, + unitIds: z.array(identity).min(1), + }) + .strict() + +/** One final dataset reserved for one adaptive decision, across processes and campaigns. */ +export type FinalEvidenceReservation = z.infer + +const measurementSchema = z + .object({ + evaluatorDigest: digest, + candidateDigests: z.array(digest).min(1), + }) + .strict() + +export type FinalEvidenceMeasurement = z.infer + +const eventSchema = z.discriminatedUnion('kind', [ + z + .object({ + kind: z.literal('reserved'), + eventId: identity, + reservation: reservationSchema, + }) + .strict(), + z + .object({ + kind: z.literal('exposed'), + eventId: identity, + requestId: identity, + measurement: measurementSchema, + }) + .strict(), +]) + +type Event = z.infer +const schema = 'agent-eval.final-evidence.v1' as const +const entrySchema = z + .object({ + schema: z.literal(schema), + sequence: z.number().int().nonnegative(), + previousHash: digest.nullable(), + event: eventSchema, + entryHash: digest, + }) + .strict() + +export interface FinalEvidenceRecord { + reservation: FinalEvidenceReservation + reservationHash: LedgerHash + exposure: { measurement: FinalEvidenceMeasurement; entryHash: LedgerHash } | null +} + +export type FinalEvidenceOutcome = + | { succeeded: true; value: T } + | { succeeded: false; error: { kind: 'conflict' | 'invalid' | 'unavailable'; message: string } } + +export interface FinalEvidenceLedger { + reserve( + input: FinalEvidenceReservation, + ): Promise> + expose( + requestId: string, + measurement: FinalEvidenceMeasurement, + ): Promise> + read(): Promise> +} + +/** Retains the distinction between invalid input, consumed evidence, and unavailable storage. */ +export class FinalEvidenceError extends Error { + constructor( + readonly kind: 'conflict' | 'invalid' | 'unavailable', + message: string, + ) { + super(message) + this.name = 'FinalEvidenceError' + } +} + +/** A consumed or conflicting dataset cannot authorize another adaptive decision. */ +export class FinalEvidenceConflictError extends FinalEvidenceError { + constructor(message: string) { + super('conflict', message) + this.name = 'FinalEvidenceConflictError' + } +} + +function invalid(message: string): Error { + return new TypeError(`final evidence: ${message}`) +} + +function normalizeReservation(input: FinalEvidenceReservation): FinalEvidenceReservation { + const parsed = reservationSchema.parse(input) + if (new Set(parsed.unitIds).size !== parsed.unitIds.length) { + throw invalid('unitIds must be unique independent source identities') + } + return { ...parsed, unitIds: [...parsed.unitIds].sort(compareCodeUnits) } +} + +function normalizeMeasurement(input: FinalEvidenceMeasurement): FinalEvidenceMeasurement { + const parsed = measurementSchema.parse(input) + return { + ...parsed, + candidateDigests: [...new Set(parsed.candidateDigests)].sort(compareCodeUnits), + } +} + +function codec(): LedgerJournalCodec<{ schema: typeof schema }, Event, FinalEvidenceRecord[]> { + return { + subject: 'final evidence ledger', + header: { schema }, + integrityError: (message, options) => new Error(message, options), + conflictError: (message) => new FinalEvidenceConflictError(message), + parseEntry: (raw, context) => { + const decoded = entrySchema.safeParse(raw) + if (!decoded.success) + throw new Error( + `final evidence ledger ${context.path}:${context.line} is invalid: ${decoded.error.message}`, + ) + const parsed = decoded.data + return { + ...parsed, + previousHash: parsed.previousHash as LedgerHash | null, + entryHash: parsed.entryHash as LedgerHash, + } + }, + checkEntryHeader: (entry) => { + if (entry.schema !== schema) throw invalid('unsupported ledger schema') + }, + createProjector: () => { + const records = new Map() + const owners = new Map() + const inputOwners = new Map() + return { + apply: (entry) => { + const event = entry.event + if (event.kind === 'reserved') { + const reservation = normalizeReservation(event.reservation) + if ( + event.eventId !== `reserve:${reservation.requestId}` || + records.has(reservation.requestId) + ) { + throw invalid('invalid or duplicate reservation identity') + } + const inputOwner = inputOwners.get(reservation.inputDigest) + if (inputOwner !== undefined) { + throw new FinalEvidenceConflictError( + `final input is already reserved by '${inputOwner}'`, + ) + } + for (const unitId of reservation.unitIds) { + const owner = owners.get(unitId) + if (owner !== undefined) { + throw new FinalEvidenceConflictError( + `final unit '${unitId}' is already reserved by '${owner}'`, + ) + } + owners.set(unitId, reservation.requestId) + } + inputOwners.set(reservation.inputDigest, reservation.requestId) + records.set(reservation.requestId, { + reservation, + reservationHash: entry.entryHash, + exposure: null, + }) + } else { + const record = records.get(event.requestId) + if ( + event.eventId !== `expose:${event.requestId}` || + !record || + record.exposure !== null + ) { + throw invalid('exposure requires one unexposed reservation') + } + record.exposure = { + measurement: normalizeMeasurement(event.measurement), + entryHash: entry.entryHash, + } + } + }, + finish: () => [...records.values()], + } + }, + } +} + +async function outcome(operation: () => Promise): Promise> { + try { + return { succeeded: true, value: await operation() } + } catch (error) { + return { + succeeded: false, + error: { + kind: + error instanceof FinalEvidenceConflictError + ? 'conflict' + : error instanceof TypeError || error instanceof z.ZodError + ? 'invalid' + : 'unavailable', + message: error instanceof Error ? error.message : String(error), + }, + } + } +} + +/** Uses the shared locked journal and requires its trusted head on every reopen. */ +export function openFinalEvidenceLedger(options: { path: string }): FinalEvidenceLedger { + if (!options.path.trim()) throw invalid('path is empty') + const journal = new FileLedgerJournal(options.path, codec(), { requireTrustedHead: true }) + return { + reserve: (input) => + outcome(async () => { + const reservation = normalizeReservation(input) + const result = await journal.append( + { + kind: 'reserved', + eventId: `reserve:${reservation.requestId}`, + reservation, + }, + { pinHead: true }, + ) + const record = result.projection.find( + (item) => item.reservation.requestId === reservation.requestId, + ) + if (!record) throw invalid('reserved record is missing after append') + return { record, replayed: !result.appended } + }), + expose: (requestId, input) => + outcome(async () => { + identity.parse(requestId) + const measurement = normalizeMeasurement(input) + const result = await journal.append( + { + kind: 'exposed', + eventId: `expose:${requestId}`, + requestId, + measurement, + }, + { pinHead: true }, + ) + const record = result.projection.find((item) => item.reservation.requestId === requestId) + if (!record) throw invalid('exposed record is missing after append') + return { record, replayed: !result.appended } + }), + read: () => outcome(async () => (await journal.replay()).projection), + } +} diff --git a/src/experiment/index.ts b/src/experiment/index.ts index 5f9b21b4..f741aa08 100644 --- a/src/experiment/index.ts +++ b/src/experiment/index.ts @@ -168,6 +168,12 @@ export { verifyMatchedBudgets, } from './budget' export { type CampaignEvidenceContext, createCampaignEvidenceReceipt } from './campaign-evidence' +export { + defineEvaluationClaim, + type EvaluationClaim, + type EvaluationUnitSummary, + summarizeEvaluationUnits, +} from './claim' // ── Define, seal, amend, execute ───────────────────────────────────── export { type ArmSpec, @@ -211,6 +217,16 @@ export { renderEvidenceIndex, validateEvidenceRegistry, } from './evidence-record' +export { + FinalEvidenceConflictError, + FinalEvidenceError, + type FinalEvidenceLedger, + type FinalEvidenceMeasurement, + type FinalEvidenceOutcome, + type FinalEvidenceRecord, + type FinalEvidenceReservation, + openFinalEvidenceLedger, +} from './final-evidence' // ── Denominator chain ──────────────────────────────────────────────── export { type AdmissionExecution, diff --git a/src/experiment/power.ts b/src/experiment/power.ts index 43d3e115..84b0df53 100644 --- a/src/experiment/power.ts +++ b/src/experiment/power.ts @@ -31,8 +31,10 @@ export class DesignRefusalError extends ValidationError {} export interface ClusteredPowerOptions { /** Rows per independent cluster, e.g. [6, 3, 3, 2]. */ clusterSizes: number[] - /** Per-row effect grid: each effect is P(win) - P(loss) added to the base rates. */ + /** P(win) − P(loss) in signal clusters; noisy clusters retain zero expected contrast. */ effects: number[] + /** Minimum worthwhile effect under the declared outcome model; must occur in effects. */ + minimumEffect: number /** Deterministic seed for outcome draws and bootstrap resampling. */ seed: number /** Simulated experiments per effect. Default 2000. */ @@ -43,11 +45,11 @@ export interface ClusteredPowerOptions { confidence?: number /** Sign-flip alpha the closed-form floor is checked against. Default 0.05. */ alpha?: number - /** Power the design must reach at some grid effect. Default 0.8. */ + /** Power required at minimumEffect. Default 0.8. */ targetPower?: number - /** Base P(row favors treatment) with no effect. Default 0.10 (tie-heavy rows). */ + /** P(row favors treatment) under the zero-effect model. Must equal baseLossRate. Default 0.10. */ baseWinRate?: number - /** Base P(row favors control). Default 0.10. */ + /** P(row favors control) under the zero-effect model. Must equal baseWinRate. Default 0.10. */ baseLossRate?: number /** * Clusters whose rows carry outcome noise instead of signal: each row wins @@ -89,11 +91,13 @@ export interface ClusteredPowerResult { seed: number confidence: number targetPower: number + minimumEffect: number + powerAtMinimumEffect: number curve: ClusteredPowerPoint[] /** Maximum simulated power across the effect grid. */ maxPower: number signFlipFloor: SignFlipFloor - /** True only when the sign-flip floor certifies AND simulation reaches target. */ + /** True when the sign-flip floor certifies and power at minimumEffect reaches target. */ adequate: boolean /** Populated exactly when `adequate` is false. The refusal lives in the artifact. */ refusal: ClusteredPowerRefusal | null @@ -101,7 +105,7 @@ export interface ClusteredPowerResult { /** * Simulate the power of a whole-cluster percentile-bootstrap design and refuse - * a structure that cannot reach the target at any registered effect. + * a structure that cannot reach target power at the declared minimum effect. */ export function clusteredPower(options: ClusteredPowerOptions): ClusteredPowerResult { const clusterSizes = options.clusterSizes @@ -113,6 +117,24 @@ export function clusteredPower(options: ClusteredPowerOptions): ClusteredPowerRe if (options.effects.length === 0) { throw new ValidationError('clusteredPower: effects grid is empty') } + if ( + options.effects.some((effect) => !Number.isFinite(effect) || effect < 0 || effect > 1) || + new Set(options.effects).size !== options.effects.length + ) { + throw new ValidationError('clusteredPower: effects must be unique finite values in [0,1]') + } + if ( + !Number.isFinite(options.minimumEffect) || + options.minimumEffect <= 0 || + options.minimumEffect > 1 + ) { + throw new ValidationError('clusteredPower: minimumEffect must be in (0,1]') + } + if (!options.effects.includes(options.minimumEffect)) { + throw new ValidationError( + 'clusteredPower: effects must contain minimumEffect exactly; no interpolation is assumed', + ) + } if (!Number.isInteger(options.seed)) { throw new ValidationError(`clusteredPower: seed must be an integer, got ${options.seed}`) } @@ -124,20 +146,56 @@ export function clusteredPower(options: ClusteredPowerOptions): ClusteredPowerRe ) } const confidence = options.confidence ?? 0.95 - if (confidence <= 0 || confidence >= 1) { + if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) { throw new ValidationError(`clusteredPower: confidence must be in (0,1), got ${confidence}`) } const alpha = options.alpha ?? 0.05 const targetPower = options.targetPower ?? 0.8 + if (!Number.isFinite(alpha) || alpha <= 0 || alpha >= 1) { + throw new ValidationError('clusteredPower: alpha must be in (0,1)') + } + if (!Number.isFinite(targetPower) || targetPower <= 0 || targetPower > 1) { + throw new ValidationError('clusteredPower: targetPower must be in (0,1]') + } const baseWinRate = options.baseWinRate ?? 0.1 const baseLossRate = options.baseLossRate ?? 0.1 + if ( + !Number.isFinite(baseWinRate) || + !Number.isFinite(baseLossRate) || + baseWinRate < 0 || + baseLossRate < 0 || + baseWinRate + baseLossRate > 1 + ) { + throw new ValidationError( + 'clusteredPower: base win/loss rates must be nonnegative and sum to at most 1', + ) + } + if (baseWinRate !== baseLossRate) { + throw new ValidationError( + 'clusteredPower: the zero-effect model requires equal baseWinRate and baseLossRate', + ) + } const noisy = new Map() for (const cluster of options.noisyClusters ?? []) { - if (cluster.index < 0 || cluster.index >= clusterSizes.length) { + if ( + !Number.isInteger(cluster.index) || + cluster.index < 0 || + cluster.index >= clusterSizes.length + ) { throw new ValidationError( `clusteredPower: noisy cluster index ${cluster.index} outside [0,${clusterSizes.length - 1}]`, ) } + if ( + noisy.has(cluster.index) || + !Number.isFinite(cluster.flipRate) || + cluster.flipRate < 0 || + cluster.flipRate > 1 + ) { + throw new ValidationError( + 'clusteredPower: noisy clusters need unique indices and flipRate in [0,1]', + ) + } noisy.set(cluster.index, cluster.flipRate) } @@ -161,6 +219,7 @@ export function clusteredPower(options: ClusteredPowerOptions): ClusteredPowerRe ) } const maxPower = Math.max(...curve.map((point) => point.power)) + const powerAtMinimumEffect = curve.find((point) => point.effect === options.minimumEffect)!.power const reasons: string[] = [] if (!signFlipFloor.certifiableAtAlpha) { @@ -171,11 +230,11 @@ export function clusteredPower(options: ClusteredPowerOptions): ClusteredPowerRe `${signFlipFloor.minClustersForAlpha} clusters are needed`, ) } - if (maxPower < targetPower) { - const best = curve.reduce((a, b) => (b.power > a.power ? b : a)) + if (powerAtMinimumEffect < targetPower) { reasons.push( - `simulated power tops out at ${maxPower.toFixed(3)} (effect ${best.effect}) across the ` + - `registered grid — below the ${targetPower} target at every effect`, + `simulated power ${powerAtMinimumEffect.toFixed(3)} at minimum worthwhile effect ` + + `${options.minimumEffect} is below target ${targetPower}; maximum grid power ` + + `${maxPower.toFixed(3)} does not establish adequacy at that effect`, ) } const adequate = reasons.length === 0 @@ -187,6 +246,8 @@ export function clusteredPower(options: ClusteredPowerOptions): ClusteredPowerRe seed: options.seed, confidence, targetPower, + minimumEffect: options.minimumEffect, + powerAtMinimumEffect, curve, maxPower, signFlipFloor, @@ -199,7 +260,8 @@ export function clusteredPower(options: ClusteredPowerOptions): ClusteredPowerRe recommendation: `Do not spend on this structure. Add independent clusters (>= ` + `${Math.max(signFlipFloor.minClustersForAlpha, clusterCount)}) or register a design ` + - `whose simulated power reaches ${targetPower}, then re-run clusteredPower.`, + `whose simulated power reaches ${targetPower} at effect ${options.minimumEffect}, ` + + `then re-run clusteredPower.`, }, } } @@ -239,9 +301,10 @@ interface SimulationConfig { } /** - * One effect point. Per row the paired contrast is +1 with probability - * min(baseWin + effect, 1), -1 with the base loss rate (capped by what - * remains), else 0. Noisy clusters draw win and loss independently at their + * One effect point. Loss probability is min(baseLoss, (1-effect)/2) and + * win probability is loss probability plus effect. Their difference equals + * effect, including near the probability boundary. Noisy clusters draw win + * and loss independently at their * flip rate. Each trial computes a whole-cluster percentile bootstrap of the * pooled row mean; the trial counts toward power when the interval excludes * zero. @@ -264,8 +327,8 @@ function simulateEffect(effect: number, config: SimulationConfig): ClusteredPowe const loss = rng() < flipRate ? 1 : 0 sum += win - loss } else { - const winRate = Math.min(1, config.baseWinRate + effect) - const lossRate = Math.min(1 - winRate, config.baseLossRate) + const lossRate = Math.min(config.baseLossRate, (1 - effect) / 2) + const winRate = lossRate + effect const u = rng() sum += u < winRate ? 1 : u < winRate + lossRate ? -1 : 0 } diff --git a/src/held-out-gate.ts b/src/held-out-gate.ts index 3ba66223..0020064a 100644 --- a/src/held-out-gate.ts +++ b/src/held-out-gate.ts @@ -601,6 +601,17 @@ export class HeldOutGate { } } + if (!decision.sufficient) { + return { + promote: false, + candidateId, + baselineId, + evidence, + reason: `few_runs: ${productiveRuns} paired holdout observations < ${decision.minimumPairs} required for ${deltaLabel} estimation`, + rejectionCode: 'few_runs', + } + } + // The exact test is a VETO on the two-point path. Redundant by construction // — `pairedRiskDifferenceExact`'s interval is dual to this p-value, so // `low > 0` already implies `pValue < 1 − confidence` (verified over 8108 diff --git a/src/hosted/schemas.ts b/src/hosted/schemas.ts index 11b1e1d1..93cb7fc4 100644 --- a/src/hosted/schemas.ts +++ b/src/hosted/schemas.ts @@ -283,6 +283,8 @@ const LiftInsightSchema = z ci95: z.tuple([finiteNumber, finiteNumber]), pValue: finiteNumber.min(0).max(1).nullable(), n: nonNegativeInteger, + pairedRunN: nonNegativeInteger.optional(), + independentUnitIds: z.array(nonEmptyString).optional(), minimumRequired: z.number().int().positive(), decisionEligible: z.boolean(), unpairedBaseline: nonNegativeInteger, diff --git a/src/index.ts b/src/index.ts index 054daddf..ee4d01d6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,10 @@ /** * @packageDocumentation * - * Root barrel, tiered: it re-exports only the symbols external consumers - * import from the root specifier, the documented front doors, and the types - * those symbols need. Everything else is reachable via its subpath only - * (`/contract`, `/campaign`, `/analyst`, `/traces`, `/reporting`, `/rl`, - * `/prm`, `/meta-eval`, `/wire`, `/testing`, ...). + * Common entry points and their public types. + * Scenario, JudgeScore, and GateDecision match the /contract campaign types. + * Specialist APIs live under /experiment, /meta-eval, /analyst, /traces, + * /reporting, /rl, and the other package exports. */ // ── contract ────────────────────────────────────────────────────────── @@ -38,38 +37,52 @@ export { toAgentProfileJson, verifyAgentProfileCell, } from './agent-profile-cell' - +export type { GateDecision, JudgeScore, Scenario } from './campaign/types' export type { AnalyzeRunsOptions } from './contract/analyze-runs' export { analyzeRuns } from './contract/analyze-runs' - -export type { DefineAgentEvalOptions, DefinedAgentEval } from './contract/define-agent-eval' +export type { + AgentEvalAgent, + AgentEvalEvaluateOptions, + AgentEvalImproveOptions, + DefineAgentEvalOptions, + DefinedAgentEval, +} from './contract/define-agent-eval' export { defineAgentEval } from './contract/define-agent-eval' - export type { InsightReport } from './contract/insight-report' - export type { + SelfImproveBudget, SelfImproveMethodOptions, SelfImproveMethodProvenance, SelfImproveMethodResult, SelfImproveOptions, + SelfImproveProgressEvent, SelfImproveProposerOptions, SelfImproveProposerResult, SelfImproveResult, } from './contract/self-improve' export { selfImprove } from './contract/self-improve' - export type { DatasetManifest, DatasetScenario, DatasetSplit } from './dataset' export * as profile from './profile/index' export type { + ArtifactCheck, + ArtifactResult, CheckResult, + CollectedArtifacts, CompletionCriterion, DriverState, + JudgeFn, + JudgeInput, JudgeRubric, - JudgeScore, + JudgeScore as DimensionJudgeScore, PersonaConfig, ProductClientConfig, RouteMap, - Scenario, + RubricDimension, + Scenario as ProductScenario, + ScenarioFile, + ScenarioResult, + Turn, + TurnResult, } from './types' // One verdict vocabulary (docs/verdicts.md): every verification path lands // in DefaultVerdict; `certification` names who certified — the strategy @@ -793,7 +806,7 @@ export type { export { deployGateLayer, viteDeployRunner, wranglerDeployRunner } from './deploy-gate-layer' export type { - GateDecision, + GateDecision as HeldOutGateDecision, GateEvidence, HeldOutGateConfig, HeldOutGateRejectionCode, diff --git a/src/meta-eval/calibration.test.ts b/src/meta-eval/calibration.test.ts new file mode 100644 index 00000000..3e46e31a --- /dev/null +++ b/src/meta-eval/calibration.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest' +import { type CalibrationPair, calibrationFromPairs } from './index' + +describe('calibrationFromPairs', () => { + it('separates calibrated controls from a confident wrong predictor without storage adapters', () => { + const calibrated: readonly CalibrationPair[] = Object.freeze([ + Object.freeze({ evalScore: 0, outcome: 0 }), + Object.freeze({ evalScore: 1, outcome: 1 }), + ]) + expect(calibrationFromPairs(calibrated, 'confidence', 'success')).toMatchObject({ + n: 2, + ece: 0, + maxGap: 0, + }) + const wrong = calibrationFromPairs( + [ + { evalScore: 1, outcome: 0 }, + { evalScore: 1, outcome: 0 }, + ], + 'confidence', + 'success', + ) + expect(wrong).toMatchObject({ n: 2, ece: 1, maxGap: 1 }) + expect(wrong?.bins).toEqual([{ lower: 1, upper: 1, n: 2, evalMean: 1, outcomeMean: 0, gap: 1 }]) + }) + + it('clips explicitly without changing caller observations or their denominator', () => { + const pairs = Object.freeze([ + Object.freeze({ evalScore: -1, outcome: 1 }), + Object.freeze({ evalScore: 2, outcome: 0 }), + ]) + const report = calibrationFromPairs(pairs, 'confidence', 'success', { + bins: 2, + range: { lo: 0, hi: 1 }, + }) + expect(report).toMatchObject({ n: 2, ece: 1 }) + expect(report?.bins.reduce((sum, bin) => sum + bin.n, 0)).toBe(2) + expect(pairs.map((pair) => pair.evalScore)).toEqual([-1, 2]) + }) + + it.each([ + { evalScore: Number.NaN, outcome: 1 }, + { evalScore: 1, outcome: Number.NaN }, + { evalScore: Number.POSITIVE_INFINITY, outcome: 1 }, + { evalScore: 1, outcome: Number.NEGATIVE_INFINITY }, + ])('refuses nonfinite observations rather than reducing n: %j', (invalid) => { + expect(() => + calibrationFromPairs( + [{ evalScore: 0, outcome: 0 }, invalid, { evalScore: 1, outcome: 1 }], + 'confidence', + 'success', + ), + ).toThrow(/calibration pair 1 must contain finite/) + }) + + it('validates pairs before treating a short input as insufficient evidence', () => { + expect(() => + calibrationFromPairs([{ evalScore: Number.NaN, outcome: 1 }], 'confidence', 'success'), + ).toThrow(/calibration pair 0/) + expect(calibrationFromPairs([], 'confidence', 'success')).toBeNull() + expect(calibrationFromPairs([{ evalScore: 0, outcome: 0 }], 'confidence', 'success')).toBeNull() + }) + + it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'validates bin count %s on direct input before returning null', + (bins) => { + expect(() => calibrationFromPairs([], 'confidence', 'success', { bins })).toThrow( + /bins must be a positive safe integer/, + ) + }, + ) + + it.each([ + { lo: 1, hi: 0 }, + { lo: Number.NaN, hi: 1 }, + { lo: 0, hi: Number.POSITIVE_INFINITY }, + { lo: -Number.MAX_VALUE, hi: Number.MAX_VALUE }, + ])('validates range %j on direct input', (range) => { + expect(() => calibrationFromPairs([], 'confidence', 'success', { range })).toThrow( + /range must have finite ordered bounds/, + ) + }) + + it.each([ + ['', 'success'], + ['confidence', ''], + [' confidence', 'success'], + ['confidence', 'success '], + ])('requires exact nonempty metric identities: %j, %j', (evalMetric, outcomeMetric) => { + expect(() => calibrationFromPairs([], evalMetric, outcomeMetric)).toThrow(/metric/) + }) +}) diff --git a/src/meta-eval/calibration.ts b/src/meta-eval/calibration.ts index e1fa7c08..e2676d47 100644 --- a/src/meta-eval/calibration.ts +++ b/src/meta-eval/calibration.ts @@ -10,6 +10,7 @@ import { runMetricExtractor } from '../trace/query' import type { TraceStore } from '../trace/store' import type { EvalMetricSpec } from './correlation-study' +import { assertUniqueObservationIds, reduceOutcomeMetric } from './outcome-observations' import type { DeploymentOutcome, OutcomeStore } from './outcome-store' export interface CalibrationBin { @@ -29,11 +30,12 @@ export interface CalibrationReport { bins: CalibrationBin[] /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */ ece: number - /** Max bin gap — upper bound on miscalibration. */ + /** Largest observed difference between a bin's mean score and mean outcome. */ maxGap: number } export interface CalibrationOptions { + /** Positive integer; empty bins are omitted. Default 10. */ bins?: number /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */ binning?: 'equal-width' | 'equal-frequency' @@ -53,7 +55,18 @@ export async function calibrationCurve( outcomeMetric: string, options: CalibrationOptions = {}, ): Promise { + const settings = { + ...options, + range: options.range === undefined ? undefined : { ...options.range }, + } + validateCalibrationRequest(evalMetric.id, outcomeMetric, settings) + const extract = evalMetric.extract ?? runMetricExtractor(evalMetric.id) + const metricId = evalMetric.id const runs = await traceStore.listRuns() + assertUniqueObservationIds( + runs.map((run) => run.runId), + 'runId', + ) const outcomes = await outcomeStore.list() const byRun = new Map() for (const o of outcomes) { @@ -62,37 +75,45 @@ export async function calibrationCurve( byRun.set(o.runId, arr) } - const extract = evalMetric.extract ?? runMetricExtractor(evalMetric.id) const pairs: Array<{ x: number; y: number }> = [] for (const run of runs) { const os = byRun.get(run.runId) if (!os?.length) continue const x = await extract(run, traceStore) if (x === null || !Number.isFinite(x)) continue - const latest = [...os].sort((a, b) => b.capturedAt - a.capturedAt)[0]! - const y = latest.metrics[outcomeMetric] - if (typeof y !== 'number' || !Number.isFinite(y)) continue + const y = reduceOutcomeMetric(os, outcomeMetric, 'latest') + if (y === null) continue pairs.push({ x, y }) } if (pairs.length < 2) return null return calibrationFromPairs( pairs.map((p) => ({ evalScore: p.x, outcome: p.y })), - evalMetric.id, + metricId, outcomeMetric, - options, + settings, ) } -function calibrationFromPairs( - inputPairs: CalibrationPair[], +/** Measure already joined observations without constructing trace and outcome stores. */ +export function calibrationFromPairs( + inputPairs: readonly CalibrationPair[], evalMetric: string, outcomeMetric: string, options: CalibrationOptions = {}, ): CalibrationReport | null { - const pairs = inputPairs.filter( - (pair) => Number.isFinite(pair.evalScore) && Number.isFinite(pair.outcome), - ) + validateCalibrationRequest(evalMetric, outcomeMetric, options) + for (const [index, pair] of inputPairs.entries()) { + if ( + pair === null || + typeof pair !== 'object' || + !Number.isFinite(pair.evalScore) || + !Number.isFinite(pair.outcome) + ) { + throw new Error(`calibration pair ${index} must contain finite evalScore and outcome values`) + } + } + const pairs = inputPairs if (pairs.length < 2) return null const numBins = options.bins ?? 10 @@ -100,25 +121,34 @@ function calibrationFromPairs( const xs = pairs.map((p) => p.evalScore) const lo = options.range?.lo ?? Math.min(...xs) const hi = options.range?.hi ?? Math.max(...xs) + const span = hi - lo + if (!Number.isFinite(span)) throw new Error('calibration range span must be finite') + const clipped = pairs.map((pair) => ({ + ...pair, + evalScore: Math.min(hi, Math.max(lo, pair.evalScore)), + })) const bins: CalibrationBin[] = [] - if (binning === 'equal-frequency') { - const sorted = [...pairs].sort((a, b) => a.evalScore - b.evalScore) - const perBin = Math.max(1, Math.floor(sorted.length / numBins)) - for (let i = 0; i < sorted.length; i += perBin) { - const chunk = sorted.slice(i, i + perBin) - if (chunk.length === 0) continue - bins.push(toBin(chunk)) + if (span === 0 || clipped.every((pair) => pair.evalScore === clipped[0]!.evalScore)) { + bins.push(toBin(clipped)) + } else if (binning === 'equal-frequency') { + const sorted = [...clipped].sort((a, b) => a.evalScore - b.evalScore) + const count = Math.min(numBins, sorted.length) + for (let i = 0; i < count; i++) { + const start = Math.floor((i * sorted.length) / count) + const end = Math.floor(((i + 1) * sorted.length) / count) + bins.push(toBin(sorted.slice(start, end))) } } else { - const width = (hi - lo) / numBins - if (width === 0) return null - for (let i = 0; i < numBins; i++) { - const binLo = lo + i * width - const binHi = i === numBins - 1 ? hi + 1e-9 : lo + (i + 1) * width - const chunk = pairs.filter((p) => p.evalScore >= binLo && p.evalScore < binHi) - if (chunk.length === 0) continue - bins.push(toBin(chunk, binLo, binHi)) + const groups = new Map() + for (const pair of clipped) { + const index = Math.min(numBins - 1, Math.floor(((pair.evalScore - lo) / span) * numBins)) + const group = groups.get(index) ?? [] + group.push(pair) + groups.set(index, group) + } + for (const [index, chunk] of [...groups].sort(([a], [b]) => a - b)) { + bins.push(toBin(chunk, lo + span * (index / numBins), lo + span * ((index + 1) / numBins))) } } @@ -145,5 +175,35 @@ function toBin(chunk: CalibrationPair[], lower?: number, upper?: number): Calibr } function mean(xs: number[]): number { - return xs.reduce((a, b) => a + b, 0) / xs.length + return xs.reduce((sum, value) => sum + value / xs.length, 0) +} + +function validateCalibrationRequest( + evalMetric: string, + outcomeMetric: string, + options: CalibrationOptions, +): void { + assertUniqueObservationIds([evalMetric], 'eval metric') + assertUniqueObservationIds([outcomeMetric], 'outcome metric') + if (evalMetric.trim() !== evalMetric || outcomeMetric.trim() !== outcomeMetric) { + throw new Error('calibration metric identities must not have surrounding whitespace') + } + if (options.bins !== undefined && (!Number.isSafeInteger(options.bins) || options.bins < 1)) { + throw new Error('calibration bins must be a positive safe integer') + } + if ( + options.binning !== undefined && + !['equal-width', 'equal-frequency'].includes(options.binning) + ) { + throw new Error('calibration binning must be equal-width or equal-frequency') + } + if ( + options.range !== undefined && + (!Number.isFinite(options.range.lo) || + !Number.isFinite(options.range.hi) || + !Number.isFinite(options.range.hi - options.range.lo) || + options.range.hi < options.range.lo) + ) { + throw new Error('calibration range must have finite ordered bounds') + } } diff --git a/src/meta-eval/correlation-study.ts b/src/meta-eval/correlation-study.ts index e71aa77d..26cbccdc 100644 --- a/src/meta-eval/correlation-study.ts +++ b/src/meta-eval/correlation-study.ts @@ -1,19 +1,21 @@ /** * Correlation study — "does our eval score predict real-world outcomes?" * - * This is the load-bearing signal. Takes a TraceStore + OutcomeStore, - * joins on runId, computes Pearson + Spearman + bootstrap CI for every - * (evalMetric, outcomeMetric) pair the caller declares. - * - * Without this number the framework is ornamental. With it and r > 0.6 - * the framework is a moat — no other agent-eval tool publishes one. + * Joins traces and outcomes by runId and reports descriptive correlations. + * Independent runs are the bootstrap observation unit. + * Association alone does not establish causation or held-out predictive performance. */ -import { pearsonR, spearmanR } from '../statistics' -import { makeRng } from '../statistics/internal' import { runMetricExtractor } from '../trace/query' import type { Run } from '../trace/schema' import type { TraceStore } from '../trace/store' +import { + assertUniqueObservationIds, + correlationSummary, + hasVariation, + reduceOutcomeMetric, + validateObservationOptions, +} from './outcome-observations' import type { DeploymentOutcome, OutcomeFilter, OutcomeStore } from './outcome-store' export interface EvalMetricSpec { @@ -35,13 +37,22 @@ export interface CorrelationResult { pearson: number spearman: number /** 95% bootstrap CI for Pearson. */ - pearsonCi95: { lower: number; upper: number } + pearsonCi95: { lower: number; upper: number } | null + /** 95% bootstrap CI for Spearman; null when no resample is estimable. */ + spearmanCi95: { lower: number; upper: number } | null /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */ verdict: 'strong' | 'moderate' | 'weak' } export interface CorrelationStudyResult { pairs: CorrelationResult[] + /** Declared pairs without an estimable correlation, including their usable sample count. */ + excludedPairs: Array< + OutcomePair & { + n: number + reason: 'insufficient_samples' | 'constant_eval_metric' | 'constant_outcome' + } + > joinedSamples: number skippedRuns: number } @@ -67,7 +78,29 @@ export async function correlationStudy( outcomeMetricNames: string[], options: CorrelationStudyOptions = {}, ): Promise { + const reduction = options.reduction ?? 'latest' + const iterations = options.bootstrapIterations ?? 500 + const seed = options.seed + validateObservationOptions(reduction, iterations, seed) + assertUniqueObservationIds( + evalMetrics.map((metric) => metric.id), + 'eval metric', + ) + assertUniqueObservationIds(outcomeMetricNames, 'outcome metric') + const maxLag = options.maxCaptureLagMs ?? Infinity + if (maxLag < 0 || Number.isNaN(maxLag)) { + throw new Error('maxCaptureLagMs must be nonnegative') + } + const extractors = evalMetrics.map((metric) => ({ + ...metric, + extract: metric.extract ?? runMetricExtractor(metric.id), + })) + const metricNames = [...outcomeMetricNames] const runs = await traceStore.listRuns() + assertUniqueObservationIds( + runs.map((run) => run.runId), + 'runId', + ) const outcomes = await outcomeStore.list(options.outcomeFilter) const outcomesByRun = new Map() for (const o of outcomes) { @@ -76,12 +109,9 @@ export async function correlationStudy( outcomesByRun.set(o.runId, arr) } - const reduction = options.reduction ?? 'latest' - const maxLag = options.maxCaptureLagMs ?? Infinity - const pairs: Array<{ evalMetric: string; outcomeMetric: string; xs: number[]; ys: number[] }> = [] - for (const em of evalMetrics) { - for (const om of outcomeMetricNames) { + for (const em of extractors) { + for (const om of metricNames) { pairs.push({ evalMetric: em.id, outcomeMetric: om, xs: [], ys: [] }) } } @@ -94,113 +124,67 @@ export async function correlationStudy( skipped++ continue } - const eligible = os.filter((o) => o.capturedAt - run.startedAt <= maxLag) + const eligible = os.filter((o) => { + const lag = o.capturedAt - run.startedAt + return lag >= 0 && lag <= maxLag + }) if (eligible.length === 0) { skipped++ continue } - for (const em of evalMetrics) { - const extract = em.extract ?? runMetricExtractor(em.id) - const x = await extract(run, traceStore) + let joinedThisRun = false + for (const em of extractors) { + const x = await em.extract(run, traceStore) if (x === null || !Number.isFinite(x)) continue - for (const om of outcomeMetricNames) { - const values = eligible - .map((o) => o.metrics[om]) - .filter((v): v is number => typeof v === 'number' && Number.isFinite(v)) - if (values.length === 0) continue - const y = reduce(values, reduction, eligible) + for (const om of metricNames) { + const y = reduceOutcomeMetric(eligible, om, reduction) if (y === null) continue const pair = pairs.find((p) => p.evalMetric === em.id && p.outcomeMetric === om)! pair.xs.push(x) pair.ys.push(y) + joinedThisRun = true } } - joined++ + if (joinedThisRun) joined++ + else skipped++ } - const results: CorrelationResult[] = pairs - .filter((p) => p.xs.length >= 3) - .map((p) => { - const pearson = pearsonR(p.xs, p.ys) - const spearman = spearmanR(p.xs, p.ys) - const pearsonCi95 = bootstrapPearsonCi( - p.xs, - p.ys, - options.bootstrapIterations ?? 500, - options.seed, - ) - const verdict: CorrelationResult['verdict'] = - Math.abs(pearson) >= 0.7 ? 'strong' : Math.abs(pearson) >= 0.4 ? 'moderate' : 'weak' - return { + const excludedPairs: CorrelationStudyResult['excludedPairs'] = [] + const results: CorrelationResult[] = [] + for (const p of pairs) { + const reason = + p.xs.length < 3 + ? 'insufficient_samples' + : !hasVariation(p.xs) + ? 'constant_eval_metric' + : !hasVariation(p.ys) + ? 'constant_outcome' + : null + if (reason !== null) { + excludedPairs.push({ evalMetric: p.evalMetric, outcomeMetric: p.outcomeMetric, n: p.xs.length, - pearson, - spearman, - pearsonCi95, - verdict, - } - }) - - return { pairs: results, joinedSamples: joined, skippedRuns: skipped } -} - -// ── Helpers ────────────────────────────────────────────────────────── - -function reduce( - values: number[], - kind: 'latest' | 'mean' | 'max', - outcomes: DeploymentOutcome[], -): number | null { - if (values.length === 0) return null - if (kind === 'mean') return values.reduce((a, b) => a + b, 0) / values.length - if (kind === 'max') return Math.max(...values) - // 'latest': pick the outcome captured last, then lookup its metric - const latest = [...outcomes].sort((a, b) => b.capturedAt - a.capturedAt)[0] - if (!latest) return null - const latestKey = Object.keys(latest.metrics)[0] - const v = latestKey !== undefined ? latest.metrics[latestKey] : undefined - // For 'latest' we already have `values` aligned; use the last-captured one - const paired = outcomes - .map((o) => { - const k = Object.keys(o.metrics)[0] - return { - at: o.capturedAt, - v: k !== undefined ? values.find((x) => o.metrics[k] === x) : undefined, - } - }) - .filter((p) => p.v !== undefined) - if (paired.length === 0) return v ?? null - return paired.sort((a, b) => b.at - a.at)[0]?.v ?? null -} - -function bootstrapPearsonCi( - xs: number[], - ys: number[], - iterations: number, - seed: number | undefined, -): { lower: number; upper: number } { - const n = xs.length - if (n < 3) return { lower: NaN, upper: NaN } - const rng = makeRng(seed, xs, ys) - const rs: number[] = [] - for (let b = 0; b < iterations; b++) { - const rx: number[] = new Array(n) - const ry: number[] = new Array(n) - for (let i = 0; i < n; i++) { - const idx = Math.floor(rng() * n) - rx[i] = xs[idx]! - ry[i] = ys[idx]! + reason, + }) + continue } - const r = pearsonR(rx, ry) - if (Number.isFinite(r)) rs.push(r) - } - rs.sort((a, b) => a - b) - if (rs.length === 0) return { lower: NaN, upper: NaN } - return { - lower: rs[Math.floor(0.025 * rs.length)]!, - upper: rs[Math.min(rs.length - 1, Math.floor(0.975 * rs.length))]!, + const summary = correlationSummary(p.xs, p.ys, iterations, seed) + const verdict: CorrelationResult['verdict'] = + Math.abs(summary.pearson) >= 0.7 + ? 'strong' + : Math.abs(summary.pearson) >= 0.4 + ? 'moderate' + : 'weak' + results.push({ + evalMetric: p.evalMetric, + outcomeMetric: p.outcomeMetric, + n: p.xs.length, + ...summary, + verdict, + }) } + return { pairs: results, excludedPairs, joinedSamples: joined, skippedRuns: skipped } } diff --git a/src/meta-eval/evaluator-admission.ts b/src/meta-eval/evaluator-admission.ts new file mode 100644 index 00000000..e1597d0b --- /dev/null +++ b/src/meta-eval/evaluator-admission.ts @@ -0,0 +1,256 @@ +import { z } from 'zod' +import { ValidationError } from '../errors' +import { type ComputedInterval, computeInterval } from '../experiment/ast' +import { compareCodeUnits, hashCanonical, type LedgerHash } from '../ledger-core/canonical' + +const identity = z + .string() + .min(1) + .refine((value) => value.trim() === value) +const policySchema = z + .object({ + confidence: z.number().finite().gt(0).lt(1), + maxFalseAcceptanceRate: z.number().finite().min(0).lt(1), + maxFalseRejectionRate: z.number().finite().min(0).lt(1), + }) + .strict() + +/** Register both error limits before inspecting audit judgments. */ +export type EvaluatorAdmissionPolicy = z.infer + +const observationSchema = z + .object({ + id: identity, + independentUnitId: identity, + evidenceRef: identity, + expected: z.enum(['accept', 'reject']), + observed: z.enum(['accept', 'reject', 'unknown']), + exposure: z.enum(['fresh', 'development']), + }) + .strict() + +/** Actual control judgments. Variants from one source retain one independent identity. */ +export type EvaluatorAuditObservation = z.infer + +const inputSchema = z + .object({ + evaluatorDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/), + population: identity, + samplingFrame: identity, + authority: z + .object({ + evaluatorAuthorId: identity, + auditorId: identity, + independenceEvidenceRef: identity, + }) + .strict(), + policy: policySchema, + observations: z.array(observationSchema), + }) + .strict() + +export type EvaluatorAuditInput = z.infer + +export interface EvaluatorErrorRate { + /** Eligible control judgments in this class, including unknown judgments. */ + cases: number + independentUnits: number + /** Units with at least one observed mistake. */ + errorUnits: number + /** Units with unknown judgments and no observed mistake yet. */ + unresolvedUnits: number + unknownCases: number + /** Missing judgments do not become measured zeros. */ + errorRate: number | null + /** Exact bounds conservatively include every possible outcome of unknown judgments. */ + interval: ComputedInterval | null + limit: number + verdict: 'pass' | 'fail' | 'inconclusive' +} + +export interface EvaluatorAdmissionReport { + evaluatorDigest: LedgerHash + policyDigest: LedgerHash + inputDigest: LedgerHash + reportDigest: LedgerHash + population: string + samplingFrame: string + authority: EvaluatorAuditInput['authority'] + policy: EvaluatorAdmissionPolicy + /** Joint coverage for the two reported error-rate intervals. */ + confidence: number + intervalConfidence: number + verdict: 'admit' | 'reject' | 'inconclusive' + reasons: string[] + observations: EvaluatorAuditObservation[] + coverage: { + cases: number + independentUnits: number + eligibleCases: number + eligibleIndependentUnits: number + excludedCases: number + unknownCases: number + } + exclusions: Array<{ id: string; independentUnitId: string; reason: 'development-exposure' }> + falseAcceptance: EvaluatorErrorRate + falseRejection: EvaluatorErrorRate +} + +function errorRate( + rows: readonly EvaluatorAuditObservation[], + expected: 'accept' | 'reject', + limit: number, + level: number, +): EvaluatorErrorRate { + const cases = rows.filter((row) => row.expected === expected) + const units = new Map() + for (const row of cases) { + const unit = units.get(row.independentUnitId) ?? { error: false, unknown: false } + unit.unknown ||= row.observed === 'unknown' + unit.error ||= row.observed !== 'unknown' && row.observed !== expected + units.set(row.independentUnitId, unit) + } + const n = units.size + const errorUnits = [...units.values()].filter((unit) => unit.error).length + const unresolvedUnits = [...units.values()].filter((unit) => unit.unknown && !unit.error).length + const unknownCases = cases.filter((row) => row.observed === 'unknown').length + // Unknown judgments contribute to n; their most favorable and adverse outcomes bound the rate. + const interval = + n === 0 + ? null + : { + lower: computeInterval( + { kind: 'clopper-pearson', level }, + { + kind: 'binomial', + successes: errorUnits, + trials: n, + }, + ).lower, + upper: computeInterval( + { kind: 'clopper-pearson', level }, + { + kind: 'binomial', + successes: errorUnits + unresolvedUnits, + trials: n, + }, + ).upper, + level, + } + const verdict = + interval && interval.lower > limit + ? 'fail' + : interval && interval.upper <= limit + ? 'pass' + : 'inconclusive' + return { + cases: cases.length, + independentUnits: n, + errorUnits, + unresolvedUnits, + unknownCases, + errorRate: n === 0 || unresolvedUnits > 0 ? null : errorUnits / n, + interval, + limit, + verdict, + } +} + +/** + * Audit frozen judgments using independent source units and exact binomial bounds. + * A unit fails a class when any control in that class is misjudged. + * The execution owner enforces auditor separation, fresh sampling, and evidence authenticity. + */ +export function auditEvaluator(input: EvaluatorAuditInput): EvaluatorAdmissionReport { + const parsed = inputSchema.safeParse(input) + if (!parsed.success) throw new ValidationError(`invalid evaluator audit: ${parsed.error.message}`) + const audit = parsed.data + if (audit.authority.evaluatorAuthorId === audit.authority.auditorId) { + throw new ValidationError('evaluator admission requires a separate declared audit authority') + } + const ids = new Set() + for (const row of audit.observations) { + if (ids.has(row.id)) + throw new ValidationError(`duplicate evaluator audit observation '${row.id}'`) + ids.add(row.id) + } + audit.observations.sort((a, b) => compareCodeUnits(a.id, b.id)) + const developmentUnits = new Set( + audit.observations + .filter((row) => row.exposure === 'development') + .map((row) => row.independentUnitId), + ) + const eligible = audit.observations.filter((row) => !developmentUnits.has(row.independentUnitId)) + const exclusions = audit.observations + .filter((row) => developmentUnits.has(row.independentUnitId)) + .map((row) => ({ + id: row.id, + independentUnitId: row.independentUnitId, + reason: 'development-exposure' as const, + })) + const intervalConfidence = 1 - (1 - audit.policy.confidence) / 2 + if (intervalConfidence >= 1) + throw new ValidationError( + 'evaluator audit confidence is too close to one for simultaneous intervals', + ) + const falseAcceptance = errorRate( + eligible, + 'reject', + audit.policy.maxFalseAcceptanceRate, + intervalConfidence, + ) + const falseRejection = errorRate( + eligible, + 'accept', + audit.policy.maxFalseRejectionRate, + intervalConfidence, + ) + const rates = [falseAcceptance, falseRejection] + const verdict = rates.some((rate) => rate.verdict === 'fail') + ? 'reject' + : rates.every((rate) => rate.verdict === 'pass') + ? 'admit' + : 'inconclusive' + const reasons: string[] = [] + for (const [name, rate] of [ + ['false acceptance', falseAcceptance], + ['false rejection', falseRejection], + ] as const) { + if (rate.independentUnits === 0) reasons.push(`${name}: no eligible independent units`) + else if (rate.unknownCases > 0) + reasons.push(`${name}: ${rate.unknownCases} judgments are unknown`) + if (rate.verdict === 'fail') reasons.push(`${name}: lower error bound exceeds ${rate.limit}`) + else if (rate.verdict === 'inconclusive' && rate.interval) + reasons.push(`${name}: upper error bound does not establish the required limit`) + } + if (verdict === 'admit') + reasons.push( + 'both error bounds meet the registered limits, including the worst case for unknown judgments', + ) + const body: Omit = { + evaluatorDigest: audit.evaluatorDigest as LedgerHash, + policyDigest: hashCanonical(audit.policy), + inputDigest: hashCanonical(audit), + population: audit.population, + samplingFrame: audit.samplingFrame, + authority: audit.authority, + policy: audit.policy, + confidence: audit.policy.confidence, + intervalConfidence, + verdict, + reasons, + observations: audit.observations, + coverage: { + cases: audit.observations.length, + independentUnits: new Set(audit.observations.map((row) => row.independentUnitId)).size, + eligibleCases: eligible.length, + eligibleIndependentUnits: new Set(eligible.map((row) => row.independentUnitId)).size, + excludedCases: exclusions.length, + unknownCases: eligible.filter((row) => row.observed === 'unknown').length, + }, + exclusions, + falseAcceptance, + falseRejection, + } + return { ...body, reportDigest: hashCanonical(body) } +} diff --git a/src/meta-eval/index.ts b/src/meta-eval/index.ts index 77bd6227..7c8fe20f 100644 --- a/src/meta-eval/index.ts +++ b/src/meta-eval/index.ts @@ -1,5 +1,33 @@ +export type { + CalibrationResult, + CandidateScore, + ContinuousAgreement, + ContinuousAgreementOptions, + ContinuousCalibrationResult, + GoldenItem, + PositionalBiasResult, + SelfPreferenceResult, + VerbosityBiasResult, +} from '../judge-calibration' +export { + calibrateJudge, + calibrateJudgeContinuous, + continuousAgreement, + positionalBias, + selfPreference, + verbosityBias, +} from '../judge-calibration' export * from './calibration' export * from './correlation-study' +export { + auditEvaluator, + type EvaluatorAdmissionPolicy, + type EvaluatorAdmissionReport, + type EvaluatorAuditInput, + type EvaluatorAuditObservation, + type EvaluatorErrorRate, +} from './evaluator-admission' +export type { CorrelationInterval, OutcomeReduction } from './outcome-observations' export * from './outcome-store' export * from './plants' export * from './rubric-predictive-validity' diff --git a/src/meta-eval/outcome-observations.ts b/src/meta-eval/outcome-observations.ts new file mode 100644 index 00000000..a2a234c5 --- /dev/null +++ b/src/meta-eval/outcome-observations.ts @@ -0,0 +1,130 @@ +import { pearsonR, spearmanR } from '../statistics' +import { makeRng } from '../statistics/internal' +import type { DeploymentOutcome } from './outcome-store' + +export type OutcomeReduction = 'latest' | 'mean' | 'max' + +export interface CorrelationInterval { + lower: number + upper: number +} + +/** Reduce observations of one named metric; unrelated keys cannot supply a value. */ +export function reduceOutcomeMetric( + outcomes: readonly DeploymentOutcome[], + metric: string, + reduction: OutcomeReduction, +): number | null { + const observations = outcomes.flatMap((outcome) => { + const value = outcome.metrics[metric] + return Number.isFinite(outcome.capturedAt) && + typeof value === 'number' && + Number.isFinite(value) + ? [{ capturedAt: outcome.capturedAt, value }] + : [] + }) + if (observations.length === 0) return null + if (reduction === 'mean') { + return observations.reduce( + (sum, observation) => sum + observation.value / observations.length, + 0, + ) + } + if (reduction === 'max') return Math.max(...observations.map((observation) => observation.value)) + return observations.reduce((a, b) => (b.capturedAt > a.capturedAt ? b : a)).value +} + +export function hasVariation(values: readonly number[]): boolean { + return values.some((value) => value !== values[0]) +} + +export function correlationSummary( + xs: number[], + ys: number[], + iterations: number, + seed: number | undefined, +): { + pearson: number + spearman: number + pearsonCi95: CorrelationInterval | null + spearmanCi95: CorrelationInterval | null +} { + const rng = makeRng(seed, xs, ys) + const pearsons: number[] = [] + const spearmans: number[] = [] + for (let b = 0; b < iterations; b++) { + const rx: number[] = [] + const ry: number[] = [] + for (let i = 0; i < xs.length; i++) { + const index = Math.floor(rng() * xs.length) + rx.push(xs[index]!) + ry.push(ys[index]!) + } + // A constant resample has no estimable correlation, even when both sides agree. + if (!hasVariation(rx) || !hasVariation(ry)) continue + const pearson = pearsonR(rx, ry) + const spearman = spearmanR(rx, ry) + if (Number.isFinite(pearson)) pearsons.push(pearson) + if (Number.isFinite(spearman)) spearmans.push(spearman) + } + return { + pearson: pearsonR(xs, ys), + spearman: spearmanR(xs, ys), + pearsonCi95: interval(pearsons), + spearmanCi95: interval(spearmans), + } +} + +export function validateObservationOptions( + reduction: OutcomeReduction, + iterations: number, + seed: number | undefined, +): void { + if (!['latest', 'mean', 'max'].includes(reduction)) { + throw new Error('outcome reduction must be latest, mean, or max') + } + if (!Number.isSafeInteger(iterations) || iterations < 1) { + throw new Error('bootstrap iterations must be a positive safe integer') + } + if (seed !== undefined && !Number.isFinite(seed)) { + throw new Error('bootstrap seed must be finite') + } +} + +export function assertUniqueObservationIds(ids: readonly string[], name: string): void { + if (ids.some((id) => typeof id !== 'string' || id.trim().length === 0)) { + throw new Error(`${name} must be a nonempty string`) + } + if (new Set(ids).size !== ids.length) throw new Error(`duplicate ${name} in outcome study`) +} + +export function validateOutcomeMetricSpecifications( + metrics: readonly { id: string; direction: string }[], +): void { + if (!Array.isArray(metrics) || metrics.length === 0) { + throw new Error('outcomeMetrics must declare at least one outcome metric and direction') + } + for (const metric of metrics) { + if (!metric || typeof metric.id !== 'string' || metric.id.trim().length === 0) { + throw new Error('each outcome metric must have a nonempty id and explicit direction') + } + if (metric.direction !== 'higher-is-better' && metric.direction !== 'lower-is-better') { + throw new Error( + `outcome metric ${metric.id} must declare higher-is-better or lower-is-better`, + ) + } + } + assertUniqueObservationIds( + metrics.map((metric) => metric.id), + 'outcome metric', + ) +} + +function interval(values: number[]): CorrelationInterval | null { + if (values.length === 0) return null + values.sort((a, b) => a - b) + return { + lower: Math.max(-1, values[Math.floor(0.025 * values.length)]!), + upper: Math.min(1, values[Math.min(values.length - 1, Math.floor(0.975 * values.length))]!), + } +} diff --git a/src/meta-eval/outcome-store.ts b/src/meta-eval/outcome-store.ts index e75538c0..bb906615 100644 --- a/src/meta-eval/outcome-store.ts +++ b/src/meta-eval/outcome-store.ts @@ -1,24 +1,14 @@ -/** - * OutcomeStore — deployment outcomes attached to Run IDs. - * - * Outcomes arrive asynchronously from production telemetry after the - * eval run completed: user ratings, retention flags, conversion events, - * revenue, support-ticket rate, anything a product team can measure. - * The store is a peer to TraceStore — separate lifecycle, same runId - * foreign key. - * - * The whole point of this module is to make the meta-eval correlation - * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`. - */ +/** Deployment observations join to evaluation records through their runId. */ + +import { z } from 'zod' export interface DeploymentOutcome { runId: string capturedAt: number - /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */ + /** Finite numeric outcomes; absence of a key means unmeasured. */ metrics: Record - /** Dimensions for stratified analysis — cohort, region, user_segment. */ labels?: Record - /** Free-form provenance (source system, pipeline version). */ + /** Source system or pipeline version. */ source?: string } @@ -32,103 +22,165 @@ export interface OutcomeFilter { export interface OutcomeStore { append(outcome: DeploymentOutcome): Promise - /** All outcomes attached to this run (a single run can have many — multiple - * capture windows over deployment time). */ forRun(runId: string): Promise list(filter?: OutcomeFilter): Promise } +const outcomeSchema = z + .object({ + runId: z.string().min(1), + capturedAt: z.number().finite(), + metrics: z.record(z.string(), z.number().finite()), + labels: z.record(z.string(), z.string()).optional(), + source: z.string().optional(), + }) + .passthrough() + export class InMemoryOutcomeStore implements OutcomeStore { private items: DeploymentOutcome[] = [] async append(outcome: DeploymentOutcome): Promise { - this.items.push({ ...outcome }) + this.items.push(structuredClone(outcomeSchema.parse(outcome))) } async forRun(runId: string): Promise { - return this.items.filter((o) => o.runId === runId).map((o) => ({ ...o })) + return this.list({ runIds: [runId] }) } async list(filter: OutcomeFilter = {}): Promise { - return this.items.filter((o) => matches(o, filter)).map((o) => ({ ...o })) + return this.items + .filter((outcome) => matches(outcome, filter)) + .map((outcome) => structuredClone(outcome)) } } export interface FileSystemOutcomeStoreOptions { dir: string + /** Rotate before a write once the active file reaches this size. Default 32 MiB. */ maxBytes?: number } +/** Storage failures retain operation, path, source line, and the original cause. */ +export class OutcomeStoreError extends Error { + constructor( + public readonly operation: 'read' | 'decode' | 'write', + public readonly path: string, + cause: unknown, + public readonly line?: number, + ) { + super(`outcome store ${operation} failed at ${path}${line === undefined ? '' : `:${line}`}`, { + cause, + }) + this.name = 'OutcomeStoreError' + } +} + +/** Local storage with serialized operations per instance; use one writer per directory. */ export class FileSystemOutcomeStore implements OutcomeStore { - private dir: string - private maxBytes: number - private memo?: InMemoryOutcomeStore - private loaded = false + private readonly dir: string + private readonly maxBytes: number + private pending: Promise = Promise.resolve() constructor(options: FileSystemOutcomeStoreOptions) { + if (!options.dir.trim()) throw new Error('outcome store dir must be nonempty') this.dir = options.dir this.maxBytes = options.maxBytes ?? 32 * 1024 * 1024 - } - - private async ensureDir(): Promise { - const fs = await import('node:fs/promises') - await fs.mkdir(this.dir, { recursive: true }) + if (!Number.isSafeInteger(this.maxBytes) || this.maxBytes < 1) { + throw new Error('outcome store maxBytes must be a positive safe integer') + } } async append(outcome: DeploymentOutcome): Promise { - await this.ensureDir() - const fs = await import('node:fs/promises') - const path = await import('node:path') - const active = path.join(this.dir, 'outcomes.ndjson') - try { - const stat = await fs.stat(active) - if (stat.size >= this.maxBytes) { - await fs.rename(active, path.join(this.dir, `outcomes.${Date.now()}.ndjson`)) + const snapshot = structuredClone(outcomeSchema.parse(outcome)) + return this.serialize(async () => { + const fs = await import('node:fs/promises') + const path = await import('node:path') + const active = path.join(this.dir, 'outcomes.ndjson') + try { + await fs.mkdir(this.dir, { recursive: true }) + let size = 0 + try { + const stat = await fs.stat(active) + if (!stat.isFile()) throw new Error('active outcome path is not a regular file') + size = stat.size + } catch (error) { + if (!isMissing(error)) throw error + } + if (size >= this.maxBytes) { + const { randomUUID } = await import('node:crypto') + await fs.rename( + active, + path.join(this.dir, `outcomes.${Date.now()}.${randomUUID()}.ndjson`), + ) + } + await fs.appendFile(active, `${JSON.stringify(snapshot)}\n`, 'utf8') + } catch (error) { + throw new OutcomeStoreError('write', active, error) } - } catch { - /* first write */ - } - await fs.appendFile(active, `${JSON.stringify(outcome)}\n`, 'utf8') - if (this.memo) await this.memo.append(outcome) + }) } - private async load(): Promise { - if (this.loaded && this.memo) return this.memo - const fs = await import('node:fs/promises') - const path = await import('node:path') - const memo = new InMemoryOutcomeStore() - try { - const entries = await fs.readdir(this.dir) - for (const file of entries) { - if (!file.endsWith('.ndjson')) continue - const content = await fs.readFile(path.join(this.dir, file), 'utf8') - for (const line of content.split('\n')) { + async forRun(runId: string): Promise { + return this.list({ runIds: [runId] }) + } + + async list(filter: OutcomeFilter = {}): Promise { + return this.serialize(async () => { + const fs = await import('node:fs/promises') + const path = await import('node:path') + let entries: string[] + try { + entries = await fs.readdir(this.dir) + } catch (error) { + if (isMissing(error)) return [] + throw new OutcomeStoreError('read', this.dir, error) + } + const outcomes: DeploymentOutcome[] = [] + // Read each snapshot from disk so later observations from another instance remain visible. + for (const file of entries.sort()) { + if (file !== 'outcomes.ndjson' && !/^outcomes\..+\.ndjson$/.test(file)) continue + const filePath = path.join(this.dir, file) + let content: string + try { + content = await fs.readFile(filePath, 'utf8') + } catch (error) { + throw new OutcomeStoreError('read', filePath, error) + } + for (const [index, line] of content.split('\n').entries()) { if (!line.trim()) continue - await memo.append(JSON.parse(line)) + let outcome: DeploymentOutcome + try { + outcome = outcomeSchema.parse(JSON.parse(line)) + } catch (error) { + throw new OutcomeStoreError('decode', filePath, error, index + 1) + } + if (matches(outcome, filter)) outcomes.push(outcome) } } - } catch { - /* empty */ - } - this.memo = memo - this.loaded = true - return memo + return outcomes + }) } - async forRun(runId: string): Promise { - return (await this.load()).forRun(runId) + private serialize(operation: () => Promise): Promise { + const result = this.pending.then(operation) + // A failed operation remains rejected to its caller, but does not disable later repaired reads or writes. + this.pending = result.then( + () => undefined, + () => undefined, + ) + return result } +} - async list(filter?: OutcomeFilter): Promise { - return (await this.load()).list(filter) - } +function isMissing(error: unknown): boolean { + return error instanceof Error && 'code' in error && error.code === 'ENOENT' } -function matches(o: DeploymentOutcome, f: OutcomeFilter): boolean { - if (f.runIds && !f.runIds.includes(o.runId)) return false - if (f.since !== undefined && o.capturedAt < f.since) return false - if (f.until !== undefined && o.capturedAt > f.until) return false - if (f.source && o.source !== f.source) return false - if (f.label && o.labels?.[f.label.key] !== f.label.value) return false +function matches(outcome: DeploymentOutcome, filter: OutcomeFilter): boolean { + if (filter.runIds && !filter.runIds.includes(outcome.runId)) return false + if (filter.since !== undefined && outcome.capturedAt < filter.since) return false + if (filter.until !== undefined && outcome.capturedAt > filter.until) return false + if (filter.source && outcome.source !== filter.source) return false + if (filter.label && outcome.labels?.[filter.label.key] !== filter.label.value) return false return true } diff --git a/src/meta-eval/rubric-predictive-validity.ts b/src/meta-eval/rubric-predictive-validity.ts index 3f52d6c2..93134383 100644 --- a/src/meta-eval/rubric-predictive-validity.ts +++ b/src/meta-eval/rubric-predictive-validity.ts @@ -1,106 +1,90 @@ /** - * Rubric predictive validity — does our eval rubric predict deployment - * outcomes? - * - * `correlationStudy` (already in this package) joins a `TraceStore` to an - * `OutcomeStore` and computes Pearson + Spearman + bootstrap CI for each - * (eval-metric, outcome-metric) pair. That answers "does X correlate with - * Y at all." `rubricPredictiveValidity` is the campaign-shaped wrapper - * around it: take a sequence of `RunRecord`s (the canonical campaign - * artifact) and a `DeploymentOutcomeStore`, join on `runId`, return a - * ranked verdict on every rubric whose dimension scores were captured in - * `outcome.raw`. - * - * The point — quoting the methodology doc — is that **without this loop - * every rubric is faith-based**. Once it's wired, you know which rubrics - * have earned their promotion power and which ones are decoration. - * - * const validity = await rubricPredictiveValidity({ - * runs: lastQuarter, - * outcomes: shipFlagOutcomeStore, - * outcomeMetrics: ['revenue_lift', 'retention_30d', 'csat'], - * rubrics: ['anti_slop', 'semantic_concept', 'tool_recovery'], - * }) - * for (const r of validity.ranked) { - * console.log(`${r.rubric} → ${r.bestOutcome}: ρ=${r.spearman.toFixed(2)}`) - * } - * - * The function is intentionally read-only. Use the verdict to deprecate - * decorative rubrics, re-weight composite scores, or trigger a - * recalibration sweep when predictive validity drops below a threshold. + * Join rubric scores to deployment outcomes and measure their association. + * Higher rubric scores always mean better evaluated behavior. + * Outcome directions are explicit because success rate and failure rate have opposite meanings. + * These descriptive associations neither establish causation nor validate a change to rubric weights. */ import type { RunRecord } from '../run-record' -import { pearsonR, spearmanR } from '../statistics' -import { makeRng } from '../statistics/internal' +import { + assertUniqueObservationIds, + type CorrelationInterval, + correlationSummary, + hasVariation, + reduceOutcomeMetric, + validateObservationOptions, + validateOutcomeMetricSpecifications, +} from './outcome-observations' import type { DeploymentOutcome, OutcomeStore } from './outcome-store' +export interface OutcomeMetricSpec { + /** Exact key in DeploymentOutcome.metrics. */ + id: string + direction: 'higher-is-better' | 'lower-is-better' +} + export interface RubricPredictiveValidityInput { - /** - * Canonical campaign output. Each record's `outcome.raw[]` - * provides the eval score; missing keys are silently skipped per pair. - */ + /** One record per independent run; rubric scores come from outcome.raw. */ runs: RunRecord[] outcomes: OutcomeStore - /** - * Outcome metric names to evaluate against. Each must appear in at - * least one `DeploymentOutcome.metrics` keyspace; pairs with too few - * joined samples are excluded from the result. - */ - outcomeMetrics: string[] - /** - * Rubric ids to evaluate. Must appear as keys in `RunRecord.outcome.raw`. - * If omitted, every numeric key in `outcome.raw` across the run set is - * treated as a rubric. - */ - rubrics?: string[] - /** Minimum joined-sample count before a pair is reported. Default 8. */ + /** Declare desired directions before inspecting associations. */ + outcomeMetrics: readonly OutcomeMetricSpec[] + /** Higher is better for each rubric. Omit to discover finite numeric outcome.raw keys. */ + rubrics?: readonly string[] + /** Minimum joined runs for an estimate; an integer at least 3. Default 8. */ minSamples?: number - /** Bootstrap resamples for CI. Default 500. */ + /** Bootstrap resamples for both correlation intervals. Default 500. */ bootstrapResamples?: number - /** Seed for the bootstrap. Absent, the seed is derived from the paired - * observations, so the same input reproduces the same interval. */ + /** Omit to derive a reproducible seed from the paired observations. */ seed?: number - /** - * Reduction when multiple outcomes attach to one runId. Default `'latest'` - * (most recently captured). - */ + /** Reduce finite observations of each named outcome within a run. Default latest. */ reduction?: 'latest' | 'mean' | 'max' } export interface RubricOutcomePair { rubric: string outcome: string + outcomeDirection: OutcomeMetricSpec['direction'] n: number + /** Raw association with the recorded outcome, before direction alignment. */ pearson: number spearman: number - ci95: { low: number; high: number } - /** - * Verdict bucket. `load_bearing` ≥ 0.7, `informative` ≥ 0.4, - * `decorative` < 0.4 in absolute correlation. A negative correlation - * with a desired outcome is also `decorative` — actively misleading - * is worse than uninformative. - */ - verdict: 'load_bearing' | 'informative' | 'decorative' + pearsonCi95: CorrelationInterval | null + spearmanCi95: CorrelationInterval | null + /** Positive values associate higher rubric scores with better outcomes. */ + alignedPearson: number + alignedSpearman: number + alignedSpearmanCi95: CorrelationInterval | null + /** Descriptive buckets at aligned Spearman +/-0.4; no causal or release authority. */ + verdict: 'aligned' | 'inverse' | 'weak' } -export interface RubricRanking { - rubric: string - /** Outcome metric this rubric correlated best with. */ +export interface RubricRanking extends Omit { + /** Outcome with the greatest direction-aligned Spearman for this rubric. */ bestOutcome: string - spearman: number - pearson: number +} + +export interface RubricOutcomeExclusion { + rubric: string + outcome: string + outcomeDirection: OutcomeMetricSpec['direction'] + /** Finite joined observations, including measured zeros. */ n: number - verdict: RubricOutcomePair['verdict'] + reason: 'insufficient_samples' | 'constant_rubric' | 'constant_outcome' } export interface RubricPredictiveValidityReport { + outcomeMetrics: OutcomeMetricSpec[] pairs: RubricOutcomePair[] - /** Per-rubric best pair, sorted descending by |spearman|. */ + /** All declared pairs lacking an estimate, with their usable observation count. */ + excludedPairs: RubricOutcomeExclusion[] + /** Exploratory ordering by aligned Spearman; never use outcome selection as confirmatory evidence. */ ranked: RubricRanking[] + /** Runs contributing at least one finite pair, including pairs below minSamples. */ joinedSamples: number + /** Runs contributing no finite pair; joinedSamples + skippedRuns equals the input run count. */ skippedRuns: number - /** Rubrics that were declared but never produced a usable score. */ + /** Declared rubrics with no finite score, distinct from too few outcomes or constant observations. */ rubricsWithoutData: string[] } @@ -110,151 +94,118 @@ export async function rubricPredictiveValidity( const minSamples = input.minSamples ?? 8 const reduction = input.reduction ?? 'latest' const resamples = input.bootstrapResamples ?? 500 + const seed = input.seed + if (!Number.isSafeInteger(minSamples) || minSamples < 3) { + throw new Error('minSamples must be a safe integer at least 3') + } + validateObservationOptions(reduction, resamples, seed) + validateOutcomeMetricSpecifications(input.outcomeMetrics) + assertUniqueObservationIds( + input.runs.map((run) => run.runId), + 'runId', + ) + + const outcomeMetrics = input.outcomeMetrics.map((metric) => ({ ...metric })) + const runs = input.runs.map((run) => ({ runId: run.runId, scores: { ...run.outcome.raw } })) + const declaredRubrics = input.rubrics === undefined ? undefined : [...input.rubrics] + if (declaredRubrics !== undefined) assertUniqueObservationIds(declaredRubrics, 'rubric') const outcomes = await input.outcomes.list() const outcomesByRun = new Map() - for (const o of outcomes) { - const arr = outcomesByRun.get(o.runId) ?? [] - arr.push(o) - outcomesByRun.set(o.runId, arr) + for (const outcome of outcomes) { + const rows = outcomesByRun.get(outcome.runId) ?? [] + rows.push(outcome) + outcomesByRun.set(outcome.runId, rows) } - // Discover rubrics: caller-declared OR every numeric key in outcome.raw - // observed across runs. const observedRubrics = new Set() - for (const r of input.runs) { - for (const k of Object.keys(r.outcome.raw)) observedRubrics.add(k) - } - const rubrics = input.rubrics ?? [...observedRubrics] - - // Collect aligned (x, y) pairs per (rubric, outcome). - type Bucket = { rubric: string; outcome: string; xs: number[]; ys: number[] } - const buckets: Bucket[] = [] - for (const r of rubrics) { - for (const o of input.outcomeMetrics) { - buckets.push({ rubric: r, outcome: o, xs: [], ys: [] }) + for (const run of runs) { + for (const [rubric, value] of Object.entries(run.scores)) { + if (typeof value === 'number' && Number.isFinite(value)) observedRubrics.add(rubric) } } + const rubrics = declaredRubrics ?? [...observedRubrics] + const buckets = rubrics.flatMap((rubric) => + outcomeMetrics.map((outcome) => ({ + rubric, + outcome, + xs: [] as number[], + ys: [] as number[], + })), + ) let joined = 0 - let skipped = 0 - for (const run of input.runs) { - const os = outcomesByRun.get(run.runId) - if (!os || os.length === 0) { - skipped++ - continue - } + for (const run of runs) { + const rows = outcomesByRun.get(run.runId) ?? [] let joinedThisRun = false - for (const r of rubrics) { - const x = run.outcome.raw[r] + for (const bucket of buckets) { + const x = run.scores[bucket.rubric] if (typeof x !== 'number' || !Number.isFinite(x)) continue - for (const o of input.outcomeMetrics) { - const values = os - .map((row) => row.metrics[o]) - .filter((v): v is number => typeof v === 'number' && Number.isFinite(v)) - if (values.length === 0) continue - const y = reduce(values, os, o, reduction) - if (y === null) continue - const bucket = buckets.find((b) => b.rubric === r && b.outcome === o)! - bucket.xs.push(x) - bucket.ys.push(y) - joinedThisRun = true - } + const y = reduceOutcomeMetric(rows, bucket.outcome.id, reduction) + if (y === null) continue + bucket.xs.push(x) + bucket.ys.push(y) + joinedThisRun = true } if (joinedThisRun) joined++ } const pairs: RubricOutcomePair[] = [] - for (const b of buckets) { - if (b.xs.length < minSamples) continue - const pearson = pearsonR(b.xs, b.ys) - const spearman = spearmanR(b.xs, b.ys) - const ci = bootstrapCi(b.xs, b.ys, resamples, input.seed) - const verdict: RubricOutcomePair['verdict'] = - Math.abs(spearman) >= 0.7 - ? 'load_bearing' - : Math.abs(spearman) >= 0.4 - ? 'informative' - : 'decorative' + const excludedPairs: RubricOutcomeExclusion[] = [] + for (const bucket of buckets) { + const identity = { + rubric: bucket.rubric, + outcome: bucket.outcome.id, + outcomeDirection: bucket.outcome.direction, + n: bucket.xs.length, + } + const reason = + bucket.xs.length < minSamples + ? 'insufficient_samples' + : !hasVariation(bucket.xs) + ? 'constant_rubric' + : !hasVariation(bucket.ys) + ? 'constant_outcome' + : null + if (reason !== null) { + excludedPairs.push({ ...identity, reason }) + continue + } + const summary = correlationSummary(bucket.xs, bucket.ys, resamples, seed) + const sign = bucket.outcome.direction === 'higher-is-better' ? 1 : -1 + const alignedSpearman = summary.spearman * sign + const alignedSpearmanCi95 = + summary.spearmanCi95 === null + ? null + : sign === 1 + ? summary.spearmanCi95 + : { lower: -summary.spearmanCi95.upper, upper: -summary.spearmanCi95.lower } pairs.push({ - rubric: b.rubric, - outcome: b.outcome, - n: b.xs.length, - pearson, - spearman, - ci95: ci, - verdict, + ...identity, + ...summary, + alignedPearson: summary.pearson * sign, + alignedSpearman, + alignedSpearmanCi95, + verdict: alignedSpearman >= 0.4 ? 'aligned' : alignedSpearman <= -0.4 ? 'inverse' : 'weak', }) } - const byRubric = new Map() - for (const p of pairs) { - const arr = byRubric.get(p.rubric) ?? [] - arr.push(p) - byRubric.set(p.rubric, arr) + const bestByRubric = new Map() + for (const pair of pairs) { + const best = bestByRubric.get(pair.rubric) + if (!best || pair.alignedSpearman > best.alignedSpearman) bestByRubric.set(pair.rubric, pair) } - const ranked: RubricRanking[] = [...byRubric.entries()] - .map(([rubric, ps]) => { - const best = ps.reduce((a, b) => (Math.abs(b.spearman) > Math.abs(a.spearman) ? b : a)) - return { - rubric, - bestOutcome: best.outcome, - spearman: best.spearman, - pearson: best.pearson, - n: best.n, - verdict: best.verdict, - } - }) - .sort((a, b) => Math.abs(b.spearman) - Math.abs(a.spearman)) - - const rubricsWithoutData = rubrics.filter((r) => !byRubric.has(r)) + const ranked = [...bestByRubric.values()] + .map(({ outcome, ...pair }) => ({ ...pair, bestOutcome: outcome })) + .sort((a, b) => b.alignedSpearman - a.alignedSpearman) - return { pairs, ranked, joinedSamples: joined, skippedRuns: skipped, rubricsWithoutData } -} - -// ── Helpers ────────────────────────────────────────────────────────────── - -function reduce( - values: number[], - outcomes: DeploymentOutcome[], - metric: string, - kind: 'latest' | 'mean' | 'max', -): number | null { - if (values.length === 0) return null - if (kind === 'mean') return values.reduce((s, v) => s + v, 0) / values.length - if (kind === 'max') return Math.max(...values) - // 'latest' - const sorted = [...outcomes] - .filter((o) => typeof o.metrics[metric] === 'number') - .sort((a, b) => b.capturedAt - a.capturedAt) - return sorted[0]?.metrics[metric] ?? null -} - -function bootstrapCi( - xs: number[], - ys: number[], - iterations: number, - seed: number | undefined, -): { low: number; high: number } { - const n = xs.length - if (n < 3) return { low: Number.NaN, high: Number.NaN } - const rng = makeRng(seed, xs, ys) - const samples: number[] = [] - for (let b = 0; b < iterations; b++) { - const rx = new Array(n) - const ry = new Array(n) - for (let i = 0; i < n; i++) { - const idx = Math.floor(rng() * n) - rx[i] = xs[idx]! - ry[i] = ys[idx]! - } - const r = pearsonR(rx, ry) - if (Number.isFinite(r)) samples.push(r) - } - samples.sort((a, b) => a - b) - if (samples.length === 0) return { low: Number.NaN, high: Number.NaN } return { - low: samples[Math.floor(0.025 * samples.length)]!, - high: samples[Math.min(samples.length - 1, Math.floor(0.975 * samples.length))]!, + outcomeMetrics, + pairs, + excludedPairs, + ranked, + joinedSamples: joined, + skippedRuns: runs.length - joined, + rubricsWithoutData: rubrics.filter((rubric) => !observedRubrics.has(rubric)), } } diff --git a/src/paired-promotion-decision.test.ts b/src/paired-promotion-decision.test.ts index d5d46f80..acbfb40c 100644 --- a/src/paired-promotion-decision.test.ts +++ b/src/paired-promotion-decision.test.ts @@ -147,6 +147,31 @@ describe('decidePairedPromotion sufficiency', () => { expect(decision.sufficient).toBe(false) expect(decision.promote).toBe(false) }) + + it('uses the actual binary minimum instead of a universal bootstrap floor', () => { + const decision = decidePairedPromotion(Array(6).fill(0), Array(6).fill(1)) + expect(decision.minimumPairs).toBe(6) + expect(decision.method).toBe('score-interval') + expect(decision.sufficient).toBe(true) + expect(decision.promote).toBe(true) + }) + + it('does not use a sign-probability result to certify a continuous mean', () => { + const before = Array(6).fill(0.1) + const after = [0.3, 0.35, 0.4, 0.45, 0.5, 0.55] + const mean = decidePairedPromotion(before, after, { minPairs: 1 }) + expect(mean.pValue).toBeLessThan(0.05) + expect(mean.minimumPairs).toBe(20) + expect(mean.sufficient).toBe(false) + expect(mean.promote).toBe(false) + expect(mean.methodDetail).toContain('does not establish a mean effect') + + const median = decidePairedPromotion(before, after, { statistic: 'median' }) + expect(median.minimumPairs).toBe(6) + expect(median.method).toBe('exact-sign') + expect(median.sufficient).toBe(true) + expect(median.promote).toBe(true) + }) }) describe('decidePairedPromotion zero-width refusal', () => { @@ -177,6 +202,29 @@ describe('decidePairedPromotion zero-width refusal', () => { expect(decision.promote).toBe(false) }) + it('does not count rounding differences as observed variation at any score scale', () => { + for (const scale of [1e-12, 1, 1e12]) { + const before = Array.from({ length: 24 }, (_, i) => [0.5, 0.6, 0.4][i % 3]! * scale) + const after = Array.from({ length: 24 }, (_, i) => [0.8, 0.9, 0.7][i % 3]! * scale) + const decision = decidePairedPromotion(before, after, { seed: 1337 }) + expect(decision.sufficient).toBe(true) + expect(decision.indeterminate).toBe(true) + expect(decision.promote).toBe(false) + } + }) + + it('retains real variation when scores use tiny units', () => { + for (const scale of [1e-12, 1, 1e12]) { + const decision = decidePairedPromotion( + continuousBaseline.map((value) => value * scale), + continuousCandidate.map((value) => value * scale), + { seed: 11 }, + ) + expect(decision.indeterminate).toBe(false) + expect(decision.promote).toBe(true) + } + }) + it('refuses a fully concordant binary comparison on the exact test, not on the interval', () => { const arm = [1, 0, 1, 0, 1, 0] const decision = decidePairedPromotion(arm, [...arm]) diff --git a/src/paired-promotion-decision.ts b/src/paired-promotion-decision.ts index f5080ee6..cd9dc09b 100644 --- a/src/paired-promotion-decision.ts +++ b/src/paired-promotion-decision.ts @@ -40,15 +40,15 @@ * bounded asymmetric null whose true mean paired delta is exactly 0: 88.50 % * false promotion at n = 6 and 65.65 % at n = 20 against a nominal 5 %. * - * Orthogonal to the small-sample switch inside {@link pairedDeltaTest}: that - * picks the TEST from the sample size (bootstrap CI at n >= 20, pre-registered - * exact sign test below it), this picks the ESTIMATOR from the outcome's shape. - * Both are needed — an exact sign test applied to a tie-pinned median is still - * blind, and a mean bootstrap CI at n = 6 is still not a valid test. + * Eligibility follows the requested target. A continuous mean needs the + * bootstrap minimum; binary outcomes and explicit median targets use the + * exact-test minimum. A small-sample sign diagnostic cannot certify a mean + * effect. These floors establish estimator eligibility, not adequate power. */ import { minimumPairsForPairedDeltaTest, pairedDeltaTest } from './paired-delta-test' import { + BOOTSTRAP_GATE_MIN_N, type PairedBootstrapResult, pairedBinaryScale, pairedDeltaTieFraction, @@ -87,8 +87,9 @@ export interface PairedPromotionDecisionOptions { resamples?: number /** Deterministic bootstrap seed. Omitted ⇒ derived from the deltas. */ seed?: number - /** Caller-required paired observations. The exact test may impose a higher - * minimum; the effective one is reported as `minimumPairs`. */ + /** Caller-required paired observations. The test also imposes its own + * minimum: bootstrap eligibility for a continuous mean, or the exact-test + * minimum for binary and explicit median targets. This is not a power guarantee. */ minPairs?: number /** * `'mean'` (default) routes by SHAPE: a two-point (pass/fail) outcome on any @@ -130,11 +131,11 @@ export interface PairedPromotionDecision { /** Exact one-sided sign-test p-value on the small-sample bootstrap path; * null otherwise. */ pValue: number | null - /** Effective observation minimum after accounting for confidence. */ + /** Effective observation minimum for the target, confidence, and caller floor. */ minimumPairs: number /** n >= minimumPairs. */ sufficient: boolean - /** The deciding interval is zero-width or non-finite — no evidence in either + /** The deciding interval is zero-width at numeric precision or non-finite — no evidence in either * direction, so it cannot clear any threshold on evidence. */ indeterminate: boolean /** McNemar's exact test refuses at a non-negative threshold. */ @@ -210,10 +211,12 @@ export function decidePairedPromotion( `decidePairedPromotion: minPairs must be a positive integer, got ${requestedMinimum}`, ) } - const minimumPairs = Math.max(requestedMinimum, exactMinimum) + const { binaryScale, tieFraction } = pairedDecisionShape(before, after, options.statistic) + const estimatorMinimum = + binaryScale === null && options.statistic !== 'median' ? BOOTSTRAP_GATE_MIN_N : exactMinimum + const minimumPairs = Math.max(requestedMinimum, exactMinimum, estimatorMinimum) const n = before.length const sufficient = n >= minimumPairs - const { binaryScale, tieFraction } = pairedDecisionShape(before, after, options.statistic) let core: { statistic: PairedDecisionStatistic @@ -270,7 +273,7 @@ export function decidePairedPromotion( statistic: bootstrapStatistic, seed: options.seed, threshold, - minPairs: options.minPairs, + minPairs: minimumPairs, }) const ci = test.bootstrap core = { @@ -286,14 +289,24 @@ export function decidePairedPromotion( label: bootstrapStatistic, methodDetail: test.method === 'exact-sign' - ? ` Below ${test.minimumPairs} pairs the interval is descriptive only;` + - ` the decision is the exact one-sided sign test, p=${fmt(test.pValue ?? 1)}.` + ? bootstrapStatistic === 'mean' + ? ` The mean requires ${minimumPairs} pairs for bootstrap eligibility;` + + ` the exact sign-test p=${fmt(test.pValue ?? 1)} does not establish a mean effect.` + : ` Below ${BOOTSTRAP_GATE_MIN_N} pairs the interval is descriptive only;` + + ` the median decision uses the exact one-sided sign test, p=${fmt(test.pValue ?? 1)}.` : '', } } + // Equivalent deltas can differ by rounding after subtraction and averaging. + // Use the interval's own scale so tiny score units retain the same decision + // as a rescaled copy of the evidence. + const intervalScale = Math.max(Math.abs(core.low), Math.abs(core.high)) + const intervalTolerance = intervalScale * Number.EPSILON * 8 const indeterminate = - !Number.isFinite(core.low) || !Number.isFinite(core.high) || core.low === core.high + !Number.isFinite(core.low) || + !Number.isFinite(core.high) || + core.high - core.low <= intervalTolerance const indeterminateCause = !indeterminate ? '' : tieFraction === 1 diff --git a/src/pre-registration.ts b/src/pre-registration.ts index 2f319ff2..05539b70 100644 --- a/src/pre-registration.ts +++ b/src/pre-registration.ts @@ -8,14 +8,11 @@ * evaluate the manifest against observed results — the library refuses * to let you re-interpret a different metric as the declared one. * - * A signed manifest is a portable record: it is written once and verified - * later, possibly by a different release. `algo` names the digest scheme it - * was signed under, and verification selects the encoder by that field, so a - * manifest signed by an earlier release still verifies. + * A signed manifest carries its required digest scheme. Verification accepts + * only RFC 8785 canonical JSON, using the same encoder as every new identity. */ -import { createHash } from 'node:crypto' -import { canonicalString, hashCanonical } from './ledger-core/canonical' +import { hashCanonical } from './ledger-core/canonical' export interface HypothesisManifest { id: string @@ -40,32 +37,14 @@ export interface HypothesisManifest { candidateLabel?: string } -/** - * Identifier for the hashing scheme used to produce `contentHash`. - * - * Both schemes are sha256 hex over the manifest with `contentHash` and `algo` - * stripped, and differ only in how that manifest is serialized: - * - * - `'sha256-rfc8785'` — RFC 8785 canonical JSON. What {@link signManifest} - * emits. - * - `'sha256-content'` — key-sorted `JSON.stringify`. Read-only: manifests - * signed by an earlier release carry it, or carry no `algo` at all, and - * {@link verifyManifest} still verifies them. - */ -export type SignedManifestAlgo = 'sha256-content' | 'sha256-rfc8785' +/** SHA-256 over RFC 8785 canonical JSON, excluding `contentHash` and `algo`. */ +export type SignedManifestAlgo = 'sha256-rfc8785' export interface SignedManifest extends HypothesisManifest { /** sha256 hex of canonicalized manifest (everything except contentHash and algo). */ contentHash: string - /** - * Algorithm string describing how `contentHash` was produced. - * - * Optional on the type so serialized manifests without it still parse, - * but ALWAYS populated by {@link signManifest}. Consumers that want to - * enforce a known algorithm should reject manifests where this field - * is missing or unrecognized. - */ - algo?: SignedManifestAlgo + /** Required digest scheme. Missing or unsupported schemes fail verification. */ + algo: SignedManifestAlgo } export interface HypothesisResult { @@ -103,40 +82,16 @@ export async function hashJson(obj: T): Promise { } /** - * Key-sorted `JSON.stringify` digest. Private and read-only: it exists so a - * manifest signed under `'sha256-content'` still verifies, and nothing that - * WRITES a digest may call it. - */ -function legacyContentDigest(value: unknown): string { - return createHash('sha256') - .update(JSON.stringify(sortKeysDeep(value)), 'utf8') - .digest('hex') -} - -function sortKeysDeep(value: unknown): unknown { - if (value === null || typeof value !== 'object') return value - if (Array.isArray(value)) return value.map(sortKeysDeep) - const out: Record = {} - for (const key of Object.keys(value as Record).sort()) { - out[key] = sortKeysDeep((value as Record)[key]) - } - return out -} - -/** - * Digest of a manifest under its own declared scheme, with `contentHash` and - * `algo` stripped. Synchronous, so a caller that must fail before consuming an - * observation does not have to await. Throws on an `algo` this release does - * not know — an unverifiable manifest must not read as a valid one. + * Digest a manifest after validating its scheme, excluding `contentHash` and + * `algo`. This synchronous check can refuse a manifest before consuming data. */ export function manifestContentDigest(manifest: SignedManifest): string { const { contentHash: _contentHash, algo, ...rest } = manifest void _contentHash - if (algo === undefined || algo === 'sha256-content') return legacyContentDigest(rest) - if (algo === 'sha256-rfc8785') { - return createHash('sha256').update(canonicalString(rest), 'utf8').digest('hex') + if (algo !== 'sha256-rfc8785') { + throw new Error(`pre-registration: unsupported manifest hash algo '${String(algo)}'`) } - throw new Error(`pre-registration: unrecognized manifest hash algo '${String(algo)}'`) + return hashCanonical(rest).slice('sha256:'.length) } /** @@ -154,7 +109,11 @@ export async function signManifest(m: HypothesisManifest): Promise { - return manifestContentDigest(m) === m.contentHash + try { + return manifestContentDigest(m) === m.contentHash + } catch { + return false + } } /** @@ -166,7 +125,7 @@ export async function evaluateHypothesis( observed: { n: number; effect: number; pValue: number }, ): Promise { if (!(await verifyManifest(manifest))) { - throw new Error('evaluateHypothesis: manifest content hash mismatch (tampered)') + throw new Error('evaluateHypothesis: unsupported manifest hash scheme or content hash mismatch') } const reasons: HypothesisResult['rejectionReasons'] = [] const directionOk = manifest.direction === 'increase' ? observed.effect > 0 : observed.effect < 0 diff --git a/src/public-integrity-api.test.ts b/src/public-integrity-api.test.ts new file mode 100644 index 00000000..b46af5c2 --- /dev/null +++ b/src/public-integrity-api.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, expectTypeOf, it } from 'vitest' +import type * as campaign from './campaign/index' +import { + defineAgentEval, + type GateDecision, + type JudgeScore, + type Scenario, +} from './contract/index' +import { defineEvaluationClaim, summarizeEvaluationUnits } from './experiment/index' +import type * as root from './index' +import { + auditEvaluator, + calibrateJudgeContinuous, + type EvaluatorAuditObservation, + positionalBias, + selfPreference, +} from './meta-eval/index' + +describe('public evaluation types', () => { + it('lets root types flow through the contract evaluation without adapters', async () => { + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf() + + const scenario: root.Scenario = { id: 'ticket', kind: 'support' } + const judgeScore: root.JudgeScore = { dimensions: { resolved: 1 }, composite: 1, notes: '' } + const kit = defineAgentEval({ + scenarios: [scenario], + baselineSurface: 'Resolve the ticket.', + agent: async () => 'Resolved.', + judge: { + name: 'resolved', + dimensions: [{ key: 'resolved', description: 'The ticket is resolved.' }], + score: () => judgeScore, + }, + expectUsage: 'off', + }) + const measured = await kit.evaluate() + expect(measured.aggregates.byJudge.resolved?.mean).toBe(1) + }) + + it('keeps the distinct product judging types explicit beside their functions', () => { + expectTypeOf().toEqualTypeOf() + expectTypeOf>>().toEqualTypeOf() + expectTypeOf< + ReturnType + >().toEqualTypeOf() + }) + + it('uses the experiment subpath to distinguish variants from independent tasks', () => { + const claim = defineEvaluationClaim({ + use: 'comparison', + population: { id: 'support', description: 'Support incidents.' }, + samplingFrame: 'Incident queue sampled before search.', + independentUnit: 'incidentId', + generalization: 'new-units', + minimumEffect: 0.05, + }) + const summary = summarizeEvaluationUnits(claim, [ + { id: 'first-variant', incidentId: 'incident-1' }, + { id: 'second-variant', incidentId: 'incident-1' }, + { id: 'third-variant', incidentId: 'incident-2' }, + ]) + expect(summary).toMatchObject({ observations: 3, independentUnits: 2 }) + }) + + it('exposes calibration, bias diagnostics, and admission together', () => { + expect(typeof calibrateJudgeContinuous).toBe('function') + expect( + positionalBias([ + { itemId: 'same-output', score: 0.9, positionOfAInput: 'first' }, + { itemId: 'same-output', score: 0.4, positionOfAInput: 'second' }, + ]), + ).toEqual({ avgDelta: 0.5, n: 1 }) + expect( + selfPreference([ + { score: 1, inFamily: true }, + { score: 0, inFamily: false }, + ]), + ).toMatchObject({ deltaMean: 1, n: 2 }) + + const observations: EvaluatorAuditObservation[] = [ + { + id: 'unknown', + independentUnitId: 'one-incident', + evidenceRef: 'artifact://unknown', + expected: 'reject', + observed: 'unknown', + exposure: 'fresh', + }, + ] + const report = auditEvaluator({ + evaluatorDigest: `sha256:${'1'.repeat(64)}`, + population: 'Support outputs.', + samplingFrame: 'Independent incident samples.', + authority: { + evaluatorAuthorId: 'author', + auditorId: 'auditor', + independenceEvidenceRef: 'artifact://access-policy', + }, + policy: { confidence: 0.95, maxFalseAcceptanceRate: 0.1, maxFalseRejectionRate: 0.1 }, + observations, + }) + expect(report.verdict).toBe('inconclusive') + expect(report.coverage.unknownCases).toBe(1) + expect(report.falseAcceptance.errorRate).toBeNull() + }) +}) diff --git a/src/rl/adaptation-eval.ts b/src/rl/adaptation-eval.ts index f73fb642..ca817c4c 100644 --- a/src/rl/adaptation-eval.ts +++ b/src/rl/adaptation-eval.ts @@ -1,32 +1,15 @@ /** * Sample-efficient adaptation evaluation. * - * For foundation-model-based agents, the load-bearing capability isn't - * raw end-state performance — it's *how fast the agent reaches that - * performance from cold start*. The same model with a worse prompt that - * adapts in 5 demonstrations beats the same model with a better prompt - * that needs 50. Standard meta-learning eval (Finn et al., MAML, RL² lit) - * reports an *adaptation curve*: score after k=0, 1, 2, 4, 8, 16, … - * in-context examples or fine-tune steps. - * - * This module ships: - * - * 1. `runAdaptationCurve` — given a runner that takes k demonstrations - * and returns a score, produce the (k, score) curve. - * 2. `compareAdaptationCurves` — paired comparison across two policies. - * Returns per-k delta with bootstrap CIs and an "area-under-curve" - * summary statistic. - * 3. `firstPassK` — for pass/fail evaluation, the minimum k at which - * the policy reliably passes (≥ pass-rate threshold over reps). - * - * Use cases: - * - Compare two prompt designs that have similar end-state performance - * but different in-context efficiency. - * - Decide between fine-tuning and prompting based on adaptation cost. - * - Detect when a policy "memorizes" k=0 inputs vs. genuinely adapts. + * An adaptation curve records scores after k demonstrations or training steps. + * Comparison pairs the same scenarios and resamples their whole curves. + * The normalized area summarizes performance over the observed k range. + * A first-pass k is descriptive and carries no separate reliability claim. */ -import { makeRng } from '../statistics/internal' +import { ValidationError } from '../errors' +import { decidePairedPromotion, type PairedPromotionDecision } from '../paired-promotion-decision' +import { type PairedBootstrapResult, pairedBootstrap } from '../statistics' export interface AdaptationRunner { /** @@ -44,7 +27,7 @@ export interface RunAdaptationCurveOptions { /** Reps per (scenario, k) cell. Default 3. */ reps?: number runner: AdaptationRunner - /** Pass-rate threshold for `firstPassK` reporting. Default 0.5. */ + /** Score threshold for a pass and pass-rate threshold for firstPassK. Default 0.5. */ passThreshold?: number } @@ -66,19 +49,26 @@ export interface AdaptationCurve { */ firstPassK: number | null /** - * Area under the (k, meanScore) curve, normalized by max-k. A - * single-number summary of "how well does this policy adapt from - * cold-start to fully-conditioned." Higher = better adapter. + * Trapezoidal area over the observed k intervals, divided by max-k. + * No performance is inferred below the first observed k. */ adaptationArea: number } -export async function runAdaptationCurve( +export async function runAdaptationCurve( opts: RunAdaptationCurveOptions, ): Promise { const ks = opts.ks ?? [0, 1, 2, 4, 8, 16] const reps = opts.reps ?? 3 const passThreshold = opts.passThreshold ?? 0.5 + assertKs(ks, 'runAdaptationCurve') + assertScenarioIds(opts.scenarios, 'runAdaptationCurve') + if (!Number.isInteger(reps) || reps < 1) { + throw new ValidationError('runAdaptationCurve: reps must be a positive integer') + } + if (!Number.isFinite(passThreshold) || passThreshold < 0 || passThreshold > 1) { + throw new ValidationError('runAdaptationCurve: passThreshold must be in [0,1]') + } const sortedKs = [...ks].sort((a, b) => a - b) const points: AdaptationPoint[] = [] @@ -88,11 +78,12 @@ export async function runAdaptationCurve( let totalPasses = 0 let totalAttempts = 0 for (const scenario of opts.scenarios) { - const sid = scenario.scenarioId ?? `scenario-${opts.scenarios.indexOf(scenario)}` + const sid = scenario.scenarioId const scores: number[] = [] let passes = 0 for (let r = 0; r < reps; r++) { const score = await opts.runner.run({ scenario, k, rep: r }) + assertScore(score, `runAdaptationCurve: scenario '${sid}', k=${k}, rep=${r}`) scores.push(score) if (score >= passThreshold) passes++ allScores.push(score) @@ -119,7 +110,7 @@ export async function runAdaptationCurve( const firstPassK = points.find((p) => p.passRate >= passThreshold)?.k ?? null const maxK = sortedKs[sortedKs.length - 1] ?? 1 - // Trapezoidal area under the (k, meanScore) curve, normalized by k-range. + // Only observed intervals contribute; an unmeasured prefix is not extrapolated. let area = 0 for (let i = 1; i < points.length; i++) { const x1 = points[i - 1]!.k @@ -136,102 +127,175 @@ export async function runAdaptationCurve( export interface CompareCurvesResult { perK: Array<{ k: number - deltaMean: number - aLow: number - aHigh: number - bLow: number - bHigh: number + /** Paired A − B score differences; intervals are descriptive across k. */ + delta: PairedBootstrapResult }> - areaDelta: number - firstPassKDelta: number | null - /** Verdict: 'a_better' | 'b_better' | 'similar'. */ - verdict: 'a_better' | 'b_better' | 'similar' + /** Descriptive paired A − B areas, computed within each scenario before resampling. */ + areaDelta: PairedBootstrapResult + /** Decisions from the shared paired inference rules; intervals may differ from the bootstrap. */ + aImprovement: PairedPromotionDecision + bImprovement: PairedPromotionDecision + /** Independent sampling units; repetitions never increase this count. */ + scenarioIds: string[] + /** Only the area decisions determine direction. Inconclusive does not mean equivalent. */ + verdict: 'a_better' | 'b_better' | 'inconclusive' | 'insufficient_evidence' /** Rationale, ready to render. */ rationale: string } /** - * Paired comparison of two adaptation curves. Per-k deltas with 95% - * bootstrap CIs (constructed from each curve's `perScenario` per-k means - * — the bootstrap unit is the scenario, not the rep). + * Compare identical scenario cohorts on identical k grids, paired by scenarioId. + * Missing pairs, duplicate identities, and cohort changes across k are refused. + * The bootstrap resamples whole scenarios, preserving dependence across k. + * Per-k intervals describe the curve; shared paired area decisions determine the verdict. + * Bootstrap eligibility is necessary but does not establish scenario independence. */ export function compareAdaptationCurves( a: AdaptationCurve, b: AdaptationCurve, - opts: { confidence?: number; bootstrapResamples?: number; seed?: number } = {}, + opts: { + confidence?: number + bootstrapResamples?: number + seed?: number + /** Minimum worthwhile difference in normalized area. Default 0. */ + minimumEffect?: number + } = {}, ): CompareCurvesResult { - const conf = opts.confidence ?? 0.95 - const resamples = opts.bootstrapResamples ?? 500 - const rng = makeRng( - opts.seed, - a.points.flatMap((point) => point.perScenario.map((cell) => cell.meanScore)), - b.points.flatMap((point) => point.perScenario.map((cell) => cell.meanScore)), - ) - - const perK: CompareCurvesResult['perK'] = [] - for (const ap of a.points) { - const bp = b.points.find((p) => p.k === ap.k) - if (!bp) continue - const aMeans = ap.perScenario.map((s) => s.meanScore) - const bMeans = bp.perScenario.map((s) => s.meanScore) - const aCi = bootstrapMeanCi(aMeans, resamples, conf, rng) - const bCi = bootstrapMeanCi(bMeans, resamples, conf, rng) - perK.push({ - k: ap.k, - deltaMean: ap.meanScore - bp.meanScore, - aLow: aCi.low, - aHigh: aCi.high, - bLow: bCi.low, - bHigh: bCi.high, - }) + const confidence = opts.confidence ?? 0.95 + const resamples = opts.bootstrapResamples ?? 2000 + const minimumEffect = opts.minimumEffect ?? 0 + if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) { + throw new ValidationError('compareAdaptationCurves: confidence must be in (0,1)') } - - const areaDelta = a.adaptationArea - b.adaptationArea - const firstPassKDelta = - a.firstPassK !== null && b.firstPassK !== null - ? b.firstPassK - a.firstPassK // smaller k for a means a adapts faster (positive delta) - : null - - // Composite verdict: positive area delta + most per-k deltas in same - // direction → that side wins. Within ε of zero on both → similar. - const meanDelta = perK.reduce((s, p) => s + p.deltaMean, 0) / Math.max(1, perK.length) + if (!Number.isInteger(resamples) || resamples < 1) { + throw new ValidationError( + 'compareAdaptationCurves: bootstrapResamples must be a positive integer', + ) + } + if (!Number.isFinite(minimumEffect) || minimumEffect < 0 || minimumEffect > 1) { + throw new ValidationError('compareAdaptationCurves: minimumEffect must be in [0,1]') + } + const aPoints = indexCurve(a, 'A') + const bPoints = indexCurve(b, 'B') + const ks = [...aPoints.keys()].sort((x, y) => x - y) + const missingInA = [...bPoints.keys()].filter((k) => !aPoints.has(k)) + const missingInB = ks.filter((k) => !bPoints.has(k)) + if (missingInA.length > 0 || missingInB.length > 0) { + throw new ValidationError( + `compareAdaptationCurves: k grids differ; missing in A=[${missingInA}], missing in B=[${missingInB}]`, + ) + } + const scenarioIds = [...aPoints.get(ks[0]!)!.keys()].sort() + const expectedIds = new Set(scenarioIds) + for (const [arm, points] of [ + ['A', aPoints], + ['B', bPoints], + ] as const) { + for (const [k, cells] of points) { + const missing = scenarioIds.filter((id) => !cells.has(id)) + const extra = [...cells.keys()].filter((id) => !expectedIds.has(id)) + if (missing.length > 0 || extra.length > 0) { + throw new ValidationError( + `compareAdaptationCurves: scenario pairs differ in ${arm} at k=${k}; missing=[${missing}], unexpected=[${extra}]`, + ) + } + } + } + const bootstrapOptions = { confidence, resamples, statistic: 'mean' as const, seed: opts.seed } + const perK = ks.map((k) => ({ + k, + delta: pairedBootstrap( + scenarioIds.map((id) => bPoints.get(k)!.get(id)!), + scenarioIds.map((id) => aPoints.get(k)!.get(id)!), + bootstrapOptions, + ), + })) + const aAreas = scenarioIds.map((id) => scenarioArea(ks, aPoints, id)) + const bAreas = scenarioIds.map((id) => scenarioArea(ks, bPoints, id)) + const decisionOptions = { + ...bootstrapOptions, + threshold: minimumEffect, + } + const aImprovement = decidePairedPromotion(bAreas, aAreas, decisionOptions) + const bImprovement = decidePairedPromotion(aAreas, bAreas, decisionOptions) + const areaDelta = aImprovement.bootstrap ?? pairedBootstrap(bAreas, aAreas, bootstrapOptions) let verdict: CompareCurvesResult['verdict'] - if (Math.abs(meanDelta) < 0.02 && Math.abs(areaDelta) < 0.02) verdict = 'similar' - else if (meanDelta > 0 && areaDelta > 0) verdict = 'a_better' - else if (meanDelta < 0 && areaDelta < 0) verdict = 'b_better' - else verdict = 'similar' + if (!aImprovement.sufficient || ks.length < 2) verdict = 'insufficient_evidence' + else if (aImprovement.promote) verdict = 'a_better' + else if (bImprovement.promote) verdict = 'b_better' + else verdict = 'inconclusive' const rationale = - `mean per-k delta=${meanDelta.toFixed(3)}, area delta=${areaDelta.toFixed(3)}` + - (firstPassKDelta !== null ? `, first-pass-k delta=${firstPassKDelta}` : '') + `paired scenarios=${scenarioIds.length}, area delta=${areaDelta.mean.toFixed(3)}, ` + + `${confidence * 100}% ${aImprovement.statistic} interval=[${aImprovement.low.toFixed(3)}, ${aImprovement.high.toFixed(3)}], ` + + `minimum effect=${minimumEffect}; ${verdict}` - return { perK, areaDelta, firstPassKDelta, verdict, rationale } + return { perK, areaDelta, aImprovement, bImprovement, scenarioIds, verdict, rationale } } -/** First k at which the curve's per-scenario pass rate reliably hits the threshold. */ +/** First observed k whose pass rate reaches the threshold; this is a descriptive summary. */ export function firstPassK(curve: AdaptationCurve, threshold = 0.5): number | null { return curve.points.find((p) => p.passRate >= threshold)?.k ?? null } // ── Helpers ────────────────────────────────────────────────────────────── -function bootstrapMeanCi( - xs: number[], - resamples: number, - confidence: number, - rng: () => number, -): { low: number; high: number } { - if (xs.length < 2) return { low: xs[0] ?? 0, high: xs[0] ?? 0 } - const samples = new Array(resamples) - for (let b = 0; b < resamples; b++) { - let sum = 0 - for (let i = 0; i < xs.length; i++) sum += xs[Math.floor(rng() * xs.length)]! - samples[b] = sum / xs.length - } - samples.sort((a, b) => a - b) - const alpha = 1 - confidence - return { - low: samples[Math.floor((alpha / 2) * resamples)]!, - high: samples[Math.min(resamples - 1, Math.ceil((1 - alpha / 2) * resamples) - 1)]!, +function assertKs(ks: number[], where: string): void { + if (ks.length === 0 || ks.some((k) => !Number.isInteger(k) || k < 0)) { + throw new ValidationError(`${where}: ks must contain nonnegative integers`) + } + if (new Set(ks).size !== ks.length) { + throw new ValidationError(`${where}: duplicate k values`) + } +} + +function assertScenarioIds(cells: Array<{ scenarioId: string }>, where: string): void { + if (cells.length === 0 || cells.some((cell) => !cell.scenarioId?.trim())) { + throw new ValidationError(`${where}: scenarios must have explicit nonempty scenarioId values`) + } + const seen = new Set() + for (const { scenarioId } of cells) { + if (seen.has(scenarioId)) + throw new ValidationError(`${where}: duplicate scenarioId '${scenarioId}'`) + seen.add(scenarioId) + } +} + +function assertScore(score: number, where: string): void { + if (!Number.isFinite(score) || score < 0 || score > 1) { + throw new ValidationError(`${where}: score must be finite and in [0,1], got ${score}`) + } +} + +function indexCurve(curve: AdaptationCurve, arm: string): Map> { + const where = `compareAdaptationCurves: ${arm}` + assertKs( + curve.points.map((point) => point.k), + where, + ) + return new Map( + curve.points.map((point) => { + assertScenarioIds(point.perScenario, `${where} at k=${point.k}`) + return [ + point.k, + new Map( + point.perScenario.map((cell) => { + assertScore(cell.meanScore, `${where}: '${cell.scenarioId}' at k=${point.k}`) + return [cell.scenarioId, cell.meanScore] + }), + ), + ] + }), + ) +} + +function scenarioArea(ks: number[], points: Map>, id: string): number { + let area = 0 + for (let i = 1; i < ks.length; i++) { + const left = ks[i - 1]! + const right = ks[i]! + area += ((points.get(left)!.get(id)! + points.get(right)!.get(id)!) * (right - left)) / 2 } + const maxK = ks[ks.length - 1]! + return maxK === 0 ? 0 : area / maxK } diff --git a/src/rl/contamination.ts b/src/rl/contamination.ts index e5c9201f..163b52f7 100644 --- a/src/rl/contamination.ts +++ b/src/rl/contamination.ts @@ -1,35 +1,16 @@ /** * Contamination probe — held-out perturbation tests. * - * The bug class: once a benchmark scenario set is published, models train - * on it, and your scores become invalid. SWE-Bench-Verified, GPQA, and - * MMLU-Pro all exist because their predecessors got contaminated within - * months. The right defense is to keep a held-out *perturbed* version of - * every scenario — same task, slightly different surface — and check - * whether scores diverge significantly. Genuine capability transfers; rote - * memorization doesn't. - * - * This module ships the probe contract: - * - * 1. A `ScenarioPerturbation` strategy type — function that produces a - * perturbed scenario from an original. - * 2. `runContaminationProbe({ originals, perturbed, scoreFn })` — runs - * both halves and reports per-scenario score divergence + a global - * contamination verdict via paired Wilcoxon. - * 3. Several stock perturbations: `renameVariables`, `shuffleOrder`, - * `paraphrasePrompt`, `injectIrrelevantClause`. Each preserves the - * task's structural difficulty while breaking surface memorization. - * - * The verdict is conservative: if the perturbed-vs-original score - * difference is statistically significant (BH-adjusted p < 0.05) AND - * the median drop is > 5 percentage points, we flag *contamination - * suspected*. False positives are possible (the perturbation might - * actually be harder); the default is to flag for review, not to - * autoreject. + * Score each scenario and its perturbation, then test the paired differences. + * A significant global Wilcoxon result plus a worthwhile median drop flags + * contamination for review. Perturbations may change difficulty, so the result + * does not identify contamination as the cause. Per-item differences have no + * calibrated sampling null and carry no p-values or q-values. */ import { ValidationError } from '../errors' -import { benjaminiHochberg, wilcoxonSignedRank } from '../statistics' +import { wilcoxonSignedRank } from '../statistics' +import { medianInPlace } from '../statistics/internal' import { mulberry32 } from '../statistics/random' export type ScenarioPerturbationKind = @@ -48,7 +29,7 @@ export interface ScenarioPerturbation { } export interface ContaminationProbeInput { - /** Identity of every scenario. The probe's `runFingerprint` keys on these. */ + /** Stable, unique identity of every original scenario. */ scenarioId: (s: S) => string /** Original scenarios. */ originals: S[] @@ -69,11 +50,8 @@ export interface ContaminationProbeInput { export interface ContaminationProbeOptions { /** Drop scores below this from the probe; treats partial failures separately. Default 0. */ scoreFloor?: number - /** - * BH-FDR threshold for declaring contamination on each per-scenario - * delta. Default 0.05. - */ - fdr?: number + /** Significance threshold for the single global paired test. Default 0.05. */ + alpha?: number /** * Minimum median per-scenario drop to flag global contamination. Default * 0.05 (5 percentage points). Smaller drops may be noise. @@ -87,26 +65,44 @@ export interface ContaminationProbeReport { originalScore: number perturbedScore: number delta: number // perturbed - original (negative = drop) - /** Per-scenario q-value (single-test BH for a single scenario). Mainly for display. */ - qValue: number }> - /** Wilcoxon paired-test on the deltas. */ - pairedTest: { w: number; p: number } - medianDelta: number - meanDelta: number + /** Global Wilcoxon paired test; null when fewer than four pairs are included. */ + pairedTest: { w: number; p: number } | null + /** Observed summaries of included pairs; null when no pairs are included. */ + medianDelta: number | null + meanDelta: number | null contaminationSuspected: boolean reason: string - /** Number of scenarios processed. */ + /** Number of pairs included after the configured score floor. */ n: number + /** Scenarios excluded by the score floor; their observed scores remain above. */ + excludedScenarioIds: string[] } export async function runContaminationProbe( input: ContaminationProbeInput, opts: ContaminationProbeOptions = {}, ): Promise { - const fdr = opts.fdr ?? 0.05 + const alpha = opts.alpha ?? 0.05 const minMedianDrop = opts.minMedianDrop ?? 0.05 const floor = opts.scoreFloor ?? 0 + if (!Number.isFinite(alpha) || alpha <= 0 || alpha >= 1) { + throw new ValidationError('runContaminationProbe: alpha must be in (0,1)') + } + for (const [name, value] of [ + ['scoreFloor', floor], + ['minMedianDrop', minMedianDrop], + ] as const) { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new ValidationError(`runContaminationProbe: ${name} must be in [0,1]`) + } + } + const ids = input.originals.map(input.scenarioId) + if (ids.some((id) => !id?.trim()) || new Set(ids).size !== ids.length) { + throw new ValidationError( + 'runContaminationProbe: original scenario IDs must be nonempty and unique', + ) + } if (!input.perturbed && !input.perturbation) { throw new ValidationError( @@ -125,63 +121,61 @@ export async function runContaminationProbe( const origScores = await Promise.all(input.originals.map((s) => input.scoreFn(s))) const pertScores = await Promise.all(perturbed.map((s) => input.scoreFn(s))) - const perScenario = input.originals.map((s, i) => ({ - scenarioId: input.scenarioId(s), + for (const score of [...origScores, ...pertScores]) { + if (!Number.isFinite(score) || score < 0 || score > 1) { + throw new ValidationError( + `runContaminationProbe: scores must be finite and in [0,1], got ${score}`, + ) + } + } + const perScenario = ids.map((scenarioId, i) => ({ + scenarioId, originalScore: origScores[i]!, perturbedScore: pertScores[i]!, delta: pertScores[i]! - origScores[i]!, - qValue: NaN, })) // Drop scenarios below the floor (partial failures we don't trust). const valid = perScenario.filter((p) => p.originalScore >= floor && p.perturbedScore >= floor) + const excludedScenarioIds = perScenario + .filter((p) => p.originalScore < floor || p.perturbedScore < floor) + .map((p) => p.scenarioId) + const deltas = valid.map((p) => p.delta) + const medianDelta = deltas.length === 0 ? null : medianInPlace(deltas) + const meanDelta = + deltas.length === 0 ? null : deltas.reduce((sum, d) => sum + d, 0) / deltas.length if (valid.length < 4) { return { perScenario, - pairedTest: { w: 0, p: 1 }, - medianDelta: 0, - meanDelta: 0, + pairedTest: null, + medianDelta, + meanDelta, contaminationSuspected: false, reason: `insufficient valid scenarios (n=${valid.length}, need ≥ 4)`, n: valid.length, + excludedScenarioIds, } } const origValid = valid.map((p) => p.originalScore) const pertValid = valid.map((p) => p.perturbedScore) const pairedTest = wilcoxonSignedRank(origValid, pertValid) - const deltas = valid.map((p) => p.delta) - const sortedDeltas = [...deltas].sort((a, b) => a - b) - const median = sortedDeltas[Math.floor(sortedDeltas.length / 2)]! - const mean = deltas.reduce((s, d) => s + d, 0) / deltas.length - - // Per-scenario q-values via BH on a synthetic per-scenario p-value - // (one-sample bootstrap; we use the absolute delta normalized by median - // as a coarse signal — this is a display aid, the load-bearing test - // is the global Wilcoxon). - const pseudoP = valid.map((p) => Math.min(1, Math.max(1e-6, 1 - Math.abs(p.delta) / 1))) - const { qValues } = benjaminiHochberg(pseudoP, fdr) - for (let i = 0; i < valid.length; i++) { - const v = valid[i]! - const idx = perScenario.findIndex((p) => p.scenarioId === v.scenarioId) - if (idx >= 0) perScenario[idx]!.qValue = qValues[i]! - } - - const contaminationSuspected = pairedTest.p < fdr && median <= -minMedianDrop + const contaminationSuspected = pairedTest.p < alpha && medianDelta! <= -minMedianDrop const reason = contaminationSuspected - ? `paired p=${pairedTest.p.toFixed(4)} < ${fdr} and median drop ${median.toFixed(4)} ≥ ${minMedianDrop}` - : pairedTest.p >= fdr + ? `paired p=${pairedTest.p.toFixed(4)} < ${alpha} and median drop ${(-medianDelta!).toFixed(4)} ≥ ${minMedianDrop}` + : pairedTest.p >= alpha ? `no significant difference (paired p=${pairedTest.p.toFixed(4)})` - : `significant but small effect (median delta ${median.toFixed(4)})` + : `significant but no qualifying drop (median delta ${medianDelta!.toFixed(4)})` return { perScenario, pairedTest, - medianDelta: median, - meanDelta: mean, + medianDelta, + meanDelta, contaminationSuspected, reason, n: valid.length, + excludedScenarioIds, } } diff --git a/src/rl/predictive-validity-researcher.ts b/src/rl/predictive-validity-researcher.ts index 2a174d4b..d3c05092 100644 --- a/src/rl/predictive-validity-researcher.ts +++ b/src/rl/predictive-validity-researcher.ts @@ -14,8 +14,13 @@ */ import type { GateDecision, SplitCoverage } from '../held-out-gate' +import { + assertUniqueObservationIds, + validateOutcomeMetricSpecifications, +} from '../meta-eval/outcome-observations' import type { OutcomeStore } from '../meta-eval/outcome-store' import { + type OutcomeMetricSpec, type RubricPredictiveValidityReport, rubricPredictiveValidity, } from '../meta-eval/rubric-predictive-validity' @@ -30,13 +35,10 @@ import { type RunRecord, runTaskScore } from '../run-record' export interface PredictiveValidityResearcherOptions { outcomes: OutcomeStore - outcomeMetrics: string[] + /** Fix one desired outcome before observing the report; recommendations never choose an outcome post hoc. */ + targetOutcome: OutcomeMetricSpec /** Score threshold below which a run counts as a "failure." Default 0.5. */ failureThreshold?: number - /** Spearman bucket below which a rubric is "decorative." Default 0.4. */ - decorativeThreshold?: number - /** Optional steering-namespace prefix for proposed changes. Default `'rubric_weight'`. */ - steeringNamespace?: string /** Override the rubric set the researcher inspects. Default: every numeric `outcome.raw` key seen. */ rubrics?: string[] /** @@ -48,15 +50,24 @@ export interface PredictiveValidityResearcherOptions { } /** - * Concrete `Researcher` driven by `rubricPredictiveValidity`. The brain: - * rubrics that don't predict deployment outcomes don't earn weight. + * Proposes rubric experiments against one declared outcome. + * A correlation supports a hypothesis; the caller must measure any resulting change. */ export class PredictiveValidityResearcher implements Researcher { - private opts: PredictiveValidityResearcherOptions + private readonly opts: PredictiveValidityResearcherOptions private lastReport: RubricPredictiveValidityReport | null = null constructor(opts: PredictiveValidityResearcherOptions) { - this.opts = opts + validateOutcomeMetricSpecifications([opts.targetOutcome]) + if (opts.rubrics !== undefined) assertUniqueObservationIds(opts.rubrics, 'rubric') + if (opts.failureThreshold !== undefined && !Number.isFinite(opts.failureThreshold)) { + throw new Error('failureThreshold must be finite') + } + this.opts = { + ...opts, + targetOutcome: { ...opts.targetOutcome }, + rubrics: opts.rubrics === undefined ? undefined : [...opts.rubrics], + } } async inspectFailures(runs: RunRecord[]): Promise { @@ -117,36 +128,48 @@ export class PredictiveValidityResearcher implements Researcher { ] } - const decorativeThreshold = this.opts.decorativeThreshold ?? 0.4 const changes: SteeringChange[] = [] - - for (const ranking of this.lastReport.ranked) { - if (ranking.verdict === 'load_bearing') continue - if (Math.abs(ranking.spearman) >= decorativeThreshold) continue - changes.push({ - kind: 'reviewer_prompt', - payload: { - rubric: ranking.rubric, - action: 'down-weight', - spearman: ranking.spearman, - bestOutcome: ranking.bestOutcome, + const target = { ...this.opts.targetOutcome } + const pairs = this.lastReport.pairs.filter( + (pair) => + pair.outcome === target.id && + pair.outcomeDirection === target.direction && + (this.opts.rubrics === undefined || this.opts.rubrics.includes(pair.rubric)), + ) + if (pairs.length === 0) { + return [ + { + kind: 'threshold', + payload: { directive: 'researcher.collect-more-outcomes', targetOutcome: target }, + rationale: `no estimable rubric association with ${target.id}; collect independent outcome observations before proposing weight changes`, }, - rationale: `predictive-validity Spearman=${ranking.spearman.toFixed(3)} vs ${ranking.bestOutcome} (decorative); recommend down-weighting`, - expectedDelta: -Math.max(0, 0.05 - Math.abs(ranking.spearman)), - }) + ] } - for (const ranking of this.lastReport.ranked.slice(0, 1)) { - if (ranking.verdict !== 'load_bearing') continue + for (const pair of pairs) { + const interval = pair.alignedSpearmanCi95 + const aligned = pair.alignedSpearman >= 0.4 && interval !== null && interval.lower > 0 + const inverse = pair.alignedSpearman <= -0.4 && interval !== null && interval.upper < 0 + const action = aligned + ? 'test-up-weight' + : inverse + ? 'test-reverse-or-replace' + : 'collect-calibration-evidence' changes.push({ kind: 'reviewer_prompt', payload: { - rubric: ranking.rubric, - action: 'up-weight', - spearman: ranking.spearman, - bestOutcome: ranking.bestOutcome, + rubric: pair.rubric, + action, + targetOutcome: target, + spearman: pair.spearman, + alignedSpearman: pair.alignedSpearman, + alignedSpearmanCi95: interval === null ? null : { ...interval }, + samples: pair.n, }, - rationale: `predictive-validity Spearman=${ranking.spearman.toFixed(3)} vs ${ranking.bestOutcome} (load-bearing); recommend up-weighting`, - expectedDelta: Math.max(0, Math.abs(ranking.spearman) - 0.5) * 0.1, + rationale: aligned + ? `higher ${pair.rubric} scores associate with better ${target.id}; test increased weight on fresh evidence before adopting it` + : inverse + ? `higher ${pair.rubric} scores associate with worse ${target.id}; test reversal or replacement on fresh evidence` + : `the association of ${pair.rubric} with desired ${target.id} does not support a direction of change; collect calibration evidence`, }) } return changes @@ -214,11 +237,11 @@ export class PredictiveValidityResearcher implements Researcher { const report = await rubricPredictiveValidity({ runs, outcomes: this.opts.outcomes, - outcomeMetrics: this.opts.outcomeMetrics, + outcomeMetrics: [this.opts.targetOutcome], rubrics: this.opts.rubrics, }) - if (this.opts.onReport) await this.opts.onReport(report) - this.lastReport = report + if (this.opts.onReport) await this.opts.onReport(structuredClone(report)) + this.setReport(report) return report } @@ -228,11 +251,17 @@ export class PredictiveValidityResearcher implements Researcher { * researcher's later proposals informed by it. */ setReport(report: RubricPredictiveValidityReport): void { - this.lastReport = report + const target = report.outcomeMetrics.find((metric) => metric.id === this.opts.targetOutcome.id) + if (target?.direction !== this.opts.targetOutcome.direction) { + throw new Error( + 'predictive validity report does not match the declared target outcome and direction', + ) + } + this.lastReport = structuredClone(report) } getLastReport(): RubricPredictiveValidityReport | null { - return this.lastReport + return this.lastReport === null ? null : structuredClone(this.lastReport) } } diff --git a/src/rl/rl-campaign.ts b/src/rl/rl-campaign.ts index fb0a5c96..365448a3 100644 --- a/src/rl/rl-campaign.ts +++ b/src/rl/rl-campaign.ts @@ -25,8 +25,10 @@ import { type FailedRun, runEvalCampaign, } from '../eval-campaign' +import { validateOutcomeMetricSpecifications } from '../meta-eval/outcome-observations' import type { OutcomeStore } from '../meta-eval/outcome-store' import { + type OutcomeMetricSpec, type RubricPredictiveValidityReport, rubricPredictiveValidity, } from '../meta-eval/rubric-predictive-validity' @@ -62,9 +64,9 @@ export interface RunRLCampaignOptions extends EvalCampaignOptions { preferences?: ExtractPreferencesOptions /** Verifiable-reward extraction options. */ verifiableReward?: VerifiableRewardExtractionOptions - /** Outcome store + metric names — when supplied, runs `rubricPredictiveValidity` post-campaign. */ + /** Outcome store and desired metric directions for post-campaign association measurement. */ outcomeStore?: OutcomeStore - outcomeMetrics?: string[] + outcomeMetrics?: readonly OutcomeMetricSpec[] /** Anytime-valid sequential evaluation options. */ sequential?: { alpha?: number @@ -151,6 +153,14 @@ export interface RLCampaignResult { } export async function runRLCampaign(opts: RunRLCampaignOptions): Promise { + const outcomeStore = opts.outcomeStore + const outcomeMetrics = opts.outcomeMetrics?.map((metric) => ({ ...metric })) + if (outcomeStore !== undefined || outcomeMetrics !== undefined) { + if (outcomeStore === undefined || outcomeMetrics === undefined) { + throw new Error('runRLCampaign requires outcomeStore and outcomeMetrics together') + } + validateOutcomeMetricSpecifications(outcomeMetrics) + } const splitTag = opts.splitTag ?? 'search' // ── 1. Run the matrix ────────────────────────────────────────────── @@ -207,11 +217,11 @@ export async function runRLCampaign(opts: RunRLCampaignOptions): Promise 0) { + if (outcomeStore && outcomeMetrics) { predictiveValidity = await rubricPredictiveValidity({ runs: campaign.runs, - outcomes: opts.outcomeStore, - outcomeMetrics: opts.outcomeMetrics, + outcomes: outcomeStore, + outcomeMetrics, }) } @@ -405,7 +415,9 @@ function buildSummary(args: { if (args.predictiveValidity) { const top = args.predictiveValidity.ranked[0] lines.push( - `top-rubric: ${top?.rubric ?? 'none'} ρ=${(top?.spearman ?? 0).toFixed(2)} (${top?.verdict ?? 'no data'})`, + top + ? `top-rubric: ${top.rubric} aligned ρ=${top.alignedSpearman.toFixed(2)} vs ${top.bestOutcome} (${top.outcomeDirection}; ${top.verdict})` + : 'top-rubric: none (no estimable outcome associations)', ) } return lines.join(' | ') diff --git a/tests/agent-profile-cell.test.ts b/tests/agent-profile-cell.test.ts index 721cef71..1c5259e1 100644 --- a/tests/agent-profile-cell.test.ts +++ b/tests/agent-profile-cell.test.ts @@ -259,7 +259,7 @@ describe('buildAgentInterfaceProfileCell', () => { }) describe('cell ids minted before the RFC 8785 scheme', () => { - it('still verifies, so a stored cell is not invalidated by the release that changed the encoder', async () => { + it('refuses a correctly hashed retired id instead of selecting another encoder', async () => { const cell = await buildAgentProfileCell(INPUT) const { cellId: _cellId, ...material } = cell void _cellId @@ -268,20 +268,14 @@ describe('cell ids minted before the RFC 8785 scheme', () => { .digest('hex') const legacyCell = { ...material, cellId: `agent-profile-cell:sha256:${legacyDigest}` } - expect(validateAgentProfileCell(legacyCell).cellId).toBe(legacyCell.cellId) - expect(await verifyAgentProfileCell(legacyCell)).toBe(true) - // A legacy id over different material is still caught. - expect( - await verifyAgentProfileCell({ - ...legacyCell, - cellId: `agent-profile-cell:sha256:${'0'.repeat(64)}`, - }), - ).toBe(false) + expect(() => validateAgentProfileCell(legacyCell)).toThrow(/sha256-rfc8785/) + await expect(verifyAgentProfileCell(legacyCell)).rejects.toThrow( + AgentProfileCellValidationError, + ) }) }) -/** Key-sorted `JSON.stringify`, the scheme cell ids were minted under before - * RFC 8785. Kept here to MINT a legacy id the verifier must still accept. */ +/** Produce a correctly hashed retired record to exercise scheme rejection. */ function sortKeysDeep(value: unknown): unknown { if (value === null || typeof value !== 'object') return value if (Array.isArray(value)) return value.map(sortKeysDeep) diff --git a/tests/boundary-integrity.test.ts b/tests/boundary-integrity.test.ts index 80a5c437..2d668ad0 100644 --- a/tests/boundary-integrity.test.ts +++ b/tests/boundary-integrity.test.ts @@ -53,10 +53,14 @@ describe('boundary integrity — agent-eval is the substrate (zero upward deps)' expect(files.length).toBeGreaterThan(50) }) - for (const pkg of FORBIDDEN) { - it(`no src/tests file imports ${pkg}`, () => { - const offenders = files.filter((f) => importsForbidden(readFileSync(f, 'utf8'), pkg)) - expect(offenders.map((f) => f.slice(ROOT.length + 1))).toEqual([]) + it('has no source or test imports from consumer packages', () => { + const offenders = files.flatMap((file) => { + const source = readFileSync(file, 'utf8') + return FORBIDDEN.filter((pkg) => importsForbidden(source, pkg)).map((pkg) => ({ + file: file.slice(ROOT.length + 1), + package: pkg, + })) }) - } + expect(offenders).toEqual([]) + }, 15_000) }) diff --git a/tests/campaign/compare-optimization-methods.test.ts b/tests/campaign/compare-optimization-methods.test.ts index 9557c03e..7d32e2c7 100644 --- a/tests/campaign/compare-optimization-methods.test.ts +++ b/tests/campaign/compare-optimization-methods.test.ts @@ -103,14 +103,14 @@ describe('compareOptimizationMethods', () => { expect(result.resamples).toBe(2000) expect(result.reps).toBe(1) - // Pairwise: strong (best) vs weak — delta 0.75, favored strong. + // Four independent scenarios rank the observed scores but cannot establish a mean win. expect(result.pairwise).toHaveLength(1) const pw = result.pairwise[0]! expect(pw.a).toBe('strong') expect(pw.b).toBe('weak') expect(pw.deltaMean).toBeCloseTo(0.75, 5) - expect(pw.favored).toBe('strong') - expect(pw.low).toBeGreaterThan(0) // CI clears zero → a real difference + expect(pw.favored).toBeNull() + expect(pw.decision).toMatchObject({ n: 4, sufficient: false, promote: false }) expect(result.testScenarioIds).toEqual(['h1', 'h2', 'h3', 'h4']) expect(strong.scenarioScores).toEqual( TEST.map((scenario) => ({ @@ -165,7 +165,7 @@ describe('compareOptimizationMethods', () => { expectUsage: 'off', }) expect(result.pairwise[0]!.deltaMean).toBeCloseTo(0, 5) - expect(result.pairwise[0]!.favored).toBe('tie') + expect(result.pairwise[0]!.favored).toBeNull() }) it('names no winner on a ZERO-WIDTH pairwise interval', async () => { @@ -184,7 +184,7 @@ describe('compareOptimizationMethods', () => { expectUsage: 'off', }) const pair = result.pairwise[0]! - if (pair.low === pair.high) expect(pair.favored).toBe('tie') + if (pair.low === pair.high) expect(pair.favored).toBeNull() }) it('a cheaper method wins a lift tie when both costs are complete', async () => { diff --git a/tests/campaign/final-evidence-integration.test.ts b/tests/campaign/final-evidence-integration.test.ts new file mode 100644 index 00000000..53185c52 --- /dev/null +++ b/tests/campaign/final-evidence-integration.test.ts @@ -0,0 +1,533 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + exposeFinalEvidence, + type FinalEvidencePolicy, + reserveFinalEvidence, +} from '../../src/campaign/final-evidence' +import { compareOptimizationMethods } from '../../src/campaign/presets/compare-optimization-methods' +import { runImprovementLoop } from '../../src/campaign/presets/run-improvement-loop' +import { inMemoryCampaignStorage } from '../../src/campaign/storage' +import type { JudgeConfig, Scenario } from '../../src/campaign/types' +import { type SelfImproveProgressEvent, selfImprove } from '../../src/contract/self-improve' +import type { EvaluationClaim } from '../../src/experiment/claim' +import { + FinalEvidenceError, + type FinalEvidenceLedger, + openFinalEvidenceLedger, +} from '../../src/experiment/final-evidence' +import { hashCanonical } from '../../src/ledger-core/canonical' + +interface Case extends Scenario { + sourceId: string +} +interface Artifact { + quality: number +} +let dir: string +let ledger: FinalEvidenceLedger +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'final-evidence-campaign-')) + ledger = openFinalEvidenceLedger({ path: join(dir, 'evidence.jsonl') }) +}) +afterEach(() => rmSync(dir, { recursive: true, force: true })) + +const source = (id: string, sourceId = id): Case => ({ id, kind: 'fixture', sourceId }) +const train = [source('train-1'), source('train-2'), source('select-1')] +const final = [source('final-1'), source('final-2')] +const judge: JudgeConfig = { + name: 'quality', + dimensions: [{ key: 'quality', description: 'Observed quality' }], + score: ({ artifact }) => ({ + composite: artifact.quality, + dimensions: { quality: artifact.quality }, + notes: '', + }), +} +const freeCost = { + totalCostUsd: 0, + costProvenance: { kind: 'observed' as const, usd: 0 }, + accountingComplete: true, + incompleteReasons: [], +} +const gate = { + name: 'fixture', + decide: async () => ({ decision: 'hold' as const, reasons: ['fixture'], contributingGates: [] }), +} +function policy(requestId = 'run'): FinalEvidencePolicy { + return { + ledger, + requestId, + evaluatorDigest: hashCanonical('fixture-evaluator'), + } +} +function claim(): EvaluationClaim { + return { + use: 'comparison', + population: { id: 'incidents', description: 'New incidents' }, + samplingFrame: 'Incident queue', + independentUnit: 'sourceId', + generalization: 'new-units', + minimumEffect: 0.05, + } +} +async function record() { + const result = await ledger.read() + if (!result.succeeded) throw new Error(result.error.message) + return result.value[0]! +} + +describe('final evidence through campaign entrypoints', () => { + it('keeps declared-unit regression data reusable and detects a large gain with eight binary pairs', async () => { + const heldout = Array.from({ length: 8 }, (_, i) => source(`holdout-${i}`)) + const options = { + scenarios: [...train, ...heldout], + judge, + baselineSurface: 'BASE', + claim: claim(), + storage: inMemoryCampaignStorage(), + runDir: dir, + model: 'fixture@2026-09-13', + expectUsage: 'off' as const, + budget: { holdoutScenarios: heldout }, + agent: async (surface: string | object) => ({ quality: surface === 'BASE' ? 0 : 1 }), + method: { name: 'fixed', optimize: async () => ({ winnerSurface: 'WIN', cost: freeCost }) }, + } + for (let run = 0; run < 2; run++) { + const result = await selfImprove(options) + expect(result.winner.surface).toBe('WIN') + expect(result.lift).toBe(1) + expect(result.gateDecision).toBe('ship') + expect(result.claim?.independentUnit).toBe('sourceId') + expect(result.finalEvidence).toBeUndefined() + } + const evidence = await ledger.read() + expect(evidence).toEqual({ succeeded: true, value: [] }) + }) + + it('returns the selected candidate and observed lift when a population-mean claim remains inconclusive', async () => { + const heldout = Array.from({ length: 4 }, (_, i) => source(`holdout-${i}`)) + const result = await selfImprove({ + scenarios: [...train, ...heldout], + judge, + baselineSurface: 'BASE', + claim: claim(), + storage: inMemoryCampaignStorage(), + runDir: dir, + model: 'fixture@2026-09-13', + expectUsage: 'off', + budget: { holdoutScenarios: heldout }, + agent: async (surface, scenario) => ({ + quality: surface === 'BASE' ? 0.3 : 0.5 + Number(scenario.id.slice(-1)) * 0.01, + }), + method: { name: 'fixed', optimize: async () => ({ winnerSurface: 'WIN', cost: freeCost }) }, + }) + expect(result.winner.surface).toBe('WIN') + expect(result.lift).toBeCloseTo(0.215) + expect(result.gateDecision).toBe('hold') + }) + + it('retains a selected candidate that loses on final tasks and reports the regression', async () => { + const heldout = Array.from({ length: 8 }, (_, i) => source(`holdout-${i}`)) + const result = await selfImprove({ + scenarios: [...train, ...heldout], + judge, + baselineSurface: 'BASE', + claim: claim(), + storage: inMemoryCampaignStorage(), + runDir: dir, + model: 'fixture@2026-09-13', + expectUsage: 'off', + budget: { holdoutScenarios: heldout }, + agent: async (surface) => ({ quality: surface === 'BASE' ? 1 : 0 }), + method: { + name: 'fixed', + optimize: async () => ({ winnerSurface: 'SELECTED', cost: freeCost }), + }, + }) + expect(result.winner.surface).toBe('SELECTED') + expect(result.lift).toBe(-1) + expect(result.gateDecision).toBe('hold') + expect(result.provenance.heldOutLift).toBe(-1) + }) + + it('returns descriptive optimizer scores when final variants come from one source unit', async () => { + const result = await compareOptimizationMethods({ + trainScenarios: train.slice(0, 2), + selectionScenarios: train.slice(2), + testScenarios: [source('variant-a', 'one-source'), source('variant-b', 'one-source')], + baselineSurface: 'BASE', + judges: [judge], + claim: claim(), + methods: [ + { name: 'fixed', optimize: async () => ({ winnerSurface: 'WIN', cost: freeCost }) }, + ], + dispatchWithSurface: async (surface) => ({ quality: surface === 'BASE' ? 0 : 1 }), + runDir: dir, + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + }) + expect(result.best.winnerSurface).toBe('WIN') + expect(result.best.winnerComposite).toBe(1) + expect(result.best.unitScores).toHaveLength(1) + expect(result.best.scenarioScores).toHaveLength(2) + expect(result.best.decision).toMatchObject({ n: 1, sufficient: false, promote: false }) + expect(result.units).toMatchObject({ observations: 2, independentUnits: 1 }) + }) + + it.each(['method', 'proposer'] as const)( + 'reserves before %s search and exposes before final dispatch', + async (mode) => { + let proposals = 0 + let finalCalls = 0 + const verifySearch = async () => { + proposals += 1 + expect((await record()).exposure).toBeNull() + } + const options = { + scenarios: [...train, ...final], + judge, + gate, + baselineSurface: 'BASE', + storage: inMemoryCampaignStorage(), + runDir: join(dir, mode), + model: 'fixture@2026-09-13', + expectUsage: 'off' as const, + budget: { + holdoutScenarios: final, + ...(mode === 'proposer' ? { generations: 1, populationSize: 1 } : {}), + }, + claim: claim(), + finalEvidence: policy(), + agent: async (surface: string | object, scenario: Case) => { + if (scenario.id.startsWith('final')) { + finalCalls += 1 + expect((await record()).exposure?.measurement.candidateDigests).toEqual( + [hashCanonical('BASE'), hashCanonical('WIN')].sort(), + ) + } + return { quality: surface === 'BASE' ? 0 : 1 } + }, + ...(mode === 'method' + ? { + method: { + name: 'fixed', + optimize: async () => { + await verifySearch() + return { winnerSurface: 'WIN', cost: freeCost } + }, + }, + } + : { + proposer: { + kind: 'fixed', + propose: async () => { + await verifySearch() + return ['WIN'] + }, + }, + }), + } + const result = await selfImprove(options) + expect(proposals).toBeGreaterThan(0) + expect(finalCalls).toBe(4) + expect(result.finalEvidence?.record.exposure).not.toBeNull() + await expect(selfImprove(options)).rejects.toThrow(/already exposed/) + expect(finalCalls).toBe(4) + }, + ) + + it('blocks source overlap before either entrypoint can invoke an optimizer', async () => { + let invoked = false + const shared = [source('variant-a', 'same-source')] + const heldout = [source('variant-b', 'same-source')] + await expect( + runImprovementLoop({ + scenarios: shared, + holdoutScenarios: heldout, + baselineSurface: 'BASE', + judges: [judge], + gate, + autoOnPromote: 'none', + runDir: dir, + storage: inMemoryCampaignStorage(), + dispatchWithSurface: async () => { + invoked = true + return { quality: 0 } + }, + proposer: { + kind: 'fixed', + propose: async () => { + invoked = true + return ['WIN'] + }, + }, + claim: claim(), + finalEvidence: policy(), + expectUsage: 'off', + }), + ).rejects.toThrow(/share independent units/) + expect(invoked).toBe(false) + expect((await ledger.read()).succeeded).toBe(true) + }) + + it('splits complete source families and snapshots caller policy during method search', async () => { + const cases = Array.from({ length: 5 }, (_, family) => + Array.from({ length: 3 }, (_, variant) => source(`${family}:${variant}`, `family-${family}`)), + ).flat() + const original = policy() + const declaredClaim = claim() + const result = await selfImprove({ + scenarios: cases, + judge, + baselineSurface: 'BASE', + storage: inMemoryCampaignStorage(), + runDir: dir, + model: 'fixture@2026-09-13', + expectUsage: 'off', + claim: declaredClaim, + finalEvidence: original, + agent: async () => ({ quality: 0 }), + method: { + name: 'same', + optimize: async (input) => { + const heldoutUnits = new Set((await record()).reservation.unitIds) + const trainUnits = new Set(input.trainScenarios.map((row) => row.sourceId)) + expect( + input.selectionScenarios.every( + (row) => !trainUnits.has(row.sourceId) && !heldoutUnits.has(row.sourceId), + ), + ).toBe(true) + expect(input.trainScenarios.every((row) => !heldoutUnits.has(row.sourceId))).toBe(true) + original.requestId = 'mutated' + declaredClaim.independentUnit = 'id' + cases[0]!.sourceId = 'mutated' + return { winnerSurface: input.baselineSurface, cost: freeCost } + }, + }, + }) + expect(result.finalEvidence?.record.reservation.requestId).toBe('run') + expect(result.finalEvidence?.claim.independentUnit).toBe('sourceId') + expect(result.finalEvidence?.record.reservation.unitIds).toHaveLength(1) + expect(result.raw.baselineOnHoldout.cells).toHaveLength(3) + }) + + it('keeps failed final execution consumed and exposes a control before scoring any arm', async () => { + const selectedPolicy = policy() + await expect( + runImprovementLoop({ + scenarios: train, + holdoutScenarios: final, + baselineSurface: 'BASE', + judges: [judge], + gate, + proposer: { kind: 'fixed', propose: async () => ['WIN'] }, + populationSize: 1, + maxGenerations: 1, + neutralize: () => 'CONTROL', + claim: claim(), + finalEvidence: selectedPolicy, + dispatchWithSurface: async (surface, scenario) => { + if (scenario.id.startsWith('final')) { + expect((await record()).exposure?.measurement.candidateDigests).toHaveLength(3) + throw new Error('final worker failed') + } + return { quality: surface === 'BASE' ? 0 : 1 } + }, + runDir: dir, + storage: inMemoryCampaignStorage(), + autoOnPromote: 'none', + expectUsage: 'off', + }), + ).rejects.toThrow(/incomplete/) + expect((await record()).exposure).not.toBeNull() + await expect(reserveFinalEvidence(policy('another-run'), claim(), final)).rejects.toThrow( + /already reserved/, + ) + }) + + it('averages final variants within registered units before comparing optimizers', async () => { + const test = [ + source('a1', 'a'), + source('a2', 'a'), + source('a3', 'a'), + source('a4', 'a'), + source('b1', 'b'), + source('c1', 'c'), + ] + const result = await compareOptimizationMethods({ + trainScenarios: train.slice(0, 2), + selectionScenarios: train.slice(2), + testScenarios: test, + baselineSurface: 'BASE', + judges: [judge], + reps: 2, + claim: claim(), + finalEvidence: policy(), + methods: [ + { + name: 'fixed', + optimize: async () => { + expect((await record()).exposure).toBeNull() + return { winnerSurface: 'WIN', cost: freeCost } + }, + }, + ], + dispatchWithSurface: async (surface, scenario) => { + expect((await record()).exposure).not.toBeNull() + return { quality: surface === 'WIN' && scenario.sourceId === 'a' ? 1 : 0 } + }, + runDir: dir, + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + }) + expect(result.best.winnerComposite).toBeCloseTo(1 / 3) + expect(result.best.scenarioScores).toHaveLength(6) + expect(result.best.unitScores).toHaveLength(3) + expect(result.best.decision).toMatchObject({ n: 3, sufficient: false, promote: false }) + expect(result.units).toMatchObject({ observations: 6, independentUnits: 3 }) + expect(result.pairedCellN).toBe(12) + expect(result.finalEvidence?.record.exposure).not.toBeNull() + }) + + it.each( + [ + { + baselinePattern: 'zero', + before: { a: 0, b: 0, c: 0 }, + after: { a: 1, b: 0, c: 0 }, + expectedBaseline: 0, + expectedWinner: 1 / 3, + expectedLift: 1 / 3, + expectedSd: 0, + }, + { + baselinePattern: 'varying', + before: { a: 0.2, b: 0.4, c: 0.6 }, + after: { a: 0.8, b: 0.4, c: 0.6 }, + expectedBaseline: 0.4, + expectedWinner: 0.6, + expectedLift: 0.2, + expectedSd: 0.2, + }, + ].flatMap((fixture) => (['method', 'proposer'] as const).map((mode) => ({ ...fixture, mode }))), + )( + 'reports $mode means over gate units with a $baselinePattern baseline', + async ({ mode, before, after, expectedBaseline, expectedWinner, expectedLift, expectedSd }) => { + const heldout = [ + source('a1', 'a'), + source('a2', 'a'), + source('a3', 'a'), + source('a4', 'a'), + source('b1', 'b'), + source('c1', 'c'), + ] + const events: SelfImproveProgressEvent[] = [] + const storage = inMemoryCampaignStorage() + const result = await selfImprove({ + scenarios: [...train, ...heldout], + judge, + baselineSurface: 'BASE', + claim: { ...claim(), minimumEffect: 0.1 }, + storage, + runDir: dir, + model: 'fixture@2026-09-13', + expectUsage: 'off', + budget: { + holdoutScenarios: heldout, + reps: 2, + ...(mode === 'proposer' ? { generations: 1, populationSize: 1 } : {}), + }, + agent: async (surface, scenario) => { + const values = surface === 'BASE' ? before : after + if (scenario.sourceId === 'a') return { quality: values.a } + if (scenario.sourceId === 'b') return { quality: values.b } + if (scenario.sourceId === 'c') return { quality: values.c } + return { quality: surface === 'BASE' ? 0 : 1 } + }, + ...(mode === 'method' + ? { + method: { + name: 'fixed', + optimize: async () => ({ winnerSurface: 'WIN', cost: freeCost }), + }, + } + : { proposer: { kind: 'fixed', propose: async () => ['WIN'] } }), + onProgress: (event) => events.push(event), + }) + + expect(result.winner.surface).toBe('WIN') + expect(result.baseline?.compositeMean).toBeCloseTo(expectedBaseline) + expect(result.winner.compositeMean).toBeCloseTo(expectedWinner) + expect(result.lift).toBeCloseTo(expectedLift) + expect(result.insight?.lift?.delta).toBeCloseTo(expectedLift) + expect(result.insight?.lift).toMatchObject({ + n: 3, + pairedRunN: 12, + independentUnitIds: ['a', 'b', 'c'], + }) + expect(result.raw.gateResult.delta).toBeCloseTo(result.lift!) + expect( + result.raw.gateResult.contributingGates.find((gate) => gate.name === 'heldout-significance') + ?.detail, + ).toMatchObject({ + n: 3, + pairedCellN: 12, + unitIds: ['a', 'b', 'c'], + }) + expect(result.winner.perScenario).toEqual({ + a1: after.a, + a2: after.a, + a3: after.a, + a4: after.a, + b1: after.b, + c1: after.c, + }) + expect(result.provenance.baselineHoldoutComposite).toBeCloseTo(expectedBaseline) + expect(result.provenance.winnerHoldoutComposite).toBeCloseTo(expectedWinner) + expect(result.provenance.heldOutLift).toBeCloseTo(result.lift!) + expect(result.provenance.claim?.independentUnit).toBe('sourceId') + const observations = + result.mode === 'method' + ? result.provenance.evidence.holdoutObservations + : result.provenance.evidence.holdout.observations + expect(observations).toEqual({ + pairedCellN: 12, + unitIds: ['a', 'b', 'c'], + unscoredCellIds: [], + independentUnitByScenarioId: { a1: 'a', a2: 'a', a3: 'a', a4: 'a', b1: 'b', c1: 'c' }, + }) + expect(events.find((event) => event.kind === 'gate.decided')).toMatchObject({ + lift: result.lift, + }) + const artifact = mode === 'method' ? 'method-provenance.json' : 'loop-provenance.json' + expect(JSON.parse(storage.read(join(dir, artifact))!)).toEqual(result.provenance) + if (result.mode === 'proposer') { + expect(result.power).toMatchObject({ n: 3, deltaThreshold: 0.1 }) + expect(result.power?.baselineMean).toBeCloseTo(expectedBaseline) + expect(result.power?.sd).toBeCloseTo(expectedSd) + expect(events.find((event) => event.kind === 'power.estimated')).toMatchObject({ n: 3 }) + } + }, + ) + + it('does not collapse an unavailable ledger into an evidence conflict', async () => { + const unavailable: FinalEvidenceLedger = { + reserve: async () => ({ + succeeded: false, + error: { kind: 'unavailable', message: 'storage offline' }, + }), + expose: async () => { + throw new Error('must not dispatch') + }, + read: async () => ({ succeeded: true, value: [] }), + } + const operation = exposeFinalEvidence({ ...policy(), ledger: unavailable }, claim(), final, [ + 'BASE', + ]) + await expect(operation).rejects.toBeInstanceOf(FinalEvidenceError) + await expect(operation).rejects.toMatchObject({ kind: 'unavailable' }) + }) +}) diff --git a/tests/campaign/heldout-gate-calibration.test.ts b/tests/campaign/heldout-gate-calibration.test.ts index 3dcaf70a..2ee7cb57 100644 --- a/tests/campaign/heldout-gate-calibration.test.ts +++ b/tests/campaign/heldout-gate-calibration.test.ts @@ -134,9 +134,9 @@ describe('heldOutGate (composable) — repeated-sampling calibration', () => { const d = await decide(before, after, 0) expect(d.ship, `zero-spread sample at n=${n}`).toBe(false) expect(d.detail.indeterminate, `indeterminate flag at n=${n}`).toBe(true) - expect(d.reason).toMatch(/carries no direction/) + expect(d.reason).toMatch(n < 20 ? /too few/ : /carries no direction/) } - }) + }, 15_000) it('refuses the 76-pair witness: 0 wins, 3 losses, 73 ties at a -0.05 margin', async () => { // Three losses and no wins out of 76 does not establish noninferiority at diff --git a/tests/campaign/judge-snapshot.test.ts b/tests/campaign/judge-snapshot.test.ts new file mode 100644 index 00000000..e4649b96 --- /dev/null +++ b/tests/campaign/judge-snapshot.test.ts @@ -0,0 +1,293 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { computeManifestHash } from '../../src/campaign/campaign-manifest' +import type { FinalEvidencePolicy } from '../../src/campaign/final-evidence' +import { defaultProductionGate } from '../../src/campaign/gates/default-production-gate' +import { + compareOptimizationMethods, + type OptimizationMethod, +} from '../../src/campaign/presets/compare-optimization-methods' +import { runFinalComparison } from '../../src/campaign/presets/run-final-comparison' +import { runImprovementLoop } from '../../src/campaign/presets/run-improvement-loop' +import { inMemoryCampaignStorage } from '../../src/campaign/storage' +import { surfaceDispatchRef } from '../../src/campaign/surface-identity' +import type { JudgeConfig, MutableSurface, Scenario } from '../../src/campaign/types' +import { selfImprove } from '../../src/contract/self-improve' +import type { EvaluationClaim } from '../../src/experiment/claim' +import { + type FinalEvidenceLedger, + openFinalEvidenceLedger, +} from '../../src/experiment/final-evidence' +import { hashCanonical } from '../../src/ledger-core/canonical' + +interface Artifact { + quality: number +} + +const train: Scenario[] = ['train-1', 'train-2', 'selection'].map((id) => ({ + id, + kind: 'fixture', +})) +const final: Scenario[] = Array.from({ length: 8 }, (_, index) => ({ + id: `final-${index}`, + kind: 'fixture', +})) +const claim: EvaluationClaim = { + use: 'comparison', + population: { id: 'incidents', description: 'Independent incidents' }, + samplingFrame: 'Incident queue', + independentUnit: 'id', + generalization: 'new-units', +} +const evaluatorDigest = hashCanonical('artifact-quality-v1') +const freeCost = { + totalCostUsd: 0, + costProvenance: { kind: 'observed' as const, usd: 0 }, + accountingComplete: true, + incompleteReasons: [], +} + +let dir: string +let ledger: FinalEvidenceLedger +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'judge-snapshot-')) + ledger = openFinalEvidenceLedger({ path: join(dir, 'evidence.jsonl') }) +}) +afterEach(() => rmSync(dir, { recursive: true, force: true })) + +function qualityJudge(): JudgeConfig { + return { + name: 'quality', + judgeVersion: 'v1', + dimensions: [{ key: 'quality', description: 'Observed quality' }], + appliesTo: () => true, + score: ({ artifact }) => ({ + composite: artifact.quality, + dimensions: { quality: artifact.quality }, + notes: '', + }), + } +} + +function policy(owner: FinalEvidenceLedger = ledger): FinalEvidencePolicy { + return { ledger: owner, requestId: 'comparison', evaluatorDigest } +} + +describe('comparison judge snapshots', () => { + it('does not promote identical outputs after the caller changes its judge between final arms', async () => { + const judge = qualityJudge() + const score = judge.score + let originalCalls = 0 + judge.score = (input) => { + originalCalls += 1 + if (originalCalls === final.length) { + judge.score = () => ({ composite: 1, dimensions: { quality: 1 }, notes: '' }) + judge.judgeVersion = 'v2' + } + return score(input) + } + + const result = await runFinalComparison({ + baselineSurface: 'BASE', + winnerSurface: 'WIN', + scenarios: final, + dispatchWithSurface: async () => ({ quality: 0 }), + judges: [judge], + maxConcurrency: 1, + gate: defaultProductionGate({ holdoutScenarios: final, deltaThreshold: 0 }), + claim, + finalEvidence: policy(), + storage: inMemoryCampaignStorage(), + runDir: dir, + expectUsage: 'off', + }) + + expect(judge.judgeVersion).toBe('v2') + expect(originalCalls).toBe(16) + for (const campaign of [result.baselineOnHoldout, result.winnerOnHoldout]) { + expect(campaign.cells).toHaveLength(8) + for (const cell of campaign.cells) { + expect(cell.artifact.quality).toBe(0) + expect(cell.judgeScores.quality?.composite).toBe(0) + } + } + expect(result.gateResult).toMatchObject({ decision: 'hold', delta: 0 }) + expect(result.finalEvidence?.record.exposure?.measurement.evaluatorDigest).toBe(evaluatorDigest) + }) + + it('preserves prototype callbacks, getter metadata, private receivers, and their original cache identity', async () => { + class Dimension { + get key() { + return 'quality' + } + + get description() { + return 'Observed quality' + } + } + class ClassJudge implements JudgeConfig { + #quality = 0 + + get name() { + return 'quality' + } + + get dimensions() { + return [new Dimension()] + } + + appliesTo() { + return this.#quality === 0 + } + + score() { + return { + composite: this.#quality, + dimensions: { quality: this.#quality }, + notes: '', + } + } + } + const judge = new ClassJudge() + const dispatchRef = 'class-judge-fixture' + const expectedManifest = computeManifestHash({ + scenarios: final, + judges: [ + { + name: judge.name, + dimensions: [{ key: 'quality', description: 'Observed quality' }], + score: judge.score, + appliesTo: judge.appliesTo, + }, + ], + dispatchRef: surfaceDispatchRef('BASE', dispatchRef), + seed: 42, + reps: 1, + }) + const result = await runFinalComparison({ + baselineSurface: 'BASE', + winnerSurface: 'WIN', + scenarios: final, + dispatchWithSurface: async () => ({ quality: 0 }), + dispatchRef, + judges: [judge], + gate: defaultProductionGate({ holdoutScenarios: final, deltaThreshold: 0 }), + storage: inMemoryCampaignStorage(), + runDir: dir, + expectUsage: 'off', + }) + + expect(result.baselineOnHoldout.manifestHash).toBe(expectedManifest) + expect(result.winnerOnHoldout.cells).toHaveLength(8) + expect( + result.winnerOnHoldout.cells.every((cell) => cell.judgeScores.quality?.composite === 0), + ).toBe(true) + expect(result.gateResult).toMatchObject({ decision: 'hold', delta: 0 }) + }) + + it.each(['method', 'proposer', 'loop', 'comparison'] as const)( + 'captures %s judges before the first reservation can yield to caller changes', + async (mode) => { + const judge = qualityJudge() + const judges = [judge] + let changed = false + const changingLedger: FinalEvidenceLedger = { + ...ledger, + reserve: async (reservation) => { + if (!changed) { + changed = true + judge.name = 'changed' + judge.judgeVersion = 'v2' + judge.dimensions[0]!.key = 'changed' + judge.dimensions[0]!.description = 'Changed description' + judge.score = () => ({ composite: 1, dimensions: { changed: 1 }, notes: '' }) + judge.appliesTo = () => false + judges.length = 0 + } + return ledger.reserve(reservation) + }, + } + const method: OptimizationMethod = { + name: 'fixed', + optimize: async (input) => { + expect(input.judges[0]).toMatchObject({ + name: 'quality', + judgeVersion: 'v1', + dimensions: [{ key: 'quality', description: 'Observed quality' }], + }) + expect(input.judges[0]?.appliesTo?.(final[0]!)).toBe(true) + return { winnerSurface: 'WIN', cost: freeCost } + }, + } + const agent = async (surface: MutableSurface, scenario: Scenario): Promise => ({ + quality: scenario.id.startsWith('final-') || surface === 'BASE' ? 0 : 1, + }) + const common = { + baselineSurface: 'BASE', + claim, + finalEvidence: policy(changingLedger), + storage: inMemoryCampaignStorage(), + runDir: dir, + expectUsage: 'off' as const, + } + + if (mode === 'comparison') { + const result = await compareOptimizationMethods({ + ...common, + trainScenarios: train.slice(0, 2), + selectionScenarios: train.slice(2), + testScenarios: final, + methods: [method], + judges, + dispatchWithSurface: agent, + }) + expect(result.best).toMatchObject({ + winnerSurface: 'WIN', + baselineComposite: 0, + winnerComposite: 0, + lift: 0, + decision: { promote: false }, + }) + } else if (mode === 'loop') { + const result = await runImprovementLoop({ + ...common, + scenarios: train, + holdoutScenarios: final, + judges, + dispatchWithSurface: agent, + proposer: { kind: 'fixed', propose: async () => ['WIN'] }, + gate: defaultProductionGate({ holdoutScenarios: final, deltaThreshold: 0 }), + populationSize: 1, + maxGenerations: 1, + autoOnPromote: 'none', + }) + expect(result.winnerSurface).toBe('WIN') + expect(result.gateResult).toMatchObject({ decision: 'hold', delta: 0 }) + } else { + const result = await selfImprove({ + ...common, + scenarios: [...train, ...final], + judge, + agent, + model: 'fixture@2026-09-13', + budget: { + holdoutScenarios: final, + ...(mode === 'proposer' ? { generations: 1, populationSize: 1 } : {}), + }, + ...(mode === 'method' + ? { method } + : { proposer: { kind: 'fixed', propose: async () => ['WIN'] } }), + }) + expect(result.winner.surface).toBe('WIN') + expect(result.lift).toBe(0) + expect(result.gateDecision).toBe('hold') + } + expect(changed).toBe(true) + const stored = await ledger.read() + if (!stored.succeeded) throw new Error(stored.error.message) + expect(stored.value[0]?.exposure?.measurement.evaluatorDigest).toBe(evaluatorDigest) + }, + ) +}) diff --git a/tests/campaign/presets.test.ts b/tests/campaign/presets.test.ts index 640eafaa..2325c7ef 100644 --- a/tests/campaign/presets.test.ts +++ b/tests/campaign/presets.test.ts @@ -230,13 +230,21 @@ describe('heldOutGate', () => { ]) it('ships when the candidate-baseline CI lower bound clears deltaThreshold', async () => { - const gate = heldOutGate({ scenarios: PROMOTION_HOLDOUT, deltaThreshold: 0.5 }) + const scenarios = Array.from({ length: 24 }, (_, index) => ({ + id: `h${index + 1}`, + kind: 'chat', + intent: `H${index + 1}`, + })) + const pairedArtifacts = new Map(scenarios.map((scenario) => [`${scenario.id}:0`, null])) + const candidateScores = [9, 8, 7, 9.5, 8.5, 7.5] + const baselineScores = [5, 4, 3, 5, 4, 3] + const gate = heldOutGate({ scenarios, deltaThreshold: 0.5 }) const result = await gate.decide({ - candidateArtifacts: artifacts as never, - baselineArtifacts: artifacts as never, - judgeScores: mk(9, 8, 7, 9.5, 8.5, 7.5), - baselineJudgeScores: mk(5, 4, 3, 5, 4, 3), - scenarios: PROMOTION_HOLDOUT, + candidateArtifacts: pairedArtifacts as never, + baselineArtifacts: pairedArtifacts as never, + judgeScores: mk(...scenarios.map((_, index) => candidateScores[index % 6]!)), + baselineJudgeScores: mk(...scenarios.map((_, index) => baselineScores[index % 6]!)), + scenarios, cost: { candidate: 0, baseline: 0 }, signal: new AbortController().signal, }) @@ -858,24 +866,22 @@ describe('defaultProductionGate', () => { string, Record; notes: string }> >(entries.map(([c, v]) => [c, { judge: { composite: v, dimensions: {}, notes: '' } }])) - // A real lift on six holdout cells clears the exact sign test. The deltas - // are deliberately NOT identical: n identical deltas give a zero-width - // interval, which is refused however large the gain. + // Six pass/fail wins support a positive success-rate delta through the paired-binary test. const judgeScores = mk([ - ['h1:0', 8], - ['h2:0', 9.5], - ['h3:0', 7], - ['h4:0', 8.5], - ['h5:0', 9], - ['h6:0', 7.5], + ['h1:0', 1], + ['h2:0', 1], + ['h3:0', 1], + ['h4:0', 1], + ['h5:0', 1], + ['h6:0', 1], ]) const baselineJudgeScores = mk([ - ['h1:0', 5], - ['h2:0', 6], - ['h3:0', 4], - ['h4:0', 5], - ['h5:0', 6], - ['h6:0', 4], + ['h1:0', 0], + ['h2:0', 0], + ['h3:0', 0], + ['h4:0', 0], + ['h5:0', 0], + ['h6:0', 0], ]) const result = await gate.decide({ candidateArtifacts: candidate, diff --git a/tests/campaign/statistical-heldout.test.ts b/tests/campaign/statistical-heldout.test.ts index 3d5e321b..aa05f94c 100644 --- a/tests/campaign/statistical-heldout.test.ts +++ b/tests/campaign/statistical-heldout.test.ts @@ -44,15 +44,24 @@ async function decide(opts: { const gateDetail = (r: Awaited>, name: string) => r.contributingGates.find((c) => c.name === name)?.detail as any +function continuousRoster(sample: (i: number) => { baseline: Cell; candidate: Cell }) { + const pairs = Array.from({ length: 24 }, (_, i) => ({ id: `h${i}`, ...sample(i) })) + return { + scenarioIds: pairs.map((pair) => pair.id), + baseline: Object.fromEntries(pairs.map((pair) => [`${pair.id}:0`, pair.baseline])), + candidate: Object.fromEntries(pairs.map((pair) => [`${pair.id}:0`, pair.candidate])), + } +} + describe('pairHoldout — full-cellId pairing (the trap that fakes a tight CI)', () => { - it('pairs by FULL cellId so reps multiply n — never averaged to one-per-scenario', () => { + it('pairs full execution cells before any independent-unit aggregation', () => { const cand = cellMap({ 'h1:0': { composite: 5 }, 'h1:1': { composite: 7 } }) const base = cellMap({ 'h1:0': { composite: 4 }, 'h1:1': { composite: 6 } }) const p = pairHoldout(cand, base, new Set(['h1']), (s) => s.composite) expect(p.cellIds).toEqual(['h1:0', 'h1:1']) expect(p.before).toEqual([4, 6]) expect(p.after).toEqual([5, 7]) - expect(p.before.length).toBe(2) // n=2 from reps, NOT collapsed to n=1 per scenario + expect(p.before.length).toBe(2) }) it('throws when candidate/baseline holdout cells do not align (load-bearing invariant)', () => { @@ -73,50 +82,24 @@ describe('defaultProductionGate — bootstrap-CI held-out (kills the point-estim it('HOLDS a noisy same-mean holdout — the exact +4 model-noise false positive', async () => { // Baseline & candidate are two noisy samples of the SAME surface: deltas // straddle zero, so no real lift. The old point-estimate gate shipped this. - const r = await decide({ - baseline: { - 'h1:0': { composite: 91 }, - 'h2:0': { composite: 88 }, - 'h3:0': { composite: 95 }, - 'h4:0': { composite: 90 }, - 'h5:0': { composite: 89 }, - 'h6:0': { composite: 93 }, - }, - candidate: { - 'h1:0': { composite: 95 }, - 'h2:0': { composite: 84 }, - 'h3:0': { composite: 93 }, - 'h4:0': { composite: 92 }, - 'h5:0': { composite: 87 }, - 'h6:0': { composite: 95 }, - }, - scenarioIds: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], - }) + const r = await decide( + continuousRoster((i) => ({ + baseline: { composite: [91, 88, 95, 90, 89, 93][i % 6]! }, + candidate: { composite: [95, 84, 93, 92, 87, 95][i % 6]! }, + })), + ) expect(r.decision).toBe('hold') expect(gateDetail(r, 'heldout-significance').fewRuns).toBe(false) expect(gateDetail(r, 'heldout-significance').ciLow).toBeLessThanOrEqual(0) }) it('SHIPS a real lift — CI.low strictly above the threshold', async () => { - const r = await decide({ - baseline: { - 'h1:0': { composite: 80 }, - 'h2:0': { composite: 82 }, - 'h3:0': { composite: 78 }, - 'h4:0': { composite: 81 }, - 'h5:0': { composite: 79 }, - 'h6:0': { composite: 83 }, - }, - candidate: { - 'h1:0': { composite: 86 }, - 'h2:0': { composite: 89 }, - 'h3:0': { composite: 83 }, - 'h4:0': { composite: 88 }, - 'h5:0': { composite: 84 }, - 'h6:0': { composite: 90 }, - }, - scenarioIds: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], - }) + const r = await decide( + continuousRoster((i) => ({ + baseline: { composite: [80, 82, 78, 81, 79, 83][i % 6]! }, + candidate: { composite: [86, 89, 83, 88, 84, 90][i % 6]! }, + })), + ) expect(r.decision).toBe('ship') expect(gateDetail(r, 'heldout-significance').ciLow).toBeGreaterThan(0) }) @@ -164,23 +147,16 @@ describe('defaultProductionGate — per-dimension regression guard (anti-Goodhar it('SHIPS when the composite rises and the critical dimension holds flat', async () => { const r = await decide({ - baseline: { - 'h1:0': { composite: 80, dimensions: { hallucination_free: 100 } }, - 'h2:0': { composite: 82, dimensions: { hallucination_free: 100 } }, - 'h3:0': { composite: 78, dimensions: { hallucination_free: 100 } }, - 'h4:0': { composite: 81, dimensions: { hallucination_free: 99 } }, - 'h5:0': { composite: 79, dimensions: { hallucination_free: 100 } }, - 'h6:0': { composite: 83, dimensions: { hallucination_free: 99 } }, - }, - candidate: { - 'h1:0': { composite: 86, dimensions: { hallucination_free: 100 } }, - 'h2:0': { composite: 89, dimensions: { hallucination_free: 100 } }, - 'h3:0': { composite: 83, dimensions: { hallucination_free: 99 } }, - 'h4:0': { composite: 88, dimensions: { hallucination_free: 100 } }, - 'h5:0': { composite: 84, dimensions: { hallucination_free: 100 } }, - 'h6:0': { composite: 90, dimensions: { hallucination_free: 100 } }, - }, - scenarioIds: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], + ...continuousRoster((i) => ({ + baseline: { + composite: [80, 82, 78, 81, 79, 83][i % 6]!, + dimensions: { hallucination_free: i % 3 === 0 ? 99 : 100 }, + }, + candidate: { + composite: [86, 89, 83, 88, 84, 90][i % 6]!, + dimensions: { hallucination_free: i % 3 === 1 ? 99 : 100 }, + }, + })), criticalDimensions: ['hallucination_free'], }) expect(r.decision).toBe('ship') diff --git a/tests/contract-analyze-runs.test.ts b/tests/contract-analyze-runs.test.ts index 040bd450..ebf35c7c 100644 --- a/tests/contract-analyze-runs.test.ts +++ b/tests/contract-analyze-runs.test.ts @@ -26,6 +26,7 @@ import { type InsightReport, summarizeExecution, } from '../src/contract' +import { InsightReportSchema } from '../src/hosted/schemas' import type { TraceSpanEvent } from '../src/hosted/types' import type { RunRecord, RunTerminalOutcome } from '../src/run-record' @@ -85,6 +86,84 @@ function makeRun(opts: { // ── analyzeRuns: lift detection ───────────────────────────────────── describe('analyzeRuns — lift detection with paired bootstrap', () => { + it('retains raw runs while one source supplies only one independent lift observation', async () => { + const baseline = [ + makeRun({ id: 'b-a', candidate: 'baseline', composite: 0.1 }), + makeRun({ id: 'b-b', candidate: 'baseline', composite: 0.3 }), + ] + const candidate = [ + makeRun({ id: 'c-a', candidate: 'candidate', composite: 0.7 }), + makeRun({ id: 'c-b', candidate: 'candidate', composite: 0.9 }), + ] + const report = await analyzeRuns({ + runs: [...baseline, ...candidate], + baselineCandidateId: 'baseline', + candidateCandidateId: 'candidate', + independentUnitByScenarioId: new Map([ + ['a', 'source'], + ['b', 'source'], + ]), + }) + expect(report.n).toBe(4) + expect(report.composite.n).toBe(4) + expect(report.lift).toMatchObject({ + baselineMean: 0.2, + candidateMean: 0.8, + n: 1, + pairedRunN: 2, + independentUnitIds: ['source'], + decisionEligible: false, + pValue: null, + requiredN: null, + }) + expect(report.lift?.delta).toBeCloseTo(0.6) + expect(report.recommendations.some((recommendation) => recommendation.kind === 'ship')).toBe( + false, + ) + expect(InsightReportSchema.parse(report).lift).toEqual(report.lift) + }) + + it('infers the lower baseline from independent-unit means when family sizes differ', async () => { + const scenarioIds = ['a1', 'a2', 'a3', 'a4', 'b', 'c'] + const runs = scenarioIds.flatMap((scenarioId) => [ + makeRun({ + id: `b-${scenarioId}`, + candidate: 'baseline', + composite: scenarioId.startsWith('a') ? 1 : 0, + }), + makeRun({ + id: `c-${scenarioId}`, + candidate: 'candidate', + composite: scenarioId.startsWith('a') ? 0 : 1, + }), + ]) + const options = { + runs, + independentUnitByScenarioId: new Map(scenarioIds.map((id) => [id, id[0]!])), + } + const inferred = await analyzeRuns(options) + const explicit = await analyzeRuns({ + ...options, + baselineCandidateId: 'baseline', + candidateCandidateId: 'candidate', + }) + expect(inferred.lift?.baselineMean).toBeCloseTo(1 / 3) + expect(inferred.lift?.candidateMean).toBeCloseTo(2 / 3) + expect(inferred.lift).toEqual(explicit.lift) + }) + + it('rejects a missing declared source instead of counting its scenario as independent', async () => { + await expect( + analyzeRuns({ + runs: [ + makeRun({ id: 'b-a', candidate: 'baseline', composite: 0 }), + makeRun({ id: 'c-a', candidate: 'candidate', composite: 1 }), + ], + independentUnitByScenarioId: new Map(), + }), + ).rejects.toThrow(/missing independent unit/) + }) + it('emits a positive lift CI when candidate beats baseline on holdout', async () => { const baseline = Array.from({ length: 20 }, (_, i) => makeRun({ id: `b-${i}`, candidate: 'baseline', composite: 0.5 + i * 0.005 }), diff --git a/tests/evaluator-admission.test.ts b/tests/evaluator-admission.test.ts new file mode 100644 index 00000000..cecd6306 --- /dev/null +++ b/tests/evaluator-admission.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from 'vitest' +import { hashCanonical } from '../src/ledger-core/canonical' +import { + auditEvaluator, + type EvaluatorAuditInput, + type EvaluatorAuditObservation, +} from '../src/meta-eval/evaluator-admission' + +function controls(n: number): EvaluatorAuditObservation[] { + return Array.from({ length: n }, (_, i) => + (['accept', 'reject'] as const).map((expected) => ({ + id: `${expected}-${i}`, + independentUnitId: `source-${i}`, + evidenceRef: `fixture:${expected}-${i}`, + expected, + observed: expected, + exposure: 'fresh' as const, + })), + ).flat() +} +function audit(observations = controls(100)): EvaluatorAuditInput { + return { + evaluatorDigest: hashCanonical('judge-v1'), + population: 'support answers', + samplingFrame: 'independent audited incidents', + authority: { + evaluatorAuthorId: 'author', + auditorId: 'auditor', + independenceEvidenceRef: 'audit:access-record', + }, + policy: { confidence: 0.95, maxFalseAcceptanceRate: 0.05, maxFalseRejectionRate: 0.05 }, + observations, + } +} + +describe('evaluator admission from independent controls', () => { + it('admits accurate judgments only when both error bounds satisfy policy', () => { + const report = auditEvaluator(audit()) + expect(report.verdict).toBe('admit') + expect(report.falseAcceptance).toMatchObject({ + cases: 100, + independentUnits: 100, + errorRate: 0, + verdict: 'pass', + }) + expect(report.falseRejection.interval!.upper).toBeGreaterThan(0.04) + expect(report.falseRejection.interval!.upper).toBeLessThan(0.05) + expect(report.confidence).toBe(0.95) + expect(report.intervalConfidence).toBe(0.975) + expect(report.coverage).toMatchObject({ + cases: 200, + independentUnits: 100, + excludedCases: 0, + unknownCases: 0, + }) + }) + + it.each(['accept', 'reject'] as const)('rejects an always-%s evaluator', (observed) => { + const report = auditEvaluator(audit(controls(100).map((row) => ({ ...row, observed })))) + expect(report.verdict).toBe('reject') + const failed = observed === 'accept' ? report.falseAcceptance : report.falseRejection + expect(failed).toMatchObject({ errorRate: 1, verdict: 'fail' }) + }) + + it('does not manufacture evidence from repeated variants or missing judgments', () => { + const duplicated = controls(100).map((row) => ({ ...row, independentUnitId: 'one-source' })) + const report = auditEvaluator(audit(duplicated)) + expect(report.verdict).toBe('inconclusive') + expect(report.falseAcceptance.independentUnits).toBe(1) + expect(report.falseAcceptance.interval!.upper).toBeGreaterThan(0.98) + const unknown = auditEvaluator( + audit(controls(100).map((row) => ({ ...row, observed: 'unknown' }))), + ) + expect(unknown.verdict).toBe('inconclusive') + expect(unknown.falseAcceptance).toMatchObject({ + errorRate: null, + unresolvedUnits: 100, + interval: { lower: 0, upper: 1 }, + }) + expect(unknown.coverage.unknownCases).toBe(200) + }) + + it('can admit incomplete judgments when their worst-case error bounds still meet policy', () => { + const input = audit(controls(200)) + input.observations[1]!.observed = 'unknown' + const report = auditEvaluator(input) + expect(report.verdict).toBe('admit') + expect(report.falseAcceptance).toMatchObject({ + errorRate: null, + unresolvedUnits: 1, + unknownCases: 1, + verdict: 'pass', + }) + expect(report.falseAcceptance.interval!.upper).toBeLessThan(0.05) + expect(report.reasons.join(' ')).toContain('worst case') + expect(report.reasons.join(' ')).not.toContain('does not establish') + }) + + it('excludes every variant of a development source and retains the exclusion', () => { + const input = audit() + input.observations[0]!.exposure = 'development' + const report = auditEvaluator(input) + expect(report.exclusions).toHaveLength(2) + expect(report.coverage).toMatchObject({ + cases: 200, + eligibleCases: 198, + eligibleIndependentUnits: 99, + excludedCases: 2, + }) + expect(auditEvaluator(audit([])).falseAcceptance.errorRate).toBeNull() + expect(auditEvaluator(audit([])).falseAcceptance.interval).toBeNull() + }) + + it('counts a source mistake even when other variants pass', () => { + const input = audit(controls(10)) + input.observations.push({ ...input.observations[0]!, id: 'bad-variant', observed: 'reject' }) + expect(auditEvaluator(input).falseRejection).toMatchObject({ + cases: 11, + independentUnits: 10, + errorUnits: 1, + errorRate: 0.1, + }) + }) + + it('binds inputs and policy without retaining caller-owned references', () => { + const input = audit() + const report = auditEvaluator(input) + expect( + auditEvaluator({ ...input, observations: [...input.observations].reverse() }).reportDigest, + ).toBe(report.reportDigest) + input.policy.maxFalseAcceptanceRate = 0.1 + input.observations[0]!.observed = 'unknown' + expect(report.policy.maxFalseAcceptanceRate).toBe(0.05) + expect(report.coverage.unknownCases).toBe(0) + expect(auditEvaluator(input).inputDigest).not.toBe(report.inputDigest) + const { reportDigest, ...body } = report + expect(hashCanonical(body)).toBe(reportDigest) + }) + + it('rejects duplicate observations and undeclared audit separation', () => { + const input = audit() + input.observations.push(input.observations[0]!) + expect(() => auditEvaluator(input)).toThrow(/duplicate/) + const sameAuthority = audit() + sameAuthority.authority.auditorId = sameAuthority.authority.evaluatorAuthorId + expect(() => auditEvaluator(sameAuthority)).toThrow(/separate declared audit authority/) + }) +}) diff --git a/tests/experiment/budget-and-seal.test.ts b/tests/experiment/budget-and-seal.test.ts index 3f6a6ae0..316556a6 100644 --- a/tests/experiment/budget-and-seal.test.ts +++ b/tests/experiment/budget-and-seal.test.ts @@ -12,6 +12,7 @@ import { MatchedBudgetError, type MatchedBudgetRule, openSealedExperiment, + type SealedExperiment, SealIntegrityError, sealExperiment, verifyMatchedBudgets, @@ -85,6 +86,7 @@ const minimalSpec: ExperimentSpec = { 'primary-95': { kind: 'cluster-bootstrap', clusterBy: 'taskName', + value: 'diff', resamples: 1000, seed: 7, level: 0.95, @@ -167,22 +169,34 @@ describe('defineExperiment validation', () => { }) describe('seal integrity', () => { - it('verifies a seal written under the previous digest scheme — regression: a sealed registration outlives the release that sealed it', async () => { + it('accepts RFC 8785 and refuses correctly hashed seals under the retired scheme', async () => { const sealed = await sealExperiment(minimalSpec) expect(sealed.algo).toBe('sha256-rfc8785') expect(await verifySealedExperiment(sealed)).toBe(true) - const legacy = { - ...sealed, - algo: 'sha256-content' as const, - digest: createHash('sha256') - .update(JSON.stringify(sortKeysDeep(sealed.spec)), 'utf8') - .digest('hex'), - } - expect(await verifySealedExperiment(legacy)).toBe(true) - expect(await verifySealedExperiment({ ...legacy, digest: 'f'.repeat(64) })).toBe(false) + const legacy: SealedExperiment = JSON.parse( + JSON.stringify({ + ...sealed, + algo: 'sha256-content', + digest: createHash('sha256') + .update(JSON.stringify(sortKeysDeep(sealed.spec)), 'utf8') + .digest('hex'), + }), + ) + expect(await verifySealedExperiment(legacy)).toBe(false) + await expect(openSealedExperiment(legacy)).rejects.toThrow(SealIntegrityError) }) + it.each([undefined, 'sha512-future'])( + 'refuses a missing or unknown seal scheme before executing (%s)', + async (algo) => { + const sealed = await sealExperiment(minimalSpec) + const input: SealedExperiment = JSON.parse(JSON.stringify({ ...sealed, algo })) + expect(await verifySealedExperiment(input)).toBe(false) + await expect(openSealedExperiment(input)).rejects.toThrow(/unsupported digest scheme/) + }, + ) + it('a tampered seal is rejected before any executor is handed out', async () => { const sealed = await sealExperiment(minimalSpec) const tampered = { @@ -269,8 +283,7 @@ describe('seal integrity', () => { }) }) -/** Key-sorted `JSON.stringify` — the scheme seals were digested under before - * RFC 8785. Kept here to MINT a legacy seal the verifier must still accept. */ +/** Produce a correctly hashed retired record to exercise scheme rejection. */ function sortKeysDeep(value: unknown): unknown { if (value === null || typeof value !== 'object') return value if (Array.isArray(value)) return value.map(sortKeysDeep) diff --git a/tests/experiment/evaluation-claim.test.ts b/tests/experiment/evaluation-claim.test.ts new file mode 100644 index 00000000..c82e800f --- /dev/null +++ b/tests/experiment/evaluation-claim.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from 'vitest' +import { + defineEvaluationClaim, + type EvaluationClaim, + summarizeEvaluationUnits, +} from '../../src/experiment/claim' +import { + amendExperiment, + defineExperiment, + type ExperimentSpec, + openSealedExperiment, + type SealedExperiment, + sealExperiment, +} from '../../src/experiment/define' + +const claim: EvaluationClaim = { + use: 'comparison', + population: { id: 'incidents', description: 'New support incidents' }, + samplingFrame: 'A random sample from the incident queue', + independentUnit: 'source.id', + generalization: 'new-units', + minimumEffect: 0.05, +} + +function spec(): ExperimentSpec { + return { + id: 'claimed-comparison', + claim, + arms: [{ id: 'candidate', role: 'treatment' }], + outcome: { kind: 'binary' }, + intervals: { + gain: { + kind: 'cluster-bootstrap', + clusterBy: 'source.id', + value: 'value', + resamples: 1000, + seed: 7, + level: 0.95, + method: 'percentile', + }, + rate: { kind: 'clopper-pearson', level: 0.95 }, + }, + decision: { kind: 'report-only', estimands: [], intervals: ['gain', 'rate'] }, + } +} + +describe('registered evaluation claims', () => { + it('distinguishes repeated observations from independent evidence', async () => { + const experiment = await openSealedExperiment(await sealExperiment(spec())) + const replicas = Array.from({ length: 100 }, () => ({ source: { id: 'incident-1' }, value: 1 })) + expect(experiment.units(replicas)).toMatchObject({ observations: 100, independentUnits: 1 }) + expect(() => experiment.interval('gain', { kind: 'rows', rows: replicas })).toThrow( + /2 clusters/, + ) + const independent = replicas.map((row, i) => ({ + ...row, + source: { id: `incident-${i}` }, + value: i % 2, + })) + const measured = experiment.interval('gain', { kind: 'rows', rows: independent }) + expect(measured.units).toMatchObject({ observations: 100, independentUnits: 100 }) + expect(measured.lower).toBeLessThan(0.5) + expect(measured.upper).toBeGreaterThan(0.5) + }) + + it('refuses an estimator that resamples variants instead of source units', () => { + const registration = spec() + registration.intervals!.gain = { + kind: 'cluster-bootstrap', + clusterBy: 'id', + value: 'value', + resamples: 1000, + seed: 7, + level: 0.95, + method: 'percentile', + } + expect(() => defineExperiment(registration)).toThrow(/must resample 'source.id'/) + }) + + it('requires power at the claimed minimum effect', () => { + const registration = spec() + registration.gates = { + power: { + kind: 'power-floor', + target: 0.8, + minimumEffect: 0.5, + effectGrid: [0.05, 0.5], + sim: { trials: 100, resamples: 1000, seed: 7 }, + }, + } + expect(() => defineExperiment(registration)).toThrow(/differs from the evaluation claim/) + }) + + it('requires one identified independent trial for a new-unit binomial interval', async () => { + const experiment = await openSealedExperiment(await sealExperiment(spec())) + for (const unitIds of [undefined, ['a'], ['a', 'a'], ['a', ' b']]) { + expect(() => + experiment.interval('rate', { kind: 'binomial', trials: 2, successes: 1, unitIds }), + ).toThrow(/unique unitId/) + } + const rate = experiment.interval('rate', { + kind: 'binomial', + trials: 2, + successes: 1, + unitIds: ['a', 'b'], + }) + expect(rate.units?.independentUnits).toBe(2) + expect(rate.lower).toBeGreaterThan(0) + expect(rate.upper).toBeLessThan(1) + }) + + it('captures the seal before any asynchronous work or caller mutation', async () => { + const input: SealedExperiment = JSON.parse(JSON.stringify(await sealExperiment(spec()))) + const opening = openSealedExperiment(input) + input.spec.claim!.independentUnit = 'variant' + input.spec.intervals!.rate = { kind: 'clopper-pearson', level: 0.5 } + const experiment = await opening + expect(experiment.sealed.spec.claim?.independentUnit).toBe('source.id') + expect( + experiment.interval('rate', { + kind: 'binomial', + trials: 2, + successes: 1, + unitIds: ['a', 'b'], + }).level, + ).toBe(0.95) + expect(Object.isFrozen(experiment.sealed.spec.intervals)).toBe(true) + }) + + it('rejects ambiguous identities instead of normalizing sealed metadata', () => { + expect(() => + defineEvaluationClaim({ ...claim, population: { ...claim.population, id: ' incidents' } }), + ).toThrow() + expect(() => summarizeEvaluationUnits(claim, [{ source: {} }])).toThrow(/source.id/) + expect(() => summarizeEvaluationUnits(claim, [{ source: { id: ' ' } }])).toThrow(/source.id/) + }) + + it('captures the prior seal and requested amendment before asynchronous verification', async () => { + const input: SealedExperiment = JSON.parse(JSON.stringify(await sealExperiment(spec()))) + const initialDigest = input.initialDigest + const requested = { spec: spec(), reason: 'registered change', blind: ['outcomes'] } + const pending = amendExperiment(input, requested) + input.initialDigest = '0'.repeat(64) + input.amendments.push({ at: 'invented', reason: 'invented', blind: [], digest: 'a'.repeat(64) }) + requested.spec.intervals!.rate = { kind: 'clopper-pearson', level: 0.5 } + requested.reason = 'changed after verification began' + requested.blind.length = 0 + const amended = await pending + expect(amended.initialDigest).toBe(initialDigest) + expect(amended.amendments).toHaveLength(1) + expect(amended.amendments[0]).toMatchObject({ + reason: 'registered change', + blind: ['outcomes'], + }) + expect(amended.spec.intervals!.rate).toEqual({ kind: 'clopper-pearson', level: 0.95 }) + }) +}) diff --git a/tests/experiment/final-evidence.test.ts b/tests/experiment/final-evidence.test.ts new file mode 100644 index 00000000..114b6dce --- /dev/null +++ b/tests/experiment/final-evidence.test.ts @@ -0,0 +1,142 @@ +import { mkdtempSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + type FinalEvidenceOutcome, + type FinalEvidenceReservation, + openFinalEvidenceLedger, +} from '../../src/experiment/final-evidence' +import { hashCanonical } from '../../src/ledger-core/canonical' + +let dir: string +let path: string +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'final-evidence-')) + path = join(dir, 'ledger.jsonl') +}) +afterEach(() => rmSync(dir, { recursive: true, force: true })) + +function value(result: FinalEvidenceOutcome): T { + if (!result.succeeded) throw new Error(result.error.message) + return result.value +} +function reservation(requestId = 'run'): FinalEvidenceReservation { + return { + requestId, + claimDigest: hashCanonical('claim'), + inputDigest: hashCanonical(requestId), + populationId: 'incidents', + unitIds: ['incident-1', 'incident-2'], + } +} +const measurement = { + evaluatorDigest: hashCanonical('evaluator'), + candidateDigests: [hashCanonical('baseline'), hashCanonical('candidate')], +} + +describe('durable final evidence lifecycle', () => { + it('reopens reservations and exact retries without creating a second exposure', async () => { + const first = openFinalEvidenceLedger({ path }) + expect(value(await first.read())).toEqual([]) + const reserved = value(await first.reserve(reservation())) + expect(reserved.replayed).toBe(false) + const second = openFinalEvidenceLedger({ path }) + expect( + value(await second.reserve({ ...reservation(), unitIds: ['incident-2', 'incident-1'] })) + .replayed, + ).toBe(true) + const exposed = value(await second.expose('run', measurement)) + expect(exposed.replayed).toBe(false) + expect(exposed.record.exposure?.measurement.candidateDigests).toHaveLength(2) + expect(value(await first.expose('run', measurement)).replayed).toBe(true) + expect(readFileSync(path, 'utf8').trim().split('\n')).toHaveLength(2) + expect(value(await first.read())[0]).toEqual(exposed.record) + }) + + it('refuses altered bindings, overlapping source units, and relabeled identical datasets', async () => { + const ledger = openFinalEvidenceLedger({ path }) + value(await ledger.reserve(reservation())) + for (const input of [ + { ...reservation(), claimDigest: hashCanonical('another claim') }, + { ...reservation('other'), populationId: 'renamed' }, + { ...reservation('other'), inputDigest: reservation().inputDigest, unitIds: ['renamed-1'] }, + ]) { + expect(await ledger.reserve(input)).toMatchObject({ + succeeded: false, + error: { kind: 'conflict' }, + }) + } + value(await ledger.expose('run', measurement)) + expect( + await ledger.expose('run', { ...measurement, candidateDigests: [hashCanonical('revised')] }), + ).toMatchObject({ succeeded: false, error: { kind: 'conflict' } }) + expect(value(await ledger.read())).toHaveLength(1) + }) + + it('serializes competing owners and reports only one first exposure', async () => { + const attempts = await Promise.all( + Array.from({ length: 12 }, (_, i) => + openFinalEvidenceLedger({ path }).reserve(reservation(`run-${i}`)), + ), + ) + const winners = attempts.filter((result) => result.succeeded) + expect(winners).toHaveLength(1) + const winner = value(winners[0]!) + const exposed = await Promise.all( + Array.from({ length: 4 }, () => + openFinalEvidenceLedger({ path }).expose(winner.record.reservation.requestId, measurement), + ), + ) + expect(exposed.map(value).filter((result) => !result.replayed)).toHaveLength(1) + }) + + it.each(['rewrite', 'truncate', 'missing-pin'] as const)( + 'refuses %s instead of opening fresh evidence', + async (damage) => { + const ledger = openFinalEvidenceLedger({ path }) + value(await ledger.reserve(reservation())) + value(await ledger.expose('run', measurement)) + const contents = readFileSync(path, 'utf8') + if (damage === 'rewrite') writeFileSync(path, contents.replace('incidents', 'tampered')) + if (damage === 'truncate') writeFileSync(path, `${contents.split('\n')[0]}\n`) + if (damage === 'missing-pin') unlinkSync(`${path}.head`) + expect(await openFinalEvidenceLedger({ path }).read()).toMatchObject({ + succeeded: false, + error: { kind: 'unavailable' }, + }) + expect((await ledger.reserve(reservation('new'))).succeeded).toBe(false) + }, + ) + + it('retains invalid input and unavailable storage as different failures', async () => { + const ledger = openFinalEvidenceLedger({ path }) + expect(await ledger.reserve({ ...reservation(), unitIds: ['same', 'same'] })).toMatchObject({ + succeeded: false, + error: { kind: 'invalid' }, + }) + expect(await ledger.expose('missing', measurement)).toMatchObject({ + succeeded: false, + error: { kind: 'invalid' }, + }) + writeFileSync(join(dir, 'file'), 'not a directory') + const unavailable = openFinalEvidenceLedger({ path: join(dir, 'file', 'ledger') }) + expect(await unavailable.reserve(reservation())).toMatchObject({ + succeeded: false, + error: { kind: 'unavailable' }, + }) + }) + + it('reports an invalid stored schema as unavailable evidence with its source location', async () => { + const ledger = openFinalEvidenceLedger({ path }) + value(await ledger.reserve(reservation())) + writeFileSync( + path, + readFileSync(path, 'utf8').replace('"unitIds":["incident-1","incident-2"]', '"unitIds":1'), + ) + expect(await ledger.read()).toMatchObject({ + succeeded: false, + error: { kind: 'unavailable', message: expect.stringContaining(`${path}:1`) }, + }) + }) +}) diff --git a/tests/experiment/interval-integrity.test.ts b/tests/experiment/interval-integrity.test.ts new file mode 100644 index 00000000..1704660a --- /dev/null +++ b/tests/experiment/interval-integrity.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from 'vitest' +import { ValidationError } from '../../src/errors' +import { computeInterval, type IntervalSpec } from '../../src/experiment/ast' +import { + defineExperiment, + type ExperimentSpec, + openSealedExperiment, + sealExperiment, +} from '../../src/experiment/define' + +const interval: Extract = { + kind: 'cluster-bootstrap', + clusterBy: 'source', + value: 'adverse', + resamples: 1_000, + seed: 7, + level: 0.95, + method: 'percentile', +} + +function registration(effect: IntervalSpec = interval): ExperimentSpec { + return { + id: 'interval-integrity', + arms: [{ id: 'candidate', role: 'treatment' }], + outcome: { kind: 'bounded-score', min: -1, max: 1, orientation: 'higher-is-better' }, + intervals: { effect }, + decision: { kind: 'report-only', estimands: [], intervals: ['effect'] }, + } +} + +const rows = Array.from({ length: 30 }, (_, index) => ({ + source: `incident-${index}`, + favourable: 0.1 + index / 1_000, + adverse: -0.1 - index / 1_000, +})) + +describe('registered interval integrity', () => { + it('binds the measured field to the seal and refuses an execution-time replacement', async () => { + const sealed = await sealExperiment(registration()) + const experiment = await openSealedExperiment(sealed) + expect(experiment.interval('effect', { kind: 'rows', rows }).upper).toBeLessThan(0) + const unsealedSelector = { kind: 'rows' as const, rows, value: 'favourable' } + expect(() => experiment.interval('effect', unsealedSelector)).toThrow( + /register value in the interval spec/, + ) + + const changed = await sealExperiment(registration({ ...interval, value: 'favourable' })) + expect(changed.digest).not.toBe(sealed.digest) + const amended = await openSealedExperiment(changed) + expect(amended.interval('effect', { kind: 'rows', rows }).lower).toBeGreaterThan(0) + }) + + it.each([ + { patch: { level: 0 }, message: 'level' }, + { patch: { level: 1 }, message: 'level' }, + { patch: { level: Number.NaN }, message: 'level' }, + { patch: { level: Number.POSITIVE_INFINITY }, message: 'level' }, + { patch: { resamples: 0 }, message: 'resamples' }, + { patch: { resamples: -1 }, message: 'resamples' }, + { patch: { resamples: 1.5 }, message: 'resamples' }, + { patch: { resamples: Number.MAX_SAFE_INTEGER }, message: 'resamples' }, + { patch: { seed: Number.NaN }, message: 'seed' }, + { patch: { seed: Number.POSITIVE_INFINITY }, message: 'seed' }, + { patch: { seed: 1.5 }, message: 'seed' }, + { patch: { clusterBy: '' }, message: 'clusterBy' }, + { patch: { clusterBy: 'source..id' }, message: 'clusterBy' }, + { patch: { clusterBy: ' source' }, message: 'clusterBy' }, + { patch: { value: undefined }, message: 'value' }, + { patch: { value: '' }, message: 'value' }, + { patch: { value: 'score.' }, message: 'value' }, + { patch: { method: 'unregistered-method' }, message: 'method' }, + ])( + 'refuses malformed $message both before sealing and at direct execution', + ({ patch, message }) => { + // Untyped JSON and JavaScript callers can supply values outside the declared union. + const invalid = { ...interval, ...patch } as IntervalSpec + expect(() => defineExperiment(registration(invalid))).toThrow(message) + expect(() => computeInterval(invalid, { kind: 'rows', rows })).toThrow(ValidationError) + }, + ) + + it.each([undefined, null, {}, [], Number.NaN, ''])( + 'refuses an unidentified cluster: %j', + (source) => { + expect(() => + computeInterval(interval, { + kind: 'rows', + rows: [...rows, { source, adverse: 0 }], + }), + ).toThrow(/cluster field 'source'/) + }, + ) + + it('keeps numeric and string cluster identities distinct', () => { + const result = computeInterval(interval, { + kind: 'rows', + rows: [ + { source: 1, adverse: 0 }, + { source: '1', adverse: 1 }, + ], + }) + expect(result).toEqual({ lower: 0, upper: 1, level: 0.95 }) + }) + + it('refuses arithmetic overflow instead of returning an infinite interval', () => { + expect(() => + computeInterval(interval, { + kind: 'rows', + rows: [ + { source: 'a', adverse: Number.MAX_VALUE }, + { source: 'b', adverse: Number.MAX_VALUE }, + ], + }), + ).toThrow(/overflowed/) + }) + + it.each([ + { successes: 1.5, trials: 2 }, + { successes: 0, trials: 0 }, + { successes: 3, trials: 2 }, + { successes: 1, trials: Number.POSITIVE_INFINITY }, + { successes: 1, trials: Number.MAX_SAFE_INTEGER + 1 }, + ])('refuses invalid binomial counts before computing tails: %j', (counts) => { + expect(() => + computeInterval( + { kind: 'clopper-pearson', level: 0.95 }, + { + kind: 'binomial', + ...counts, + }, + ), + ).toThrow(ValidationError) + }) + + it.each([0, 1, Number.NaN, Number.POSITIVE_INFINITY])( + 'refuses invalid binomial confidence %s', + (level) => { + const spec: IntervalSpec = { kind: 'clopper-pearson', level } + expect(() => defineExperiment(registration(spec))).toThrow(/level/) + expect(() => computeInterval(spec, { kind: 'binomial', successes: 1, trials: 2 })).toThrow( + /level/, + ) + }, + ) +}) diff --git a/tests/experiment/paired-binary-outcome.test.ts b/tests/experiment/paired-binary-outcome.test.ts index 98184ca3..3e456396 100644 --- a/tests/experiment/paired-binary-outcome.test.ts +++ b/tests/experiment/paired-binary-outcome.test.ts @@ -48,6 +48,16 @@ const gatedStopSpec: ExperimentSpec = { pairedContrast95: { kind: 'cluster-bootstrap', clusterBy: 'taskName', + value: 'diff', + resamples: 2_000, + seed: 20260814, + level: 0.95, + method: 'percentile', + }, + treatmentRate95: { + kind: 'cluster-bootstrap', + clusterBy: 'taskName', + value: 'passed', resamples: 2_000, seed: 20260814, level: 0.95, @@ -193,6 +203,35 @@ describe('paired-mean-diff over a binary outcome', () => { }) describe('the sealed gated-stop path, from boolean rows to a verdict', () => { + it('does not turn equally successful arms into a positive contrast interval', async () => { + const registered = await openSealedExperiment(await sealExperiment(gatedStopSpec)) + const equalArms = booleanRows().map((row) => ({ ...row, passed: true })) + const byPair = new Map() + for (const row of equalArms) { + const id = String(row.rowId) + const pair = byPair.get(id) ?? [] + pair.push(row) + byPair.set(id, pair) + } + const differences = [...byPair.values()].map((pair) => ({ + taskName: pair[0]!.taskName, + diff: registered.estimate('pairedContrast', pair).value, + })) + expect(registered.estimate('pairedContrast', equalArms).value).toBe(0) + const interval = registered.interval('pairedContrast95', { kind: 'rows', rows: differences }) + expect(interval).toEqual({ lower: 0, upper: 0, level: 0.95 }) + expect( + registered.decide({ + intervals: { pairedContrast95: interval }, + quantities: {}, + obligationsMet: { 'matched-realized-tokens': true }, + }).verdict, + ).toBe('no-effect-resolved-at-this-n') + expect(() => + registered.interval('pairedContrast95', { kind: 'rows', rows: equalArms }), + ).toThrow(/value field 'diff'/) + }) + it('estimates, brackets, and decides without the caller re-encoding anything', async () => { const sealed = await sealExperiment(gatedStopSpec, { sealedAt: '2026-08-14T00:00:00Z' }) const registered = await openSealedExperiment(sealed) @@ -203,7 +242,6 @@ describe('the sealed gated-stop path, from boolean rows to a verdict', () => { const interval = registered.interval('pairedContrast95', { kind: 'rows', rows: pairDifferenceRows(), - value: 'diff', }) // Every task cluster carries at least one improved pair and no regression, // so no resample of whole clusters can reach zero. @@ -224,10 +262,9 @@ describe('the sealed gated-stop path, from boolean rows to a verdict', () => { const sealed = await sealExperiment(gatedStopSpec, { sealedAt: '2026-08-14T00:00:00Z' }) const registered = await openSealedExperiment(sealed) const treatmentRows = booleanRows().filter((row) => row.arm === 'gated-continue') - const interval = registered.interval('pairedContrast95', { + const interval = registered.interval('treatmentRate95', { kind: 'rows', rows: treatmentRows, - value: 'passed', }) // 11 of 16 treatment rows pass, so the bootstrap sits inside (0, 1). expect(interval.lower).toBeGreaterThan(0) diff --git a/tests/experiment/power-floor.test.ts b/tests/experiment/power-floor.test.ts new file mode 100644 index 00000000..446880aa --- /dev/null +++ b/tests/experiment/power-floor.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from 'vitest' +import { evaluatePowerFloorGate, type ValidityGate } from '../../src/experiment/ast' +import { defineExperiment } from '../../src/experiment/define' +import { killtestSpec } from './preregistrations' + +const gate: Extract = { + kind: 'power-floor', + target: 0.8, + minimumEffect: 0.01, + effectGrid: [0.01, 1], + sim: { trials: 2000, resamples: 4000, seed: 1 }, +} + +describe('registered power at the minimum worthwhile effect', () => { + it('refuses an underpowered worthwhile effect even when a larger effect has perfect power', () => { + const result = evaluatePowerFloorGate('power', gate, [ + { effect: 0.01, power: 0.1 }, + { effect: 1, power: 1 }, + ]) + expect(result.passed).toBe(false) + expect(result.evidence).toMatchObject({ + target: 0.8, + minimumEffect: 0.01, + powerAtMinimumEffect: 0.1, + maxPower: 1, + }) + }) + + it('accepts target power at the registered effect independently of other grid points', () => { + const result = evaluatePowerFloorGate('power', gate, [ + { effect: 0.01, power: 0.8 }, + { effect: 1, power: 0.7 }, + ]) + expect(result.passed).toBe(true) + }) + + it.each([ + { minimumEffect: 0.02 }, + { minimumEffect: NaN }, + { target: 0 }, + { target: NaN }, + { effectGrid: [] }, + { effectGrid: [0.01, 0.01, 1] }, + { effectGrid: [0.01, Infinity] }, + { sim: { trials: 0, resamples: 1, seed: 1 } }, + ])('rejects malformed power registration before sealing: %j', (change) => { + const spec = { + ...killtestSpec, + gates: { ...killtestSpec.gates, 'power-floor': { ...gate, ...change } }, + } + expect(() => defineExperiment(spec)).toThrow() + }) + + it.each([ + [{ effect: 1, power: 1 }], + [ + { effect: 0.01, power: 0.9 }, + { effect: 1, power: 1 }, + { effect: 2, power: 1 }, + ], + [ + { effect: 0.01, power: 0.1 }, + { effect: 0.01, power: 1 }, + { effect: 1, power: 1 }, + ], + [ + { effect: 0.01, power: NaN }, + { effect: 1, power: 1 }, + ], + [ + { effect: 0.01, power: 1.1 }, + { effect: 1, power: 1 }, + ], + ])('refuses missing, duplicate, extra, or uncalibrated curve values: %j', (...curve) => { + expect(() => evaluatePowerFloorGate('power', gate, curve)).toThrow() + }) +}) diff --git a/tests/experiment/power.test.ts b/tests/experiment/power.test.ts index 227f4b4b..c62db515 100644 --- a/tests/experiment/power.test.ts +++ b/tests/experiment/power.test.ts @@ -19,6 +19,7 @@ describe('clusteredPower refusals (the recorded burns)', () => { const result = clusteredPower({ clusterSizes: [6, 3, 3, 2], effects: [0, 0.3, 0.9, 1], + minimumEffect: 0.3, seed: 20260810, trials: 300, resamples: 500, @@ -40,6 +41,7 @@ describe('clusteredPower refusals (the recorded burns)', () => { const result = clusteredPower({ clusterSizes: [5, 5, 5], effects: [1], + minimumEffect: 1, seed: 7, trials: 200, resamples: 400, @@ -53,6 +55,7 @@ describe('clusteredPower refusals (the recorded burns)', () => { const result = clusteredPower({ clusterSizes: [4, 4, 4, 4], effects: [1], + minimumEffect: 1, seed: 7, trials: 100, resamples: 300, @@ -64,6 +67,7 @@ describe('clusteredPower refusals (the recorded burns)', () => { const result = clusteredPower({ clusterSizes: Array.from({ length: 12 }, () => 4), effects: [0.5], + minimumEffect: 0.5, seed: 11, trials: 300, resamples: 400, @@ -79,6 +83,7 @@ describe('clusteredPower refusals (the recorded burns)', () => { const result = clusteredPower({ clusterSizes: Array.from({ length: 8 }, () => 2), effects: [0.05], + minimumEffect: 0.05, seed: 13, trials: 300, resamples: 400, @@ -86,7 +91,7 @@ describe('clusteredPower refusals (the recorded burns)', () => { expect(result.signFlipFloor.certifiableAtAlpha).toBe(true) expect(result.maxPower).toBeLessThan(0.8) expect(result.adequate).toBe(false) - expect(result.refusal!.reasons.join(' ')).toContain('simulated power tops out') + expect(result.refusal!.reasons.join(' ')).toContain('at minimum worthwhile effect 0.05') }) }) @@ -96,6 +101,7 @@ describe('clusteredPower simulation sanity', () => { clusteredPower({ clusterSizes: [6, 3, 3, 2], effects: [0.5], + minimumEffect: 0.5, seed: 42, trials: 200, resamples: 300, @@ -107,6 +113,7 @@ describe('clusteredPower simulation sanity', () => { const result = clusteredPower({ clusterSizes: Array.from({ length: 10 }, () => 4), effects: [0.1, 0.5], + minimumEffect: 0.1, seed: 21, trials: 300, resamples: 400, @@ -120,6 +127,7 @@ describe('clusteredPower simulation sanity', () => { const simulated = clusteredPower({ clusterSizes: Array.from({ length: 40 }, () => 1), effects: [0.3], + minimumEffect: 0.3, seed: 99, trials: 400, resamples: 500, @@ -129,22 +137,76 @@ describe('clusteredPower simulation sanity', () => { }) it('rejects invalid structures loudly', () => { - expect(() => clusteredPower({ clusterSizes: [], effects: [0.5], seed: 1 })).toThrow( - /positive integers/, - ) - expect(() => clusteredPower({ clusterSizes: [3, 0], effects: [0.5], seed: 1 })).toThrow( - /positive integers/, - ) - expect(() => clusteredPower({ clusterSizes: [3, 3], effects: [], seed: 1 })).toThrow( - /grid is empty/, - ) + expect(() => + clusteredPower({ clusterSizes: [], effects: [0.5], minimumEffect: 0.5, seed: 1 }), + ).toThrow(/positive integers/) + expect(() => + clusteredPower({ clusterSizes: [3, 0], effects: [0.5], minimumEffect: 0.5, seed: 1 }), + ).toThrow(/positive integers/) + expect(() => + clusteredPower({ clusterSizes: [3, 3], effects: [], minimumEffect: 0.5, seed: 1 }), + ).toThrow(/grid is empty/) expect(() => clusteredPower({ clusterSizes: [3, 3], effects: [0.5], + minimumEffect: 0.5, seed: 1, noisyClusters: [{ index: 5, flipRate: 0.3 }], }), ).toThrow(/outside/) }) + + it('refuses a design powerful only at effects larger than the worthwhile effect', () => { + const result = clusteredPower({ + clusterSizes: Array.from({ length: 12 }, () => 4), + effects: [0.01, 1], + minimumEffect: 0.01, + seed: 31, + trials: 200, + resamples: 300, + }) + expect(result.maxPower).toBe(1) + expect(result.powerAtMinimumEffect).toBeLessThan(0.8) + expect(result.adequate).toBe(false) + expect(() => assertDesignAdequate(result)).toThrow(/minimum worthwhile effect 0.01/) + }) + + it('requires the worthwhile effect and finite probability parameters before simulation', () => { + const options = { + clusterSizes: [4, 4, 4, 4, 4, 4], + effects: [0.5], + minimumEffect: 0.5, + seed: 1, + } + expect(() => clusteredPower({ ...options, minimumEffect: 0.1 })).toThrow( + /contain minimumEffect/, + ) + expect(() => clusteredPower({ ...options, minimumEffect: NaN })).toThrow(/minimumEffect/) + expect(() => clusteredPower({ ...options, effects: [0.5, 0.5] })).toThrow(/unique/) + expect(() => clusteredPower({ ...options, alpha: 0 })).toThrow(/alpha/) + expect(() => clusteredPower({ ...options, confidence: NaN })).toThrow(/confidence/) + expect(() => clusteredPower({ ...options, targetPower: 2 })).toThrow(/targetPower/) + expect(() => clusteredPower({ ...options, baseWinRate: 0.9, baseLossRate: 0.9 })).toThrow( + /sum to at most/, + ) + expect(() => clusteredPower({ ...options, baseWinRate: 0.2, baseLossRate: 0.1 })).toThrow( + /zero-effect model/, + ) + expect(() => + clusteredPower({ ...options, noisyClusters: [{ index: 1, flipRate: -0.1 }] }), + ).toThrow(/flipRate/) + }) + + it('does not replace a large non-deterministic effect with perfect wins by clipping probabilities', () => { + const result = clusteredPower({ + clusterSizes: Array.from({ length: 40 }, () => 1), + effects: [0.9], + minimumEffect: 0.9, + seed: 39, + trials: 200, + resamples: 300, + }) + expect(result.curve[0]!.medianCiWidth).toBeGreaterThan(0.05) + }) }) diff --git a/tests/experiment/preregistrations.ts b/tests/experiment/preregistrations.ts index 82f66a8b..b4ba1bae 100644 --- a/tests/experiment/preregistrations.ts +++ b/tests/experiment/preregistrations.ts @@ -132,6 +132,7 @@ export const killtestSpec: ExperimentSpec = { 'task-clustered-95': { kind: 'cluster-bootstrap', clusterBy: 'taskName', + value: 'diff', resamples: 4000, seed: 20260810, level: 0.95, @@ -169,6 +170,8 @@ export const killtestSpec: ExperimentSpec = { 'power-floor': { kind: 'power-floor', target: 0.8, + // This fixture preserves the original structural ceiling check at effect 1. + minimumEffect: 1, effectGrid: [0, 0.1, 0.3, 0.5, 0.9, 1], sim: { trials: 2000, resamples: 4000, seed: 20260810 }, }, @@ -264,6 +267,7 @@ export function freelunchSpec(ledger: { id: string; usd: number }[]): Experiment 'task-clustered-95': { kind: 'cluster-bootstrap', clusterBy: 'taskName', + value: 'passed', resamples: 10000, seed: 7, level: 0.95, @@ -272,6 +276,7 @@ export function freelunchSpec(ledger: { id: string; usd: number }[]): Experiment 'row-clustered-95': { kind: 'cluster-bootstrap', clusterBy: 'rowId', + value: 'passed', resamples: 10000, seed: 7, level: 0.95, @@ -380,6 +385,7 @@ export const milestone2Spec: ExperimentSpec = { 'task-clustered-95': { kind: 'cluster-bootstrap', clusterBy: 'taskName', + value: 'diff', resamples: 10000, seed: 7, level: 0.95, diff --git a/tests/held-out-gate-calibration.test.ts b/tests/held-out-gate-calibration.test.ts index 6412cc0b..0328d997 100644 --- a/tests/held-out-gate-calibration.test.ts +++ b/tests/held-out-gate-calibration.test.ts @@ -150,7 +150,7 @@ describe('HeldOutGate — repeated-sampling calibration', () => { expect(d.promote, `zero-spread sample at n=${n}`).toBe(false) expect(d.rejectionCode).toBe('indeterminate_delta') } - }) + }, 15_000) it('is deterministic: one identical input, 500 evaluations, one verdict', () => { // #457 was closed partly because 500 evaluations of one input produced 233 diff --git a/tests/held-out-gate.test.ts b/tests/held-out-gate.test.ts index 238bcec2..a25abc83 100644 --- a/tests/held-out-gate.test.ts +++ b/tests/held-out-gate.test.ts @@ -129,7 +129,7 @@ describe('HeldOutGate — rejection paths', () => { expect(d.reason).toMatch(/few_runs/) }) - it('rejects on negative paired delta on holdout', () => { + it('retains a negative observed delta while refusing an ineligible mean claim', () => { const g = new HeldOutGate({ baselineKey: 'baseline', minProductiveRuns: 3, seed: 1 }) // Candidate worse than baseline on holdout. const pairs = joinPairs( @@ -142,7 +142,7 @@ describe('HeldOutGate — rejection paths', () => { ) const d = g.evaluate(pairs.candidate, pairs.baseline) expect(d.promote).toBe(false) - expect(d.rejectionCode).toBe('negative_delta') + expect(d.rejectionCode).toBe('few_runs') expect(d.evidence.medianPairedDelta!).toBeLessThan(0) expect(d.evidence.pairedCI!.high).toBeLessThanOrEqual(0) }) @@ -189,13 +189,13 @@ describe('HeldOutGate — rejection paths', () => { seed: 1, }) // search≈0.95, holdout≈0.55 (gap≈0.40); baseline search=0.55, holdout=0.50 - // (gap=0.05). The holdout scores carry a little spread on purpose: six + // (gap=0.05). The holdout scores carry observed spread across 24 pairs; // pairs improving by an identical amount give a zero-width CI, which the // gate refuses as `indeterminate_delta` before it ever reaches the overfit // check, and this test is about the overfit check. const pairs = joinPairs( - ...[0.54, 0.55, 0.56, 0.55, 0.54, 0.56].map((holdout, i) => - makePair('cand', i, 0.95, holdout, 0.55, 0.5), + ...Array.from({ length: 24 }, (_, i) => + makePair('cand', i, 0.95, 0.54 + (i % 3) * 0.01, 0.55, 0.5), ), ) const d = g.evaluate(pairs.candidate, pairs.baseline) @@ -253,8 +253,8 @@ describe('HeldOutGate — rejection paths', () => { // identical paired deltas are refused for zero CI width, which would mask // the overfit rejection this test is asserting. const pairs = joinPairs( - ...[0.59, 0.6, 0.61, 0.6, 0.59, 0.61].map((holdout, i) => - makePair('cand', i, 0.9, holdout, 0.5, 0.5), + ...Array.from({ length: 24 }, (_, i) => + makePair('cand', i, 0.9, 0.59 + (i % 3) * 0.01, 0.5, 0.5), ), ) const matchedOnly = g.evaluate(pairs.candidate, pairs.baseline) @@ -269,7 +269,7 @@ describe('HeldOutGate — rejection paths', () => { ) // At the default coverage requirement the 10 unmatched rows are themselves - // disqualifying — 3 of 13 dealt holdout items scored on both arms. + // disqualifying — 24 of 34 dealt holdout items scored on both arms. expect( g.evaluate([...pairs.candidate, ...unmatchedCandidateRows], pairs.baseline), ).toMatchObject({ promote: false, rejectionCode: 'incomplete_coverage' }) @@ -309,6 +309,9 @@ describe('HeldOutGate — promotion path', () => { makePair('cand', 5, 0.75, 0.75, 0.5, 0.51), makePair('cand', 6, 0.76, 0.76, 0.51, 0.5), makePair('cand', 7, 0.74, 0.74, 0.5, 0.51), + ...Array.from({ length: 16 }, (_, i) => + makePair('cand', 8 + i, 0.73 + (i % 4) * 0.01, 0.73 + (i % 4) * 0.01, 0.5, 0.5), + ), ) const d = g.evaluate(pairs.candidate, pairs.baseline) expect(d.promote).toBe(true) @@ -489,12 +492,12 @@ describe('HeldOutGate — cost ceiling', () => { }) // Candidate is strictly better on quality but costs 4x baseline. const pairs = joinPairs( - makePair('cand', 0, 0.7, 0.7, 0.5, 0.5, { candidate: 0.08, baseline: 0.02 }), - makePair('cand', 1, 0.72, 0.72, 0.5, 0.5, { candidate: 0.08, baseline: 0.02 }), - makePair('cand', 2, 0.71, 0.71, 0.5, 0.5, { candidate: 0.08, baseline: 0.02 }), - makePair('cand', 3, 0.73, 0.73, 0.5, 0.5, { candidate: 0.08, baseline: 0.02 }), - makePair('cand', 4, 0.74, 0.74, 0.5, 0.5, { candidate: 0.08, baseline: 0.02 }), - makePair('cand', 5, 0.75, 0.75, 0.5, 0.5, { candidate: 0.08, baseline: 0.02 }), + ...Array.from({ length: 24 }, (_, i) => + makePair('cand', i, 0.7 + (i % 6) * 0.01, 0.7 + (i % 6) * 0.01, 0.5, 0.5, { + candidate: 0.08, + baseline: 0.02, + }), + ), ) const d = g.evaluate(pairs.candidate, pairs.baseline) expect(d.promote).toBe(false) @@ -512,12 +515,12 @@ describe('HeldOutGate — cost ceiling', () => { costPerTaskCeiling: 0.05, }) const pairs = joinPairs( - makePair('cand', 0, 0.7, 0.7, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 1, 0.72, 0.72, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 2, 0.71, 0.71, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 3, 0.73, 0.73, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 4, 0.74, 0.74, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 5, 0.75, 0.75, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), + ...Array.from({ length: 24 }, (_, i) => + makePair('cand', i, 0.7 + (i % 6) * 0.01, 0.7 + (i % 6) * 0.01, 0.5, 0.5, { + candidate: 0.03, + baseline: 0.02, + }), + ), ) const d = g.evaluate(pairs.candidate, pairs.baseline) expect(d.promote).toBe(true) @@ -547,12 +550,12 @@ describe('HeldOutGate — cost ceiling', () => { costPerTaskCeiling: 0.05, }) const pairs = joinPairs( - makePair('cand', 0, 0.7, 0.7, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 1, 0.72, 0.72, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 2, 0.71, 0.71, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 3, 0.73, 0.73, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 4, 0.74, 0.74, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), - makePair('cand', 5, 0.75, 0.75, 0.5, 0.5, { candidate: 0.03, baseline: 0.02 }), + ...Array.from({ length: 24 }, (_, i) => + makePair('cand', i, 0.7 + (i % 6) * 0.01, 0.7 + (i % 6) * 0.01, 0.5, 0.5, { + candidate: 0.03, + baseline: 0.02, + }), + ), ) pairs.candidate[0] = record({ ...pairs.candidate[0], @@ -804,15 +807,15 @@ describe('HeldOutGate — held-out coverage', () => { }) it('a caller may accept a shrunken denominator only by DECLARING it', () => { - const runs = partialCoverage({ n: 26, answered: 6, mode: 'no-row' }) + const runs = partialCoverage({ n: 26, answered: 20, mode: 'no-row' }) const d = new HeldOutGate({ baselineKey: 'baseline', seed: 1337, - minCoverage: 6 / 26, + minCoverage: 20 / 26, }).evaluate(runs.candidate, runs.baseline) expect(d.promote).toBe(true) // …and the declaration does not hide the shrink: it is still in the evidence. - expect(d.evidence.holdoutCoverage).toMatchObject({ dealt: 26, answered: 6, baselineOnly: 20 }) + expect(d.evidence.holdoutCoverage).toMatchObject({ dealt: 26, answered: 20, baselineOnly: 6 }) }) it('rejects an out-of-range minCoverage instead of silently clamping', () => { @@ -1021,16 +1024,17 @@ describe('HeldOutGate — coverage cannot be laundered', () => { }) }) -describe('HeldOutGate — the coverage check does not move a COMPLETE verdict', () => { - it('pins the pre-coverage numbers byte-for-byte on a fully covered comparison', () => { +describe('HeldOutGate — complete coverage and estimator eligibility', () => { + it('retains measured values while refusing an ineligible small-sample mean verdict', () => { // Every number asserted here was produced by the gate on `origin/main` // (2789970, published 0.133.3) with the identical fixture and seed. The - // coverage check must not move any of them. + // coverage check must not move the measured values. Eight continuous + // observations do not meet the mean estimator's eligibility minimum. // // ONE number moved since, deliberately: `pairedCI` is now the interval on // the MEAN paired delta, not the median, so its upper bound reads 0.195 // instead of 0.190. The deltas here are continuous (0.17-0.21) and the - // verdict, the lower bound and the p-value are all unchanged — the switch + // lower bound and the p-value remain unchanged — the switch // exists for the shapes where the median is pinned at 0, and this pin is // what proves it does not disturb the shapes where it was not. const pairs = joinPairs( @@ -1052,8 +1056,8 @@ describe('HeldOutGate — the coverage check does not move a COMPLETE verdict', seed: 42, }).evaluate(pairs.candidate, pairs.baseline) - expect(d.promote).toBe(true) - expect(d.rejectionCode).toBeNull() + expect(d.promote).toBe(false) + expect(d.rejectionCode).toBe('few_runs') expect(d.evidence.productiveRuns).toBe(8) expect(d.evidence.medianPairedDelta).toBe(0.19) expect(d.evidence.deltaStatistic).toBe('mean_bootstrap') @@ -1147,7 +1151,7 @@ describe('HeldOutGate — the cost median has a denominator too', () => { } { const candidate: RunRecord[] = [] const baseline: RunRecord[] = [] - for (let i = 0; i < 12; i += 1) { + for (let i = 0; i < 24; i += 1) { const scenarioId = `s${String(i).padStart(2, '0')}` baseline.push( record({ @@ -1342,7 +1346,7 @@ describe('HeldOutGate — binary (pass/fail) held-out outcomes', () => { expect(d.promote).toBe(true) }) - it('decides CONTINUOUS outcomes on the mean by default, same verdict', () => { + it('does not use the small-sample sign result to certify the default continuous mean', () => { const g = new HeldOutGate({ baselineKey: 'baseline', minProductiveRuns: 3, @@ -1367,7 +1371,8 @@ describe('HeldOutGate — binary (pass/fail) held-out outcomes', () => { expect(d.evidence.medianPairedDelta).toBe(0.22999999999999998) expect(d.evidence.decidingDelta).toBeCloseTo(0.22749999999999998, 12) expect(d.evidence.pairedCI!.low).toBeGreaterThan(0) - expect(d.promote).toBe(true) + expect(d.promote).toBe(false) + expect(d.rejectionCode).toBe('few_runs') }) }) diff --git a/tests/meta-eval.test.ts b/tests/meta-eval.test.ts index 6aa8a736..dac83d56 100644 --- a/tests/meta-eval.test.ts +++ b/tests/meta-eval.test.ts @@ -55,7 +55,39 @@ describe('InMemoryOutcomeStore', () => { }) describe('correlationStudy', () => { - it('returns strong positive correlation when eval score predicts outcome — regression: framework without this is ornamental', async () => { + it.each([ + { + evalMetrics: [{ id: 'score' }, { id: 'score' }], + outcomeMetrics: ['y'], + duplicate: 'eval metric', + }, + { evalMetrics: [{ id: 'score' }], outcomeMetrics: ['y', 'y'], duplicate: 'outcome metric' }, + ])( + 'refuses duplicate $duplicate declarations before they can multiply observations', + async ({ evalMetrics, outcomeMetrics, duplicate }) => { + await expect( + correlationStudy( + new InMemoryTraceStore(), + new InMemoryOutcomeStore(), + evalMetrics, + outcomeMetrics, + ), + ).rejects.toThrow(`duplicate ${duplicate}`) + }, + ) + + it('refuses repeated run identities from a custom trace store', async () => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + await seedRun(trace, 1, out, 1) + const runs = await trace.listRuns() + trace.listRuns = async () => [...runs, ...runs] + await expect(correlationStudy(trace, out, [{ id: 'score' }], ['retention_7d'])).rejects.toThrow( + /duplicate runId/, + ) + }) + + it('returns strong positive association between score and outcome', async () => { const trace = new InMemoryTraceStore() const out = new InMemoryOutcomeStore() // Strongly correlated: high score → high retention @@ -69,7 +101,7 @@ describe('correlationStudy', () => { expect(report.pairs).toHaveLength(1) expect(report.pairs[0].pearson).toBeGreaterThan(0.85) expect(report.pairs[0].verdict).toBe('strong') - expect(report.pairs[0].pearsonCi95.lower).toBeGreaterThan(0) + expect(report.pairs[0].pearsonCi95?.lower).toBeGreaterThan(0) }) it('returns weak verdict when uncorrelated', async () => { @@ -96,9 +128,100 @@ describe('correlationStudy', () => { expect(report.skippedRuns).toBe(1) expect(report.joinedSamples).toBe(5) }) + + it.each(['latest', 'mean', 'max'] as const)( + 'reduces only the requested outcome metric with %s', + async (reduction) => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + for (let i = 0; i < 10; i++) { + const runId = await seedRun(trace, i, out, 10 - i) + await out.append({ + runId, + capturedAt: Date.now() + 2_000, + metrics: { retention_7d: 10 - i, csat: i }, + }) + await out.append({ runId, capturedAt: Date.now() + 3_000, metrics: { retention_7d: i } }) + } + const rows = await out.list() + out.list = async () => [ + ...rows, + ...rows.map((row) => ({ + ...row, + capturedAt: Date.now() + 4_000, + metrics: { csat: Number.NaN }, + })), + ] + const report = await correlationStudy(trace, out, [{ id: 'score' }], ['csat'], { reduction }) + expect(report.pairs).toHaveLength(1) + expect(report.pairs[0]).toMatchObject({ n: 10, pearson: 1, spearman: 1 }) + expect(report.joinedSamples).toBe(10) + }, + ) + + it('selects the last finite observation of each metric independently of insertion order', async () => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + for (let i = 0; i < 10; i++) { + const runId = await seedRun(trace, i, out, i) + await out.append({ runId, capturedAt: Date.now() + 3_000, metrics: { other: -i, csat: i } }) + await out.append({ runId, capturedAt: Date.now() + 2_000, metrics: { csat: -i } }) + } + const report = await correlationStudy(trace, out, [{ id: 'score' }], ['csat']) + expect(report.pairs[0]?.pearson).toBe(1) + }) + + it('accounts for unusable joins and excludes outcomes captured before the run', async () => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + await seedRun(trace, 0, out, 0) + const missing = await seedRun(trace, 1, new InMemoryOutcomeStore(), 1) + await out.append({ runId: missing, capturedAt: Date.now() + 1_000, metrics: { unrelated: 5 } }) + const before = await seedRun(trace, 2, new InMemoryOutcomeStore(), 2) + await out.append({ runId: before, capturedAt: 1, metrics: { retention_7d: 2 } }) + const report = await correlationStudy(trace, out, [{ id: 'score' }], ['retention_7d']) + expect(report.joinedSamples).toBe(1) + expect(report.skippedRuns).toBe(2) + expect(report.excludedPairs).toEqual([ + { evalMetric: 'score', outcomeMetric: 'retention_7d', n: 1, reason: 'insufficient_samples' }, + ]) + }) + + it('reports constant observations as unestimable rather than perfect correlation', async () => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + for (let i = 0; i < 8; i++) await seedRun(trace, 0, out, 0) + const report = await correlationStudy(trace, out, [{ id: 'score' }], ['retention_7d']) + expect(report.joinedSamples).toBe(8) + expect(report.pairs).toEqual([]) + expect(report.excludedPairs[0]).toMatchObject({ n: 8, reason: 'constant_eval_metric' }) + }) }) describe('calibrationCurve', () => { + it('uses the latest finite requested metric despite later unrelated or invalid observations', async () => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + for (let i = 0; i < 10; i++) { + const runId = await seedRun(trace, i / 10, out, i / 10) + await out.append({ runId, capturedAt: Date.now() + 2_000, metrics: { other: 1 } }) + } + const rows = await out.list() + out.list = async () => [ + ...rows, + ...rows.map((row) => ({ + ...row, + capturedAt: Date.now() + 3_000, + metrics: { retention_7d: Number.NaN }, + })), + ] + + const report = await calibrationCurve(trace, out, { id: 'score' }, 'retention_7d', { bins: 5 }) + expect(report).toMatchObject({ n: 10, ece: 0 }) + expect(report!.bins.reduce((sum, bin) => sum + bin.n, 0)).toBe(10) + expect(report!.bins[0]?.outcomeMean).toBe(0.05) + }) + it('produces bins with ECE near 0 when eval = outcome identically', async () => { const trace = new InMemoryTraceStore() const out = new InMemoryOutcomeStore() @@ -129,6 +252,78 @@ describe('calibrationCurve', () => { const report = await calibrationCurve(trace, out, { id: 'score' }, 'retention_7d') expect(report).toBeNull() }) + + it('clips out-of-range scores while retaining every joined observation in the calibration error', async () => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + await seedRun(trace, -1, out, 1) + await seedRun(trace, 2, out, 0) + const report = await calibrationCurve(trace, out, { id: 'score' }, 'retention_7d', { + bins: 5, + range: { lo: 0, hi: 1 }, + }) + expect(report).toMatchObject({ n: 2, ece: 1, maxGap: 1 }) + expect(report!.bins.reduce((sum, bin) => sum + bin.n, 0)).toBe(report!.n) + expect(report!.bins.map((bin) => bin.evalMean)).toEqual([0, 1]) + }) + + it.each([{}, { binning: 'equal-frequency', range: { lo: 0, hi: 1 } }] as const)( + 'measures a constant confident predictor with options %j', + async (options) => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + await seedRun(trace, 1, out, 0) + await seedRun(trace, 1, out, 0) + const report = await calibrationCurve(trace, out, { id: 'score' }, 'retention_7d', options) + expect(report).toMatchObject({ n: 2, ece: 1, maxGap: 1 }) + expect(report!.bins).toEqual([ + { lower: 1, upper: 1, n: 2, evalMean: 1, outcomeMean: 0, gap: 1 }, + ]) + }, + ) + + it('makes the requested equal-frequency bins with balanced counts', async () => { + const trace = new InMemoryTraceStore() + const out = new InMemoryOutcomeStore() + for (let i = 0; i < 23; i++) await seedRun(trace, i / 23, out, i / 23) + const report = await calibrationCurve(trace, out, { id: 'score' }, 'retention_7d', { + bins: 10, + binning: 'equal-frequency', + }) + expect(report).toMatchObject({ n: 23, ece: 0 }) + expect(report!.bins).toHaveLength(10) + expect(report!.bins.reduce((sum, bin) => sum + bin.n, 0)).toBe(23) + expect(report!.bins.every((bin) => bin.n === 2 || bin.n === 3)).toBe(true) + }) + + it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'refuses invalid bin count %s before treating evidence as empty', + async (bins) => { + await expect( + calibrationCurve( + new InMemoryTraceStore(), + new InMemoryOutcomeStore(), + { id: 'score' }, + 'y', + { + bins, + }, + ), + ).rejects.toThrow(/bins must be a positive safe integer/) + }, + ) + + it.each([ + { lo: 1, hi: 0 }, + { lo: Number.NaN, hi: 1 }, + { lo: 0, hi: Number.POSITIVE_INFINITY }, + ])('refuses invalid range %j before treating evidence as empty', async (range) => { + await expect( + calibrationCurve(new InMemoryTraceStore(), new InMemoryOutcomeStore(), { id: 'score' }, 'y', { + range, + }), + ).rejects.toThrow(/range must have finite ordered bounds/) + }) }) /** diff --git a/tests/outcome-store.test.ts b/tests/outcome-store.test.ts new file mode 100644 index 00000000..7d70c6ea --- /dev/null +++ b/tests/outcome-store.test.ts @@ -0,0 +1,146 @@ +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + type DeploymentOutcome, + FileSystemOutcomeStore, + InMemoryOutcomeStore, + OutcomeStoreError, +} from '../src/meta-eval/outcome-store' + +const observation = (runId = 'run'): DeploymentOutcome => ({ + runId, + capturedAt: 1, + metrics: { success: 0 }, + labels: { cohort: 'test' }, +}) + +describe('outcome evidence storage', () => { + let dir: string + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'agent-eval-outcome-store-')) + }) + + afterEach(async () => { + vi.restoreAllMocks() + await rm(dir, { recursive: true, force: true }) + }) + + it.each(['memory', 'filesystem'] as const)( + 'protects %s records from input and read-result mutation', + async (kind) => { + const store = + kind === 'memory' ? new InMemoryOutcomeStore() : new FileSystemOutcomeStore({ dir }) + const input = observation() + await store.append(input) + input.metrics.success = 1 + input.labels!.cohort = 'changed' + const first = await store.forRun('run') + expect(first[0]).toEqual(observation()) + first[0]!.metrics.success = 100 + first[0]!.labels!.cohort = 'changed-again' + expect(await store.list()).toEqual([observation()]) + }, + ) + + it.each(['memory', 'filesystem'] as const)( + 'refuses non-finite %s observations instead of serializing them as null', + async (kind) => { + const store = + kind === 'memory' ? new InMemoryOutcomeStore() : new FileSystemOutcomeStore({ dir }) + await expect( + store.append({ ...observation(), metrics: { success: Number.NaN } }), + ).rejects.toThrow() + expect(await store.list()).toEqual([]) + await store.append(observation()) + expect((await store.list())[0]?.metrics.success).toBe(0) + }, + ) + + it('treats a missing directory as empty and observes subsequent writes from another instance', async () => { + const missing = join(dir, 'new-store') + const reader = new FileSystemOutcomeStore({ dir: missing }) + expect(await reader.list()).toEqual([]) + const writer = new FileSystemOutcomeStore({ dir: missing }) + await writer.append(observation('first')) + expect((await reader.list()).map((row) => row.runId)).toEqual(['first']) + await writer.append(observation('second')) + expect((await reader.list()).map((row) => row.runId)).toEqual(['first', 'second']) + }) + + it('reports the corrupt file and line instead of returning partial or empty evidence', async () => { + const file = join(dir, 'outcomes.ndjson') + await writeFile(file, `${JSON.stringify(observation())}\n{broken\n`) + const store = new FileSystemOutcomeStore({ dir }) + await expect(store.list()).rejects.toMatchObject({ + name: 'OutcomeStoreError', + operation: 'decode', + path: file, + line: 2, + }) + await expect(store.forRun('unrelated')).rejects.toBeInstanceOf(OutcomeStoreError) + await writeFile(file, `${JSON.stringify(observation())}\n`) + expect(await store.list()).toEqual([observation()]) + }) + + it.each([ + { runId: 'run', capturedAt: 1, metrics: { success: null } }, + { runId: 'run', capturedAt: 1, metrics: null }, + { runId: '', capturedAt: 1, metrics: { success: 0 } }, + { runId: 'run', capturedAt: 'yesterday', metrics: { success: 0 } }, + ])('refuses a decoded record outside the outcome contract: %j', async (invalid) => { + const file = join(dir, 'outcomes.ndjson') + await writeFile(file, `${JSON.stringify(invalid)}\n`) + await expect(new FileSystemOutcomeStore({ dir }).list()).rejects.toMatchObject({ + operation: 'decode', + path: file, + line: 1, + }) + }) + + it('surfaces an unreadable store path as a read error', async () => { + const file = join(dir, 'file-not-directory') + await writeFile(file, 'existing data') + await expect(new FileSystemOutcomeStore({ dir: file }).list()).rejects.toMatchObject({ + operation: 'read', + path: file, + }) + }) + + it('surfaces an unreadable evidence file instead of calling it an empty store', async () => { + const file = join(dir, 'outcomes.ndjson') + await mkdir(file) + await expect(new FileSystemOutcomeStore({ dir }).list()).rejects.toMatchObject({ + operation: 'read', + path: file, + }) + }) + + it('retains write failure diagnostics and accepts a repaired path on the next operation', async () => { + const file = join(dir, 'outcomes.ndjson') + await mkdir(file) + const store = new FileSystemOutcomeStore({ dir }) + await expect(store.append(observation())).rejects.toMatchObject({ + name: 'OutcomeStoreError', + operation: 'write', + path: file, + cause: expect.any(Error), + }) + await rm(file, { recursive: true }) + await store.append(observation()) + expect(await store.list()).toEqual([observation()]) + }) + + it('serializes concurrent writes and keeps every rotation even within one millisecond', async () => { + vi.spyOn(Date, 'now').mockReturnValue(1_234) + const store = new FileSystemOutcomeStore({ dir, maxBytes: 1 }) + await Promise.all(Array.from({ length: 20 }, (_, i) => store.append(observation(`run-${i}`)))) + const rows = await store.list() + expect(rows).toHaveLength(20) + expect(new Set(rows.map((row) => row.runId)).size).toBe(20) + expect(await readdir(dir)).toHaveLength(20) + expect(await readFile(join(dir, 'outcomes.ndjson'), 'utf8')).toContain('run-19') + }) +}) diff --git a/tests/rl-adaptation-eval.test.ts b/tests/rl-adaptation-eval.test.ts index 79d89db7..2aa01d92 100644 --- a/tests/rl-adaptation-eval.test.ts +++ b/tests/rl-adaptation-eval.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import type { AdaptationRunner } from '../src/rl/adaptation-eval' import { compareAdaptationCurves, firstPassK, runAdaptationCurve } from '../src/rl/adaptation-eval' +import { pairedBootstrap } from '../src/statistics' interface Scen { scenarioId: string @@ -87,39 +88,46 @@ describe('runAdaptationCurve', () => { }) describe('compareAdaptationCurves', () => { + const pairedScenarios = Array.from({ length: 30 }, (_, i) => ({ + scenarioId: `scenario-${i}`, + difficulty: 0.1 + (0.8 * i) / 29, + })) + it('flags a_better when curve A dominates', async () => { const fast = await runAdaptationCurve({ - scenarios: stockScenarios, + scenarios: pairedScenarios, ks: [0, 4, 16], reps: 1, runner: makeRunner(2), }) const slow = await runAdaptationCurve({ - scenarios: stockScenarios, + scenarios: pairedScenarios, ks: [0, 4, 16], reps: 1, runner: makeRunner(0.3), }) const out = compareAdaptationCurves(fast, slow, { seed: 1 }) expect(out.verdict).toBe('a_better') - expect(out.areaDelta).toBeGreaterThan(0) + expect(out.areaDelta.low).toBeGreaterThan(0) + expect(out.areaDelta.n).toBe(pairedScenarios.length) + expect(compareAdaptationCurves(slow, fast, { seed: 1 }).verdict).toBe('b_better') }) - it('flags similar when curves overlap', async () => { + it('reports inconclusive instead of claiming equivalence when curves overlap', async () => { const a = await runAdaptationCurve({ - scenarios: stockScenarios, + scenarios: pairedScenarios, ks: [0, 4, 16], reps: 1, runner: makeRunner(1), }) const b = await runAdaptationCurve({ - scenarios: stockScenarios, + scenarios: pairedScenarios, ks: [0, 4, 16], reps: 1, runner: makeRunner(1), }) const out = compareAdaptationCurves(a, b, { seed: 1 }) - expect(out.verdict).toBe('similar') + expect(out.verdict).toBe('inconclusive') }) it('returns one perK entry per matched k', async () => { @@ -138,4 +146,147 @@ describe('compareAdaptationCurves', () => { const out = compareAdaptationCurves(a, b, { seed: 1 }) expect(out.perK).toHaveLength(3) }) + + it('joins IDs before bootstrapping and retains within-scenario covariance across k', async () => { + const exactScores = pairedScenarios.map((scenario, i) => ({ ...scenario, difficulty: i / 32 })) + const a = await runAdaptationCurve({ + scenarios: exactScores, + ks: [0, 1], + reps: 3, + runner: { run: async ({ scenario }) => scenario.difficulty / 2 + 0.125 }, + }) + const b = await runAdaptationCurve({ + scenarios: [...exactScores].reverse(), + ks: [1, 0], + reps: 3, + runner: { run: async ({ scenario }) => scenario.difficulty / 2 }, + }) + const out = compareAdaptationCurves(a, b, { seed: 17 }) + for (const point of out.perK) { + expect(point.delta.n).toBe(30) + expect(point.delta.low).toBe(0.125) + expect(point.delta.high).toBe(0.125) + } + expect(out.areaDelta.low).toBe(0.125) + expect(out.areaDelta.high).toBe(0.125) + expect(out.aImprovement.indeterminate).toBe(true) + expect(out.verdict).toBe('inconclusive') + expect(compareAdaptationCurves(a, b, { minimumEffect: 0.2 }).verdict).toBe('inconclusive') + }) + + it('resamples per-scenario areas rather than independent k points', async () => { + const scenarios = pairedScenarios.map((scenario, i) => ({ + ...scenario, + gain: i < 15 ? 0.2 : -0.1, + })) + const a = await runAdaptationCurve({ + scenarios, + ks: [0, 2, 4], + runner: { run: async ({ scenario, k }) => 0.5 + (scenario.gain * k) / 4 }, + }) + const b = await runAdaptationCurve({ + scenarios, + ks: [0, 2, 4], + runner: { run: async () => 0.5 }, + }) + const out = compareAdaptationCurves(a, b, { seed: 19 }) + const sorted = [...scenarios].sort((left, right) => + left.scenarioId.localeCompare(right.scenarioId), + ) + const reference = pairedBootstrap( + sorted.map(() => 0.5), + sorted.map((scenario) => 0.5 + scenario.gain / 2), + { seed: 19, statistic: 'mean' }, + ) + expect(out.areaDelta.mean).toBeCloseTo(reference.mean, 10) + expect(out.areaDelta.low).toBeCloseTo(reference.low, 10) + expect(out.areaDelta.high).toBeCloseTo(reference.high, 10) + }) + + it('keeps repeated measurements of three scenarios descriptive', async () => { + const a = await runAdaptationCurve({ + scenarios: stockScenarios, + ks: [0, 2], + reps: 30, + runner: { run: async () => 0.9 }, + }) + const b = await runAdaptationCurve({ + scenarios: stockScenarios, + ks: [0, 2], + reps: 30, + runner: { run: async () => 0.1 }, + }) + const out = compareAdaptationCurves(a, b) + expect(out.verdict).toBe('insufficient_evidence') + expect(out.areaDelta.n).toBe(3) + expect(out.areaDelta.gateEligible).toBe(false) + }) + + it('can decide binary area outcomes below the continuous bootstrap minimum', async () => { + const scenarios = pairedScenarios.slice(0, 6) + const a = await runAdaptationCurve({ + scenarios, + ks: [0, 1], + reps: 1, + runner: { run: async () => 1 }, + }) + const b = await runAdaptationCurve({ + scenarios, + ks: [0, 1], + reps: 1, + runner: { run: async () => 0 }, + }) + const result = compareAdaptationCurves(a, b) + expect(result.aImprovement.method).toBe('score-interval') + expect(result.aImprovement.n).toBe(6) + expect(result.aImprovement.sufficient).toBe(true) + expect(result.verdict).toBe('a_better') + }) + + it('refuses disjoint cohorts, missing pairs, mismatched k grids, and repeated identities', async () => { + const a = await runAdaptationCurve({ + scenarios: stockScenarios, + ks: [0, 1], + runner: makeRunner(1), + }) + const disjoint = await runAdaptationCurve({ + scenarios: stockScenarios.map((scenario) => ({ + ...scenario, + scenarioId: `other-${scenario.scenarioId}`, + })), + ks: [0, 1], + runner: makeRunner(1), + }) + expect(() => compareAdaptationCurves(a, disjoint)).toThrow( + /scenario pairs differ.*missing=\[s-easy/, + ) + const missing = structuredClone(a) + missing.points[1]!.perScenario.pop() + expect(() => compareAdaptationCurves(a, missing)).toThrow(/at k=1.*missing=\[s-hard\]/) + const wrongGrid = { ...a, points: a.points.slice(1) } + expect(() => compareAdaptationCurves(a, wrongGrid)).toThrow(/k grids differ/) + const duplicate = structuredClone(a) + duplicate.points[0]!.perScenario.push(duplicate.points[0]!.perScenario[0]!) + expect(() => compareAdaptationCurves(a, duplicate)).toThrow(/duplicate scenarioId/) + }) + + it('refuses malformed designs and scoring failures instead of inventing measurements', async () => { + const options = { scenarios: stockScenarios, ks: [0, 1], runner: makeRunner(1) } + await expect( + runAdaptationCurve({ ...options, scenarios: [{ scenarioId: '', difficulty: 0.1 }] }), + ).rejects.toThrow(/explicit nonempty/) + await expect( + runAdaptationCurve({ ...options, scenarios: [stockScenarios[0]!, stockScenarios[0]!] }), + ).rejects.toThrow(/duplicate scenarioId/) + await expect(runAdaptationCurve({ ...options, reps: 0 })).rejects.toThrow(/positive integer/) + await expect(runAdaptationCurve({ ...options, ks: [0, 0] })).rejects.toThrow(/duplicate k/) + await expect( + runAdaptationCurve({ ...options, runner: { run: async () => NaN } }), + ).rejects.toThrow(/score must be finite/) + const curve = await runAdaptationCurve(options) + expect(() => compareAdaptationCurves(curve, curve, { bootstrapResamples: 0 })).toThrow( + /positive integer/, + ) + expect(() => compareAdaptationCurves(curve, curve, { confidence: NaN })).toThrow(/confidence/) + }) }) diff --git a/tests/rl-contamination.test.ts b/tests/rl-contamination.test.ts index ca5169b1..fa9e1f55 100644 --- a/tests/rl-contamination.test.ts +++ b/tests/rl-contamination.test.ts @@ -26,7 +26,13 @@ describe('runContaminationProbe', () => { }) expect(out.contaminationSuspected).toBe(true) expect(out.medianDelta).toBeLessThan(-0.05) - expect(out.pairedTest.p).toBeLessThan(0.05) + expect(out.pairedTest!.p).toBeLessThan(0.05) + expect(out.perScenario[0]).toEqual({ + scenarioId: 's-0', + originalScore: 1, + perturbedScore: 0.4, + delta: -0.6, + }) }) it('does not flag contamination when scores are similar', async () => { @@ -56,6 +62,9 @@ describe('runContaminationProbe', () => { }) expect(out.contaminationSuspected).toBe(false) expect(out.reason).toMatch(/insufficient/) + expect(out.pairedTest).toBeNull() + expect(out.meanDelta).toBe(0) + expect(out.medianDelta).toBe(0) }) it('synthesizes perturbations via the strategy callback', async () => { @@ -71,6 +80,53 @@ describe('runContaminationProbe', () => { }) expect(out.n).toBeGreaterThanOrEqual(4) }) + + it('keeps exclusion and missing evidence distinct from measured zero', async () => { + const originals = [{ id: 'excluded', prompt: 'one' }] + const report = await runContaminationProbe( + { + scenarioId: id, + originals, + perturbed: originals, + scoreFn: async () => 0.1, + }, + { scoreFloor: 0.5 }, + ) + expect(report).toMatchObject({ + n: 0, + excludedScenarioIds: ['excluded'], + pairedTest: null, + meanDelta: null, + medianDelta: null, + }) + expect(report.perScenario).toHaveLength(1) + }) + + it('computes the median across the two central differences for even cohorts', async () => { + const originals = [0.1, 0.2, 0.6, 0.7].map((drop, i) => ({ id: `s${i}`, prompt: '', drop })) + const report = await runContaminationProbe({ + scenarioId: id, + originals, + perturbed: originals.map((scenario) => ({ ...scenario, prompt: 'perturbed' })), + scoreFn: async (scenario) => (scenario.prompt ? 1 - scenario.drop : 1), + }) + expect(report.medianDelta).toBeCloseTo(-0.4, 10) + }) + + it('rejects malformed options, duplicate identities, and nonfinite scores', async () => { + const originals = [{ id: 'one', prompt: '' }] + const input = { scenarioId: id, originals, perturbed: originals, scoreFn: async () => 0.5 } + await expect(runContaminationProbe(input, { alpha: 0 })).rejects.toThrow(/alpha/) + await expect(runContaminationProbe(input, { minMedianDrop: NaN })).rejects.toThrow( + /minMedianDrop/, + ) + await expect( + runContaminationProbe({ ...input, originals: [...originals, ...originals] }), + ).rejects.toThrow(/unique/) + await expect(runContaminationProbe({ ...input, scoreFn: async () => NaN })).rejects.toThrow( + /finite/, + ) + }) }) describe('renameVariables perturbation', () => { diff --git a/tests/rl-predictive-validity-researcher.test.ts b/tests/rl-predictive-validity-researcher.test.ts index 8ba930f1..19269a8c 100644 --- a/tests/rl-predictive-validity-researcher.test.ts +++ b/tests/rl-predictive-validity-researcher.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' import { InMemoryOutcomeStore } from '../src/meta-eval/outcome-store' +import { + type OutcomeMetricSpec, + rubricPredictiveValidity, +} from '../src/meta-eval/rubric-predictive-validity' import type { ExperimentPlan } from '../src/researcher' import { PredictiveValidityResearcher } from '../src/rl/predictive-validity-researcher' import type { RunRecord } from '../src/run-record' @@ -32,7 +36,10 @@ function rec(args: { describe('PredictiveValidityResearcher', () => { it('inspects failures by candidateId, packaging RunRecord ids as evidence', async () => { const outcomes = new InMemoryOutcomeStore() - const researcher = new PredictiveValidityResearcher({ outcomes, outcomeMetrics: ['revenue'] }) + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'revenue', direction: 'higher-is-better' }, + }) const runs = [ rec({ runId: 'a-1', candidateId: 'A', scenarioId: 's', score: 0.2 }), rec({ runId: 'a-2', candidateId: 'A', scenarioId: 's', score: 0.3 }), @@ -47,7 +54,10 @@ describe('PredictiveValidityResearcher', () => { it('proposes "collect-more-outcomes" until the first runValidityCheck has been done', async () => { const outcomes = new InMemoryOutcomeStore() - const researcher = new PredictiveValidityResearcher({ outcomes, outcomeMetrics: ['x'] }) + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'x', direction: 'higher-is-better' }, + }) const failures = [{ code: 'f', description: 'd', evidence: { runIds: ['r'], samples: 1 } }] const changes = await researcher.proposeChange(failures) expect(changes).toHaveLength(1) @@ -55,7 +65,7 @@ describe('PredictiveValidityResearcher', () => { expect(changes[0]?.rationale).toMatch(/no prior report/) }) - it('proposes down-weight changes for decorative rubrics after a validity report', async () => { + it('proposes calibration evidence for weak rubrics instead of inventing a weight benefit', async () => { const outcomes = new InMemoryOutcomeStore() const runs: RunRecord[] = [] for (let i = 0; i < 12; i++) { @@ -76,7 +86,7 @@ describe('PredictiveValidityResearcher', () => { } const researcher = new PredictiveValidityResearcher({ outcomes, - outcomeMetrics: ['revenue'], + targetOutcome: { id: 'revenue', direction: 'higher-is-better' }, rubrics: ['load_bearing', 'decorative'], }) const report = await researcher.runValidityCheck(runs) @@ -86,14 +96,18 @@ describe('PredictiveValidityResearcher', () => { ]) const downweight = changes.find((c) => { const p = c.payload as { action?: string; rubric?: string } - return p.action === 'down-weight' + return p.action === 'collect-calibration-evidence' }) expect(downweight).toBeDefined() + expect(changes.every((change) => change.expectedDelta === undefined)).toBe(true) }) it('applyChange merges proposed changes into the plan', async () => { const outcomes = new InMemoryOutcomeStore() - const researcher = new PredictiveValidityResearcher({ outcomes, outcomeMetrics: ['x'] }) + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'x', direction: 'higher-is-better' }, + }) const baseline: ExperimentPlan = { baselineCandidateId: 'A', proposedCandidateId: 'B', @@ -112,7 +126,10 @@ describe('PredictiveValidityResearcher', () => { it('evaluateChange returns a no-op gate decision (caller drives the sweep)', async () => { const outcomes = new InMemoryOutcomeStore() - const researcher = new PredictiveValidityResearcher({ outcomes, outcomeMetrics: ['x'] }) + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'x', direction: 'higher-is-better' }, + }) const plan: ExperimentPlan = { baselineCandidateId: 'A', proposedCandidateId: 'B', @@ -146,11 +163,179 @@ describe('PredictiveValidityResearcher', () => { } const researcher = new PredictiveValidityResearcher({ outcomes, - outcomeMetrics: ['y'], + targetOutcome: { id: 'y', direction: 'higher-is-better' }, rubrics: ['rA'], }) expect(researcher.getLastReport()).toBeNull() await researcher.runValidityCheck(runs) expect(researcher.getLastReport()).not.toBeNull() }) + + it.each([ + { direction: 'higher-is-better', sign: -1, action: 'test-reverse-or-replace' }, + { direction: 'lower-is-better', sign: -1, action: 'test-up-weight' }, + { direction: 'higher-is-better', sign: 1, action: 'test-up-weight' }, + { direction: 'lower-is-better', sign: 1, action: 'test-reverse-or-replace' }, + ] as const)( + 'proposes $action for $direction with raw association sign $sign', + async ({ direction, sign, action }) => { + const outcomes = new InMemoryOutcomeStore() + const runs = Array.from({ length: 12 }, (_, i) => + rec({ + runId: `r-${i}`, + candidateId: 'A', + scenarioId: `s-${i}`, + score: i / 12, + rubrics: { quality: i / 12 }, + }), + ) + for (const [i, run] of runs.entries()) { + await outcomes.append({ runId: run.runId, capturedAt: 1, metrics: { target: sign * i } }) + } + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'target', direction }, + }) + const report = await researcher.runValidityCheck(runs) + expect(report.pairs[0]?.spearman).toBe(sign) + const changes = await researcher.proposeChange(await researcher.inspectFailures(runs)) + expect(changes).toHaveLength(1) + expect(changes[0]?.payload).toMatchObject({ + action, + targetOutcome: { id: 'target', direction }, + samples: 12, + }) + expect(changes[0]?.expectedDelta).toBeUndefined() + expect(changes[0]?.rationale).toMatch(/fresh evidence/) + }, + ) + + it('uses the declared target even when another outcome produces a better exploratory ranking', async () => { + const outcomes = new InMemoryOutcomeStore() + const runs = Array.from({ length: 12 }, (_, i) => + rec({ + runId: `r-${i}`, + candidateId: 'A', + scenarioId: `s-${i}`, + score: i / 12, + rubrics: { quality: i }, + }), + ) + for (const [i, run] of runs.entries()) { + await outcomes.append({ + runId: run.runId, + capturedAt: 1, + metrics: { success: -i, distraction: i }, + }) + } + const report = await rubricPredictiveValidity({ + runs, + outcomes, + outcomeMetrics: [ + { id: 'success', direction: 'higher-is-better' }, + { id: 'distraction', direction: 'higher-is-better' }, + ], + }) + expect(report.ranked[0]?.bestOutcome).toBe('distraction') + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'success', direction: 'higher-is-better' }, + }) + researcher.setReport(report) + const changes = await researcher.proposeChange(await researcher.inspectFailures(runs)) + expect(changes[0]?.payload).toMatchObject({ + action: 'test-reverse-or-replace', + targetOutcome: { id: 'success', direction: 'higher-is-better' }, + }) + }) + + it('refuses a cached report for a different target direction', async () => { + const outcomes = new InMemoryOutcomeStore() + const report = await rubricPredictiveValidity({ + runs: [], + outcomes, + outcomeMetrics: [{ id: 'failure_rate', direction: 'higher-is-better' }], + }) + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'failure_rate', direction: 'lower-is-better' }, + }) + expect(() => researcher.setReport(report)).toThrow(/declared target outcome and direction/) + expect(researcher.getLastReport()).toBeNull() + }) + + it('keeps the declared target and cached evidence isolated from callers, hooks, and proposal payloads', async () => { + const outcomes = new InMemoryOutcomeStore() + const runs = Array.from({ length: 12 }, (_, i) => + rec({ + runId: `r-${i}`, + candidateId: 'A', + scenarioId: `s-${i}`, + score: 0, + rubrics: { quality: i }, + }), + ) + for (const [i, run] of runs.entries()) { + await outcomes.append({ runId: run.runId, capturedAt: 1, metrics: { success: i } }) + } + const targetOutcome: OutcomeMetricSpec = { id: 'success', direction: 'higher-is-better' } + const rubrics = ['quality'] + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome, + rubrics, + onReport: (report) => { + report.pairs.length = 0 + report.outcomeMetrics[0]!.direction = 'lower-is-better' + }, + }) + targetOutcome.id = 'caller-mutated-target' + rubrics[0] = 'caller-mutated-rubric' + const report = await researcher.runValidityCheck(runs) + report.pairs.length = 0 + researcher.getLastReport()!.pairs.length = 0 + const first = await researcher.proposeChange(await researcher.inspectFailures(runs)) + expect(first[0]?.payload).toMatchObject({ + action: 'test-up-weight', + rubric: 'quality', + targetOutcome: { id: 'success', direction: 'higher-is-better' }, + }) + const payload = first[0]!.payload as { + targetOutcome: OutcomeMetricSpec + alignedSpearmanCi95: { lower: number; upper: number } + } + payload.targetOutcome.id = 'payload-mutated-target' + payload.alignedSpearmanCi95.lower = -1 + const second = await researcher.proposeChange(await researcher.inspectFailures(runs)) + expect(second[0]?.payload).toMatchObject({ + action: 'test-up-weight', + targetOutcome: { id: 'success', direction: 'higher-is-better' }, + alignedSpearmanCi95: { lower: 1, upper: 1 }, + }) + expect(researcher.getLastReport()!.pairs).toHaveLength(1) + }) + + it('requests evidence after an unestimable report instead of treating constant data as a perfect predictor', async () => { + const outcomes = new InMemoryOutcomeStore() + const runs = Array.from({ length: 12 }, (_, i) => + rec({ + runId: `r-${i}`, + candidateId: 'A', + scenarioId: `s-${i}`, + score: 0, + rubrics: { quality: 0 }, + }), + ) + for (const run of runs) + await outcomes.append({ runId: run.runId, capturedAt: 1, metrics: { success: 0 } }) + const researcher = new PredictiveValidityResearcher({ + outcomes, + targetOutcome: { id: 'success', direction: 'higher-is-better' }, + }) + await researcher.runValidityCheck(runs) + const changes = await researcher.proposeChange(await researcher.inspectFailures(runs)) + expect(changes).toHaveLength(1) + expect(changes[0]?.payload).toMatchObject({ directive: 'researcher.collect-more-outcomes' }) + expect(changes[0]?.expectedDelta).toBeUndefined() + }) }) diff --git a/tests/rl-rl-campaign.test.ts b/tests/rl-rl-campaign.test.ts index db02d337..f00e7c33 100644 --- a/tests/rl-rl-campaign.test.ts +++ b/tests/rl-rl-campaign.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest' import { createChatClient } from '../src/analyst/chat-client' import type { CampaignRunner } from '../src/eval-campaign' -import { runRLCampaign } from '../src/rl/rl-campaign' +import { InMemoryOutcomeStore } from '../src/meta-eval/outcome-store' +import type { OutcomeMetricSpec } from '../src/meta-eval/rubric-predictive-validity' +import { type RunRLCampaignOptions, runRLCampaign } from '../src/rl/rl-campaign' import { InMemoryRawProviderSink } from '../src/trace/raw-provider-sink' import { InMemoryTraceStore } from '../src/trace/store' @@ -59,6 +61,80 @@ const defaultRunner: CampaignRunner = async (ctx) => { } describe('runRLCampaign', () => { + const invalidOutcomeOptions: Array< + Pick, 'outcomeStore' | 'outcomeMetrics'> + > = [ + { outcomeStore: new InMemoryOutcomeStore() }, + { outcomeMetrics: [{ id: 'success', direction: 'higher-is-better' }] }, + { outcomeStore: new InMemoryOutcomeStore(), outcomeMetrics: [] }, + ] + + it.each(invalidOutcomeOptions)( + 'refuses incomplete outcome declarations before executing a campaign: %j', + async (outcomeOptions) => { + let executions = 0 + await expect( + runRLCampaign({ + campaignId: 'rl-outcome-options', + commitSha: 'cafebabe', + variants: [{ id: 'baseline', payload: { prompt: 'baseline' } }], + scenarios: [{ scenarioId: 'task' }], + seeds: [0], + chatFactory, + executionRef: EXECUTION_REF, + storeFactory: () => new InMemoryTraceStore(), + rawSinkFactory: () => new InMemoryRawProviderSink(), + runner: async (context) => { + executions++ + return defaultRunner(context) + }, + ...outcomeOptions, + }), + ).rejects.toThrow(/requires outcomeStore and outcomeMetrics|must declare/) + expect(executions).toBe(0) + }, + ) + + it('reports the outcome direction declared before execution and retains signed association', async () => { + const outcomes = new InMemoryOutcomeStore() + const outcomeMetrics: OutcomeMetricSpec[] = [ + { id: 'failure_rate', direction: 'lower-is-better' }, + ] + const result = await runRLCampaign({ + campaignId: 'rl-outcomes', + commitSha: 'cafebabe', + variants: [{ id: 'baseline', payload: { prompt: 'baseline' } }], + scenarios: Array.from({ length: 10 }, (_, i) => ({ scenarioId: `task-${i}` })), + seeds: [0], + chatFactory, + executionRef: EXECUTION_REF, + storeFactory: () => new InMemoryTraceStore(), + rawSinkFactory: () => new InMemoryRawProviderSink(), + outcomeStore: outcomes, + outcomeMetrics, + runner: async (context) => { + outcomeMetrics[0]!.direction = 'higher-is-better' + const quality = Number(context.scenarioId.split('-')[1]) / 10 + await outcomes.append({ + runId: context.runId, + capturedAt: 1, + metrics: { failure_rate: 1 - quality }, + }) + return { ...(await defaultRunner(context)), raw: { quality } } + }, + }) + expect(result.predictiveValidity?.pairs[0]).toMatchObject({ + rubric: 'quality', + outcome: 'failure_rate', + outcomeDirection: 'lower-is-better', + spearman: -1, + alignedSpearman: 1, + verdict: 'aligned', + n: 10, + }) + expect(result.summary).toContain('aligned ρ=1.00 vs failure_rate (lower-is-better; aligned)') + }) + it('runs the matrix, extracts preferences, computes interim confidence, and reports rewardHacking verdict', async () => { const result = await runRLCampaign({ campaignId: 'rl-test', diff --git a/tests/rubric-predictive-validity.test.ts b/tests/rubric-predictive-validity.test.ts index 45d9d8c2..5b80b1c1 100644 --- a/tests/rubric-predictive-validity.test.ts +++ b/tests/rubric-predictive-validity.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest' import { InMemoryOutcomeStore } from '../src/meta-eval/outcome-store' -import { rubricPredictiveValidity } from '../src/meta-eval/rubric-predictive-validity' +import { + type OutcomeMetricSpec, + rubricPredictiveValidity, +} from '../src/meta-eval/rubric-predictive-validity' import type { RunRecord } from '../src/run-record' function rec(runId: string, rubrics: Record, score = 0.5): RunRecord { @@ -25,7 +28,7 @@ function rec(runId: string, rubrics: Record, score = 0.5): RunRe } describe('rubricPredictiveValidity', () => { - it('ranks load-bearing rubrics above decorative ones by Spearman magnitude', async () => { + it('ranks aligned rubrics above weak ones by desired outcome direction', async () => { const runs: RunRecord[] = [] const outcomes = new InMemoryOutcomeStore() // load-bearing rubric: monotonic with revenue. decorative rubric: random. @@ -37,20 +40,20 @@ describe('rubricPredictiveValidity', () => { await outcomes.append({ runId: id, capturedAt: Date.now(), - metrics: { revenue: loadBearing * 100 + (Math.random() - 0.5) * 5 }, + metrics: { revenue: loadBearing * 100 + ((i * 7) % 5) / 5 }, }) } const report = await rubricPredictiveValidity({ runs, outcomes, - outcomeMetrics: ['revenue'], + outcomeMetrics: [{ id: 'revenue', direction: 'higher-is-better' }], rubrics: ['load_bearing', 'decorative'], seed: 1, }) expect(report.ranked[0]?.rubric).toBe('load_bearing') - expect(report.ranked[0]?.verdict).toBe('load_bearing') + expect(report.ranked[0]?.verdict).toBe('aligned') expect(report.ranked[1]?.rubric).toBe('decorative') - expect(report.ranked[1]?.verdict).toBe('decorative') + expect(report.ranked[1]?.verdict).toBe('weak') }) it('discovers rubrics from outcome.raw when the caller does not declare them', async () => { @@ -75,7 +78,7 @@ describe('rubricPredictiveValidity', () => { const report = await rubricPredictiveValidity({ runs, outcomes, - outcomeMetrics: ['csat'], + outcomeMetrics: [{ id: 'csat', direction: 'higher-is-better' }], seed: 1, }) const rubrics = report.ranked.map((r) => r.rubric).sort() @@ -93,13 +96,17 @@ describe('rubricPredictiveValidity', () => { const report = await rubricPredictiveValidity({ runs, outcomes, - outcomeMetrics: ['x'], + outcomeMetrics: [{ id: 'x', direction: 'higher-is-better' }], rubrics: ['thin', 'absent'], minSamples: 4, seed: 1, }) expect(report.pairs).toEqual([]) // thin had only 2 samples; absent had none - expect(report.rubricsWithoutData).toEqual(expect.arrayContaining(['thin', 'absent'])) + expect(report.rubricsWithoutData).toEqual(['absent']) + expect(report.excludedPairs).toEqual([ + expect.objectContaining({ rubric: 'thin', n: 2, reason: 'insufficient_samples' }), + expect.objectContaining({ rubric: 'absent', n: 0, reason: 'insufficient_samples' }), + ]) }) it('skips runs with no joined outcome', async () => { @@ -110,16 +117,16 @@ describe('rubricPredictiveValidity', () => { const report = await rubricPredictiveValidity({ runs, outcomes, - outcomeMetrics: ['y'], + outcomeMetrics: [{ id: 'y', direction: 'higher-is-better' }], rubrics: ['x'], - minSamples: 1, + minSamples: 3, seed: 1, }) expect(report.joinedSamples).toBe(1) expect(report.skippedRuns).toBe(2) }) - it('verdict is decorative for negative or near-zero correlation', async () => { + it('marks negative association with a desired increase as inverse', async () => { const runs: RunRecord[] = [] const outcomes = new InMemoryOutcomeStore() // anti-correlated rubric: rubric=i, revenue=-i. @@ -130,16 +137,169 @@ describe('rubricPredictiveValidity', () => { const report = await rubricPredictiveValidity({ runs, outcomes, - outcomeMetrics: ['revenue'], + outcomeMetrics: [{ id: 'revenue', direction: 'higher-is-better' }], rubrics: ['antiCorrelated'], seed: 1, }) - // Spearman magnitude is high but the verdict bucketing is on |ρ|, so an - // anti-correlated rubric still buckets as load_bearing — that's correct; - // it's load-bearing in the wrong direction. The caller's job is to inspect - // the sign before promoting it. const r = report.ranked[0]! expect(r.spearman).toBeLessThan(-0.9) - expect(r.verdict).toBe('load_bearing') + expect(r.verdict).toBe('inverse') + expect(r.alignedSpearman).toBe(-1) + expect(r.alignedSpearmanCi95?.upper).toBeLessThan(0) + }) + + it('aligns a lower-is-better outcome while preserving raw negative association', async () => { + const outcomes = new InMemoryOutcomeStore() + const runs = Array.from({ length: 12 }, (_, i) => rec(`r-${i}`, { quality: i })) + for (const [i, run] of runs.entries()) { + await outcomes.append({ + runId: run.runId, + capturedAt: 1, + metrics: { failure_rate: 1 - i / 12 }, + }) + } + const report = await rubricPredictiveValidity({ + runs, + outcomes, + outcomeMetrics: [{ id: 'failure_rate', direction: 'lower-is-better' }], + }) + expect(report.pairs[0]).toMatchObject({ + spearman: -1, + alignedSpearman: 1, + outcomeDirection: 'lower-is-better', + verdict: 'aligned', + }) + expect(report.pairs[0]?.spearmanCi95?.upper).toBeLessThan(0) + expect(report.pairs[0]?.alignedSpearmanCi95?.lower).toBeGreaterThan(0) + }) + + it('distinguishes missing scores, missing outcomes, measured zeros, and constant observations', async () => { + const outcomes = new InMemoryOutcomeStore() + const runs = Array.from({ length: 8 }, (_, i) => rec(`r-${i}`, { varying: i, constant: 0 })) + for (const [i, run] of runs.entries()) { + await outcomes.append({ runId: run.runId, capturedAt: 1, metrics: { measured: i, fixed: 0 } }) + } + const report = await rubricPredictiveValidity({ + runs, + outcomes, + rubrics: ['varying', 'constant', 'missing'], + outcomeMetrics: [ + { id: 'measured', direction: 'higher-is-better' }, + { id: 'fixed', direction: 'higher-is-better' }, + { id: 'absent', direction: 'higher-is-better' }, + ], + }) + expect(report.joinedSamples).toBe(8) + expect(report.skippedRuns).toBe(0) + expect(report.rubricsWithoutData).toEqual(['missing']) + expect(report.pairs).toHaveLength(1) + expect(report.pairs[0]).toMatchObject({ rubric: 'varying', outcome: 'measured', n: 8 }) + expect(report.excludedPairs).toHaveLength(8) + expect(report.excludedPairs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rubric: 'varying', + outcome: 'absent', + reason: 'insufficient_samples', + n: 0, + }), + expect.objectContaining({ + rubric: 'varying', + outcome: 'fixed', + reason: 'constant_outcome', + n: 8, + }), + expect.objectContaining({ + rubric: 'constant', + outcome: 'measured', + reason: 'constant_rubric', + n: 8, + }), + ]), + ) + }) + + it('excludes later non-finite and unrelated metrics without losing the last valid zero', async () => { + const outcomes = new InMemoryOutcomeStore() + const runs = Array.from({ length: 8 }, (_, i) => rec(`r-${i}`, { quality: i })) + for (const [i, run] of runs.entries()) { + await outcomes.append({ runId: run.runId, capturedAt: 1, metrics: { error: -i, success: i } }) + await outcomes.append({ runId: run.runId, capturedAt: 2, metrics: { error: 100 } }) + } + const rows = await outcomes.list() + outcomes.list = async () => [ + ...rows, + ...rows.map((row) => ({ + ...row, + capturedAt: 3, + metrics: { success: Number.NaN }, + })), + ] + const report = await rubricPredictiveValidity({ + runs, + outcomes, + outcomeMetrics: [{ id: 'success', direction: 'higher-is-better' }], + }) + expect(report.pairs[0]).toMatchObject({ n: 8, spearman: 1, verdict: 'aligned' }) + }) + + it('counts a run with only unrelated outcome keys as skipped', async () => { + const outcomes = new InMemoryOutcomeStore() + await outcomes.append({ runId: 'a', capturedAt: 1, metrics: { other: 1 } }) + const report = await rubricPredictiveValidity({ + runs: [rec('a', { quality: 0 })], + outcomes, + outcomeMetrics: [{ id: 'success', direction: 'higher-is-better' }], + }) + expect(report.joinedSamples).toBe(0) + expect(report.skippedRuns).toBe(1) + expect(report.excludedPairs[0]?.n).toBe(0) + expect(report.rubricsWithoutData).toEqual([]) + }) + + it('rejects duplicated run ids instead of treating copied observations as independent runs', async () => { + await expect( + rubricPredictiveValidity({ + runs: [rec('a', { quality: 0 }), rec('a', { quality: 0 })], + outcomes: new InMemoryOutcomeStore(), + outcomeMetrics: [{ id: 'success', direction: 'higher-is-better' }], + }), + ).rejects.toThrow(/duplicate runId/) + }) + + it('propagates outcome-store failure rather than returning an empty successful study', async () => { + const outcomes = new InMemoryOutcomeStore() + outcomes.list = async () => { + throw new Error('outcome store unavailable') + } + await expect( + rubricPredictiveValidity({ + runs: [], + outcomes, + outcomeMetrics: [{ id: 'success', direction: 'higher-is-better' }], + }), + ).rejects.toThrow('outcome store unavailable') + }) + + it('keeps the declared metric direction and run observations fixed while reading outcomes', async () => { + const outcomes = new InMemoryOutcomeStore() + const runs = Array.from({ length: 8 }, (_, i) => rec(`r-${i}`, { quality: i })) + for (const [i, run] of runs.entries()) { + await outcomes.append({ runId: run.runId, capturedAt: 1, metrics: { success: i } }) + } + const outcomeMetrics: OutcomeMetricSpec[] = [{ id: 'success', direction: 'higher-is-better' }] + const rubrics = ['quality'] + const rows = await outcomes.list() + outcomes.list = async () => { + outcomeMetrics[0]!.direction = 'lower-is-better' + rubrics[0] = 'changed' + for (const run of runs) run.outcome.raw.quality = 0 + runs.push(rec('new-run', { quality: 0 })) + return rows + } + const report = await rubricPredictiveValidity({ runs, outcomes, outcomeMetrics, rubrics }) + expect(report.outcomeMetrics).toEqual([{ id: 'success', direction: 'higher-is-better' }]) + expect(report.pairs[0]).toMatchObject({ rubric: 'quality', verdict: 'aligned', n: 8 }) + expect(report.joinedSamples + report.skippedRuns).toBe(8) }) }) diff --git a/tests/tier2.test.ts b/tests/tier2.test.ts index 3fcd89bc..8000d578 100644 --- a/tests/tier2.test.ts +++ b/tests/tier2.test.ts @@ -17,8 +17,7 @@ import { import type { ToolSpan } from '../src/trace' import { InMemoryTraceStore, TraceEmitter } from '../src/trace' -/** Key-sorted `JSON.stringify`, the scheme manifests were signed under before - * RFC 8785. Kept here to MINT a legacy manifest the verifier must still accept. */ +/** Produce a correctly hashed retired record to exercise scheme rejection. */ function sortKeysDeep(value: unknown): unknown { if (value === null || typeof value !== 'object') return value if (Array.isArray(value)) return value.map(sortKeysDeep) @@ -160,32 +159,37 @@ describe('pre-registration', () => { expect(await verifyManifest(a)).toBe(true) }) - it('signs under RFC 8785 and still verifies a manifest signed by the previous scheme — regression: a durable manifest outlives the release that signed it', async () => { + it('accepts RFC 8785 and refuses correctly hashed records under the retired scheme', async () => { const signed = await signManifest(base) expect(signed.algo).toBe('sha256-rfc8785') expect(await verifyManifest(signed)).toBe(true) - // A manifest signed by the previous release: key-sorted JSON.stringify, - // tagged 'sha256-content' — and the same manifest with no `algo` at all, - // which is how the oldest serialized manifests look. const legacyDigest = createHash('sha256') .update(JSON.stringify(sortKeysDeep(base)), 'utf8') .digest('hex') - const tagged: SignedManifest = { ...base, contentHash: legacyDigest, algo: 'sha256-content' } - const untagged = { ...base, contentHash: legacyDigest } as SignedManifest - expect(await verifyManifest(tagged)).toBe(true) - expect(await verifyManifest(untagged)).toBe(true) + const tagged: SignedManifest = JSON.parse( + JSON.stringify({ ...base, contentHash: legacyDigest, algo: 'sha256-content' }), + ) + const untagged: SignedManifest = JSON.parse( + JSON.stringify({ ...base, contentHash: legacyDigest }), + ) + expect(await verifyManifest(tagged)).toBe(false) + expect(await verifyManifest(untagged)).toBe(false) + await expect(evaluateHypothesis(tagged, { n: 30, effect: 0.08, pValue: 0.01 })).rejects.toThrow( + /unsupported manifest hash scheme/, + ) - // Tampering is still caught under either scheme. - expect(await verifyManifest({ ...tagged, minEffect: base.minEffect + 1 })).toBe(false) expect(await verifyManifest({ ...signed, minEffect: base.minEffect + 1 })).toBe(false) }) - it('refuses a manifest whose algo this release cannot verify instead of reading it as valid', () => { - const alien = { ...base, contentHash: 'x'.repeat(64), algo: 'sha512-future' } as unknown - expect(() => manifestContentDigest(alien as SignedManifest)).toThrow( - /unrecognized manifest hash algo 'sha512-future'/, + it('refuses an unknown scheme in both synchronous and asynchronous verification', async () => { + const alien: SignedManifest = JSON.parse( + JSON.stringify({ ...base, contentHash: 'x'.repeat(64), algo: 'sha512-future' }), + ) + expect(() => manifestContentDigest(alien)).toThrow( + /unsupported manifest hash algo 'sha512-future'/, ) + expect(await verifyManifest(alien)).toBe(false) }) it('evaluateHypothesis confirms when all conditions met', async () => {