feat(evaluation)!: add reusable claims and final evidence tracking

.evolve/skill-runs.jsonl

@@ -62,3 +62,13 @@
 {"skill":"/agent-eval","ts":"2026-09-08T19:57:11Z","project":"agent-eval-composable-search","target":"configurable scoped and sequential optimization methods","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
 {"skill":"/agent-eval","ts":"2026-09-10T01:01:49Z","project":"agent-eval-search-projector","target":"search ledger projector extraction","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
 {"skill":"/simplify","ts":"2026-09-10T01:01:49Z","project":"agent-eval-search-projector","target":"search ledger module ownership; 89 preserved declarations and unchanged public API","operatorPrompt":"","durationMin":null,"verdict":"KEEP","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/reconcile","ts":"2026-09-13T01:32:41Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"ML benchmark book against agent-eval: source-backed adoption decisions and offline boundary probes","operatorPrompt":"","durationMin":null,"verdict":"ADAPT","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/report","ts":"2026-09-13T01:32:42Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Complete linked ML benchmark book review and agent-eval capability map at fe1cc511","operatorPrompt":"","durationMin":null,"verdict":"COMPLETE","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/review-to-green","ts":"2026-09-13T01:47:58Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"PR #748: source identity reproduces from the reviewed archive and detects local source changes; 11 focused checks passed","operatorPrompt":"","durationMin":null,"verdict":"FIXED","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/report","ts":"2026-09-14T01:27:02Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Book-inspired methodology and historical selfImprove evidence: 10 registry entries, 20 notebook records, 4 raw analyst results; efficacy limits retained in docs/design/self-improvement-evidence-audit.md","operatorPrompt":"","durationMin":null,"verdict":"COMPLETE","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/dont-collapse-the-architecture","ts":"2026-09-14T01:27:02Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"SelfImprove usefulness: fixed-roster development remains reusable; candidates survive inconclusive or negative final results; historical nulls do not isolate optimizer mechanisms or prove cross-domain failure","operatorPrompt":"","durationMin":null,"verdict":"OPEN","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/deep-clean","ts":"2026-09-14T02:09:17Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Evaluation APIs: removed retired digest readers and canonical-JSON waiver paths; root types align with contract; breaking upgrades documented; caller coverage 139/201 refs with 62 exclusions","operatorPrompt":"","durationMin":null,"verdict":"IMPLEMENTED","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/polish","ts":"2026-09-14T02:09:17Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Evaluation integrity: shared units, outcome validity, evaluator admission, reusable claims, README and public exports; final package checks and 107 affected tests passed; full dedicated CI pending","operatorPrompt":"","durationMin":null,"verdict":"OPEN","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/verify","ts":"2026-09-14T02:09:17Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Final judge snapshot: 107/107 affected tests; typecheck, examples, scripts, lint, build, package verification and public offline example passed; full local suite has timing failures; dedicated CI pending","operatorPrompt":"","durationMin":null,"verdict":"OPEN","dispatchedTo":"/review-to-green","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/review-to-green","ts":"2026-09-14T02:09:18Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"Local Standards/Spec review: fixed mutable-evaluator false promotion and matrix docs; six adversarial regressions and independent fix review passed; hosted review quota exhausted, formal approval remains required","operatorPrompt":"","durationMin":null,"verdict":"LOCAL_FIXED","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
+{"skill":"/converge","ts":"2026-09-14T02:16:07Z","project":"agent-eval-mlbenchmarks-review-20260912","target":"PR #749: clean CI checkout cannot resolve public example imports before build; moved example typecheck after package build without altering check coverage","operatorPrompt":"","durationMin":null,"verdict":"LOCAL_FIXED","dispatchedTo":"/review-to-green","operatorOverride":null,"transcriptPath":null,"traceDir":null}

.github/workflows/ci.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
 
 jobs:
   ci:
@@ -43,9 +42,6 @@ jobs:
       - name: Typecheck
         run: pnpm typecheck
 
-      - name: Typecheck examples
-        run: pnpm typecheck:examples
-
       - name: Typecheck scripts
         run: pnpm typecheck:scripts
 
@@ -57,6 +53,9 @@ jobs:
           NODE_OPTIONS: --max-old-space-size=8192
         run: pnpm build
 
+      - name: Typecheck examples
+        run: pnpm typecheck:examples
+
       - name: Verify packed package exports
         run: pnpm run verify:package
 

CHANGELOG.md

@@ -6,8 +6,47 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ## Unreleased
 
+### Changed
+
+- **Breaking:** Root `Scenario`, `JudgeScore`, and `GateDecision` now match `/contract`.
+  Product workflows use `ProductScenario`, `DimensionJudgeScore`, and `HeldOutGateDecision`.
+- **Breaking:** Current canonical envelopes and algorithm identifiers are required for seals, attestations, and profile identities.
+  Retired digest readers and canonical-JSON waiver paths are removed.
+  Historical evidence retains its original identity.
+- **Breaking:** Cluster interval registrations bind their measured `value` field.
+  Row execution evidence supplies rows; it cannot select another metric under the same seal.
+- **Breaking:** Power calculations and power-floor gates require `minimumEffect` and assess adequacy at that effect.
+- **Breaking:** Predictive validity requires a declared outcome direction and uses descriptive `aligned`, `inverse`, and `weak` associations.
+  Research proposals retain hypotheses instead of invented expected gains.
+- **Breaking:** Adaptation comparisons require matched identified scenario cohorts and report paired uncertainty and inconclusive results.
+  Contamination diagnostics use `alpha`; heuristic per-item `qValue` values are removed.
+- Method comparisons expose unit-level scores, raw paired-cell counts, and the deciding statistical evidence.
+  `favored: null` replaces the ambiguous `tie` sentinel for inconclusive comparisons.
+  Continuous mean decisions cannot use a small-sample sign test as evidence about the mean.
+  Binary and explicit median decisions retain their appropriate confidence-dependent observation requirements.
+
+### Added
+
+- Optional top-level `claim` metadata declares populations and independent source units without consuming reusable regression evidence.
+  New-unit claims keep source families together across automatic partitions.
+  Self-improvement retains the selected candidate when release evidence is negative or inconclusive.
+- Optional `finalEvidence` reserves fresh final units before search and records exposure before final measurement.
+  The shared journal detects conflicting use, concurrent ownership, corrupted history, and deleted trusted heads.
+- `/meta-eval` exports evaluator admission from actual controls, simultaneous error bounds, and explicit unknown and excluded evidence.
+  Existing position and self-preference audits are public alongside calibration and verbosity diagnostics.
+- `calibrationFromPairs()` accepts direct measured rows without requiring trace and outcome stores.
+- The [evaluation-integrity guide](docs/evaluation-integrity.md) explains methodology and limits.
+  Its offline example composes public imports and exports actual fixture results.
+
 ### Fixed
 
+- Outcome queries select the latest finite requested metric instead of an unrelated latest observation.
+  Outcome-store corruption and unavailable evidence remain visible failures.
+- Calibration preserves clipped observations, measures constant predictors, and honors the requested bin count.
+- Registered-unit gates pair complete cells before aggregation; repetitions and source variants cannot multiply independent evidence.
+- Opened experiments and outcome research retain validated snapshots instead of mutable caller-owned rules.
+- Comparisons capture judge configuration and callbacks before asynchronous work.
+  Replacing a caller's judge between arms cannot create artificial lift under the original evaluator identity.
 - Pareto promotion applies regression floors to the deciding confidence interval.
   Tied binary outcomes cannot bypass a safety floor through a zero-width diagnostic bootstrap.
   Zero-width or non-finite deciding intervals now produce an `indeterminate` axis and `not_evaluated` check.