From 1742f95c7c0de6ca65753e6d2b7a81fc8744a54f Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <Timothy.Gregg@complete.tech>
Date: Fri, 18 Sep 2026 10:15:59 -0400
Subject: [PATCH 1/4] Check live benchmark credentials inside CI without
 exposing secrets

---
 .github/workflows/dspy-jev-preflight.yml | 73 ++++++++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 .github/workflows/dspy-jev-preflight.yml

diff --git a/.github/workflows/dspy-jev-preflight.yml b/.github/workflows/dspy-jev-preflight.yml
new file mode 100644
index 0000000..5191e45
--- /dev/null
+++ b/.github/workflows/dspy-jev-preflight.yml
@@ -0,0 +1,73 @@
+name: DSPy Jev live preflight
+on:
+  push:
+    branches: [research/dspy-jev-benchmark]
+    paths: ['.github/workflows/dspy-jev-preflight.yml']
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  preflight:
+    if: github.repository == 'CompleteDotTech/paper-package'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: '3.12'
+      - run: python -m pip install -r graph_synthesis/dspy_jev_optimizer/requirements.txt
+      - name: Bounded live connectivity tests
+        env:
+          TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY || secrets.JEV_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          DSPY_PROPOSER_MODEL: ${{ vars.DSPY_PROPOSER_MODEL }}
+          LITELLM_LOCAL_MODEL_COST_MAP: 'True'
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          from graph_synthesis.dspy_jev_optimizer.core import JevConfig
+          from graph_synthesis.dspy_jev_optimizer.providers import TypeSafeBackend, DSPyProposer
+          presence = {k: bool(os.environ.get(k, '').strip()) for k in ('TYPESAFE_API_KEY','OPENROUTER_API_KEY','OPENAI_API_KEY','ANTHROPIC_API_KEY')}
+          result = {'credential_present': presence, 'status': 'started'}
+          out = Path(os.environ['RUNNER_TEMP']) / 'preflight.json'
+          cfg = JevConfig('Evidence classification', 'Does the evidence support the claim?', {'supports':'Evidence establishes the claim.', 'refutes':'Evidence contradicts the claim.', 'unknown':'Evidence is insufficient.'})
+          try:
+              if not presence['TYPESAFE_API_KEY']:
+                  raise ValueError('TypeSafe credential is not mapped')
+              model = os.environ.get('DSPY_PROPOSER_MODEL','').strip()
+              if not model:
+                  model = ('openrouter/anthropic/claude-sonnet-4.5' if presence['OPENROUTER_API_KEY'] else 'anthropic/claude-sonnet-4-5-20250929' if presence['ANTHROPIC_API_KEY'] else 'openai/gpt-4.1' if presence['OPENAI_API_KEY'] else '')
+              if not model:
+                  raise ValueError('No proposal-provider credential is mapped')
+              result['proposer_model'] = model
+              backend = TypeSafeBackend('jev-1.13.0', retries=0)
+              try:
+                  answer = backend.predict(cfg, {'claim':'A precedes B.', 'evidence':['A precedes B.']})
+              finally:
+                  backend.close()
+              result['jev'] = {'model':answer['model'], 'choice':answer['choice'], 'usage':answer['usage']}
+              proposal = DSPyProposer(model).propose(cfg, [{'state':{'claim':'A precedes B.', 'evidence':['A precedes B.']}, 'gold':'supports', 'choice':'unknown', 'probabilities':{'supports':0.3,'refutes':0.1,'unknown':0.6}}], 1, [])
+              JevConfig.from_dict(proposal, cfg)
+              result['status'] = 'passed'
+          except Exception as exc:
+              result['status'] = 'failed'
+              result['error_type'] = type(exc).__name__
+          finally:
+              out.write_text(json.dumps(result, indent=2)+'\n')
+              print(json.dumps(result))
+          if result['status'] != 'passed':
+              raise SystemExit(1)
+          PY
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        if: always()
+        with:
+          name: dspy-jev-live-preflight
+          path: ${{ runner.temp }}/preflight.json
+          retention-days: 14
+          if-no-files-found: warn

From f8778489e8e3ae1d8aeec4affb4eb6af476083cf Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <Timothy.Gregg@complete.tech>
Date: Fri, 18 Sep 2026 10:19:16 -0400
Subject: [PATCH 2/4] Separate TypeSafe connectivity from proposer-key
 diagnostics

---
 .github/workflows/dspy-jev-preflight.yml | 30 +++++++++++++-----------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/dspy-jev-preflight.yml b/.github/workflows/dspy-jev-preflight.yml
index 5191e45..853e250 100644
--- a/.github/workflows/dspy-jev-preflight.yml
+++ b/.github/workflows/dspy-jev-preflight.yml
@@ -19,12 +19,13 @@ jobs:
         with:
           python-version: '3.12'
       - run: python -m pip install -r graph_synthesis/dspy_jev_optimizer/requirements.txt
-      - name: Bounded live connectivity tests
+      - name: Bounded live connectivity and secret-name diagnostics
         env:
           TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY || secrets.JEV_API_KEY }}
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          CI_SECRET_NAMES_INPUT: ${{ toJSON(secrets) }}
           DSPY_PROPOSER_MODEL: ${{ vars.DSPY_PROPOSER_MODEL }}
           LITELLM_LOCAL_MODEL_COST_MAP: 'True'
         run: |
@@ -33,35 +34,36 @@ jobs:
           from pathlib import Path
           from graph_synthesis.dspy_jev_optimizer.core import JevConfig
           from graph_synthesis.dspy_jev_optimizer.providers import TypeSafeBackend, DSPyProposer
+          # Discard values immediately. Never print, persist, or send secret values.
+          names = sorted(json.loads(os.environ.pop('CI_SECRET_NAMES_INPUT')).keys())
           presence = {k: bool(os.environ.get(k, '').strip()) for k in ('TYPESAFE_API_KEY','OPENROUTER_API_KEY','OPENAI_API_KEY','ANTHROPIC_API_KEY')}
-          result = {'credential_present': presence, 'status': 'started'}
+          result = {'configured_secret_names': names, 'credential_present': presence, 'status': 'started'}
           out = Path(os.environ['RUNNER_TEMP']) / 'preflight.json'
           cfg = JevConfig('Evidence classification', 'Does the evidence support the claim?', {'supports':'Evidence establishes the claim.', 'refutes':'Evidence contradicts the claim.', 'unknown':'Evidence is insufficient.'})
           try:
-              if not presence['TYPESAFE_API_KEY']:
-                  raise ValueError('TypeSafe credential is not mapped')
-              model = os.environ.get('DSPY_PROPOSER_MODEL','').strip()
-              if not model:
-                  model = ('openrouter/anthropic/claude-sonnet-4.5' if presence['OPENROUTER_API_KEY'] else 'anthropic/claude-sonnet-4-5-20250929' if presence['ANTHROPIC_API_KEY'] else 'openai/gpt-4.1' if presence['OPENAI_API_KEY'] else '')
-              if not model:
-                  raise ValueError('No proposal-provider credential is mapped')
-              result['proposer_model'] = model
               backend = TypeSafeBackend('jev-1.13.0', retries=0)
               try:
                   answer = backend.predict(cfg, {'claim':'A precedes B.', 'evidence':['A precedes B.']})
               finally:
                   backend.close()
               result['jev'] = {'model':answer['model'], 'choice':answer['choice'], 'usage':answer['usage']}
-              proposal = DSPyProposer(model).propose(cfg, [{'state':{'claim':'A precedes B.', 'evidence':['A precedes B.']}, 'gold':'supports', 'choice':'unknown', 'probabilities':{'supports':0.3,'refutes':0.1,'unknown':0.6}}], 1, [])
-              JevConfig.from_dict(proposal, cfg)
-              result['status'] = 'passed'
+              model = os.environ.get('DSPY_PROPOSER_MODEL','').strip()
+              if not model:
+                  model = ('openrouter/anthropic/claude-sonnet-4.5' if presence['OPENROUTER_API_KEY'] else 'anthropic/claude-sonnet-4-5-20250929' if presence['ANTHROPIC_API_KEY'] else 'openai/gpt-4.1' if presence['OPENAI_API_KEY'] else '')
+              if model:
+                  result['proposer_model'] = model
+                  proposal = DSPyProposer(model).propose(cfg, [{'state':{'claim':'A precedes B.', 'evidence':['A precedes B.']}, 'gold':'supports', 'choice':'unknown', 'probabilities':{'supports':0.3,'refutes':0.1,'unknown':0.6}}], 1, [])
+                  JevConfig.from_dict(proposal, cfg)
+                  result['status'] = 'passed'
+              else:
+                  result['status'] = 'jev_passed_proposer_not_configured'
           except Exception as exc:
               result['status'] = 'failed'
               result['error_type'] = type(exc).__name__
           finally:
               out.write_text(json.dumps(result, indent=2)+'\n')
               print(json.dumps(result))
-          if result['status'] != 'passed':
+          if result['status'] == 'failed':
               raise SystemExit(1)
           PY
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02

From 73ec5595fa5611327657f9e9a5a3a3e2575bd101 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <Timothy.Gregg@complete.tech>
Date: Fri, 18 Sep 2026 10:22:20 -0400
Subject: [PATCH 3/4] Keep live preflight manual and restrict it to explicitly
 named provider keys

---
 .github/workflows/dspy-jev-preflight.yml | 35 ++++++++++--------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/dspy-jev-preflight.yml b/.github/workflows/dspy-jev-preflight.yml
index 853e250..4b99b54 100644
--- a/.github/workflows/dspy-jev-preflight.yml
+++ b/.github/workflows/dspy-jev-preflight.yml
@@ -1,8 +1,6 @@
 name: DSPy Jev live preflight
+# Manual only. Do not bypass an Actions approval or environment protection gate.
 on:
-  push:
-    branches: [research/dspy-jev-benchmark]
-    paths: ['.github/workflows/dspy-jev-preflight.yml']
   workflow_dispatch:
 permissions:
   contents: read
@@ -19,13 +17,12 @@ jobs:
         with:
           python-version: '3.12'
       - run: python -m pip install -r graph_synthesis/dspy_jev_optimizer/requirements.txt
-      - name: Bounded live connectivity and secret-name diagnostics
+      - name: Bounded live connectivity test
         env:
-          TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY || secrets.JEV_API_KEY }}
+          TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          CI_SECRET_NAMES_INPUT: ${{ toJSON(secrets) }}
           DSPY_PROPOSER_MODEL: ${{ vars.DSPY_PROPOSER_MODEL }}
           LITELLM_LOCAL_MODEL_COST_MAP: 'True'
         run: |
@@ -34,36 +31,34 @@ jobs:
           from pathlib import Path
           from graph_synthesis.dspy_jev_optimizer.core import JevConfig
           from graph_synthesis.dspy_jev_optimizer.providers import TypeSafeBackend, DSPyProposer
-          # Discard values immediately. Never print, persist, or send secret values.
-          names = sorted(json.loads(os.environ.pop('CI_SECRET_NAMES_INPUT')).keys())
           presence = {k: bool(os.environ.get(k, '').strip()) for k in ('TYPESAFE_API_KEY','OPENROUTER_API_KEY','OPENAI_API_KEY','ANTHROPIC_API_KEY')}
-          result = {'configured_secret_names': names, 'credential_present': presence, 'status': 'started'}
+          result = {'credential_present': presence, 'status': 'started'}
           out = Path(os.environ['RUNNER_TEMP']) / 'preflight.json'
           cfg = JevConfig('Evidence classification', 'Does the evidence support the claim?', {'supports':'Evidence establishes the claim.', 'refutes':'Evidence contradicts the claim.', 'unknown':'Evidence is insufficient.'})
           try:
-              backend = TypeSafeBackend('jev-1.13.0', retries=0)
-              try:
-                  answer = backend.predict(cfg, {'claim':'A precedes B.', 'evidence':['A precedes B.']})
-              finally:
-                  backend.close()
-              result['jev'] = {'model':answer['model'], 'choice':answer['choice'], 'usage':answer['usage']}
               model = os.environ.get('DSPY_PROPOSER_MODEL','').strip()
               if not model:
                   model = ('openrouter/anthropic/claude-sonnet-4.5' if presence['OPENROUTER_API_KEY'] else 'anthropic/claude-sonnet-4-5-20250929' if presence['ANTHROPIC_API_KEY'] else 'openai/gpt-4.1' if presence['OPENAI_API_KEY'] else '')
-              if model:
+              if not presence['TYPESAFE_API_KEY'] or not model:
+                  result['status'] = 'credentials_missing'
+              else:
                   result['proposer_model'] = model
-                  proposal = DSPyProposer(model).propose(cfg, [{'state':{'claim':'A precedes B.', 'evidence':['A precedes B.']}, 'gold':'supports', 'choice':'unknown', 'probabilities':{'supports':0.3,'refutes':0.1,'unknown':0.6}}], 1, [])
+                  backend = TypeSafeBackend('jev-1.13.0', retries=0)
+                  try:
+                      answer = backend.predict(cfg, {'claim':'A precedes B.', 'evidence':['A precedes B.']})
+                  finally:
+                      backend.close()
+                  result['jev'] = {'model':answer['model'], 'choice':answer['choice'], 'usage':answer['usage']}
+                  proposal = DSPyProposer(model).propose(cfg, [], 1, [])
                   JevConfig.from_dict(proposal, cfg)
                   result['status'] = 'passed'
-              else:
-                  result['status'] = 'jev_passed_proposer_not_configured'
           except Exception as exc:
               result['status'] = 'failed'
               result['error_type'] = type(exc).__name__
           finally:
               out.write_text(json.dumps(result, indent=2)+'\n')
               print(json.dumps(result))
-          if result['status'] == 'failed':
+          if result['status'] != 'passed':
               raise SystemExit(1)
           PY
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02

From 2db06433d5a73532862d55791d50e827be7a7e06 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <Timothy.Gregg@complete.tech>
Date: Fri, 18 Sep 2026 10:51:55 -0400
Subject: [PATCH 4/4] Add repeated DSPy/Jev benchmark matrix and audited
 offline results

Prepare five-seed live comparisons with separate calibration and held-out panels. Preserve native baseline formulations and immutable research evidence. Record 1092 passing local tests, six skips, eleven successful archived replays, and exploratory saved-output recalibration. Live inference remains blocked by missing mapped proposal credentials and an Actions approval requirement; no DSPy performance gain is claimed.
---
 .github/workflows/dspy-benchmark.yml          |  62 ++++
 DSPY_BENCHMARKS.md                            |  14 +
 .../dspy-comparison-20260918/STATUS.md        |  59 ++++
 .../archive-calibration/RESULTS.md            |  24 ++
 .../archive-calibration/summary.json          |   1 +
 .../artifact-manifest.json                    |  26 ++
 .../dspy-comparison-20260918/live-status.json |  16 ++
 .../offline-validation-summary.json           |   1 +
 graph_synthesis/dspy_benchmark/.gitignore     |   3 +
 graph_synthesis/dspy_benchmark/README.md      | 109 +++++++
 graph_synthesis/dspy_benchmark/__init__.py    |   1 +
 .../dspy_benchmark/archive_calibration.py     |  55 ++++
 graph_synthesis/dspy_benchmark/calibration.py |  99 +++++++
 graph_synthesis/dspy_benchmark/data.py        |  86 ++++++
 graph_synthesis/dspy_benchmark/report.py      |  70 +++++
 .../dspy_benchmark/requirements.txt           |   3 +
 graph_synthesis/dspy_benchmark/run.py         | 269 ++++++++++++++++++
 .../dspy_benchmark/tests/test_benchmark.py    | 147 ++++++++++
 18 files changed, 1045 insertions(+)
 create mode 100644 .github/workflows/dspy-benchmark.yml
 create mode 100644 DSPY_BENCHMARKS.md
 create mode 100644 experiments/dspy-comparison-20260918/STATUS.md
 create mode 100644 experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md
 create mode 100644 experiments/dspy-comparison-20260918/archive-calibration/summary.json
 create mode 100644 experiments/dspy-comparison-20260918/artifact-manifest.json
 create mode 100644 experiments/dspy-comparison-20260918/live-status.json
 create mode 100644 experiments/dspy-comparison-20260918/offline-validation-summary.json
 create mode 100644 graph_synthesis/dspy_benchmark/.gitignore
 create mode 100644 graph_synthesis/dspy_benchmark/README.md
 create mode 100644 graph_synthesis/dspy_benchmark/__init__.py
 create mode 100644 graph_synthesis/dspy_benchmark/archive_calibration.py
 create mode 100644 graph_synthesis/dspy_benchmark/calibration.py
 create mode 100644 graph_synthesis/dspy_benchmark/data.py
 create mode 100644 graph_synthesis/dspy_benchmark/report.py
 create mode 100644 graph_synthesis/dspy_benchmark/requirements.txt
 create mode 100644 graph_synthesis/dspy_benchmark/run.py
 create mode 100644 graph_synthesis/dspy_benchmark/tests/test_benchmark.py

diff --git a/.github/workflows/dspy-benchmark.yml b/.github/workflows/dspy-benchmark.yml
new file mode 100644
index 0000000..f2263e5
--- /dev/null
+++ b/.github/workflows/dspy-benchmark.yml
@@ -0,0 +1,62 @@
+name: Repeated DSPy Jev benchmarks
+on:
+  pull_request:
+    paths: ['graph_synthesis/dspy_benchmark/**', '.github/workflows/dspy-benchmark.yml']
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  offline:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: '3.12'
+      - run: python -m pip install numpy==2.3.5 scipy==1.17.0
+      - run: python -B -m unittest discover -s graph_synthesis/dspy_benchmark/tests -v
+      - run: python -B -m graph_synthesis.dspy_jev_optimizer.verify
+  live:
+    if: github.event_name == 'workflow_dispatch' && github.repository == 'CompleteDotTech/paper-package'
+    needs: offline
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        seed: [11, 23, 37, 53, 71]
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: '3.12'
+      - run: python -m pip install -r graph_synthesis/dspy_benchmark/requirements.txt
+      - name: Pin resolved dependency record
+        run: python -m pip freeze > '${{ runner.temp }}/environment.txt'
+      - name: Live repeated comparison
+        env:
+          TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          DSPY_PROPOSER_MODEL: ${{ vars.DSPY_PROPOSER_MODEL }}
+          LITELLM_LOCAL_MODEL_COST_MAP: 'True'
+        run: python -B -m graph_synthesis.dspy_benchmark.run --seed ${{ matrix.seed }} --iterations 8 --workers 4 --output '${{ runner.temp }}/comparison/seed-${{ matrix.seed }}'
+      - name: Generate report without inventing missing runs
+        if: always()
+        run: python -B -m graph_synthesis.dspy_benchmark.report --directory '${{ runner.temp }}/comparison'
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        if: always()
+        with:
+          name: dspy-jev-seed-${{ matrix.seed }}
+          path: |
+            ${{ runner.temp }}/comparison
+            ${{ runner.temp }}/environment.txt
+          retention-days: 14
+          if-no-files-found: warn
diff --git a/DSPY_BENCHMARKS.md b/DSPY_BENCHMARKS.md
new file mode 100644
index 0000000..6ee5c8a
--- /dev/null
+++ b/DSPY_BENCHMARKS.md
@@ -0,0 +1,14 @@
+# DSPy versus Jev benchmark execution
+
+[Comparison protocol and runner](graph_synthesis/dspy_benchmark/README.md) ·
+[Actual execution status](experiments/dspy-comparison-20260918/STATUS.md) ·
+[Archived-output calibration check](experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md)
+
+The live comparison has not completed: CI identified a missing mapped generative
+proposal-provider key, then marked the next diagnostic `action_required` before
+starting jobs. No approval restriction was bypassed and no DSPy gain is claimed.
+
+The comparison code registers five isolated repeats, two search objectives, eight
+native legacy formulations and seven calibration settings per frozen arm. New
+local regression and archived-replay checks are recorded separately from model
+performance. Historical evidence and the root immutable manifest are unchanged.
diff --git a/experiments/dspy-comparison-20260918/STATUS.md b/experiments/dspy-comparison-20260918/STATUS.md
new file mode 100644
index 0000000..3950ccb
--- /dev/null
+++ b/experiments/dspy-comparison-20260918/STATUS.md
@@ -0,0 +1,59 @@
+# DSPy / Jev comparison: actual execution status
+
+**Live with/without-DSPy comparison: NOT COMPLETED. No new Jev inference calls and no new DSPy inference runs were completed.**
+
+## Live execution blockers
+
+[Preflight run 35355158096](https://github.com/CompleteDotTech/paper-package/actions/runs/35355158096) found the mapped TypeSafe key populated, but none of `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, or `ANTHROPIC_API_KEY`. It stopped before calling Jev. This does not establish whether differently named or environment-scoped provider credentials exist, or whether the TypeSafe key is valid.
+
+[Diagnostic run 35355491970](https://github.com/CompleteDotTech/paper-package/actions/runs/35355491970) was marked `action_required` with zero jobs. Its gate was left intact. Broad secret-context diagnostics were removed; preflight and live runs are manual-only and use explicitly named keys. Secret values are not in these reports.
+
+## Executed offline checks
+
+**1,092 tests passed; 6 skipped; 16 test suites completed. All 11 deterministic experiment replays passed.** The original 161-file inventory and the existing optimizer inventory also passed verification.
+
+Four SDK contract tests were skipped because SDK packages were not installed in this local runtime; two archived reproduction tests also reported skips. Skips are not counted as passes. An initial reproduction invocation lacked PYTHONPATH; the corrected invocation passed. A later combined wrapper timed out on two replay processes; the individual replay commands recorded below had already passed. Neither failed attempt is treated as an additional successful test.
+
+| Check | Tests discovered | Skipped | Result |
+|---|---:|---:|---|
+| replay-adaptive | 0 | 0 | passed |
+| replay-assumption_aware | 0 | 0 | passed |
+| replay-followup | 0 | 0 | passed |
+| replay-frontier | 0 | 0 | passed |
+| replay-novel_mechanisms | 0 | 0 | passed |
+| replay-reliability | 0 | 0 | passed |
+| replay-risk_control | 0 | 0 | passed |
+| replay-source_structural | 0 | 0 | passed |
+| replay-structural | 0 | 0 | passed |
+| replay-theory_suite | 0 | 0 | passed |
+| replay-uncertainty | 0 | 0 | passed |
+| test-graph_synthesis-adaptive-tests | 64 | 0 | passed |
+| test-graph_synthesis-assumption_aware-tests | 47 | 0 | passed |
+| test-graph_synthesis-corpus-tests | 32 | 0 | passed |
+| test-graph_synthesis-dspy_benchmark-tests | 26 | 0 | passed |
+| test-graph_synthesis-dspy_jev_optimizer-tests | 35 | 4 | passed |
+| test-graph_synthesis-followup-tests | 67 | 0 | passed |
+| test-graph_synthesis-frontier-tests | 75 | 0 | passed |
+| test-graph_synthesis-novel_mechanisms-tests | 92 | 0 | passed |
+| test-graph_synthesis-reliability-tests | 63 | 0 | passed |
+| test-graph_synthesis-risk_control-tests | 66 | 0 | passed |
+| test-graph_synthesis-source_structural-tests | 86 | 0 | passed |
+| test-graph_synthesis-structural-tests | 36 | 0 | passed |
+| test-graph_synthesis-tests | 200 | 0 | passed |
+| test-graph_synthesis-theory_suite-tests | 37 | 0 | passed |
+| test-graph_synthesis-uncertainty-tests | 36 | 0 | passed |
+| test-reproduction-tests | 136 | 2 | passed |
+| verify-frozen | 0 | 0 | passed |
+| verify-optimizer | 0 | 0 | passed |
+
+The machine-readable [validation record](offline-validation-summary.json) includes per-suite status and counts; the downloadable audit bundle also includes commands, durations and log hashes. These runs disabled socket connections and used no inference keys. They reproduce saved observations and algorithm fixtures, not newly optimized model predictions.
+
+## Separate archived calibration analysis
+
+[Archived calibration results](archive-calibration/RESULTS.md) refit fixed calibrators on the original calibration split and score saved test responses. They contain zero fresh inference and zero DSPy runs. Invalid archived response exclusions and denominators are explicit. Do not attribute these results to DSPy.
+
+## Ready-to-run live comparison
+
+[Protocol and commands](../../graph_synthesis/dspy_benchmark/README.md) cover five repeats, two DSPy search objectives, eight original native formulations, seven calibration settings and seven task/panel combinations. The code was tested with controlled doubles, not substituted for live inference.
+
+To execute live, map an authorized generative-provider credential and model into the workflow, resolve the repository Actions approval requirement, and manually run the live workflow. Original manuscripts, predictions and the immutable root manifest remain unchanged.
diff --git a/experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md b/experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md
new file mode 100644
index 0000000..73362f9
--- /dev/null
+++ b/experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md
@@ -0,0 +1,24 @@
+# Archived-output calibration check
+
+**Zero fresh model calls. Zero DSPy runs. This is not the requested live comparison.** Invalid archived responses are excluded from probability metrics and listed in summary.json; accuracy is conditional on a valid response, not end-to-end accuracy.
+
+Calibrators were refit using only the archived calibration split. All three fixed methods are shown; none was selected using test performance. The archived evaluation split was historically examined, so these are exploratory results.
+
+| Task | Historical arm | Calibration | n | Accuracy | Macro-F1 | Brier | Log loss | ECE |
+|---|---|---|---:|---:|---:|---:|---:|---:|
+| relation_support | baseline_choice | raw | 338 | 0.846154 | 0.844335 | 0.234192 | 1.240206 | 0.074408 |
+| relation_support | baseline_choice | temperature | 338 | 0.846154 | 0.844335 | 0.272597 | 0.569980 | 0.120948 |
+| relation_support | baseline_choice | bias_temperature | 338 | 0.828402 | 0.817346 | 0.261256 | 0.532340 | 0.096751 |
+| relation_support | fewshot_contract | raw | 337 | 0.860534 | 0.859685 | 0.224429 | 1.320742 | 0.068843 |
+| relation_support | fewshot_contract | temperature | 337 | 0.860534 | 0.859685 | 0.256077 | 0.632292 | 0.131532 |
+| relation_support | fewshot_contract | bias_temperature | 337 | 0.759644 | 0.756568 | 0.282491 | 0.574257 | 0.107833 |
+| entity_resolution | baseline_noul | raw | 413 | 0.980630 | 0.960452 | 0.040651 | 0.106744 | 0.064504 |
+| entity_resolution | baseline_noul | temperature | 413 | 0.980630 | 0.960452 | 0.031287 | 0.065248 | 0.013925 |
+| entity_resolution | baseline_noul | bias_temperature | 413 | 0.983051 | 0.965638 | 0.028751 | 0.059969 | 0.013186 |
+| entity_resolution | fewshot_contract | raw | 413 | 0.987893 | 0.976434 | 0.018150 | 0.033291 | 0.015036 |
+| entity_resolution | fewshot_contract | temperature | 413 | 0.987893 | 0.976434 | 0.015705 | 0.026191 | 0.009182 |
+| entity_resolution | fewshot_contract | bias_temperature | 413 | 0.987893 | 0.975793 | 0.017257 | 0.028161 | 0.003360 |
+
+Temperature scaling preserves decisions; its accuracy differences must be zero. Bias/temperature can change decisions. A probability-quality improvement is not necessarily an accuracy improvement.
+
+Source SHA-256: `9fc0f0a345f6fbbd311759a5143c1d1be62313b72e26acd54e3ae289e040d76f`
diff --git a/experiments/dspy-comparison-20260918/archive-calibration/summary.json b/experiments/dspy-comparison-20260918/archive-calibration/summary.json
new file mode 100644
index 0000000..66330c1
--- /dev/null
+++ b/experiments/dspy-comparison-20260918/archive-calibration/summary.json
@@ -0,0 +1 @@
+{"evidence":"exploratory_saved_response_reanalysis","fresh_model_calls":0,"dspy_runs":0,"source":"experiments/jev-rerun-20260918/predictions.jsonl","source_sha256":"9fc0f0a345f6fbbd311759a5143c1d1be62313b72e26acd54e3ae289e040d76f","full_results_sha256":"8fe4c82a4b2d0807c883b4d4abbec830403f3386a14c8d96530f436253635c9f","excluded_invalid_responses":[{"task":"relation_support","arm":"baseline_choice","split":"evaluation","id":"scifact-213-13625993","error":"Probabilities must sum to one"},{"task":"relation_support","arm":"fewshot_contract","split":"calibration","id":"scifact-1258-12040627","error":"choice must be a highest-probability option"},{"task":"relation_support","arm":"fewshot_contract","split":"evaluation","id":"scifact-1202-3475317","error":"Probabilities must sum to one"},{"task":"relation_support","arm":"fewshot_contract","split":"evaluation","id":"scifact-1088-37549932","error":"choice must be a highest-probability option"}],"columns":["task","arm","calibration","n","accuracy","macro_f1","brier","log_loss","ece"],"results":[["relation_support","baseline_choice","raw",338,0.846154,0.844335,0.234192,1.240206,0.074408],["relation_support","baseline_choice","temperature",338,0.846154,0.844335,0.272597,0.56998,0.120948],["relation_support","baseline_choice","bias_temperature",338,0.828402,0.817346,0.261256,0.53234,0.096751],["relation_support","fewshot_contract","raw",337,0.860534,0.859685,0.224429,1.320742,0.068843],["relation_support","fewshot_contract","temperature",337,0.860534,0.859685,0.256077,0.632292,0.131532],["relation_support","fewshot_contract","bias_temperature",337,0.759644,0.756568,0.282491,0.574257,0.107833],["entity_resolution","baseline_noul","raw",413,0.98063,0.960452,0.040651,0.106744,0.064504],["entity_resolution","baseline_noul","temperature",413,0.98063,0.960452,0.031287,0.065248,0.013925],["entity_resolution","baseline_noul","bias_temperature",413,0.983051,0.965638,0.028751,0.059969,0.013186],["entity_resolution","fewshot_contract","raw",413,0.987893,0.976434,0.01815,0.033291,0.015036],["entity_resolution","fewshot_contract","temperature",413,0.987893,0.976434,0.015705,0.026191,0.009182],["entity_resolution","fewshot_contract","bias_temperature",413,0.987893,0.975793,0.017257,0.028161,0.00336]]}
diff --git a/experiments/dspy-comparison-20260918/artifact-manifest.json b/experiments/dspy-comparison-20260918/artifact-manifest.json
new file mode 100644
index 0000000..cf24f79
--- /dev/null
+++ b/experiments/dspy-comparison-20260918/artifact-manifest.json
@@ -0,0 +1,26 @@
+{
+  "schema_version": 1,
+  "scope": "New benchmark implementation, workflow and reports; excludes this inventory. Original research inventory is unchanged.",
+  "baseline_commit": "c726b66c3a07a115eb928b3f2629de972c728083",
+  "original_manifest_git_blob": "a8d9f447c266fbedf8837003ceb9639e00778b0e",
+  "preflight_git_blob": "4b99b54a351d0d927656aa6052e4db509ac41a9e",
+  "sha256": {
+    ".github/workflows/dspy-benchmark.yml": "43a1261432d541c42dc2056868b561f0578b3910d376ef73c049dc94884bd0f3",
+    "DSPY_BENCHMARKS.md": "3846accfc911b784ba44545d4910807b69eb45bb4b16fa5f6ed43878d86870f8",
+    "experiments/dspy-comparison-20260918/STATUS.md": "1560e13f922ae70a9fb7e8036f8943ec9c258e5236986c90d24c22b9293c6d45",
+    "experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md": "129ff8e48148e6de19c931760488d8e26c1af7816786cd89f961ec2aa7eda9de",
+    "experiments/dspy-comparison-20260918/archive-calibration/summary.json": "c18e3b3c7d40dab043193d0fe4e6601d69fea511b77e3f2fca54ab1fa0a23bdb",
+    "experiments/dspy-comparison-20260918/live-status.json": "d5d085d71219fea23bd08882de0df72e26588e78dfb0007566d22915c39c9aa7",
+    "experiments/dspy-comparison-20260918/offline-validation-summary.json": "ba061c00f7fc44b08bb02a26ed705f6e38badc756d03eb6754defec2d67bbfbd",
+    "graph_synthesis/dspy_benchmark/.gitignore": "130b65c1033d6acb35e41f551f2aac6601947e5bc14f954a65b5154fa40716a1",
+    "graph_synthesis/dspy_benchmark/README.md": "b5dfac1161bc13ffd53587dfebe3c4f281d419fc27f4dcc5ef35676306b5349a",
+    "graph_synthesis/dspy_benchmark/__init__.py": "13d9369fd42ab6800e4d3fb99ad40f8deed29ce1acefc662d17d4b489e485abf",
+    "graph_synthesis/dspy_benchmark/archive_calibration.py": "7154a8a2818c65fb94291f4201106b6287e20b74e86e38ea6d629bc32af556a4",
+    "graph_synthesis/dspy_benchmark/calibration.py": "8c256a74fbd3a0820a7c64d42ac04c75150f9c9b2bab8c680264ea7dae18b96d",
+    "graph_synthesis/dspy_benchmark/data.py": "1933b9d51e2901f4f78bea6ca4cca38d8b2099b27dccc88975c5a8f1ce3bc001",
+    "graph_synthesis/dspy_benchmark/report.py": "3d9b7c3025565b951320c0a36390c0ae660d3beb97f8194dc7e94cf256d4bc86",
+    "graph_synthesis/dspy_benchmark/requirements.txt": "b8fab64f96b3a873a0feb9fae9f0eb1308d07ca3cfbea6620a27f6831a3fbcf4",
+    "graph_synthesis/dspy_benchmark/run.py": "21ae6b66e654dfb0f35c2d3f7d865c7ed0509b61d13292fedab301582e17ad43",
+    "graph_synthesis/dspy_benchmark/tests/test_benchmark.py": "f0d86ad6b47ea70008b1dc7e565acb2cc72826b6feed15640292eb92a4cce1c9"
+  }
+}
diff --git a/experiments/dspy-comparison-20260918/live-status.json b/experiments/dspy-comparison-20260918/live-status.json
new file mode 100644
index 0000000..0e592f5
--- /dev/null
+++ b/experiments/dspy-comparison-20260918/live-status.json
@@ -0,0 +1,16 @@
+{
+  "status": "blocked",
+  "completed_live_dspy_runs": 0,
+  "new_jev_calls": 0,
+  "preflight_run": 35355158096,
+  "credential_present": {
+    "TYPESAFE_API_KEY": true,
+    "OPENROUTER_API_KEY": false,
+    "OPENAI_API_KEY": false,
+    "ANTHROPIC_API_KEY": false
+  },
+  "approval_run": 35355491970,
+  "approval_conclusion": "action_required",
+  "approval_jobs": 0,
+  "scope": "The four explicit mappings were checked; other names or environment-scoped keys were not inspected. TypeSafe key validity was not tested."
+}
diff --git a/experiments/dspy-comparison-20260918/offline-validation-summary.json b/experiments/dspy-comparison-20260918/offline-validation-summary.json
new file mode 100644
index 0000000..ed590f9
--- /dev/null
+++ b/experiments/dspy-comparison-20260918/offline-validation-summary.json
@@ -0,0 +1 @@
+{"source_commit":"c726b66c3a07a115eb928b3f2629de972c728083","execution":"individually_executed_offline_commands","fresh_service_calls":0,"python":"3.13.5","numpy":"2.3.5","scipy":"1.17.0","test_suites":16,"tests_discovered":1098,"tests_skipped":6,"replay_suites":11,"tests_passed":1092,"columns":["check","tests","skipped","result"],"checks":[["replay-adaptive",0,0,"passed"],["replay-assumption_aware",0,0,"passed"],["replay-followup",0,0,"passed"],["replay-frontier",0,0,"passed"],["replay-novel_mechanisms",0,0,"passed"],["replay-reliability",0,0,"passed"],["replay-risk_control",0,0,"passed"],["replay-source_structural",0,0,"passed"],["replay-structural",0,0,"passed"],["replay-theory_suite",0,0,"passed"],["replay-uncertainty",0,0,"passed"],["test-graph_synthesis-adaptive-tests",64,0,"passed"],["test-graph_synthesis-assumption_aware-tests",47,0,"passed"],["test-graph_synthesis-corpus-tests",32,0,"passed"],["test-graph_synthesis-dspy_benchmark-tests",26,0,"passed"],["test-graph_synthesis-dspy_jev_optimizer-tests",35,4,"passed"],["test-graph_synthesis-followup-tests",67,0,"passed"],["test-graph_synthesis-frontier-tests",75,0,"passed"],["test-graph_synthesis-novel_mechanisms-tests",92,0,"passed"],["test-graph_synthesis-reliability-tests",63,0,"passed"],["test-graph_synthesis-risk_control-tests",66,0,"passed"],["test-graph_synthesis-source_structural-tests",86,0,"passed"],["test-graph_synthesis-structural-tests",36,0,"passed"],["test-graph_synthesis-tests",200,0,"passed"],["test-graph_synthesis-theory_suite-tests",37,0,"passed"],["test-graph_synthesis-uncertainty-tests",36,0,"passed"],["test-reproduction-tests",136,2,"passed"],["verify-frozen",0,0,"passed"],["verify-optimizer",0,0,"passed"]],"full_validation_sha256":"deb69ab26e253f6047017bfdf8ad2205a3368fb7a2f423b07bf4b6b3999c6df0"}
diff --git a/graph_synthesis/dspy_benchmark/.gitignore b/graph_synthesis/dspy_benchmark/.gitignore
new file mode 100644
index 0000000..145667d
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/.gitignore
@@ -0,0 +1,3 @@
+/runs/
+/.cache/
+__pycache__/
diff --git a/graph_synthesis/dspy_benchmark/README.md b/graph_synthesis/dspy_benchmark/README.md
new file mode 100644
index 0000000..5790a10
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/README.md
@@ -0,0 +1,109 @@
+# Repeated DSPy versus Jev benchmark comparison
+
+**Live comparison status: blocked, not completed.** The first CI preflight found
+`TYPESAFE_API_KEY` populated, but none of the three explicitly mapped generative
+provider keys (`OPENROUTER_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`). It stopped
+before inference. The next diagnostic was marked `action_required` before any job
+ran. The approval gate has not been bypassed. The diagnostic was reduced to
+explicit named keys and made manual-only.
+
+## Registered comparison
+
+| Dimension | Fixed configuration |
+|---|---|
+| Jev target | `jev-1.13.0` |
+| Repeats | Seeds 11, 23, 37, 53, 71; fresh isolated cache namespace per repeat |
+| Tasks | SciFact relation support; DBLP-ACM entity resolution |
+| No-DSPy controls | All four original native formulations for each task, including Noul and few-shot controls |
+| DSPy searches | Accuracy and composite objectives; five runs per objective per task = 20 searches |
+| Search budget | Eight candidate proposals per search; no test-based early stopping |
+| Calibration | Raw; temperature; regularized bias + temperature |
+| Calibration-size sensitivity | 25%, 50%, 100% source-group prefixes; raw is reported once |
+| Selection data | 29 training and 31 validation examples per task, split from original development only |
+| Calibration data | Original 150 relation / 390 entity calibration examples |
+| Primary test | Original 339 relation / 413 entity evaluation examples |
+| Transfer panels | 50 relation fixtures, 100 entity fixtures, 48 semantic challenges, and the 73/263 multicall input panels |
+
+The full matrix contains 1,470 metric records when every run succeeds. These are
+not 1,470 independent experiments. Each frozen prompt is evaluated once per input
+panel per repeat; alternative calibration transforms reuse those probabilities.
+Twelve entity fixtures labeled `uncertain` remain in prediction files but cannot
+be scored against the binary label schema. Their exclusion count is explicit.
+
+The original eight formulations are issued with their original Choice/Noul
+question types and demonstrations. DSPy optimizes the contracted Choice baseline
+for each task. It does not optimize Jev weights, evolve graph topology, or run
+GEPA/MIPROv2. The seven historical multicall execution policies are not replaced
+by seven fake DSPy arms: this matrix applies the frozen direct-decision prompts
+to their input panels. Existing graph-policy/algorithm suites are separately
+replayed; those results are not new model comparisons.
+
+## Boundaries and interpretation
+
+All candidates use training states/labels only in proposal feedback. Validation
+selects the winner. All prompt searches for a task are frozen before fitting any
+calibrator, and all calibrators for that task are frozen before test inference.
+Split hashes and source/entity identity separation are checked. Historical plans
+are public and their evaluation sets were previously examined; "held out" means
+held out from this optimization, not a never-seen external benchmark.
+
+Temperature scaling leaves decisions unchanged. Bias/temperature can change
+labels. The latter uses fixed regularization 0.01 and bounded parameters, never
+test-tuned regularization. Probability zeros are clipped to 1e-15 before logs.
+Confidence metrics use class probabilities, not Jev's distribution-concentration
+field. Lower Brier/log loss/ECE is better; higher accuracy/F1/MCC is better.
+
+Report all seeds, including unsuccessful searches and null/negative gains. Never
+pick the best seed using test accuracy. Report means and sample standard deviations;
+repeated runs on one test set are not independent datasets. `paired_interval` is
+an optional fixed-fit, provided-group bootstrap helper; its intervals are not
+simultaneous or multiplicity-adjusted. No significance claim is produced by the
+aggregate report.
+
+## Execute, after resolving CI credentials and approval
+
+Use Python 3.12 or newer. From the repository root:
+
+```bash
+python -m pip install -r graph_synthesis/dspy_benchmark/requirements.txt
+python -B -m unittest discover -s graph_synthesis/dspy_benchmark/tests -v
+python -B -m graph_synthesis.dspy_benchmark.run \
+  --seed 11 --iterations 8 --workers 4 \
+  --output graph_synthesis/dspy_benchmark/runs/seed-11
+python -B -m graph_synthesis.dspy_benchmark.report \
+  --directory graph_synthesis/dspy_benchmark/runs
+```
+
+Set `TYPESAFE_API_KEY`, `DSPY_PROPOSER_MODEL`, and the provider credential required
+by that DSPy model. Keys must stay in the CI secret store; do not commit or paste
+them into reports. The manual [workflow](../../.github/workflows/dspy-benchmark.yml)
+has a five-seed matrix with at most two simultaneous jobs. It never injects keys
+into pull-request tests. Select the desired branch when manually dispatching.
+Approval/environment protections remain applicable. `--baseline-only` is an
+explicit partial-run mode and never counts as a completed DSPy comparison.
+
+The registered eight-iteration matrix has an upper bound of 65,000 logical Jev
+requests across five seeds before cache savings, with at most two SDK retries per
+request, and 160 DSPy proposal invocations. This is not a dollar-denominated budget
+or a guarantee of provider determinism. Each CI seed job has a 120-minute timeout.
+
+Every run writes protocol/config/source hashes, proposals, raw requests and
+responses, probability files, calibration parameters, metrics and usage. HTTP
+credentials and complete LM configuration dictionaries are not serialized. Each
+run directory is new; accidental overwriting is refused. Missing/malformed model
+responses terminate the run rather than being silently assigned an accuracy.
+
+## What was actually executed in this change
+
+See [the execution record](../../experiments/dspy-comparison-20260918/STATUS.md).
+It distinguishes fresh local regressions, deterministic archived replay, the
+blocked live preflight, and [exploratory archived-output recalibration](../../experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md).
+The latter contains **zero new model calls and zero DSPy runs**. It must not be
+presented as evidence for DSPy gains.
+
+The root immutable `MANIFEST.json` and all archived research files are unchanged.
+New files have their own integrity inventory. SDK dependency versions are pinned;
+CI also records its complete resolved environment. Method references:
+[temperature scaling](https://proceedings.mlr.press/v70/guo17a.html),
+[TypeSafe probabilities](https://docs.typesafe.ai/primitives/choice),
+[DSPy programs](https://dspy.ai/).
diff --git a/graph_synthesis/dspy_benchmark/__init__.py b/graph_synthesis/dspy_benchmark/__init__.py
new file mode 100644
index 0000000..a32c448
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/__init__.py
@@ -0,0 +1 @@
+"""Repeated DSPy/Jev comparison; historical evidence is read-only."""
diff --git a/graph_synthesis/dspy_benchmark/archive_calibration.py b/graph_synthesis/dspy_benchmark/archive_calibration.py
new file mode 100644
index 0000000..200cb8a
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/archive_calibration.py
@@ -0,0 +1,55 @@
+"""Exploratory recalibration of SAVED responses; never presented as a live DSPy run."""
+import argparse
+import hashlib
+from pathlib import Path
+from .data import ROOT, load_task
+from .calibration import fit, apply, evaluation
+from graph_synthesis.dspy_jev_optimizer.core import parse_json, write_json
+
+SOURCE = 'experiments/jev-rerun-20260918/predictions.jsonl'
+
+def execute(output):
+    output.mkdir(parents=True,exist_ok=False)
+    source=ROOT/SOURCE
+    rows=[parse_json(line) for line in source.read_text(encoding='utf-8').splitlines() if line]
+    result=[]
+    exclusions=[]
+    for task in ('relation_support','entity_resolution'):
+        cfg,splits,_,_=load_task(task)
+        groups={r.id:r.group_id for name in ('calibration','test') for r in splits[name]}
+        arms=sorted({r['arm'] for r in rows if r['task']==task and r['split']=='calibration'})
+        for arm in arms:
+            def get(split):
+                selected=[r for r in rows if r['task']==task and r['arm']==arm and r['split']==split and r['repeat']==0]
+                bad=[r for r in selected if r.get('error')]
+                exclusions.extend({'task':task,'arm':arm,'split':split,'id':r['id'],'error':r['error']} for r in bad)
+                selected=[r for r in selected if not r.get('error')]
+                return [{'id':r['id'],'group_id':groups[r['id']],'gold':r['gold_label'],
+                         'choice':max(cfg.criteria,key=lambda k:r['distribution'][k]),'probabilities':r['distribution']} for r in selected]
+            calibration,test=get('calibration'),get('evaluation')
+            if not calibration or not test:
+                raise ValueError('No valid archived responses')
+            for method in ('raw','temperature','bias_temperature'):
+                fitted=fit(calibration,list(cfg.criteria),method)
+                write_json(output/(task+'-'+arm+'-'+method+'-fit.json'),fitted)
+                result.append({'task':task,'arm':arm,'calibration':method,'temperature':fitted['temperature'],'planned_test_n':len(splits['test']),'response_coverage':len(test)/len(splits['test']),
+                               **evaluation(apply(test,fitted),cfg.criteria)})
+    report={'evidence':'exploratory_saved_response_reanalysis','fresh_model_calls':0,'dspy_runs':0,
+            'source':SOURCE,'source_sha256':hashlib.sha256(source.read_bytes()).hexdigest(),
+            'method':'Fit only archived calibration labels; score historically examined archived evaluation labels. No method selected on test scores.',
+            'excluded_invalid_responses':exclusions,'results':result}
+    write_json(output/'results.json',report)
+    text=['# Archived-output calibration check','',
+          '**Zero fresh model calls. Zero DSPy runs. This is not the requested live comparison.** Invalid archived responses are excluded from probability metrics and listed in results.json; accuracy is conditional on a valid response, not end-to-end accuracy.','',
+          'Calibrators were refit using only the archived calibration split. All three fixed methods are shown; none was selected using test performance. The archived evaluation split was historically examined, so these are exploratory results.', '',
+          '| Task | Historical arm | Calibration | n | Accuracy | Macro-F1 | Brier | Log loss | ECE |',
+          '|---|---|---|---:|---:|---:|---:|---:|---:|']
+    for r in result:
+        text.append('| '+' | '.join([r['task'],r['arm'],r['calibration'],str(r['n'])]+[f"{r[m]:.6f}" for m in ('accuracy','macro_f1','brier','log_loss','ece')])+' |')
+    text.extend(['','Temperature scaling preserves decisions; its accuracy differences must be zero. Bias/temperature can change decisions. A probability-quality improvement is not necessarily an accuracy improvement.','',f"Source SHA-256: `{report['source_sha256']}`",''])
+    (output/'RESULTS.md').write_text('\n'.join(text),encoding='utf-8')
+    return report
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--output',type=Path,required=True)
+    execute(parser.parse_args().output)
diff --git a/graph_synthesis/dspy_benchmark/calibration.py b/graph_synthesis/dspy_benchmark/calibration.py
new file mode 100644
index 0000000..fdc2494
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/calibration.py
@@ -0,0 +1,99 @@
+"""Post-hoc calibration. Fit on calibration labels, never on test labels."""
+import math
+import numpy as np
+from scipy.optimize import minimize, minimize_scalar
+from graph_synthesis.dspy_jev_optimizer.core import metrics, digest
+
+
+def matrix(predictions, labels):
+    return np.asarray([[r['probabilities'][k] for k in labels] for r in predictions], dtype=float)
+
+
+def softmax(values):
+    values = values - values.max(axis=1, keepdims=True)
+    out = np.exp(values)
+    return out / out.sum(axis=1, keepdims=True)
+
+
+def fit(predictions, labels, method='temperature'):
+    if not predictions or method not in ('raw','temperature','bias_temperature'):
+        raise ValueError('Invalid calibration request')
+    p = matrix(predictions, labels)
+    if not np.isfinite(p).all() or (p < 0).any() or not np.allclose(p.sum(1), 1, atol=1e-6):
+        raise ValueError('Invalid calibration probabilities')
+    y = np.asarray([list(labels).index(r['gold']) for r in predictions])
+    logp = np.log(np.clip(p, 1e-15, 1))
+    def loss(log_t, bias):
+        q = softmax(logp / np.exp(log_t) + bias)
+        return float(-np.log(np.clip(q[np.arange(len(y)), y], 1e-15, 1)).mean())
+    log_t, bias = 0., np.zeros(len(labels))
+    if method == 'temperature':
+        fitted = minimize_scalar(lambda t: loss(t, bias), bounds=(-2.3, 2.3), method='bounded')
+        if not fitted.success:
+            raise RuntimeError('Temperature fit failed')
+        log_t = float(fitted.x)
+    elif method == 'bias_temperature':
+        # Fixed shrinkage; never choose its strength using test outcomes.
+        def objective(x):
+            b = x[1:] - np.mean(x[1:])
+            return loss(x[0], b) + .01 * float(np.mean(b*b))
+        fitted = minimize(objective, np.zeros(len(labels)+1), method='L-BFGS-B',
+                          bounds=[(-2.3,2.3)]+[(-3.,3.)]*len(labels))
+        if not fitted.success:
+            raise RuntimeError('Bias/temperature fit failed')
+        log_t, bias = float(fitted.x[0]), fitted.x[1:]-np.mean(fitted.x[1:])
+    return {'method':method, 'temperature':math.exp(log_t), 'bias':bias.tolist(),
+            'labels':list(labels), 'n_calibration':len(predictions),
+            'calibration_ids_sha256':digest([r['id'] for r in predictions])}
+
+
+def apply(predictions, fitted):
+    labels = fitted['labels']
+    if fitted['method'] == 'raw':
+        return [dict(r) for r in predictions]
+    p = matrix(predictions, labels)
+    q = softmax(np.log(np.clip(p,1e-15,1))/fitted['temperature']+np.asarray(fitted['bias']))
+    # Temperature preserves selected labels, including provider tie-breaking.
+    return [{**r, 'choice':r['choice'] if fitted['method']=='temperature' else labels[int(np.argmax(q[i]))],
+             'probabilities':dict(zip(labels, map(float,q[i])))} for i,r in enumerate(predictions)]
+
+
+def evaluation(predictions, labels):
+    valid = [r for r in predictions if r['gold'] in labels]
+    result = metrics(valid, labels)
+    result['unscorable_count'] = len(predictions)-len(valid)
+    result['selective'] = []
+    for threshold in (.5,.7,.9,.95,.99):
+        selected = [r for r in valid if r['probabilities'][r['choice']] >= threshold]
+        result['selective'].append({'threshold':threshold, 'accepted':len(selected),
+            'coverage':len(selected)/len(valid), 'errors':sum(r['choice']!=r['gold'] for r in selected),
+            'risk':sum(r['choice']!=r['gold'] for r in selected)/len(selected) if selected else None})
+    if 'same' in labels:
+        result['false_merges'] = sum(r['choice']=='same' and r['gold']!='same' for r in valid)
+        result['missed_merges'] = sum(r['choice']!='same' and r['gold']=='same' for r in valid)
+    else:
+        edge_labels = {'SUPPORTS','REFUTES'}
+        tp = sum(r['choice'] in edge_labels and r['gold']==r['choice'] for r in valid)
+        fp = sum(r['choice'] in edge_labels and r['gold']!=r['choice'] for r in valid)
+        fn = sum(r['gold'] in edge_labels and r['gold']!=r['choice'] for r in valid)
+        result.update(correct_edges=tp, wrong_edges=fp, missed_edges=fn,
+                      edge_f1=2*tp/(2*tp+fp+fn) if 2*tp+fp+fn else 0.)
+    return result
+
+
+def paired_interval(baseline, candidate, labels, metric, seed=812, repeats=1000):
+    if [r['id'] for r in baseline] != [r['id'] for r in candidate]:
+        raise ValueError('Unpaired examples')
+    groups = {}
+    for i,r in enumerate(baseline):
+        groups.setdefault(r.get('group_id',r['id']), []).append(i)
+    keys = list(groups)
+    rng = np.random.default_rng(seed)
+    values = []
+    for _ in range(repeats):
+        indices = [i for g in rng.choice(len(keys),len(keys)) for i in groups[keys[g]]]
+        a, b = ([rows[i] for i in indices] for rows in (baseline,candidate))
+        values.append(metrics(b,labels)[metric]-metrics(a,labels)[metric])
+    return {'delta':metrics(candidate,labels)[metric]-metrics(baseline,labels)[metric],
+            'low':float(np.quantile(values,.025)), 'high':float(np.quantile(values,.975)),
+            'resampling_unit':'provided group_id', 'replicates':repeats, 'multiplicity_adjusted':False}
diff --git a/graph_synthesis/dspy_benchmark/data.py b/graph_synthesis/dspy_benchmark/data.py
new file mode 100644
index 0000000..d497c53
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/data.py
@@ -0,0 +1,86 @@
+"""Deterministic benchmark splits from the original frozen experiment plans."""
+from dataclasses import asdict
+from pathlib import Path
+from collections import Counter
+from graph_synthesis.dspy_jev_optimizer.core import Example, JevConfig, digest, read_json, fingerprints, disjoint
+
+ROOT = Path(__file__).resolve().parents[2]
+PLAN = 'reproduction/results/jev/run-20260918/plan.json'
+MULTICALL = 'experiments/jev-multicall-20260918/plan.json'
+CHALLENGE = 'experiments/falsification/challenge.json'
+SEEDS = [11, 23, 37, 53, 71]
+OBJECTIVES = ['accuracy', 'composite']
+
+
+def state(task, row):
+    if task == 'relation_support':
+        return {'claim': row['claim'], 'evidence': row['evidence']}
+    result = {k: row[k] for k in ('record_1', 'record_2')}
+    if row.get('context'):
+        result['context'] = row['context']
+    return result
+
+
+def identities(task, row):
+    return {row['group']} if task == 'relation_support' else set(row['identity_groups'])
+
+
+def example(task, row):
+    group = row.get('group') if task == 'relation_support' else digest(sorted(identities(task, row)))
+    return Example(row['id'], state(task, row), row['gold_label'], group)
+
+
+def load_task(task):
+    plan = read_json(ROOT / PLAN)
+    source = plan['tasks'][task]
+    # Stratification uses development labels only; the partition never changes between seeds.
+    train, validation = [], []
+    for label in sorted({r['gold_label'] for r in source['development']}):
+        rows = sorted([r for r in source['development'] if r['gold_label'] == label],
+                      key=lambda r: digest(['dspy-development-v1', r['id']]))
+        train.extend(rows[:len(rows)//2])
+        validation.extend(rows[len(rows)//2:])
+    raw = {'train': train, 'validation': validation,
+           'calibration': source['calibration'], 'test': source['evaluation'],
+           'demonstrations': source['demonstrations']}
+    # Pair hashes alone cannot detect a reused entity; check every constituent identity.
+    pools = {s: set().union(*(identities(task, r) for r in rows)) for s, rows in raw.items()}
+    for i, a in enumerate(pools):
+        for b in list(pools)[i+1:]:
+            if pools[a] & pools[b]:
+                raise ValueError('Source/entity leakage: '+a+' / '+b)
+    splits = {s: [example(task, r) for r in rows] for s, rows in raw.items()}
+    for i, a in enumerate(splits):
+        for b in list(splits)[i+1:]:
+            disjoint(fingerprints(splits[a]), fingerprints(splits[b]))
+    arm = 'evidence_contract' if task == 'relation_support' else 'identity_contract'
+    question = next(iter(plan['question_specs'][task][arm].values()))
+    config = JevConfig(task, question['instructions'], question['criteria'])
+    return config, splits, plan['question_specs'][task], source
+
+
+def panels(task):
+    """Return panel definitions; they are never supplied to optimizer feedback."""
+    config, splits, _, source = load_task(task)
+    fixtures = [Example(r['id'], state(task, r), r['gold_label'], r.get('group', r['id']))
+                for r in source['fixtures']]
+    result = {'test': splits['test'], 'fixtures': fixtures}
+    if task == 'relation_support':
+        result['semantic_challenge'] = [Example(r['id'], r['state'], r['gold'], r['pair'])
+                                       for r in read_json(ROOT / CHALLENGE)]
+        mc = read_json(ROOT / MULTICALL)['rows']
+        for split in ('development', 'test'):
+            result['multicall_'+split] = [example(task, r) for r in mc if r['split'] == split]
+    return result
+
+
+def inventory():
+    output = {}
+    for task in ('relation_support', 'entity_resolution'):
+        config, splits, specs, source = load_task(task)
+        output[task] = {'labels': list(config.criteria), 'legacy_arms': list(specs),
+                        'splits': {s: {'n':len(rows), 'labels':dict(Counter(r.label for r in rows)),
+                        'sha256':digest([asdict(r) for r in rows])} for s, rows in splits.items()},
+                        'panels': {s: {'n':len(rows), 'scorable':sum(r.label in config.criteria for r in rows)}
+                                   for s, rows in panels(task).items()}}
+    return output
diff --git a/graph_synthesis/dspy_benchmark/report.py b/graph_synthesis/dspy_benchmark/report.py
new file mode 100644
index 0000000..3b56fe0
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/report.py
@@ -0,0 +1,70 @@
+"""Aggregate completed runs only; no inference and no selection on test scores."""
+import argparse
+import csv
+import json
+import statistics
+from pathlib import Path
+from collections import defaultdict
+from .data import SEEDS
+from graph_synthesis.dspy_jev_optimizer.core import read_json, write_json
+
+
+def summarize(root):
+    rows, status = [], []
+    for seed in SEEDS:
+        directory = root/('seed-'+str(seed))
+        item = read_json(directory/'status.json') if (directory/'status.json').exists() else {'status':'not_run'}
+        status.append({'seed':seed,**item})
+        if item['status'] in ('completed','completed_baseline_only'):
+            rows.extend(read_json(directory/'metrics.json'))
+    write_json(root/'run-status.json',status)
+    groups = defaultdict(list)
+    for row in rows:
+        groups[(row['task'],row['panel'],row['arm'],row['calibration'])].append(row)
+    summary = []
+    for key,values in sorted(groups.items()):
+        record = dict(zip(('task','panel','arm','calibration'),key))
+        record['runs'] = len(values)
+        record['n_per_run'] = values[0]['n']
+        for metric in ('accuracy','macro_f1','mcc','brier','log_loss','ece'):
+            numbers = [r[metric] for r in values]
+            record[metric+'_mean'] = statistics.mean(numbers)
+            record[metric+'_sd'] = statistics.stdev(numbers) if len(numbers)>1 else None
+        summary.append(record)
+    write_json(root/'summary.json',summary)
+    if summary:
+        with (root/'summary.csv').open('w',newline='',encoding='utf-8') as f:
+            writer = csv.DictWriter(f,fieldnames=list(summary[0]));writer.writeheader();writer.writerows(summary)
+    complete = sum(s['status']=='completed' for s in status)
+    lines = ['# Repeated DSPy / Jev benchmark comparison','',
+             f'Completed full DSPy runs: **{complete}/{len(SEEDS)}**.','',
+             'Previously published observations are not relabeled as fresh runs. '
+             'A missing or failed run is not an accuracy score. Five evaluations on the same test set '
+             'are repeated model/search runs, not five independent datasets.','',
+             '| Seed | Status |','|---|---|']
+    lines += [f"| {s['seed']} | {s['status']} |" for s in status]
+    lines += ['', '## Primary test results', '',
+              '| Task | Arm | Calibration | Runs | Accuracy | Macro-F1 | Brier | Log loss | ECE |',
+              '|---|---|---|---:|---:|---:|---:|---:|---:|']
+    for r in summary:
+        if r['panel']=='test' and r['calibration'].endswith('_1.0'):
+            lines.append('| '+ ' | '.join([r['task'],r['arm'],r['calibration'],str(r['runs'])]+[
+                f"{r[k+'_mean']:.6f}" for k in ('accuracy','macro_f1','brier','log_loss','ece')])+' |')
+    if not summary:
+        lines += ['','**No completed live comparison is available.**']
+    lines += ['','## Interpretation boundaries','',
+        'Accuracy/composite search and post-hoc calibration are separate interventions. '
+        'Temperature scaling leaves predicted labels unchanged; bias/temperature may change labels. '
+        'Calibration uses only the original calibration split. Published synthetic fixtures and '
+        'the semantic challenge are exploratory transfer panels, not new independent gold labels. '
+        'Twelve uncertain entity fixtures have no label in the binary schema and are excluded from '
+        'accuracy denominators, but their predictions remain in the raw records.','',
+        'The implementation is the repository\'s custom DSPy proposal loop, not GEPA or MIPROv2. '
+        'No test-score winner is selected across seeds. Broad superiority or a global accuracy optimum '
+        'cannot be inferred from this finite search.']
+    (root/'RESULTS.md').write_text('\n'.join(lines)+'\n',encoding='utf-8')
+    return summary
+
+if __name__=='__main__':
+    p=argparse.ArgumentParser(description=__doc__);p.add_argument('--directory',type=Path,required=True)
+    summarize(p.parse_args().directory)
diff --git a/graph_synthesis/dspy_benchmark/requirements.txt b/graph_synthesis/dspy_benchmark/requirements.txt
new file mode 100644
index 0000000..dd6a367
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/requirements.txt
@@ -0,0 +1,3 @@
+-r ../dspy_jev_optimizer/requirements.txt
+numpy==2.3.5
+scipy==1.17.0
diff --git a/graph_synthesis/dspy_benchmark/run.py b/graph_synthesis/dspy_benchmark/run.py
new file mode 100644
index 0000000..9f62436
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/run.py
@@ -0,0 +1,269 @@
+"""Fresh, repeated benchmark execution. Never treats a replay as live evidence."""
+from __future__ import annotations
+import argparse
+import gzip
+import json
+import os
+import random
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import asdict
+from pathlib import Path
+import numpy as np
+from graph_synthesis.dspy_jev_optimizer.core import (
+    JevConfig, Evaluator, Cache, BudgetExhausted, digest, canonical, parse_json,
+    validate_answer, metrics, optimize, write_json, read_json, frozen_run)
+from graph_synthesis.dspy_jev_optimizer.providers import TypeSafeBackend, DSPyProposer
+from .data import ROOT, SEEDS, OBJECTIVES, load_task, panels, state, inventory
+from .calibration import fit, apply, evaluation
+
+
+class ParallelEvaluator(Evaluator):
+    """Only the caller thread touches SQLite and audit files."""
+    def __init__(self, *args, workers=4, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not 1 <= workers <= 8:
+            raise ValueError('workers must be 1..8')
+        self.workers = workers
+
+    def evaluate(self, config, rows, phase):
+        keys = [digest({'schema':1, 'backend':self.backend.identity,
+                        'config':config.fingerprint(), 'state':r.state}) for r in rows]
+        responses, missing = {}, {}
+        for key, row in zip(keys, rows):
+            cached = self.cache.get(key)
+            if cached is None:
+                missing.setdefault(key, row)
+            else:
+                responses[key] = cached
+        if len(missing) > self.max_calls-self.calls:
+            raise BudgetExhausted('Insufficient budget for a complete evaluation')
+        with ThreadPoolExecutor(max_workers=self.workers) as pool:
+            jobs = []
+            for key, row in missing.items():
+                self.calls += 1
+                self.event({'event':'request', 'key':key, 'phase':phase, 'state':row.state,
+                            'question':config.question(), 'backend':self.backend.identity})
+                jobs.append((key, pool.submit(self.backend.predict, config, parse_json(canonical(row.state)))))
+            errors = []
+            for key, job in jobs:
+                try:
+                    response = job.result()
+                    validate_answer(response,config.criteria)
+                    if response.get('model') != self.backend.identity['model']:
+                        raise ValueError('Pinned model drift')
+                    for token in self.usage:
+                        count = response.get('usage',{}).get(token,0)
+                        if type(count) is not int or count < 0:
+                            raise ValueError('Invalid usage')
+                        self.usage[token] += count
+                    self.cache.put(key,response)
+                    responses[key] = response
+                    self.event({'event':'response','key':key,'phase':phase,'response':response})
+                except Exception as exc:
+                    errors.append(type(exc).__name__)
+                    self.event({'event':'request_failed','key':key,'phase':phase,'error_type':type(exc).__name__})
+        if errors:
+            raise RuntimeError('Inference failures: '+','.join(sorted(set(errors))))
+        predictions = []
+        for key,row in zip(keys,rows):
+            response = responses[key]
+            validate_answer(response,config.criteria)
+            if response.get('model') != self.backend.identity['model']:
+                raise ValueError('Cached model drift')
+            if key not in missing:
+                self.cache_hits += 1
+                self.event({'event':'cache_hit','key':key,'phase':phase,'response':response})
+            predictions.append({'id':row.id, 'group_id':row.group_id or row.id, 'gold':row.label,
+                                'choice':response['choice'],'probabilities':response['probabilities'],
+                                'request_sha256':key})
+        scorable = [r for r in predictions if r['gold'] in config.criteria]
+        return metrics(scorable,config.criteria) if scorable else {}, predictions
+
+
+class LegacyBackend(TypeSafeBackend):
+    """Preserve native Choice/Noul formulations, not a rewritten surrogate."""
+    def __init__(self, model, task, arm, spec, demos, seed):
+        super().__init__(model, retries=2)
+        self.task, self.arm, self.spec, self.demos = task,arm,spec,demos
+        self.identity.update(legacy_arm=arm, spec_sha256=digest(spec), repeat_seed=seed,
+                             demonstration_sha256=digest(demos) if arm=='fewshot_contract' else None)
+
+    def predict(self, config, input_state):
+        from typesafe_sdk import Choice, Noul
+        questions = {k: (Choice(instructions=v['instructions'],criteria=v['criteria'])
+                         if v['type']=='choice' else Noul(instructions=v['instructions'])) for k,v in self.spec.items()}
+        current = input_state
+        if self.arm == 'fewshot_contract':
+            current = {'labeled_examples':self.demos, 'input':current}
+        start = time.perf_counter()
+        raw = self.client.system_one(model=self.identity['model'],state=current,questions=questions).model_dump(mode='json')
+        a = raw['answers']
+        if self.arm == 'conditional_nouls':
+            p = float(a[self.arm+'__support']['noul'])
+            r = float(a[self.arm+'__refute_given_not_support']['noul'])
+            probs = {'SUPPORTS':p,'REFUTES':(1-p)*r,'NOT_ENOUGH_INFO':(1-p)*(1-r)}
+        elif self.arm in ('baseline_noul','identity_noul'):
+            p = float(a[self.arm+'__decision']['noul'])
+            probs = {'same':p,'different':1-p}
+        else:
+            probs = a[self.arm+'__decision']['probabilities']
+        # Match the original research decoder's argmax probability decision.
+        choice = max(config.criteria, key=lambda k:probs[k])
+        return {'choice':choice,'probabilities':probs,'model':raw['model'], 'usage':raw.get('usage',{}),
+                'latency_seconds':time.perf_counter()-start,'raw_response':raw,
+                'request':{'model':self.identity['model'],'state':current,'questions':self.spec}}
+
+
+class LoggedProposer:
+    def __init__(self, model, output, seed):
+        self.inner = DSPyProposer(model)
+        self.output = output
+        self.identity = {**self.inner.identity,'repeat_seed':seed,
+                         'seed_scope':'Python/NumPy and feedback order; remote sampling is not guaranteed deterministic'}
+
+    def propose(self, config, feedback, iteration, history):
+        proposal = self.inner.propose(config,feedback,iteration,history)
+        records = self.inner.lm.history
+        last = records[-1] if records else {}
+        # Whitelist metadata; never serialize provider arguments or authentication headers.
+        row = {'iteration':iteration,'input_config':asdict(config),'training_feedback':feedback,
+               'candidate':proposal,'usage':last.get('usage'), 'cost_usd':last.get('cost')}
+        self.output.parent.mkdir(parents=True,exist_ok=True)
+        with self.output.open('a',encoding='utf-8') as f:
+            f.write(canonical(row)+'\n')
+        return proposal
+
+
+def provider_model():
+    configured = os.environ.get('DSPY_PROPOSER_MODEL','').strip()
+    if configured:
+        return configured
+    for key,model in [('OPENROUTER_API_KEY','openrouter/anthropic/claude-sonnet-4.5'),
+                      ('ANTHROPIC_API_KEY','anthropic/claude-sonnet-4-5-20250929'),
+                      ('OPENAI_API_KEY','openai/gpt-4.1')]:
+        if os.environ.get(key,'').strip():
+            return model
+    return None
+
+
+def save_predictions(path, rows):
+    path.parent.mkdir(parents=True,exist_ok=True)
+    path.write_bytes(gzip.compress(canonical(rows).encode(),mtime=0))
+
+
+def execute(output: Path, seed: int, *, iterations=8, baseline_only=False, model='jev-1.13.0', workers=4):
+    if seed not in SEEDS or not 0 <= iterations <= 12:
+        raise ValueError('Unregistered seed or iteration budget')
+    if output.exists():
+        raise FileExistsError('Use a new run directory; do not overwrite an experiment')
+    proposer_model = provider_model()
+    output.mkdir(parents=True)
+    if not os.environ.get('TYPESAFE_API_KEY') or (not proposer_model and not baseline_only):
+        write_json(output/'status.json', {'status':'blocked_missing_credentials',
+            'typesafe_key_present':bool(os.environ.get('TYPESAFE_API_KEY')),
+            'proposer_model_configured':bool(proposer_model),'live_calls':0})
+        raise RuntimeError('Live benchmark requires TypeSafe and a generative proposal provider')
+    random.seed(seed)
+    np.random.seed(seed)
+    protocol = {'seed':seed,'iterations':iterations,'objectives':OBJECTIVES,'model':model,
+                'proposer_model':proposer_model,'baseline_only':baseline_only, 'workers':workers,
+                'inventory':inventory(),'source_sha256':{
+                    p.name:__import__('hashlib').sha256(p.read_bytes()).hexdigest() for p in Path(__file__).parent.glob('*.py')},
+                'execution':'live_provider_calls',
+                'note':'Historically examined public test sets; held out only from the current optimization.'}
+    write_json(output/'protocol.json',protocol)
+    all_metrics,usage = [],[]
+    try:
+        for task in ('relation_support','entity_resolution'):
+            cfg,splits,specs,source = load_task(task)
+            train = list(splits['train'])
+            random.Random(seed).shuffle(train)
+            task_dir = output/task
+            task_dir.mkdir()
+            cache = Cache(task_dir/'calls.sqlite3')
+            backend = TypeSafeBackend(model,retries=2)
+            backend.identity['repeat_seed'] = seed
+            targets = []
+            try:
+                # Freeze every search before this task's calibration or test inference.
+                if not baseline_only:
+                    for objective in OBJECTIVES:
+                        search = task_dir/('search-'+objective)
+                        ev = ParallelEvaluator(backend,cache,search,2000,workers=workers)
+                        proposer = LoggedProposer(proposer_model, task_dir/('proposals-'+objective+'.jsonl'),seed)
+                        freeze = optimize(cfg,train,splits['validation'],proposer,ev,search,
+                                          iterations=iterations,selection_metric=objective,failure_sample=12,patience=0)
+                        frozen_run(search)
+                        targets.append(('dspy_'+objective,JevConfig.from_dict(freeze['champion']),backend))
+                        usage.append({'task':task,'stage':'search_'+objective,**ev.summary()})
+                demos = [{'input':state(task,r),'answer':r['gold_label']} for r in source['demonstrations']]
+                controls = []
+                for arm,spec in specs.items():
+                    control = LegacyBackend(model,task,arm,spec,demos,seed)
+                    controls.append(control)
+                    targets.append(('without_dspy_'+arm,cfg,control))
+                write_json(task_dir/'frozen-targets.json', {'targets':{name:asdict(c) for name,c,_ in targets},
+                                                         'protocol_sha256':digest(protocol)})
+                frozen_calibration = []
+                # All calibration parameters are persisted before ANY test prediction in this task.
+                for name,config,target_backend in targets:
+                    target_dir = task_dir/name
+                    ev = ParallelEvaluator(target_backend,cache,target_dir,12000,workers=workers)
+                    _,cal = ev.evaluate(config,splits['calibration'],'calibration')
+                    fits = {}
+                    for fraction in (.25,.5,1.):
+                        ranked = sorted(cal,key=lambda r:digest(['calibration-prefix-v1',seed,r['group_id']]))
+                        groups = list(dict.fromkeys(r['group_id'] for r in ranked))
+                        selected = set(groups[:max(1,int(len(groups)*fraction))])
+                        subset = [r for r in ranked if r['group_id'] in selected]
+                        for method in ('raw','temperature','bias_temperature'):
+                            if method=='raw' and fraction!=1.:
+                                continue
+                            fits[method+'_'+str(fraction)] = fit(subset,list(config.criteria),method)
+                    write_json(target_dir/'calibration.json',fits)
+                    save_predictions(target_dir/'calibration-predictions.json.gz',cal)
+                    frozen_calibration.append((name,config,ev,fits))
+                write_json(task_dir/'calibration-freeze.json',{
+                    name:digest(fits) for name,_,_,fits in frozen_calibration})
+                for name,config,ev,fits in frozen_calibration:
+                    for panel,rows in panels(task).items():
+                        _,predictions = ev.evaluate(config,rows,panel)
+                        save_predictions(task_dir/name/(panel+'-predictions.json.gz'),predictions)
+                        for fit_name,fitted in fits.items():
+                            transformed = apply(predictions,fitted)
+                            all_metrics.append({'seed':seed,'task':task,'arm':name,'panel':panel,
+                                'calibration':fit_name,'n_calibration':fitted['n_calibration'],
+                                **evaluation(transformed,config.criteria)})
+                        write_json(output/'metrics.json',all_metrics)
+                    usage.append({'task':task,'stage':name,**ev.summary()})
+                    print(json.dumps({'seed':seed,'task':task,'arm':name,'status':'evaluated'}),flush=True)
+                for control in controls:
+                    control.close()
+            finally:
+                backend.close()
+                cache.close()
+            write_json(output/'usage.json',usage)
+        write_json(output/'status.json',{'status':'completed_baseline_only' if baseline_only else 'completed',
+            'live_logical_calls':sum(r['logical_calls'] for r in usage), 'protocol_sha256':digest(protocol)})
+        return all_metrics
+    except Exception as exc:
+        write_json(output/'usage.json',usage)
+        write_json(output/'status.json',{'status':'failed','error_type':type(exc).__name__,
+                                       'note':'Partial observations are not complete benchmark results.'})
+        raise
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--output',type=Path,required=True)
+    parser.add_argument('--seed',type=int,choices=SEEDS,required=True)
+    parser.add_argument('--iterations',type=int,default=8)
+    parser.add_argument('--workers',type=int,default=4)
+    parser.add_argument('--baseline-only',action='store_true')
+    args = parser.parse_args()
+    execute(args.output,args.seed,iterations=args.iterations,workers=args.workers,baseline_only=args.baseline_only)
+
+if __name__=='__main__':
+    main()
diff --git a/graph_synthesis/dspy_benchmark/tests/test_benchmark.py b/graph_synthesis/dspy_benchmark/tests/test_benchmark.py
new file mode 100644
index 0000000..9199bd2
--- /dev/null
+++ b/graph_synthesis/dspy_benchmark/tests/test_benchmark.py
@@ -0,0 +1,147 @@
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+import numpy as np
+from graph_synthesis.dspy_jev_optimizer.core import Example,JevConfig,Cache,BudgetExhausted,metrics,read_json
+from graph_synthesis.dspy_benchmark.data import inventory,load_task,panels,SEEDS
+from graph_synthesis.dspy_benchmark.calibration import fit,apply,evaluation,paired_interval
+from graph_synthesis.dspy_benchmark.run import ParallelEvaluator,provider_model,execute
+from graph_synthesis.dspy_benchmark.report import summarize
+
+LABELS={'a':'first','b':'second'}
+CFG=JevConfig('task','classify',LABELS)
+
+def sample(n=20):
+    return [{'id':str(i),'group_id':str(i//2),'gold':'a' if i%4 else 'b',
+             'choice':'a','probabilities':{'a':.98,'b':.02}} for i in range(n)]
+
+class FakeBackend:
+    def __init__(self,seed=1,bad=False):
+        self.identity={'model':'mock-test-only','seed':seed};self.count=0;self.bad=bad
+    def predict(self,config,state):
+        self.count+=1
+        return {'model':'drift' if self.bad else self.identity['model'],'choice':'a',
+                'probabilities':{'a':.8,'b':.2},'usage':{'input_tokens':10,'output_tokens':0}}
+
+class Tests(unittest.TestCase):
+    def test_inventory_all_registered_panels(self):
+        v=inventory()
+        self.assertEqual(v['relation_support']['splits']['test']['n'],339)
+        self.assertEqual(v['entity_resolution']['splits']['test']['n'],413)
+        self.assertEqual(v['relation_support']['panels']['semantic_challenge']['n'],48)
+        self.assertEqual(v['relation_support']['panels']['multicall_test']['n'],263)
+    def test_train_validation_calibration_are_distinct(self):
+        for task in ('relation_support','entity_resolution'):
+            _,s,_,_=load_task(task)
+            self.assertEqual((len(s['train']),len(s['validation'])),(29,31))
+            for a in s:
+                for b in s:
+                    if a!=b:self.assertFalse({r.id for r in s[a]} & {r.id for r in s[b]})
+    def test_all_legacy_formulations_are_retained(self):
+        self.assertEqual(len(load_task('relation_support')[2]),4)
+        self.assertEqual(len(load_task('entity_resolution')[2]),4)
+    def test_uncertain_fixtures_are_not_silently_removed(self):
+        cfg,_,_,_=load_task('entity_resolution')
+        rows=panels('entity_resolution')['fixtures']
+        self.assertEqual(len(rows),100)
+        self.assertEqual(sum(r.label not in cfg.criteria for r in rows),12)
+    def test_inventory_deterministic(self):
+        self.assertEqual(inventory(),inventory())
+    def test_state_does_not_include_labels(self):
+        for task in ('relation_support','entity_resolution'):
+            for rows in load_task(task)[1].values():
+                for r in rows:self.assertFalse({'gold_label','rationale_sentence_ids','label'} & set(r.state))
+    def test_temperature_preserves_labels(self):
+        rows=sample();model=fit(rows,LABELS)
+        self.assertEqual([r['choice'] for r in apply(rows,model)],[r['choice'] for r in rows])
+    def test_temperature_reduces_overconfident_calibration_loss(self):
+        rows=sample();after=apply(rows,fit(rows,LABELS))
+        self.assertLess(metrics(after,LABELS)['log_loss'],metrics(rows,LABELS)['log_loss'])
+    def test_raw_is_unchanged(self):
+        rows=sample();self.assertEqual(rows,apply(rows,fit(rows,LABELS,'raw')))
+    def test_bias_temperature_is_normalized(self):
+        for row in apply(sample(),fit(sample(),LABELS,'bias_temperature')):
+            self.assertAlmostEqual(sum(row['probabilities'].values()),1)
+    def test_fit_serializes(self):
+        model=fit(sample(),LABELS)
+        self.assertEqual(model,json.loads(json.dumps(model)))
+    def test_invalid_calibration_distribution(self):
+        rows=sample();rows[0]['probabilities']['a']=4
+        with self.assertRaises(ValueError):fit(rows,LABELS)
+    def test_empty_calibration_rejected(self):
+        with self.assertRaises(ValueError):fit([],LABELS)
+    def test_unscorable_excluded_from_denominator(self):
+        rows=sample()+[{'id':'u','gold':'uncertain','choice':'a','probabilities':{'a':.5,'b':.5}}]
+        r=evaluation(rows,LABELS)
+        self.assertEqual((r['n'],r['unscorable_count']),(20,1))
+    def test_risk_empty_acceptance_is_not_zero_error_claim(self):
+        rows=sample();r=evaluation(rows,LABELS)
+        self.assertIsNone(r['selective'][-1]['risk'])
+    def test_identical_paired_interval_zero(self):
+        r=paired_interval(sample(),sample(),LABELS,'brier',repeats=10)
+        self.assertEqual((r['delta'],r['low'],r['high']),(0.,0.,0.))
+    def test_unpaired_intervals_rejected(self):
+        with self.assertRaises(ValueError):paired_interval(sample(),list(reversed(sample())),LABELS,'accuracy')
+    def test_five_registered_seeds(self):
+        self.assertEqual(len(SEEDS),5);self.assertEqual(len(set(SEEDS)),5)
+    def test_cache_never_crosses_repeat_namespace(self):
+        with tempfile.TemporaryDirectory() as d:
+            p=Path(d);cache=Cache(p/'c.sqlite');r=[Example('one',{'x':1},'a')]
+            a=FakeBackend(1);e=ParallelEvaluator(a,cache,p/'a',10)
+            e.evaluate(CFG,r,'test');e.evaluate(CFG,r,'test');self.assertEqual(a.count,1)
+            b=FakeBackend(2);ParallelEvaluator(b,cache,p/'b',10).evaluate(CFG,r,'test')
+            self.assertEqual(b.count,1);cache.close()
+    def test_budget_prevents_partial_evaluation(self):
+        with tempfile.TemporaryDirectory() as d:
+            p=Path(d);c=Cache(p/'c');b=FakeBackend();e=ParallelEvaluator(b,c,p,1)
+            with self.assertRaises(BudgetExhausted):e.evaluate(CFG,[Example('1',{'x':1},'a'),Example('2',{'x':2},'a')],'test')
+            self.assertEqual(b.count,0);c.close()
+    def test_model_drift_fails_closed(self):
+        with tempfile.TemporaryDirectory() as d:
+            p=Path(d);c=Cache(p/'c');e=ParallelEvaluator(FakeBackend(bad=True),c,p,2)
+            with self.assertRaises(RuntimeError):e.evaluate(CFG,[Example('1',{'x':1},'a')],'test')
+            c.close()
+    def test_missing_keys_block_before_model_calls(self):
+        with tempfile.TemporaryDirectory() as d,patch.dict(os.environ,{},clear=True):
+            p=Path(d)/'run'
+            with self.assertRaises(RuntimeError):execute(p,11)
+            self.assertEqual(read_json(p/'status.json')['live_calls'],0)
+    def test_missing_results_are_not_zero_accuracy(self):
+        with tempfile.TemporaryDirectory() as d:
+            self.assertEqual(summarize(Path(d)),[])
+            self.assertEqual(read_json(Path(d)/'run-status.json')[0]['status'],'not_run')
+    def test_provider_is_explicit_not_magic_fallback(self):
+        with patch.dict(os.environ,{},clear=True):self.assertIsNone(provider_model())
+    def test_existing_output_refused(self):
+        with tempfile.TemporaryDirectory() as d:
+            with self.assertRaises(FileExistsError):execute(Path(d),11)
+
+    def test_full_comparison_orchestration_with_test_doubles(self):
+        events=[]
+        class Backend:
+            def __init__(self,*args,**kwargs): self.identity={'model':'jev-1.13.0','test_double':True}
+            def close(self): pass
+            def predict(self,config,state):
+                events.append(state['phase'])
+                return {'model':'jev-1.13.0','choice':'a','probabilities':{'a':.7,'b':.3},'usage':{'input_tokens':2,'output_tokens':0}}
+        class Proposer:
+            def __init__(self,*args,**kwargs): self.identity={'test_double':True}
+            def propose(self,cfg,feedback,iteration,history):
+                return {'task':cfg.task,'instructions':cfg.instructions+' revised','criteria':cfg.criteria}
+        def rows(phase,n):
+            return [Example(phase+str(i),{'phase':phase,'i':i},'a' if i%2 else 'b',phase+str(i)) for i in range(n)]
+        splits={s:rows(s,6) for s in ('train','validation','calibration')}
+        from graph_synthesis.dspy_benchmark import run
+        with tempfile.TemporaryDirectory() as d, patch.dict(os.environ,{'TYPESAFE_API_KEY':'test-only','DSPY_PROPOSER_MODEL':'test-only'},clear=True), patch.object(run,'load_task',return_value=(CFG,splits,{'baseline_choice':{}},{'demonstrations':[]})), patch.object(run,'panels',return_value={'test':rows('test',6)}), patch.object(run,'TypeSafeBackend',Backend), patch.object(run,'LegacyBackend',Backend), patch.object(run,'LoggedProposer',Proposer):
+            output=Path(d)/'seed-11'
+            result=execute(output,11,iterations=1,workers=2)
+            self.assertEqual(read_json(output/'status.json')['status'],'completed')
+            self.assertEqual(len(result),2*3*7)
+            self.assertEqual({r['arm'] for r in result},{'dspy_accuracy','dspy_composite','without_dspy_baseline_choice'})
+            self.assertTrue((output/'relation_support'/'calibration-freeze.json').exists())
+            self.assertGreater(events.index('test'),events.index('calibration'))
+
+if __name__=='__main__':unittest.main()