Prepare repeated DSPy/Jev comparisons; record offline reruns and live blockers

.github/workflows/dspy-comparison.yml

@@ -0,0 +1,93 @@
+name: Repeated DSPy Jev benchmarks
+on:
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  offline:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: '3.12'
+      - run: python -m pip install numpy==2.3.5 scipy==1.17.0
+      - run: python -B -m unittest discover -s graph_synthesis/dspy_benchmark/tests -v
+      - run: python -B -m graph_synthesis.dspy_jev_optimizer.verify
+  live:
+    if: github.event_name == 'workflow_dispatch' && github.repository == 'CompleteDotTech/paper-package'
+    needs: offline
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        seed: [11, 23, 37, 53, 71]
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: '3.12'
+      - run: python -m pip install -r graph_synthesis/dspy_benchmark/requirements.txt
+      - name: Pin resolved dependency record
+        run: |
+          mkdir -p '${{ runner.temp }}/comparison'
+          python -m pip freeze > '${{ runner.temp }}/comparison/environment-seed-${{ matrix.seed }}.txt'
+      - name: Live repeated comparison
+        env:
+          TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          DSPY_PROPOSER_MODEL: ${{ vars.DSPY_PROPOSER_MODEL }}
+          LITELLM_LOCAL_MODEL_COST_MAP: 'True'
+        run: python -B -m graph_synthesis.dspy_benchmark.run --seed ${{ matrix.seed }} --iterations 8 --workers 4 --output '${{ runner.temp }}/comparison/seed-${{ matrix.seed }}'
+      - name: Generate report without inventing missing runs
+        if: always()
+        run: python -B -m graph_synthesis.dspy_benchmark.report --directory '${{ runner.temp }}/comparison'
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        if: always()
+        with:
+          name: dspy-jev-seed-${{ matrix.seed }}
+          path: ${{ runner.temp }}/comparison/
+          retention-days: 14
+          if-no-files-found: warn
+  aggregate:
+    if: always() && needs.offline.result == 'success' && github.repository == 'CompleteDotTech/paper-package'
+    needs: [offline, live]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: '3.12'
+      - run: mkdir -p '${{ runner.temp }}/comparison'
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        continue-on-error: true
+        with:
+          pattern: dspy-jev-seed-*
+          merge-multiple: true
+          path: ${{ runner.temp }}/comparison
+      - name: Aggregate all available seed results
+        run: python -B -m graph_synthesis.dspy_benchmark.report --directory '${{ runner.temp }}/comparison'
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: dspy-jev-aggregate
+          path: |
+            ${{ runner.temp }}/comparison/RESULTS.md
+            ${{ runner.temp }}/comparison/run-status.json
+            ${{ runner.temp }}/comparison/summary.json
+            ${{ runner.temp }}/comparison/summary.csv
+            ${{ runner.temp }}/comparison/baseline-only-summary.json
+            ${{ runner.temp }}/comparison/baseline-only-summary.csv
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/dspy-jev-preflight.yml

@@ -0,0 +1,74 @@
+name: DSPy Jev live preflight
+# Manual only. Do not bypass an Actions approval or environment protection gate.
+on:
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  preflight:
+    if: github.repository == 'CompleteDotTech/paper-package'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: '3.12'
+      - run: python -m pip install -r graph_synthesis/dspy_jev_optimizer/requirements.txt
+      - name: Bounded live connectivity test
+        env:
+          TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          DSPY_PROPOSER_MODEL: ${{ vars.DSPY_PROPOSER_MODEL }}
+          LITELLM_LOCAL_MODEL_COST_MAP: 'True'
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          from graph_synthesis.dspy_jev_optimizer.core import JevConfig
+          from graph_synthesis.dspy_jev_optimizer.providers import TypeSafeBackend, DSPyProposer
+          presence = {k: bool(os.environ.get(k, '').strip()) for k in ('TYPESAFE_API_KEY','OPENROUTER_API_KEY','OPENAI_API_KEY','ANTHROPIC_API_KEY')}
+          result = {'credential_present': presence, 'status': 'started'}
+          out = Path(os.environ['RUNNER_TEMP']) / 'preflight.json'
+          cfg = JevConfig('Evidence classification', 'Does the evidence support the claim?', {'supports':'Evidence establishes the claim.', 'refutes':'Evidence contradicts the claim.', 'unknown':'Evidence is insufficient.'})
+          try:
+              model = os.environ.get('DSPY_PROPOSER_MODEL','').strip()
+              if not model:
+                  model = ('openrouter/anthropic/claude-sonnet-4.5' if presence['OPENROUTER_API_KEY'] else 'anthropic/claude-sonnet-4-5-20250929' if presence['ANTHROPIC_API_KEY'] else 'openai/gpt-4.1' if presence['OPENAI_API_KEY'] else '')
+              proposer_key = next((key for prefix, key in (
+                  ('openrouter/', 'OPENROUTER_API_KEY'),
+                  ('anthropic/', 'ANTHROPIC_API_KEY'),
+                  ('openai/', 'OPENAI_API_KEY')) if model.startswith(prefix)), None)
+              if not presence['TYPESAFE_API_KEY'] or not proposer_key or not presence[proposer_key]:
+                  result['status'] = 'credentials_missing'
+              else:
+                  result['proposer_model'] = model
+                  backend = TypeSafeBackend('jev-1.13.0', retries=0)
+                  try:
+                      answer = backend.predict(cfg, {'claim':'A precedes B.', 'evidence':['A precedes B.']})
+                  finally:
+                      backend.close()
+                  result['jev'] = {'model':answer['model'], 'choice':answer['choice'], 'usage':answer['usage']}
+                  proposal = DSPyProposer(model).propose(cfg, [], 1, [])
+                  JevConfig.from_dict(proposal, cfg)
+                  result['status'] = 'passed'
+          except Exception as exc:
+              result['status'] = 'failed'
+              result['error_type'] = type(exc).__name__
+          finally:
+              out.write_text(json.dumps(result, indent=2)+'\n')
+              print(json.dumps(result))
+          if result['status'] != 'passed':
+              raise SystemExit(1)
+          PY
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        if: always()
+        with:
+          name: dspy-jev-live-preflight
+          path: ${{ runner.temp }}/preflight.json
+          retention-days: 14
+          if-no-files-found: warn

DSPY_BENCHMARKS.md

@@ -0,0 +1,14 @@
+# DSPy versus Jev benchmark execution
+
+[Comparison protocol and runner](graph_synthesis/dspy_benchmark/README.md) ·
+[Actual execution status](experiments/dspy-comparison-20260918/STATUS.md) ·
+[Archived-output calibration check](experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md)
+
+The live comparison has not completed: CI identified a missing mapped generative
+proposal-provider key, then marked the next diagnostic `action_required` before
+starting jobs. No approval restriction was bypassed and no DSPy gain is claimed.
+
+The comparison code registers five isolated repeats, two search objectives, eight
+native legacy formulations and seven calibration settings per frozen arm. New
+local regression and archived-replay checks are recorded separately from model
+performance. Historical evidence and the root immutable manifest are unchanged.

experiments/dspy-comparison-20260918/STATUS.md

@@ -0,0 +1,72 @@
+# DSPy / Jev comparison: actual execution status
+
+**Live with/without-DSPy comparison: NOT COMPLETED. No new Jev inference calls and no new DSPy inference runs were completed.**
+
+## Live execution blockers
+
+[Preflight run 35355158096](https://github.com/CompleteDotTech/paper-package/actions/runs/35355158096) found the mapped TypeSafe key populated, but none of `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, or `ANTHROPIC_API_KEY`. It stopped before calling Jev. This does not establish whether differently named or environment-scoped provider credentials exist, or whether the TypeSafe key is valid.
+
+[Diagnostic run 35355491970](https://github.com/CompleteDotTech/paper-package/actions/runs/35355491970) was marked `action_required` with zero jobs. Its gate was left intact. Broad secret-context diagnostics were removed; preflight and live runs are manual-only and use explicitly named keys. Secret values are not in these reports.
+
+## Executed offline checks
+
+**1,092 tests passed; 6 skipped; 16 test suites completed. All 11 deterministic experiment replays passed.** The original 161-file inventory and the existing optimizer inventory also passed verification.
+
+Four SDK contract tests were skipped because SDK packages were not installed in this local runtime; two archived reproduction tests also reported skips. Skips are not counted as passes. An initial reproduction invocation lacked PYTHONPATH; the corrected invocation passed. A later combined wrapper timed out on two replay processes; the individual replay commands recorded below had already passed. Neither failed attempt is treated as an additional successful test.
+
+| Check | Tests discovered | Skipped | Result |
+|---|---:|---:|---|
+| replay-adaptive | 0 | 0 | passed |
+| replay-assumption_aware | 0 | 0 | passed |
+| replay-followup | 0 | 0 | passed |
+| replay-frontier | 0 | 0 | passed |
+| replay-novel_mechanisms | 0 | 0 | passed |
+| replay-reliability | 0 | 0 | passed |
+| replay-risk_control | 0 | 0 | passed |
+| replay-source_structural | 0 | 0 | passed |
+| replay-structural | 0 | 0 | passed |
+| replay-theory_suite | 0 | 0 | passed |
+| replay-uncertainty | 0 | 0 | passed |
+| test-graph_synthesis-adaptive-tests | 64 | 0 | passed |
+| test-graph_synthesis-assumption_aware-tests | 47 | 0 | passed |
+| test-graph_synthesis-corpus-tests | 32 | 0 | passed |
+| test-graph_synthesis-dspy_benchmark-tests | 26 | 0 | passed |
+| test-graph_synthesis-dspy_jev_optimizer-tests | 35 | 4 | passed |
+| test-graph_synthesis-followup-tests | 67 | 0 | passed |
+| test-graph_synthesis-frontier-tests | 75 | 0 | passed |
+| test-graph_synthesis-novel_mechanisms-tests | 92 | 0 | passed |
+| test-graph_synthesis-reliability-tests | 63 | 0 | passed |
+| test-graph_synthesis-risk_control-tests | 66 | 0 | passed |
+| test-graph_synthesis-source_structural-tests | 86 | 0 | passed |
+| test-graph_synthesis-structural-tests | 36 | 0 | passed |
+| test-graph_synthesis-tests | 200 | 0 | passed |
+| test-graph_synthesis-theory_suite-tests | 37 | 0 | passed |
+| test-graph_synthesis-uncertainty-tests | 36 | 0 | passed |
+| test-reproduction-tests | 136 | 2 | passed |
+| verify-frozen | 0 | 0 | passed |
+| verify-optimizer | 0 | 0 | passed |
+
+The machine-readable [validation record](offline-validation-summary.json) includes per-suite status and counts; the downloadable audit bundle also includes commands, durations and log hashes. These runs disabled socket connections and used no inference keys. They reproduce saved observations and algorithm fixtures, not newly optimized model predictions.
+
+## Separate archived calibration analysis
+
+[Archived calibration results](archive-calibration/RESULTS.md) refit fixed calibrators on the original calibration split and score saved test responses. They contain zero fresh inference and zero DSPy runs. Invalid archived response exclusions and denominators are explicit. Do not attribute these results to DSPy.
+
+## Ready-to-run live comparison
+
+[Protocol and commands](../../graph_synthesis/dspy_benchmark/README.md) cover five repeats, two DSPy search objectives, eight original native formulations, seven calibration settings and seven task/panel combinations. The code was tested with controlled doubles, not substituted for live inference.
+
+To execute live, map an authorized generative-provider credential and model into the workflow, resolve the repository Actions approval requirement, and manually run the live workflow. Original manuscripts, predictions and the immutable root manifest remain unchanged.
+
+## Offline integration correction
+
+The comparison workflow is now manual-only and has its own filename so it can
+coexist with the separate DSPy optimizer study. A full comparison requires at
+least one proposal iteration and verifies the configured proposer's matching
+credential before constructing a Jev target backend. Each completed seed writes
+a hash inventory of its run files; the downstream report verifies those files,
+retains missing/invalid seed statuses, and marks the five-seed matrix incomplete
+until all five completed artifacts validate. Baseline-only observations have a
+separate summary and cannot fill the primary DSPy table. These corrections were
+checked with offline tests and test doubles. The live comparison remains blocked
+and no new provider calls or DSPy results were produced by this correction.

experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md

@@ -0,0 +1,24 @@
+# Archived-output calibration check
+
+**Zero fresh model calls. Zero DSPy runs. This is not the requested live comparison.** Invalid archived responses are excluded from probability metrics and listed in summary.json; accuracy is conditional on a valid response, not end-to-end accuracy.
+
+Calibrators were refit using only the archived calibration split. All three fixed methods are shown; none was selected using test performance. The archived evaluation split was historically examined, so these are exploratory results.
+
+| Task | Historical arm | Calibration | n | Accuracy | Macro-F1 | Brier | Log loss | ECE |
+|---|---|---|---:|---:|---:|---:|---:|---:|
+| relation_support | baseline_choice | raw | 338 | 0.846154 | 0.844335 | 0.234192 | 1.240206 | 0.074408 |
+| relation_support | baseline_choice | temperature | 338 | 0.846154 | 0.844335 | 0.272597 | 0.569980 | 0.120948 |
+| relation_support | baseline_choice | bias_temperature | 338 | 0.828402 | 0.817346 | 0.261256 | 0.532340 | 0.096751 |
+| relation_support | fewshot_contract | raw | 337 | 0.860534 | 0.859685 | 0.224429 | 1.320742 | 0.068843 |
+| relation_support | fewshot_contract | temperature | 337 | 0.860534 | 0.859685 | 0.256077 | 0.632292 | 0.131532 |
+| relation_support | fewshot_contract | bias_temperature | 337 | 0.759644 | 0.756568 | 0.282491 | 0.574257 | 0.107833 |
+| entity_resolution | baseline_noul | raw | 413 | 0.980630 | 0.960452 | 0.040651 | 0.106744 | 0.064504 |
+| entity_resolution | baseline_noul | temperature | 413 | 0.980630 | 0.960452 | 0.031287 | 0.065248 | 0.013925 |
+| entity_resolution | baseline_noul | bias_temperature | 413 | 0.983051 | 0.965638 | 0.028751 | 0.059969 | 0.013186 |
+| entity_resolution | fewshot_contract | raw | 413 | 0.987893 | 0.976434 | 0.018150 | 0.033291 | 0.015036 |
+| entity_resolution | fewshot_contract | temperature | 413 | 0.987893 | 0.976434 | 0.015705 | 0.026191 | 0.009182 |
+| entity_resolution | fewshot_contract | bias_temperature | 413 | 0.987893 | 0.975793 | 0.017257 | 0.028161 | 0.003360 |
+
+Temperature scaling preserves decisions; its accuracy differences must be zero. Bias/temperature can change decisions. A probability-quality improvement is not necessarily an accuracy improvement.
+
+Source SHA-256: `9fc0f0a345f6fbbd311759a5143c1d1be62313b72e26acd54e3ae289e040d76f`

experiments/dspy-comparison-20260918/archive-calibration/summary.json

@@ -0,0 +1 @@
+{"evidence":"exploratory_saved_response_reanalysis","fresh_model_calls":0,"dspy_runs":0,"source":"experiments/jev-rerun-20260918/predictions.jsonl","source_sha256":"9fc0f0a345f6fbbd311759a5143c1d1be62313b72e26acd54e3ae289e040d76f","full_results_sha256":"8fe4c82a4b2d0807c883b4d4abbec830403f3386a14c8d96530f436253635c9f","excluded_invalid_responses":[{"task":"relation_support","arm":"baseline_choice","split":"evaluation","id":"scifact-213-13625993","error":"Probabilities must sum to one"},{"task":"relation_support","arm":"fewshot_contract","split":"calibration","id":"scifact-1258-12040627","error":"choice must be a highest-probability option"},{"task":"relation_support","arm":"fewshot_contract","split":"evaluation","id":"scifact-1202-3475317","error":"Probabilities must sum to one"},{"task":"relation_support","arm":"fewshot_contract","split":"evaluation","id":"scifact-1088-37549932","error":"choice must be a highest-probability option"}],"columns":["task","arm","calibration","n","accuracy","macro_f1","brier","log_loss","ece"],"results":[["relation_support","baseline_choice","raw",338,0.846154,0.844335,0.234192,1.240206,0.074408],["relation_support","baseline_choice","temperature",338,0.846154,0.844335,0.272597,0.56998,0.120948],["relation_support","baseline_choice","bias_temperature",338,0.828402,0.817346,0.261256,0.53234,0.096751],["relation_support","fewshot_contract","raw",337,0.860534,0.859685,0.224429,1.320742,0.068843],["relation_support","fewshot_contract","temperature",337,0.860534,0.859685,0.256077,0.632292,0.131532],["relation_support","fewshot_contract","bias_temperature",337,0.759644,0.756568,0.282491,0.574257,0.107833],["entity_resolution","baseline_noul","raw",413,0.98063,0.960452,0.040651,0.106744,0.064504],["entity_resolution","baseline_noul","temperature",413,0.98063,0.960452,0.031287,0.065248,0.013925],["entity_resolution","baseline_noul","bias_temperature",413,0.983051,0.965638,0.028751,0.059969,0.013186],["entity_resolution","fewshot_contract","raw",413,0.987893,0.976434,0.01815,0.033291,0.015036],["entity_resolution","fewshot_contract","temperature",413,0.987893,0.976434,0.015705,0.026191,0.009182],["entity_resolution","fewshot_contract","bias_temperature",413,0.987893,0.975793,0.017257,0.028161,0.00336]]}