diff --git a/.github/workflows/dspy-comparison.yml b/.github/workflows/dspy-comparison.yml new file mode 100644 index 0000000..40a5047 --- /dev/null +++ b/.github/workflows/dspy-comparison.yml @@ -0,0 +1,93 @@ +name: Repeated DSPy Jev benchmarks +on: + workflow_dispatch: +permissions: + contents: read +jobs: + offline: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + - run: python -m pip install numpy==2.3.5 scipy==1.17.0 + - run: python -B -m unittest discover -s graph_synthesis/dspy_benchmark/tests -v + - run: python -B -m graph_synthesis.dspy_jev_optimizer.verify + live: + if: github.event_name == 'workflow_dispatch' && github.repository == 'CompleteDotTech/paper-package' + needs: offline + strategy: + fail-fast: false + max-parallel: 2 + matrix: + seed: [11, 23, 37, 53, 71] + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + - run: python -m pip install -r graph_synthesis/dspy_benchmark/requirements.txt + - name: Pin resolved dependency record + run: | + mkdir -p '${{ runner.temp }}/comparison' + python -m pip freeze > '${{ runner.temp }}/comparison/environment-seed-${{ matrix.seed }}.txt' + - name: Live repeated comparison + env: + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + DSPY_PROPOSER_MODEL: ${{ vars.DSPY_PROPOSER_MODEL }} + LITELLM_LOCAL_MODEL_COST_MAP: 'True' + run: python -B -m graph_synthesis.dspy_benchmark.run --seed ${{ matrix.seed }} --iterations 8 --workers 4 --output '${{ runner.temp }}/comparison/seed-${{ matrix.seed }}' + - name: Generate report without inventing missing runs + if: always() + run: python -B -m graph_synthesis.dspy_benchmark.report --directory '${{ runner.temp }}/comparison' + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: dspy-jev-seed-${{ matrix.seed }} + path: ${{ runner.temp }}/comparison/ + retention-days: 14 + if-no-files-found: warn + aggregate: + if: always() && needs.offline.result == 'success' && github.repository == 'CompleteDotTech/paper-package' + needs: [offline, live] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + - run: mkdir -p '${{ runner.temp }}/comparison' + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + continue-on-error: true + with: + pattern: dspy-jev-seed-* + merge-multiple: true + path: ${{ runner.temp }}/comparison + - name: Aggregate all available seed results + run: python -B -m graph_synthesis.dspy_benchmark.report --directory '${{ runner.temp }}/comparison' + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: dspy-jev-aggregate + path: | + ${{ runner.temp }}/comparison/RESULTS.md + ${{ runner.temp }}/comparison/run-status.json + ${{ runner.temp }}/comparison/summary.json + ${{ runner.temp }}/comparison/summary.csv + ${{ runner.temp }}/comparison/baseline-only-summary.json + ${{ runner.temp }}/comparison/baseline-only-summary.csv + retention-days: 14 + if-no-files-found: error diff --git a/.github/workflows/dspy-jev-preflight.yml b/.github/workflows/dspy-jev-preflight.yml new file mode 100644 index 0000000..1c684c9 --- /dev/null +++ b/.github/workflows/dspy-jev-preflight.yml @@ -0,0 +1,74 @@ +name: DSPy Jev live preflight +# Manual only. Do not bypass an Actions approval or environment protection gate. +on: + workflow_dispatch: +permissions: + contents: read +jobs: + preflight: + if: github.repository == 'CompleteDotTech/paper-package' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + - run: python -m pip install -r graph_synthesis/dspy_jev_optimizer/requirements.txt + - name: Bounded live connectivity test + env: + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + DSPY_PROPOSER_MODEL: ${{ vars.DSPY_PROPOSER_MODEL }} + LITELLM_LOCAL_MODEL_COST_MAP: 'True' + run: | + python - <<'PY' + import json, os + from pathlib import Path + from graph_synthesis.dspy_jev_optimizer.core import JevConfig + from graph_synthesis.dspy_jev_optimizer.providers import TypeSafeBackend, DSPyProposer + presence = {k: bool(os.environ.get(k, '').strip()) for k in ('TYPESAFE_API_KEY','OPENROUTER_API_KEY','OPENAI_API_KEY','ANTHROPIC_API_KEY')} + result = {'credential_present': presence, 'status': 'started'} + out = Path(os.environ['RUNNER_TEMP']) / 'preflight.json' + cfg = JevConfig('Evidence classification', 'Does the evidence support the claim?', {'supports':'Evidence establishes the claim.', 'refutes':'Evidence contradicts the claim.', 'unknown':'Evidence is insufficient.'}) + try: + model = os.environ.get('DSPY_PROPOSER_MODEL','').strip() + if not model: + model = ('openrouter/anthropic/claude-sonnet-4.5' if presence['OPENROUTER_API_KEY'] else 'anthropic/claude-sonnet-4-5-20250929' if presence['ANTHROPIC_API_KEY'] else 'openai/gpt-4.1' if presence['OPENAI_API_KEY'] else '') + proposer_key = next((key for prefix, key in ( + ('openrouter/', 'OPENROUTER_API_KEY'), + ('anthropic/', 'ANTHROPIC_API_KEY'), + ('openai/', 'OPENAI_API_KEY')) if model.startswith(prefix)), None) + if not presence['TYPESAFE_API_KEY'] or not proposer_key or not presence[proposer_key]: + result['status'] = 'credentials_missing' + else: + result['proposer_model'] = model + backend = TypeSafeBackend('jev-1.13.0', retries=0) + try: + answer = backend.predict(cfg, {'claim':'A precedes B.', 'evidence':['A precedes B.']}) + finally: + backend.close() + result['jev'] = {'model':answer['model'], 'choice':answer['choice'], 'usage':answer['usage']} + proposal = DSPyProposer(model).propose(cfg, [], 1, []) + JevConfig.from_dict(proposal, cfg) + result['status'] = 'passed' + except Exception as exc: + result['status'] = 'failed' + result['error_type'] = type(exc).__name__ + finally: + out.write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps(result)) + if result['status'] != 'passed': + raise SystemExit(1) + PY + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: dspy-jev-live-preflight + path: ${{ runner.temp }}/preflight.json + retention-days: 14 + if-no-files-found: warn diff --git a/DSPY_BENCHMARKS.md b/DSPY_BENCHMARKS.md new file mode 100644 index 0000000..6ee5c8a --- /dev/null +++ b/DSPY_BENCHMARKS.md @@ -0,0 +1,14 @@ +# DSPy versus Jev benchmark execution + +[Comparison protocol and runner](graph_synthesis/dspy_benchmark/README.md) · +[Actual execution status](experiments/dspy-comparison-20260918/STATUS.md) · +[Archived-output calibration check](experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md) + +The live comparison has not completed: CI identified a missing mapped generative +proposal-provider key, then marked the next diagnostic `action_required` before +starting jobs. No approval restriction was bypassed and no DSPy gain is claimed. + +The comparison code registers five isolated repeats, two search objectives, eight +native legacy formulations and seven calibration settings per frozen arm. New +local regression and archived-replay checks are recorded separately from model +performance. Historical evidence and the root immutable manifest are unchanged. diff --git a/experiments/dspy-comparison-20260918/STATUS.md b/experiments/dspy-comparison-20260918/STATUS.md new file mode 100644 index 0000000..844cbc5 --- /dev/null +++ b/experiments/dspy-comparison-20260918/STATUS.md @@ -0,0 +1,72 @@ +# DSPy / Jev comparison: actual execution status + +**Live with/without-DSPy comparison: NOT COMPLETED. No new Jev inference calls and no new DSPy inference runs were completed.** + +## Live execution blockers + +[Preflight run 35355158096](https://github.com/CompleteDotTech/paper-package/actions/runs/35355158096) found the mapped TypeSafe key populated, but none of `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, or `ANTHROPIC_API_KEY`. It stopped before calling Jev. This does not establish whether differently named or environment-scoped provider credentials exist, or whether the TypeSafe key is valid. + +[Diagnostic run 35355491970](https://github.com/CompleteDotTech/paper-package/actions/runs/35355491970) was marked `action_required` with zero jobs. Its gate was left intact. Broad secret-context diagnostics were removed; preflight and live runs are manual-only and use explicitly named keys. Secret values are not in these reports. + +## Executed offline checks + +**1,092 tests passed; 6 skipped; 16 test suites completed. All 11 deterministic experiment replays passed.** The original 161-file inventory and the existing optimizer inventory also passed verification. + +Four SDK contract tests were skipped because SDK packages were not installed in this local runtime; two archived reproduction tests also reported skips. Skips are not counted as passes. An initial reproduction invocation lacked PYTHONPATH; the corrected invocation passed. A later combined wrapper timed out on two replay processes; the individual replay commands recorded below had already passed. Neither failed attempt is treated as an additional successful test. + +| Check | Tests discovered | Skipped | Result | +|---|---:|---:|---| +| replay-adaptive | 0 | 0 | passed | +| replay-assumption_aware | 0 | 0 | passed | +| replay-followup | 0 | 0 | passed | +| replay-frontier | 0 | 0 | passed | +| replay-novel_mechanisms | 0 | 0 | passed | +| replay-reliability | 0 | 0 | passed | +| replay-risk_control | 0 | 0 | passed | +| replay-source_structural | 0 | 0 | passed | +| replay-structural | 0 | 0 | passed | +| replay-theory_suite | 0 | 0 | passed | +| replay-uncertainty | 0 | 0 | passed | +| test-graph_synthesis-adaptive-tests | 64 | 0 | passed | +| test-graph_synthesis-assumption_aware-tests | 47 | 0 | passed | +| test-graph_synthesis-corpus-tests | 32 | 0 | passed | +| test-graph_synthesis-dspy_benchmark-tests | 26 | 0 | passed | +| test-graph_synthesis-dspy_jev_optimizer-tests | 35 | 4 | passed | +| test-graph_synthesis-followup-tests | 67 | 0 | passed | +| test-graph_synthesis-frontier-tests | 75 | 0 | passed | +| test-graph_synthesis-novel_mechanisms-tests | 92 | 0 | passed | +| test-graph_synthesis-reliability-tests | 63 | 0 | passed | +| test-graph_synthesis-risk_control-tests | 66 | 0 | passed | +| test-graph_synthesis-source_structural-tests | 86 | 0 | passed | +| test-graph_synthesis-structural-tests | 36 | 0 | passed | +| test-graph_synthesis-tests | 200 | 0 | passed | +| test-graph_synthesis-theory_suite-tests | 37 | 0 | passed | +| test-graph_synthesis-uncertainty-tests | 36 | 0 | passed | +| test-reproduction-tests | 136 | 2 | passed | +| verify-frozen | 0 | 0 | passed | +| verify-optimizer | 0 | 0 | passed | + +The machine-readable [validation record](offline-validation-summary.json) includes per-suite status and counts; the downloadable audit bundle also includes commands, durations and log hashes. These runs disabled socket connections and used no inference keys. They reproduce saved observations and algorithm fixtures, not newly optimized model predictions. + +## Separate archived calibration analysis + +[Archived calibration results](archive-calibration/RESULTS.md) refit fixed calibrators on the original calibration split and score saved test responses. They contain zero fresh inference and zero DSPy runs. Invalid archived response exclusions and denominators are explicit. Do not attribute these results to DSPy. + +## Ready-to-run live comparison + +[Protocol and commands](../../graph_synthesis/dspy_benchmark/README.md) cover five repeats, two DSPy search objectives, eight original native formulations, seven calibration settings and seven task/panel combinations. The code was tested with controlled doubles, not substituted for live inference. + +To execute live, map an authorized generative-provider credential and model into the workflow, resolve the repository Actions approval requirement, and manually run the live workflow. Original manuscripts, predictions and the immutable root manifest remain unchanged. + +## Offline integration correction + +The comparison workflow is now manual-only and has its own filename so it can +coexist with the separate DSPy optimizer study. A full comparison requires at +least one proposal iteration and verifies the configured proposer's matching +credential before constructing a Jev target backend. Each completed seed writes +a hash inventory of its run files; the downstream report verifies those files, +retains missing/invalid seed statuses, and marks the five-seed matrix incomplete +until all five completed artifacts validate. Baseline-only observations have a +separate summary and cannot fill the primary DSPy table. These corrections were +checked with offline tests and test doubles. The live comparison remains blocked +and no new provider calls or DSPy results were produced by this correction. diff --git a/experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md b/experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md new file mode 100644 index 0000000..73362f9 --- /dev/null +++ b/experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md @@ -0,0 +1,24 @@ +# Archived-output calibration check + +**Zero fresh model calls. Zero DSPy runs. This is not the requested live comparison.** Invalid archived responses are excluded from probability metrics and listed in summary.json; accuracy is conditional on a valid response, not end-to-end accuracy. + +Calibrators were refit using only the archived calibration split. All three fixed methods are shown; none was selected using test performance. The archived evaluation split was historically examined, so these are exploratory results. + +| Task | Historical arm | Calibration | n | Accuracy | Macro-F1 | Brier | Log loss | ECE | +|---|---|---|---:|---:|---:|---:|---:|---:| +| relation_support | baseline_choice | raw | 338 | 0.846154 | 0.844335 | 0.234192 | 1.240206 | 0.074408 | +| relation_support | baseline_choice | temperature | 338 | 0.846154 | 0.844335 | 0.272597 | 0.569980 | 0.120948 | +| relation_support | baseline_choice | bias_temperature | 338 | 0.828402 | 0.817346 | 0.261256 | 0.532340 | 0.096751 | +| relation_support | fewshot_contract | raw | 337 | 0.860534 | 0.859685 | 0.224429 | 1.320742 | 0.068843 | +| relation_support | fewshot_contract | temperature | 337 | 0.860534 | 0.859685 | 0.256077 | 0.632292 | 0.131532 | +| relation_support | fewshot_contract | bias_temperature | 337 | 0.759644 | 0.756568 | 0.282491 | 0.574257 | 0.107833 | +| entity_resolution | baseline_noul | raw | 413 | 0.980630 | 0.960452 | 0.040651 | 0.106744 | 0.064504 | +| entity_resolution | baseline_noul | temperature | 413 | 0.980630 | 0.960452 | 0.031287 | 0.065248 | 0.013925 | +| entity_resolution | baseline_noul | bias_temperature | 413 | 0.983051 | 0.965638 | 0.028751 | 0.059969 | 0.013186 | +| entity_resolution | fewshot_contract | raw | 413 | 0.987893 | 0.976434 | 0.018150 | 0.033291 | 0.015036 | +| entity_resolution | fewshot_contract | temperature | 413 | 0.987893 | 0.976434 | 0.015705 | 0.026191 | 0.009182 | +| entity_resolution | fewshot_contract | bias_temperature | 413 | 0.987893 | 0.975793 | 0.017257 | 0.028161 | 0.003360 | + +Temperature scaling preserves decisions; its accuracy differences must be zero. Bias/temperature can change decisions. A probability-quality improvement is not necessarily an accuracy improvement. + +Source SHA-256: `9fc0f0a345f6fbbd311759a5143c1d1be62313b72e26acd54e3ae289e040d76f` diff --git a/experiments/dspy-comparison-20260918/archive-calibration/summary.json b/experiments/dspy-comparison-20260918/archive-calibration/summary.json new file mode 100644 index 0000000..66330c1 --- /dev/null +++ b/experiments/dspy-comparison-20260918/archive-calibration/summary.json @@ -0,0 +1 @@ +{"evidence":"exploratory_saved_response_reanalysis","fresh_model_calls":0,"dspy_runs":0,"source":"experiments/jev-rerun-20260918/predictions.jsonl","source_sha256":"9fc0f0a345f6fbbd311759a5143c1d1be62313b72e26acd54e3ae289e040d76f","full_results_sha256":"8fe4c82a4b2d0807c883b4d4abbec830403f3386a14c8d96530f436253635c9f","excluded_invalid_responses":[{"task":"relation_support","arm":"baseline_choice","split":"evaluation","id":"scifact-213-13625993","error":"Probabilities must sum to one"},{"task":"relation_support","arm":"fewshot_contract","split":"calibration","id":"scifact-1258-12040627","error":"choice must be a highest-probability option"},{"task":"relation_support","arm":"fewshot_contract","split":"evaluation","id":"scifact-1202-3475317","error":"Probabilities must sum to one"},{"task":"relation_support","arm":"fewshot_contract","split":"evaluation","id":"scifact-1088-37549932","error":"choice must be a highest-probability option"}],"columns":["task","arm","calibration","n","accuracy","macro_f1","brier","log_loss","ece"],"results":[["relation_support","baseline_choice","raw",338,0.846154,0.844335,0.234192,1.240206,0.074408],["relation_support","baseline_choice","temperature",338,0.846154,0.844335,0.272597,0.56998,0.120948],["relation_support","baseline_choice","bias_temperature",338,0.828402,0.817346,0.261256,0.53234,0.096751],["relation_support","fewshot_contract","raw",337,0.860534,0.859685,0.224429,1.320742,0.068843],["relation_support","fewshot_contract","temperature",337,0.860534,0.859685,0.256077,0.632292,0.131532],["relation_support","fewshot_contract","bias_temperature",337,0.759644,0.756568,0.282491,0.574257,0.107833],["entity_resolution","baseline_noul","raw",413,0.98063,0.960452,0.040651,0.106744,0.064504],["entity_resolution","baseline_noul","temperature",413,0.98063,0.960452,0.031287,0.065248,0.013925],["entity_resolution","baseline_noul","bias_temperature",413,0.983051,0.965638,0.028751,0.059969,0.013186],["entity_resolution","fewshot_contract","raw",413,0.987893,0.976434,0.01815,0.033291,0.015036],["entity_resolution","fewshot_contract","temperature",413,0.987893,0.976434,0.015705,0.026191,0.009182],["entity_resolution","fewshot_contract","bias_temperature",413,0.987893,0.975793,0.017257,0.028161,0.00336]]} diff --git a/experiments/dspy-comparison-20260918/artifact-manifest.json b/experiments/dspy-comparison-20260918/artifact-manifest.json new file mode 100644 index 0000000..5deb1e1 --- /dev/null +++ b/experiments/dspy-comparison-20260918/artifact-manifest.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "scope": "New benchmark implementation, workflow and reports; excludes this inventory. Original research inventory is unchanged.", + "baseline_commit": "c726b66c3a07a115eb928b3f2629de972c728083", + "original_manifest_git_blob": "a8d9f447c266fbedf8837003ceb9639e00778b0e", + "preflight_git_blob": "4b99b54a351d0d927656aa6052e4db509ac41a9e", + "sha256": { + ".github/workflows/dspy-comparison.yml": "379ff23defc01446fd648ae56f07b4a649c3f3ae7735c84f0ed5710642e45491", + ".github/workflows/dspy-jev-preflight.yml": "11d74371173b42c4ee71972d0a5b8a2f9fba3e9bc3ac85bc2bd44471007736dc", + "DSPY_BENCHMARKS.md": "3846accfc911b784ba44545d4910807b69eb45bb4b16fa5f6ed43878d86870f8", + "experiments/dspy-comparison-20260918/STATUS.md": "6e0b20b0f906d5f448e51eabe994f0f54e4566833ac8203aaf1aa08cb8119551", + "experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md": "129ff8e48148e6de19c931760488d8e26c1af7816786cd89f961ec2aa7eda9de", + "experiments/dspy-comparison-20260918/archive-calibration/summary.json": "c18e3b3c7d40dab043193d0fe4e6601d69fea511b77e3f2fca54ab1fa0a23bdb", + "experiments/dspy-comparison-20260918/live-status.json": "d5d085d71219fea23bd08882de0df72e26588e78dfb0007566d22915c39c9aa7", + "experiments/dspy-comparison-20260918/offline-validation-summary.json": "ba061c00f7fc44b08bb02a26ed705f6e38badc756d03eb6754defec2d67bbfbd", + "graph_synthesis/dspy_benchmark/.gitignore": "130b65c1033d6acb35e41f551f2aac6601947e5bc14f954a65b5154fa40716a1", + "graph_synthesis/dspy_benchmark/README.md": "f0fdb5c1290de34dd1c1033789be7d2ced7e195391527d8266fdec16ca897eda", + "graph_synthesis/dspy_benchmark/__init__.py": "13d9369fd42ab6800e4d3fb99ad40f8deed29ce1acefc662d17d4b489e485abf", + "graph_synthesis/dspy_benchmark/archive_calibration.py": "7154a8a2818c65fb94291f4201106b6287e20b74e86e38ea6d629bc32af556a4", + "graph_synthesis/dspy_benchmark/calibration.py": "8c256a74fbd3a0820a7c64d42ac04c75150f9c9b2bab8c680264ea7dae18b96d", + "graph_synthesis/dspy_benchmark/data.py": "1933b9d51e2901f4f78bea6ca4cca38d8b2099b27dccc88975c5a8f1ce3bc001", + "graph_synthesis/dspy_benchmark/report.py": "43ad935b7552430cc4d5b8f5972b95871999e0b549eca2b9acab5db216e4ec52", + "graph_synthesis/dspy_benchmark/requirements.txt": "b8fab64f96b3a873a0feb9fae9f0eb1308d07ca3cfbea6620a27f6831a3fbcf4", + "graph_synthesis/dspy_benchmark/run.py": "5206511ba90a53141025bcf1b212e5b1087b508ae7950a4b8cb73248395d1dfe", + "graph_synthesis/dspy_benchmark/tests/test_benchmark.py": "390ac4e378e95ab26cae343eb42b1531e0e682d7affbeac753017473dd0e3038" + } +} diff --git a/experiments/dspy-comparison-20260918/live-status.json b/experiments/dspy-comparison-20260918/live-status.json new file mode 100644 index 0000000..0e592f5 --- /dev/null +++ b/experiments/dspy-comparison-20260918/live-status.json @@ -0,0 +1,16 @@ +{ + "status": "blocked", + "completed_live_dspy_runs": 0, + "new_jev_calls": 0, + "preflight_run": 35355158096, + "credential_present": { + "TYPESAFE_API_KEY": true, + "OPENROUTER_API_KEY": false, + "OPENAI_API_KEY": false, + "ANTHROPIC_API_KEY": false + }, + "approval_run": 35355491970, + "approval_conclusion": "action_required", + "approval_jobs": 0, + "scope": "The four explicit mappings were checked; other names or environment-scoped keys were not inspected. TypeSafe key validity was not tested." +} diff --git a/experiments/dspy-comparison-20260918/offline-validation-summary.json b/experiments/dspy-comparison-20260918/offline-validation-summary.json new file mode 100644 index 0000000..ed590f9 --- /dev/null +++ b/experiments/dspy-comparison-20260918/offline-validation-summary.json @@ -0,0 +1 @@ +{"source_commit":"c726b66c3a07a115eb928b3f2629de972c728083","execution":"individually_executed_offline_commands","fresh_service_calls":0,"python":"3.13.5","numpy":"2.3.5","scipy":"1.17.0","test_suites":16,"tests_discovered":1098,"tests_skipped":6,"replay_suites":11,"tests_passed":1092,"columns":["check","tests","skipped","result"],"checks":[["replay-adaptive",0,0,"passed"],["replay-assumption_aware",0,0,"passed"],["replay-followup",0,0,"passed"],["replay-frontier",0,0,"passed"],["replay-novel_mechanisms",0,0,"passed"],["replay-reliability",0,0,"passed"],["replay-risk_control",0,0,"passed"],["replay-source_structural",0,0,"passed"],["replay-structural",0,0,"passed"],["replay-theory_suite",0,0,"passed"],["replay-uncertainty",0,0,"passed"],["test-graph_synthesis-adaptive-tests",64,0,"passed"],["test-graph_synthesis-assumption_aware-tests",47,0,"passed"],["test-graph_synthesis-corpus-tests",32,0,"passed"],["test-graph_synthesis-dspy_benchmark-tests",26,0,"passed"],["test-graph_synthesis-dspy_jev_optimizer-tests",35,4,"passed"],["test-graph_synthesis-followup-tests",67,0,"passed"],["test-graph_synthesis-frontier-tests",75,0,"passed"],["test-graph_synthesis-novel_mechanisms-tests",92,0,"passed"],["test-graph_synthesis-reliability-tests",63,0,"passed"],["test-graph_synthesis-risk_control-tests",66,0,"passed"],["test-graph_synthesis-source_structural-tests",86,0,"passed"],["test-graph_synthesis-structural-tests",36,0,"passed"],["test-graph_synthesis-tests",200,0,"passed"],["test-graph_synthesis-theory_suite-tests",37,0,"passed"],["test-graph_synthesis-uncertainty-tests",36,0,"passed"],["test-reproduction-tests",136,2,"passed"],["verify-frozen",0,0,"passed"],["verify-optimizer",0,0,"passed"]],"full_validation_sha256":"deb69ab26e253f6047017bfdf8ad2205a3368fb7a2f423b07bf4b6b3999c6df0"} diff --git a/graph_synthesis/dspy_benchmark/.gitignore b/graph_synthesis/dspy_benchmark/.gitignore new file mode 100644 index 0000000..145667d --- /dev/null +++ b/graph_synthesis/dspy_benchmark/.gitignore @@ -0,0 +1,3 @@ +/runs/ +/.cache/ +__pycache__/ diff --git a/graph_synthesis/dspy_benchmark/README.md b/graph_synthesis/dspy_benchmark/README.md new file mode 100644 index 0000000..98d7cbb --- /dev/null +++ b/graph_synthesis/dspy_benchmark/README.md @@ -0,0 +1,116 @@ +# Repeated DSPy versus Jev benchmark comparison + +**Live comparison status: blocked, not completed.** The first CI preflight found +`TYPESAFE_API_KEY` populated, but none of the three explicitly mapped generative +provider keys (`OPENROUTER_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`). It stopped +before inference. The next diagnostic was marked `action_required` before any job +ran. The approval gate has not been bypassed. The diagnostic was reduced to +explicit named keys and made manual-only. + +## Registered comparison + +| Dimension | Fixed configuration | +|---|---| +| Jev target | `jev-1.13.0` | +| Repeats | Seeds 11, 23, 37, 53, 71; fresh isolated cache namespace per repeat | +| Tasks | SciFact relation support; DBLP-ACM entity resolution | +| No-DSPy controls | All four original native formulations for each task, including Noul and few-shot controls | +| DSPy searches | Accuracy and composite objectives; five runs per objective per task = 20 searches | +| Search budget | Eight candidate proposals per search; no test-based early stopping | +| Calibration | Raw; temperature; regularized bias + temperature | +| Calibration-size sensitivity | 25%, 50%, 100% source-group prefixes; raw is reported once | +| Selection data | 29 training and 31 validation examples per task, split from original development only | +| Calibration data | Original 150 relation / 390 entity calibration examples | +| Primary test | Original 339 relation / 413 entity evaluation examples | +| Transfer panels | 50 relation fixtures, 100 entity fixtures, 48 semantic challenges, and the 73/263 multicall input panels | + +The full matrix contains 1,470 metric records when every run succeeds. These are +not 1,470 independent experiments. Each frozen prompt is evaluated once per input +panel per repeat; alternative calibration transforms reuse those probabilities. +Twelve entity fixtures labeled `uncertain` remain in prediction files but cannot +be scored against the binary label schema. Their exclusion count is explicit. + +The original eight formulations are issued with their original Choice/Noul +question types and demonstrations. DSPy optimizes the contracted Choice baseline +for each task. It does not optimize Jev weights, evolve graph topology, or run +GEPA/MIPROv2. The seven historical multicall execution policies are not replaced +by seven fake DSPy arms: this matrix applies the frozen direct-decision prompts +to their input panels. Existing graph-policy/algorithm suites are separately +replayed; those results are not new model comparisons. + +## Boundaries and interpretation + +All candidates use training states/labels only in proposal feedback. Validation +selects the winner. All prompt searches for a task are frozen before fitting any +calibrator, and all calibrators for that task are frozen before test inference. +Split hashes and source/entity identity separation are checked. Historical plans +are public and their evaluation sets were previously examined; "held out" means +held out from this optimization, not a never-seen external benchmark. + +Temperature scaling leaves decisions unchanged. Bias/temperature can change +labels. The latter uses fixed regularization 0.01 and bounded parameters, never +test-tuned regularization. Probability zeros are clipped to 1e-15 before logs. +Confidence metrics use class probabilities, not Jev's distribution-concentration +field. Lower Brier/log loss/ECE is better; higher accuracy/F1/MCC is better. + +Report all seeds, including unsuccessful searches and null/negative gains. Never +pick the best seed using test accuracy. Report means and sample standard deviations; +repeated runs on one test set are not independent datasets. `paired_interval` is +an optional fixed-fit, provided-group bootstrap helper; its intervals are not +simultaneous or multiplicity-adjusted. No significance claim is produced by the +aggregate report. + +## Execute, after resolving CI credentials and approval + +Use Python 3.12 or newer. From the repository root: + +```bash +python -m pip install -r graph_synthesis/dspy_benchmark/requirements.txt +python -B -m unittest discover -s graph_synthesis/dspy_benchmark/tests -v +python -B -m graph_synthesis.dspy_benchmark.run \ + --seed 11 --iterations 8 --workers 4 \ + --output graph_synthesis/dspy_benchmark/runs/seed-11 +python -B -m graph_synthesis.dspy_benchmark.report \ + --directory graph_synthesis/dspy_benchmark/runs +``` + +Set `TYPESAFE_API_KEY`, `DSPY_PROPOSER_MODEL`, and the provider credential required +by that DSPy model. Keys must stay in the CI secret store; do not commit or paste +them into reports. The manual [comparison workflow](../../.github/workflows/dspy-comparison.yml) +has a five-seed matrix with at most two simultaneous jobs. It never injects keys +into pull-request tests. Select the desired branch when manually dispatching. +Approval/environment protections remain applicable. `--baseline-only` is an +explicit partial-run mode and never counts as a completed DSPy comparison. Full +comparison runs require at least one proposal iteration and the credential for +the configured OpenRouter, Anthropic or OpenAI proposer model before any Jev +target inference. The workflow downloads all available seed artifacts into one +directory, generates a five-seed aggregate report, and publishes that report as +`dspy-jev-aggregate`; missing or failed seeds remain visible in its status table. +Baseline-only measurements appear in a separate summary and cannot populate +the primary DSPy comparison table. + +The registered eight-iteration matrix has an upper bound of 65,000 logical Jev +requests across five seeds before cache savings, with at most two SDK retries per +request, and 160 DSPy proposal invocations. This is not a dollar-denominated budget +or a guarantee of provider determinism. Each CI seed job has a 120-minute timeout. + +Every run writes protocol/config/source hashes, proposals, raw requests and +responses, probability files, calibration parameters, metrics and usage. HTTP +credentials and complete LM configuration dictionaries are not serialized. Each +run directory is new; accidental overwriting is refused. Missing/malformed model +responses terminate the run rather than being silently assigned an accuracy. + +## What was actually executed in this change + +See [the execution record](../../experiments/dspy-comparison-20260918/STATUS.md). +It distinguishes fresh local regressions, deterministic archived replay, the +blocked live preflight, and [exploratory archived-output recalibration](../../experiments/dspy-comparison-20260918/archive-calibration/RESULTS.md). +The latter contains **zero new model calls and zero DSPy runs**. It must not be +presented as evidence for DSPy gains. + +The root immutable `MANIFEST.json` and all archived research files are unchanged. +New files have their own integrity inventory. SDK dependency versions are pinned; +CI also records its complete resolved environment. Method references: +[temperature scaling](https://proceedings.mlr.press/v70/guo17a.html), +[TypeSafe probabilities](https://docs.typesafe.ai/primitives/choice), +[DSPy programs](https://dspy.ai/). diff --git a/graph_synthesis/dspy_benchmark/__init__.py b/graph_synthesis/dspy_benchmark/__init__.py new file mode 100644 index 0000000..a32c448 --- /dev/null +++ b/graph_synthesis/dspy_benchmark/__init__.py @@ -0,0 +1 @@ +"""Repeated DSPy/Jev comparison; historical evidence is read-only.""" diff --git a/graph_synthesis/dspy_benchmark/archive_calibration.py b/graph_synthesis/dspy_benchmark/archive_calibration.py new file mode 100644 index 0000000..200cb8a --- /dev/null +++ b/graph_synthesis/dspy_benchmark/archive_calibration.py @@ -0,0 +1,55 @@ +"""Exploratory recalibration of SAVED responses; never presented as a live DSPy run.""" +import argparse +import hashlib +from pathlib import Path +from .data import ROOT, load_task +from .calibration import fit, apply, evaluation +from graph_synthesis.dspy_jev_optimizer.core import parse_json, write_json + +SOURCE = 'experiments/jev-rerun-20260918/predictions.jsonl' + +def execute(output): + output.mkdir(parents=True,exist_ok=False) + source=ROOT/SOURCE + rows=[parse_json(line) for line in source.read_text(encoding='utf-8').splitlines() if line] + result=[] + exclusions=[] + for task in ('relation_support','entity_resolution'): + cfg,splits,_,_=load_task(task) + groups={r.id:r.group_id for name in ('calibration','test') for r in splits[name]} + arms=sorted({r['arm'] for r in rows if r['task']==task and r['split']=='calibration'}) + for arm in arms: + def get(split): + selected=[r for r in rows if r['task']==task and r['arm']==arm and r['split']==split and r['repeat']==0] + bad=[r for r in selected if r.get('error')] + exclusions.extend({'task':task,'arm':arm,'split':split,'id':r['id'],'error':r['error']} for r in bad) + selected=[r for r in selected if not r.get('error')] + return [{'id':r['id'],'group_id':groups[r['id']],'gold':r['gold_label'], + 'choice':max(cfg.criteria,key=lambda k:r['distribution'][k]),'probabilities':r['distribution']} for r in selected] + calibration,test=get('calibration'),get('evaluation') + if not calibration or not test: + raise ValueError('No valid archived responses') + for method in ('raw','temperature','bias_temperature'): + fitted=fit(calibration,list(cfg.criteria),method) + write_json(output/(task+'-'+arm+'-'+method+'-fit.json'),fitted) + result.append({'task':task,'arm':arm,'calibration':method,'temperature':fitted['temperature'],'planned_test_n':len(splits['test']),'response_coverage':len(test)/len(splits['test']), + **evaluation(apply(test,fitted),cfg.criteria)}) + report={'evidence':'exploratory_saved_response_reanalysis','fresh_model_calls':0,'dspy_runs':0, + 'source':SOURCE,'source_sha256':hashlib.sha256(source.read_bytes()).hexdigest(), + 'method':'Fit only archived calibration labels; score historically examined archived evaluation labels. No method selected on test scores.', + 'excluded_invalid_responses':exclusions,'results':result} + write_json(output/'results.json',report) + text=['# Archived-output calibration check','', + '**Zero fresh model calls. Zero DSPy runs. This is not the requested live comparison.** Invalid archived responses are excluded from probability metrics and listed in results.json; accuracy is conditional on a valid response, not end-to-end accuracy.','', + 'Calibrators were refit using only the archived calibration split. All three fixed methods are shown; none was selected using test performance. The archived evaluation split was historically examined, so these are exploratory results.', '', + '| Task | Historical arm | Calibration | n | Accuracy | Macro-F1 | Brier | Log loss | ECE |', + '|---|---|---|---:|---:|---:|---:|---:|---:|'] + for r in result: + text.append('| '+' | '.join([r['task'],r['arm'],r['calibration'],str(r['n'])]+[f"{r[m]:.6f}" for m in ('accuracy','macro_f1','brier','log_loss','ece')])+' |') + text.extend(['','Temperature scaling preserves decisions; its accuracy differences must be zero. Bias/temperature can change decisions. A probability-quality improvement is not necessarily an accuracy improvement.','',f"Source SHA-256: `{report['source_sha256']}`",'']) + (output/'RESULTS.md').write_text('\n'.join(text),encoding='utf-8') + return report + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--output',type=Path,required=True) + execute(parser.parse_args().output) diff --git a/graph_synthesis/dspy_benchmark/calibration.py b/graph_synthesis/dspy_benchmark/calibration.py new file mode 100644 index 0000000..fdc2494 --- /dev/null +++ b/graph_synthesis/dspy_benchmark/calibration.py @@ -0,0 +1,99 @@ +"""Post-hoc calibration. Fit on calibration labels, never on test labels.""" +import math +import numpy as np +from scipy.optimize import minimize, minimize_scalar +from graph_synthesis.dspy_jev_optimizer.core import metrics, digest + + +def matrix(predictions, labels): + return np.asarray([[r['probabilities'][k] for k in labels] for r in predictions], dtype=float) + + +def softmax(values): + values = values - values.max(axis=1, keepdims=True) + out = np.exp(values) + return out / out.sum(axis=1, keepdims=True) + + +def fit(predictions, labels, method='temperature'): + if not predictions or method not in ('raw','temperature','bias_temperature'): + raise ValueError('Invalid calibration request') + p = matrix(predictions, labels) + if not np.isfinite(p).all() or (p < 0).any() or not np.allclose(p.sum(1), 1, atol=1e-6): + raise ValueError('Invalid calibration probabilities') + y = np.asarray([list(labels).index(r['gold']) for r in predictions]) + logp = np.log(np.clip(p, 1e-15, 1)) + def loss(log_t, bias): + q = softmax(logp / np.exp(log_t) + bias) + return float(-np.log(np.clip(q[np.arange(len(y)), y], 1e-15, 1)).mean()) + log_t, bias = 0., np.zeros(len(labels)) + if method == 'temperature': + fitted = minimize_scalar(lambda t: loss(t, bias), bounds=(-2.3, 2.3), method='bounded') + if not fitted.success: + raise RuntimeError('Temperature fit failed') + log_t = float(fitted.x) + elif method == 'bias_temperature': + # Fixed shrinkage; never choose its strength using test outcomes. + def objective(x): + b = x[1:] - np.mean(x[1:]) + return loss(x[0], b) + .01 * float(np.mean(b*b)) + fitted = minimize(objective, np.zeros(len(labels)+1), method='L-BFGS-B', + bounds=[(-2.3,2.3)]+[(-3.,3.)]*len(labels)) + if not fitted.success: + raise RuntimeError('Bias/temperature fit failed') + log_t, bias = float(fitted.x[0]), fitted.x[1:]-np.mean(fitted.x[1:]) + return {'method':method, 'temperature':math.exp(log_t), 'bias':bias.tolist(), + 'labels':list(labels), 'n_calibration':len(predictions), + 'calibration_ids_sha256':digest([r['id'] for r in predictions])} + + +def apply(predictions, fitted): + labels = fitted['labels'] + if fitted['method'] == 'raw': + return [dict(r) for r in predictions] + p = matrix(predictions, labels) + q = softmax(np.log(np.clip(p,1e-15,1))/fitted['temperature']+np.asarray(fitted['bias'])) + # Temperature preserves selected labels, including provider tie-breaking. + return [{**r, 'choice':r['choice'] if fitted['method']=='temperature' else labels[int(np.argmax(q[i]))], + 'probabilities':dict(zip(labels, map(float,q[i])))} for i,r in enumerate(predictions)] + + +def evaluation(predictions, labels): + valid = [r for r in predictions if r['gold'] in labels] + result = metrics(valid, labels) + result['unscorable_count'] = len(predictions)-len(valid) + result['selective'] = [] + for threshold in (.5,.7,.9,.95,.99): + selected = [r for r in valid if r['probabilities'][r['choice']] >= threshold] + result['selective'].append({'threshold':threshold, 'accepted':len(selected), + 'coverage':len(selected)/len(valid), 'errors':sum(r['choice']!=r['gold'] for r in selected), + 'risk':sum(r['choice']!=r['gold'] for r in selected)/len(selected) if selected else None}) + if 'same' in labels: + result['false_merges'] = sum(r['choice']=='same' and r['gold']!='same' for r in valid) + result['missed_merges'] = sum(r['choice']!='same' and r['gold']=='same' for r in valid) + else: + edge_labels = {'SUPPORTS','REFUTES'} + tp = sum(r['choice'] in edge_labels and r['gold']==r['choice'] for r in valid) + fp = sum(r['choice'] in edge_labels and r['gold']!=r['choice'] for r in valid) + fn = sum(r['gold'] in edge_labels and r['gold']!=r['choice'] for r in valid) + result.update(correct_edges=tp, wrong_edges=fp, missed_edges=fn, + edge_f1=2*tp/(2*tp+fp+fn) if 2*tp+fp+fn else 0.) + return result + + +def paired_interval(baseline, candidate, labels, metric, seed=812, repeats=1000): + if [r['id'] for r in baseline] != [r['id'] for r in candidate]: + raise ValueError('Unpaired examples') + groups = {} + for i,r in enumerate(baseline): + groups.setdefault(r.get('group_id',r['id']), []).append(i) + keys = list(groups) + rng = np.random.default_rng(seed) + values = [] + for _ in range(repeats): + indices = [i for g in rng.choice(len(keys),len(keys)) for i in groups[keys[g]]] + a, b = ([rows[i] for i in indices] for rows in (baseline,candidate)) + values.append(metrics(b,labels)[metric]-metrics(a,labels)[metric]) + return {'delta':metrics(candidate,labels)[metric]-metrics(baseline,labels)[metric], + 'low':float(np.quantile(values,.025)), 'high':float(np.quantile(values,.975)), + 'resampling_unit':'provided group_id', 'replicates':repeats, 'multiplicity_adjusted':False} diff --git a/graph_synthesis/dspy_benchmark/data.py b/graph_synthesis/dspy_benchmark/data.py new file mode 100644 index 0000000..d497c53 --- /dev/null +++ b/graph_synthesis/dspy_benchmark/data.py @@ -0,0 +1,86 @@ +"""Deterministic benchmark splits from the original frozen experiment plans.""" +from dataclasses import asdict +from pathlib import Path +from collections import Counter +from graph_synthesis.dspy_jev_optimizer.core import Example, JevConfig, digest, read_json, fingerprints, disjoint + +ROOT = Path(__file__).resolve().parents[2] +PLAN = 'reproduction/results/jev/run-20260918/plan.json' +MULTICALL = 'experiments/jev-multicall-20260918/plan.json' +CHALLENGE = 'experiments/falsification/challenge.json' +SEEDS = [11, 23, 37, 53, 71] +OBJECTIVES = ['accuracy', 'composite'] + + +def state(task, row): + if task == 'relation_support': + return {'claim': row['claim'], 'evidence': row['evidence']} + result = {k: row[k] for k in ('record_1', 'record_2')} + if row.get('context'): + result['context'] = row['context'] + return result + + +def identities(task, row): + return {row['group']} if task == 'relation_support' else set(row['identity_groups']) + + +def example(task, row): + group = row.get('group') if task == 'relation_support' else digest(sorted(identities(task, row))) + return Example(row['id'], state(task, row), row['gold_label'], group) + + +def load_task(task): + plan = read_json(ROOT / PLAN) + source = plan['tasks'][task] + # Stratification uses development labels only; the partition never changes between seeds. + train, validation = [], [] + for label in sorted({r['gold_label'] for r in source['development']}): + rows = sorted([r for r in source['development'] if r['gold_label'] == label], + key=lambda r: digest(['dspy-development-v1', r['id']])) + train.extend(rows[:len(rows)//2]) + validation.extend(rows[len(rows)//2:]) + raw = {'train': train, 'validation': validation, + 'calibration': source['calibration'], 'test': source['evaluation'], + 'demonstrations': source['demonstrations']} + # Pair hashes alone cannot detect a reused entity; check every constituent identity. + pools = {s: set().union(*(identities(task, r) for r in rows)) for s, rows in raw.items()} + for i, a in enumerate(pools): + for b in list(pools)[i+1:]: + if pools[a] & pools[b]: + raise ValueError('Source/entity leakage: '+a+' / '+b) + splits = {s: [example(task, r) for r in rows] for s, rows in raw.items()} + for i, a in enumerate(splits): + for b in list(splits)[i+1:]: + disjoint(fingerprints(splits[a]), fingerprints(splits[b])) + arm = 'evidence_contract' if task == 'relation_support' else 'identity_contract' + question = next(iter(plan['question_specs'][task][arm].values())) + config = JevConfig(task, question['instructions'], question['criteria']) + return config, splits, plan['question_specs'][task], source + + +def panels(task): + """Return panel definitions; they are never supplied to optimizer feedback.""" + config, splits, _, source = load_task(task) + fixtures = [Example(r['id'], state(task, r), r['gold_label'], r.get('group', r['id'])) + for r in source['fixtures']] + result = {'test': splits['test'], 'fixtures': fixtures} + if task == 'relation_support': + result['semantic_challenge'] = [Example(r['id'], r['state'], r['gold'], r['pair']) + for r in read_json(ROOT / CHALLENGE)] + mc = read_json(ROOT / MULTICALL)['rows'] + for split in ('development', 'test'): + result['multicall_'+split] = [example(task, r) for r in mc if r['split'] == split] + return result + + +def inventory(): + output = {} + for task in ('relation_support', 'entity_resolution'): + config, splits, specs, source = load_task(task) + output[task] = {'labels': list(config.criteria), 'legacy_arms': list(specs), + 'splits': {s: {'n':len(rows), 'labels':dict(Counter(r.label for r in rows)), + 'sha256':digest([asdict(r) for r in rows])} for s, rows in splits.items()}, + 'panels': {s: {'n':len(rows), 'scorable':sum(r.label in config.criteria for r in rows)} + for s, rows in panels(task).items()}} + return output diff --git a/graph_synthesis/dspy_benchmark/report.py b/graph_synthesis/dspy_benchmark/report.py new file mode 100644 index 0000000..a789e5c --- /dev/null +++ b/graph_synthesis/dspy_benchmark/report.py @@ -0,0 +1,155 @@ +"""Aggregate completed runs only; no inference and no selection on test scores.""" +import argparse +import csv +import hashlib +import json +import statistics +from pathlib import Path +from collections import defaultdict +from .data import SEEDS +from graph_synthesis.dspy_jev_optimizer.core import read_json, write_json, digest + + +def validate_artifact(directory, seed, item): + manifest_path = directory/'artifact-inventory.json' + if not manifest_path.is_file(): + return 'missing artifact inventory' + manifest = read_json(manifest_path) + hashes = manifest.get('sha256') if isinstance(manifest, dict) else None + if not isinstance(manifest, dict) or manifest.get('schema_version') != 1 or not isinstance(hashes, dict): + return 'malformed artifact inventory' + actual = {p.relative_to(directory).as_posix() for p in directory.rglob('*') + if p.is_file() and p != manifest_path} + if actual != set(hashes): + return 'artifact file set differs from inventory' + for relative, expected in hashes.items(): + sha = hashlib.sha256() + with (directory/relative).open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + sha.update(chunk) + if sha.hexdigest() != expected: + return 'artifact hash mismatch: '+relative + for name in ('protocol.json','status.json','metrics.json','usage.json', + 'relation_support/frozen-targets.json','relation_support/calibration-freeze.json', + 'entity_resolution/frozen-targets.json','entity_resolution/calibration-freeze.json'): + if name not in hashes: + return 'required artifact missing: '+name + protocol = read_json(directory/'protocol.json') + if (protocol.get('seed') != seed or item.get('protocol_sha256') != digest(protocol) + or bool(protocol.get('baseline_only')) != (item['status'] == 'completed_baseline_only') + or (item['status'] == 'completed' and protocol.get('iterations', 0) < 1)): + return 'protocol and completion status disagree' + metrics = read_json(directory/'metrics.json') + if not isinstance(metrics, list) or not metrics: + return 'missing metric records' + for task in ('relation_support','entity_resolution'): + arms = set(read_json(directory/task/'frozen-targets.json')['targets']) + observed = {row['arm'] for row in metrics if row['task'] == task and row['panel'] == 'test'} + if not arms or not arms.issubset(observed): + return 'missing test measurements for '+task + if item['status'] == 'completed' and not any(a.startswith('dspy_') for a in arms): + return 'full run has no DSPy arms' + return None + + +def aggregate(rows): + groups = defaultdict(list) + for row in rows: + groups[(row['task'],row['panel'],row['arm'],row['calibration'])].append(row) + summary = [] + for key,values in sorted(groups.items()): + record = dict(zip(('task','panel','arm','calibration'),key)) + record['runs'] = len(values) + record['n_per_run'] = values[0]['n'] + for metric in ('accuracy','macro_f1','mcc','brier','log_loss','ece'): + numbers = [r[metric] for r in values] + record[metric+'_mean'] = statistics.mean(numbers) + record[metric+'_sd'] = statistics.stdev(numbers) if len(numbers)>1 else None + summary.append(record) + return summary + + +def write_summary_csv(path, summary): + if not summary: + path.unlink(missing_ok=True) + return + with path.open('w',newline='',encoding='utf-8') as f: + writer = csv.DictWriter(f,fieldnames=list(summary[0])) + writer.writeheader();writer.writerows(summary) + + +def summarize(root): + rows, baseline_rows, status = [], [], [] + for seed in SEEDS: + directory = root/('seed-'+str(seed)) + try: + item = read_json(directory/'status.json') if (directory/'status.json').exists() else {'status':'not_run'} + if not isinstance(item, dict) or 'status' not in item: + raise ValueError('malformed status') + except (OSError, ValueError, TypeError): + item = {'status':'invalid_artifact','artifact_error':'unreadable status'} + if item['status'] in ('completed','completed_baseline_only'): + try: + error = validate_artifact(directory,seed,item) + except (OSError, ValueError, KeyError, TypeError): + error = 'unreadable run artifact' + if error: + item = {'status':'invalid_artifact','reported_status':item['status'], + 'artifact_error':error} + status.append({'seed':seed,**item}) + if item['status'] == 'completed': + rows.extend(read_json(directory/'metrics.json')) + elif item['status'] == 'completed_baseline_only': + baseline_rows.extend(read_json(directory/'metrics.json')) + write_json(root/'run-status.json',status) + baseline_summary = aggregate(baseline_rows) + write_json(root/'baseline-only-summary.json',baseline_summary) + write_summary_csv(root/'baseline-only-summary.csv',baseline_summary) + summary = aggregate(rows) + write_json(root/'summary.json',summary) + write_summary_csv(root/'summary.csv',summary) + complete = sum(s['status']=='completed' for s in status) + baseline_complete = sum(s['status']=='completed_baseline_only' for s in status) + lines = ['# Repeated DSPy / Jev benchmark comparison','', + f'Completed full DSPy runs: **{complete}/{len(SEEDS)}**.', + f'Full five-seed matrix: **{"complete" if complete == len(SEEDS) else "incomplete"}**.', + f'Completed baseline-only runs: **{baseline_complete}/{len(SEEDS)}**.','', + 'Previously published observations are not relabeled as fresh runs. ' + 'A missing or failed run is not an accuracy score. Five evaluations on the same test set ' + 'are repeated model/search runs, not five independent datasets.','', + '| Seed | Status |','|---|---|'] + lines += [f"| {s['seed']} | {s['status']} |" for s in status] + lines += ['', '## Primary test results', '', + '| Task | Arm | Calibration | Runs | Accuracy | Macro-F1 | Brier | Log loss | ECE |', + '|---|---|---|---:|---:|---:|---:|---:|---:|'] + for r in summary: + if r['panel']=='test' and r['calibration'].endswith('_1.0'): + lines.append('| '+ ' | '.join([r['task'],r['arm'],r['calibration'],str(r['runs'])]+[ + f"{r[k+'_mean']:.6f}" for k in ('accuracy','macro_f1','brier','log_loss','ece')])+' |') + if not complete: + lines += ['','**No completed live DSPy comparison is available.**'] + if baseline_summary: + lines += ['', '## Baseline-only results (separate partial run)', '', + 'These measurements contain no DSPy search and are excluded from the primary comparison.', '', + '| Task | Arm | Calibration | Runs | Accuracy | Macro-F1 | Brier | Log loss | ECE |', + '|---|---|---|---:|---:|---:|---:|---:|---:|'] + for r in baseline_summary: + if r['panel']=='test' and r['calibration'].endswith('_1.0'): + lines.append('| '+ ' | '.join([r['task'],r['arm'],r['calibration'],str(r['runs'])]+[ + f"{r[k+'_mean']:.6f}" for k in ('accuracy','macro_f1','brier','log_loss','ece')])+' |') + lines += ['','## Interpretation boundaries','', + 'Accuracy/composite search and post-hoc calibration are separate interventions. ' + 'Temperature scaling leaves predicted labels unchanged; bias/temperature may change labels. ' + 'Calibration uses only the original calibration split. Published synthetic fixtures and ' + 'the semantic challenge are exploratory transfer panels, not new independent gold labels. ' + 'Twelve uncertain entity fixtures have no label in the binary schema and are excluded from ' + 'accuracy denominators, but their predictions remain in the raw records.','', + 'The implementation is the repository\'s custom DSPy proposal loop, not GEPA or MIPROv2. ' + 'No test-score winner is selected across seeds. Broad superiority or a global accuracy optimum ' + 'cannot be inferred from this finite search.'] + (root/'RESULTS.md').write_text('\n'.join(lines)+'\n',encoding='utf-8') + return summary + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--directory',type=Path,required=True) + summarize(p.parse_args().directory) diff --git a/graph_synthesis/dspy_benchmark/requirements.txt b/graph_synthesis/dspy_benchmark/requirements.txt new file mode 100644 index 0000000..dd6a367 --- /dev/null +++ b/graph_synthesis/dspy_benchmark/requirements.txt @@ -0,0 +1,3 @@ +-r ../dspy_jev_optimizer/requirements.txt +numpy==2.3.5 +scipy==1.17.0 diff --git a/graph_synthesis/dspy_benchmark/run.py b/graph_synthesis/dspy_benchmark/run.py new file mode 100644 index 0000000..b95b983 --- /dev/null +++ b/graph_synthesis/dspy_benchmark/run.py @@ -0,0 +1,300 @@ +"""Fresh, repeated benchmark execution. Never treats a replay as live evidence.""" +from __future__ import annotations +import argparse +import gzip +import hashlib +import json +import os +import random +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict +from pathlib import Path +import numpy as np +from graph_synthesis.dspy_jev_optimizer.core import ( + JevConfig, Evaluator, Cache, BudgetExhausted, digest, canonical, parse_json, + validate_answer, metrics, optimize, write_json, read_json, frozen_run) +from graph_synthesis.dspy_jev_optimizer.providers import TypeSafeBackend, DSPyProposer +from .data import ROOT, SEEDS, OBJECTIVES, load_task, panels, state, inventory +from .calibration import fit, apply, evaluation + + +class ParallelEvaluator(Evaluator): + """Only the caller thread touches SQLite and audit files.""" + def __init__(self, *args, workers=4, **kwargs): + super().__init__(*args, **kwargs) + if not 1 <= workers <= 8: + raise ValueError('workers must be 1..8') + self.workers = workers + + def evaluate(self, config, rows, phase): + keys = [digest({'schema':1, 'backend':self.backend.identity, + 'config':config.fingerprint(), 'state':r.state}) for r in rows] + responses, missing = {}, {} + for key, row in zip(keys, rows): + cached = self.cache.get(key) + if cached is None: + missing.setdefault(key, row) + else: + responses[key] = cached + if len(missing) > self.max_calls-self.calls: + raise BudgetExhausted('Insufficient budget for a complete evaluation') + with ThreadPoolExecutor(max_workers=self.workers) as pool: + jobs = [] + for key, row in missing.items(): + self.calls += 1 + self.event({'event':'request', 'key':key, 'phase':phase, 'state':row.state, + 'question':config.question(), 'backend':self.backend.identity}) + jobs.append((key, pool.submit(self.backend.predict, config, parse_json(canonical(row.state))))) + errors = [] + for key, job in jobs: + try: + response = job.result() + validate_answer(response,config.criteria) + if response.get('model') != self.backend.identity['model']: + raise ValueError('Pinned model drift') + for token in self.usage: + count = response.get('usage',{}).get(token,0) + if type(count) is not int or count < 0: + raise ValueError('Invalid usage') + self.usage[token] += count + self.cache.put(key,response) + responses[key] = response + self.event({'event':'response','key':key,'phase':phase,'response':response}) + except Exception as exc: + errors.append(type(exc).__name__) + self.event({'event':'request_failed','key':key,'phase':phase,'error_type':type(exc).__name__}) + if errors: + raise RuntimeError('Inference failures: '+','.join(sorted(set(errors)))) + predictions = [] + for key,row in zip(keys,rows): + response = responses[key] + validate_answer(response,config.criteria) + if response.get('model') != self.backend.identity['model']: + raise ValueError('Cached model drift') + if key not in missing: + self.cache_hits += 1 + self.event({'event':'cache_hit','key':key,'phase':phase,'response':response}) + predictions.append({'id':row.id, 'group_id':row.group_id or row.id, 'gold':row.label, + 'choice':response['choice'],'probabilities':response['probabilities'], + 'request_sha256':key}) + scorable = [r for r in predictions if r['gold'] in config.criteria] + return metrics(scorable,config.criteria) if scorable else {}, predictions + + +class LegacyBackend(TypeSafeBackend): + """Preserve native Choice/Noul formulations, not a rewritten surrogate.""" + def __init__(self, model, task, arm, spec, demos, seed): + super().__init__(model, retries=2) + self.task, self.arm, self.spec, self.demos = task,arm,spec,demos + self.identity.update(legacy_arm=arm, spec_sha256=digest(spec), repeat_seed=seed, + demonstration_sha256=digest(demos) if arm=='fewshot_contract' else None) + + def predict(self, config, input_state): + from typesafe_sdk import Choice, Noul + questions = {k: (Choice(instructions=v['instructions'],criteria=v['criteria']) + if v['type']=='choice' else Noul(instructions=v['instructions'])) for k,v in self.spec.items()} + current = input_state + if self.arm == 'fewshot_contract': + current = {'labeled_examples':self.demos, 'input':current} + start = time.perf_counter() + raw = self.client.system_one(model=self.identity['model'],state=current,questions=questions).model_dump(mode='json') + a = raw['answers'] + if self.arm == 'conditional_nouls': + p = float(a[self.arm+'__support']['noul']) + r = float(a[self.arm+'__refute_given_not_support']['noul']) + probs = {'SUPPORTS':p,'REFUTES':(1-p)*r,'NOT_ENOUGH_INFO':(1-p)*(1-r)} + elif self.arm in ('baseline_noul','identity_noul'): + p = float(a[self.arm+'__decision']['noul']) + probs = {'same':p,'different':1-p} + else: + probs = a[self.arm+'__decision']['probabilities'] + # Match the original research decoder's argmax probability decision. + choice = max(config.criteria, key=lambda k:probs[k]) + return {'choice':choice,'probabilities':probs,'model':raw['model'], 'usage':raw.get('usage',{}), + 'latency_seconds':time.perf_counter()-start,'raw_response':raw, + 'request':{'model':self.identity['model'],'state':current,'questions':self.spec}} + + +class LoggedProposer: + def __init__(self, model, output, seed): + self.inner = DSPyProposer(model) + self.output = output + self.identity = {**self.inner.identity,'repeat_seed':seed, + 'seed_scope':'Python/NumPy and feedback order; remote sampling is not guaranteed deterministic'} + + def propose(self, config, feedback, iteration, history): + proposal = self.inner.propose(config,feedback,iteration,history) + records = self.inner.lm.history + last = records[-1] if records else {} + # Whitelist metadata; never serialize provider arguments or authentication headers. + row = {'iteration':iteration,'input_config':asdict(config),'training_feedback':feedback, + 'candidate':proposal,'usage':last.get('usage'), 'cost_usd':last.get('cost')} + self.output.parent.mkdir(parents=True,exist_ok=True) + with self.output.open('a',encoding='utf-8') as f: + f.write(canonical(row)+'\n') + return proposal + + +def provider_model(): + configured = os.environ.get('DSPY_PROPOSER_MODEL','').strip() + if configured: + return configured + for key,model in [('OPENROUTER_API_KEY','openrouter/anthropic/claude-sonnet-4.5'), + ('ANTHROPIC_API_KEY','anthropic/claude-sonnet-4-5-20250929'), + ('OPENAI_API_KEY','openai/gpt-4.1')]: + if os.environ.get(key,'').strip(): + return model + return None + + +def proposer_credential(model): + """Resolve only the proposal providers explicitly exposed by the workflow.""" + for prefix, key in (('openrouter/', 'OPENROUTER_API_KEY'), + ('anthropic/', 'ANTHROPIC_API_KEY'), + ('openai/', 'OPENAI_API_KEY')): + if model.startswith(prefix): + return key + return None + + +def save_predictions(path, rows): + path.parent.mkdir(parents=True,exist_ok=True) + path.write_bytes(gzip.compress(canonical(rows).encode(),mtime=0)) + + +def write_artifact_inventory(output: Path): + files = {} + for path in sorted(output.rglob('*')): + if path.is_file() and path.name != 'artifact-inventory.json': + digest_file = hashlib.sha256() + with path.open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest_file.update(chunk) + files[path.relative_to(output).as_posix()] = digest_file.hexdigest() + write_json(output/'artifact-inventory.json', {'schema_version':1,'sha256':files}) + + +def execute(output: Path, seed: int, *, iterations=8, baseline_only=False, model='jev-1.13.0', workers=4): + if seed not in SEEDS or not 0 <= iterations <= 12 or (iterations == 0 and not baseline_only): + raise ValueError('Unregistered seed or iteration budget') + if output.exists(): + raise FileExistsError('Use a new run directory; do not overwrite an experiment') + proposer_model = provider_model() if not baseline_only else None + if baseline_only: + iterations = 0 + proposer_key = proposer_credential(proposer_model) if proposer_model else None + target_ready = bool(os.environ.get('TYPESAFE_API_KEY', '').strip()) + proposal_ready = baseline_only or bool(proposer_key and os.environ.get(proposer_key, '').strip()) + output.mkdir(parents=True) + if not target_ready or not proposal_ready: + write_json(output/'status.json', {'status':'blocked_missing_credentials', + 'typesafe_key_present':target_ready, + 'proposer_model_configured':bool(proposer_model), + 'proposer_credential_present':bool(proposer_key and os.environ.get(proposer_key, '').strip()), + 'live_calls':0}) + raise RuntimeError('Live benchmark requires TypeSafe and a generative proposal provider') + random.seed(seed) + np.random.seed(seed) + protocol = {'seed':seed,'iterations':iterations,'objectives':OBJECTIVES,'model':model, + 'proposer_model':proposer_model,'baseline_only':baseline_only, 'workers':workers, + 'inventory':inventory(),'source_sha256':{ + p.name:__import__('hashlib').sha256(p.read_bytes()).hexdigest() for p in Path(__file__).parent.glob('*.py')}, + 'execution':'live_provider_calls', + 'note':'Historically examined public test sets; held out only from the current optimization.'} + write_json(output/'protocol.json',protocol) + all_metrics,usage = [],[] + try: + for task in ('relation_support','entity_resolution'): + cfg,splits,specs,source = load_task(task) + train = list(splits['train']) + random.Random(seed).shuffle(train) + task_dir = output/task + task_dir.mkdir() + cache = Cache(task_dir/'calls.sqlite3') + backend = TypeSafeBackend(model,retries=2) + backend.identity['repeat_seed'] = seed + targets = [] + try: + # Freeze every search before this task's calibration or test inference. + if not baseline_only: + for objective in OBJECTIVES: + search = task_dir/('search-'+objective) + ev = ParallelEvaluator(backend,cache,search,2000,workers=workers) + proposer = LoggedProposer(proposer_model, task_dir/('proposals-'+objective+'.jsonl'),seed) + freeze = optimize(cfg,train,splits['validation'],proposer,ev,search, + iterations=iterations,selection_metric=objective,failure_sample=12,patience=0) + frozen_run(search) + targets.append(('dspy_'+objective,JevConfig.from_dict(freeze['champion']),backend)) + usage.append({'task':task,'stage':'search_'+objective,**ev.summary()}) + demos = [{'input':state(task,r),'answer':r['gold_label']} for r in source['demonstrations']] + controls = [] + for arm,spec in specs.items(): + control = LegacyBackend(model,task,arm,spec,demos,seed) + controls.append(control) + targets.append(('without_dspy_'+arm,cfg,control)) + write_json(task_dir/'frozen-targets.json', {'targets':{name:asdict(c) for name,c,_ in targets}, + 'protocol_sha256':digest(protocol)}) + frozen_calibration = [] + # All calibration parameters are persisted before ANY test prediction in this task. + for name,config,target_backend in targets: + target_dir = task_dir/name + ev = ParallelEvaluator(target_backend,cache,target_dir,12000,workers=workers) + _,cal = ev.evaluate(config,splits['calibration'],'calibration') + fits = {} + for fraction in (.25,.5,1.): + ranked = sorted(cal,key=lambda r:digest(['calibration-prefix-v1',seed,r['group_id']])) + groups = list(dict.fromkeys(r['group_id'] for r in ranked)) + selected = set(groups[:max(1,int(len(groups)*fraction))]) + subset = [r for r in ranked if r['group_id'] in selected] + for method in ('raw','temperature','bias_temperature'): + if method=='raw' and fraction!=1.: + continue + fits[method+'_'+str(fraction)] = fit(subset,list(config.criteria),method) + write_json(target_dir/'calibration.json',fits) + save_predictions(target_dir/'calibration-predictions.json.gz',cal) + frozen_calibration.append((name,config,ev,fits)) + write_json(task_dir/'calibration-freeze.json',{ + name:digest(fits) for name,_,_,fits in frozen_calibration}) + for name,config,ev,fits in frozen_calibration: + for panel,rows in panels(task).items(): + _,predictions = ev.evaluate(config,rows,panel) + save_predictions(task_dir/name/(panel+'-predictions.json.gz'),predictions) + for fit_name,fitted in fits.items(): + transformed = apply(predictions,fitted) + all_metrics.append({'seed':seed,'task':task,'arm':name,'panel':panel, + 'calibration':fit_name,'n_calibration':fitted['n_calibration'], + **evaluation(transformed,config.criteria)}) + write_json(output/'metrics.json',all_metrics) + usage.append({'task':task,'stage':name,**ev.summary()}) + print(json.dumps({'seed':seed,'task':task,'arm':name,'status':'evaluated'}),flush=True) + for control in controls: + control.close() + finally: + backend.close() + cache.close() + write_json(output/'usage.json',usage) + write_json(output/'status.json',{'status':'completed_baseline_only' if baseline_only else 'completed', + 'live_logical_calls':sum(r['logical_calls'] for r in usage), 'protocol_sha256':digest(protocol)}) + write_artifact_inventory(output) + return all_metrics + except Exception as exc: + write_json(output/'usage.json',usage) + write_json(output/'status.json',{'status':'failed','error_type':type(exc).__name__, + 'note':'Partial observations are not complete benchmark results.'}) + raise + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output',type=Path,required=True) + parser.add_argument('--seed',type=int,choices=SEEDS,required=True) + parser.add_argument('--iterations',type=int,default=8) + parser.add_argument('--workers',type=int,default=4) + parser.add_argument('--baseline-only',action='store_true') + args = parser.parse_args() + execute(args.output,args.seed,iterations=args.iterations,workers=args.workers,baseline_only=args.baseline_only) + +if __name__=='__main__': + main() diff --git a/graph_synthesis/dspy_benchmark/tests/test_benchmark.py b/graph_synthesis/dspy_benchmark/tests/test_benchmark.py new file mode 100644 index 0000000..0292e92 --- /dev/null +++ b/graph_synthesis/dspy_benchmark/tests/test_benchmark.py @@ -0,0 +1,225 @@ +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +import numpy as np +from graph_synthesis.dspy_jev_optimizer.core import Example,JevConfig,Cache,BudgetExhausted,metrics,read_json,write_json,digest +from graph_synthesis.dspy_benchmark.data import inventory,load_task,panels,SEEDS +from graph_synthesis.dspy_benchmark.calibration import fit,apply,evaluation,paired_interval +from graph_synthesis.dspy_benchmark.run import ParallelEvaluator,provider_model,execute,write_artifact_inventory +from graph_synthesis.dspy_benchmark.report import summarize + +LABELS={'a':'first','b':'second'} +CFG=JevConfig('task','classify',LABELS) + +def sample(n=20): + return [{'id':str(i),'group_id':str(i//2),'gold':'a' if i%4 else 'b', + 'choice':'a','probabilities':{'a':.98,'b':.02}} for i in range(n)] + + +def completed_fixture(root, seed, *, baseline_only=False): + run_dir=root/f'seed-{seed}' + run_dir.mkdir() + arm='without_dspy_baseline_choice' if baseline_only else 'dspy_accuracy' + protocol={'seed':seed,'iterations':0 if baseline_only else 1,'baseline_only':baseline_only} + write_json(run_dir/'protocol.json',protocol) + write_json(run_dir/'status.json',{'status':'completed_baseline_only' if baseline_only else 'completed', + 'protocol_sha256':digest(protocol)}) + write_json(run_dir/'usage.json',[]) + rows=[] + for task in ('relation_support','entity_resolution'): + task_dir=run_dir/task;task_dir.mkdir() + write_json(task_dir/'frozen-targets.json',{'targets':{arm:{}}}) + write_json(task_dir/'calibration-freeze.json',{}) + rows.append({'task':task,'panel':'test','arm':arm,'calibration':'raw_1.0', + 'n':2,'accuracy':.5,'macro_f1':.5,'mcc':0.,'brier':.25, + 'log_loss':.69,'ece':.1}) + write_json(run_dir/'metrics.json',rows) + write_artifact_inventory(run_dir) + return run_dir + +class FakeBackend: + def __init__(self,seed=1,bad=False): + self.identity={'model':'mock-test-only','seed':seed};self.count=0;self.bad=bad + def predict(self,config,state): + self.count+=1 + return {'model':'drift' if self.bad else self.identity['model'],'choice':'a', + 'probabilities':{'a':.8,'b':.2},'usage':{'input_tokens':10,'output_tokens':0}} + +class Tests(unittest.TestCase): + def test_inventory_all_registered_panels(self): + v=inventory() + self.assertEqual(v['relation_support']['splits']['test']['n'],339) + self.assertEqual(v['entity_resolution']['splits']['test']['n'],413) + self.assertEqual(v['relation_support']['panels']['semantic_challenge']['n'],48) + self.assertEqual(v['relation_support']['panels']['multicall_test']['n'],263) + def test_train_validation_calibration_are_distinct(self): + for task in ('relation_support','entity_resolution'): + _,s,_,_=load_task(task) + self.assertEqual((len(s['train']),len(s['validation'])),(29,31)) + for a in s: + for b in s: + if a!=b:self.assertFalse({r.id for r in s[a]} & {r.id for r in s[b]}) + def test_all_legacy_formulations_are_retained(self): + self.assertEqual(len(load_task('relation_support')[2]),4) + self.assertEqual(len(load_task('entity_resolution')[2]),4) + def test_uncertain_fixtures_are_not_silently_removed(self): + cfg,_,_,_=load_task('entity_resolution') + rows=panels('entity_resolution')['fixtures'] + self.assertEqual(len(rows),100) + self.assertEqual(sum(r.label not in cfg.criteria for r in rows),12) + def test_inventory_deterministic(self): + self.assertEqual(inventory(),inventory()) + def test_state_does_not_include_labels(self): + for task in ('relation_support','entity_resolution'): + for rows in load_task(task)[1].values(): + for r in rows:self.assertFalse({'gold_label','rationale_sentence_ids','label'} & set(r.state)) + def test_temperature_preserves_labels(self): + rows=sample();model=fit(rows,LABELS) + self.assertEqual([r['choice'] for r in apply(rows,model)],[r['choice'] for r in rows]) + def test_temperature_reduces_overconfident_calibration_loss(self): + rows=sample();after=apply(rows,fit(rows,LABELS)) + self.assertLess(metrics(after,LABELS)['log_loss'],metrics(rows,LABELS)['log_loss']) + def test_raw_is_unchanged(self): + rows=sample();self.assertEqual(rows,apply(rows,fit(rows,LABELS,'raw'))) + def test_bias_temperature_is_normalized(self): + for row in apply(sample(),fit(sample(),LABELS,'bias_temperature')): + self.assertAlmostEqual(sum(row['probabilities'].values()),1) + def test_fit_serializes(self): + model=fit(sample(),LABELS) + self.assertEqual(model,json.loads(json.dumps(model))) + def test_invalid_calibration_distribution(self): + rows=sample();rows[0]['probabilities']['a']=4 + with self.assertRaises(ValueError):fit(rows,LABELS) + def test_empty_calibration_rejected(self): + with self.assertRaises(ValueError):fit([],LABELS) + def test_unscorable_excluded_from_denominator(self): + rows=sample()+[{'id':'u','gold':'uncertain','choice':'a','probabilities':{'a':.5,'b':.5}}] + r=evaluation(rows,LABELS) + self.assertEqual((r['n'],r['unscorable_count']),(20,1)) + def test_risk_empty_acceptance_is_not_zero_error_claim(self): + rows=sample();r=evaluation(rows,LABELS) + self.assertIsNone(r['selective'][-1]['risk']) + def test_identical_paired_interval_zero(self): + r=paired_interval(sample(),sample(),LABELS,'brier',repeats=10) + self.assertEqual((r['delta'],r['low'],r['high']),(0.,0.,0.)) + def test_unpaired_intervals_rejected(self): + with self.assertRaises(ValueError):paired_interval(sample(),list(reversed(sample())),LABELS,'accuracy') + def test_five_registered_seeds(self): + self.assertEqual(len(SEEDS),5);self.assertEqual(len(set(SEEDS)),5) + def test_cache_never_crosses_repeat_namespace(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d);cache=Cache(p/'c.sqlite');r=[Example('one',{'x':1},'a')] + a=FakeBackend(1);e=ParallelEvaluator(a,cache,p/'a',10) + e.evaluate(CFG,r,'test');e.evaluate(CFG,r,'test');self.assertEqual(a.count,1) + b=FakeBackend(2);ParallelEvaluator(b,cache,p/'b',10).evaluate(CFG,r,'test') + self.assertEqual(b.count,1);cache.close() + def test_budget_prevents_partial_evaluation(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d);c=Cache(p/'c');b=FakeBackend();e=ParallelEvaluator(b,c,p,1) + with self.assertRaises(BudgetExhausted):e.evaluate(CFG,[Example('1',{'x':1},'a'),Example('2',{'x':2},'a')],'test') + self.assertEqual(b.count,0);c.close() + def test_model_drift_fails_closed(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d);c=Cache(p/'c');e=ParallelEvaluator(FakeBackend(bad=True),c,p,2) + with self.assertRaises(RuntimeError):e.evaluate(CFG,[Example('1',{'x':1},'a')],'test') + c.close() + def test_missing_keys_block_before_model_calls(self): + with tempfile.TemporaryDirectory() as d,patch.dict(os.environ,{},clear=True): + p=Path(d)/'run' + with self.assertRaises(RuntimeError):execute(p,11) + self.assertEqual(read_json(p/'status.json')['live_calls'],0) + def test_missing_matching_proposer_key_blocks_before_target_inference(self): + from graph_synthesis.dspy_benchmark import run + with tempfile.TemporaryDirectory() as d, patch.dict(os.environ,{ + 'TYPESAFE_API_KEY':'test-only', 'DSPY_PROPOSER_MODEL':'openai/test-only'},clear=True), \ + patch.object(run,'TypeSafeBackend',side_effect=AssertionError('target backend constructed')): + output=Path(d)/'seed-11' + with self.assertRaises(RuntimeError): execute(output,11) + status=read_json(output/'status.json') + self.assertEqual(status['status'],'blocked_missing_credentials') + self.assertEqual(status['live_calls'],0) + self.assertFalse((output/'protocol.json').exists()) + def test_zero_proposal_iterations_cannot_complete_comparison(self): + with tempfile.TemporaryDirectory() as d, patch.dict(os.environ,{ + 'TYPESAFE_API_KEY':'test-only', 'DSPY_PROPOSER_MODEL':'openai/test-only', + 'OPENAI_API_KEY':'test-only'},clear=True): + output=Path(d)/'seed-11' + with self.assertRaises(ValueError): execute(output,11,iterations=0) + self.assertFalse(output.exists()) + def test_missing_results_are_not_zero_accuracy(self): + with tempfile.TemporaryDirectory() as d: + self.assertEqual(summarize(Path(d)),[]) + self.assertEqual(read_json(Path(d)/'run-status.json')[0]['status'],'not_run') + def test_baseline_only_results_cannot_fill_primary_comparison(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);completed_fixture(root,11,baseline_only=True) + self.assertEqual(summarize(root),[]) + self.assertEqual(read_json(root/'summary.json'),[]) + self.assertEqual(read_json(root/'baseline-only-summary.json')[0]['runs'],1) + report=(root/'RESULTS.md').read_text(encoding='utf-8') + self.assertIn('No completed live DSPy comparison is available',report) + self.assertIn('Baseline-only results',report) + def test_aggregate_keeps_all_completed_seeds_and_missing_status(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d) + for seed in (11,23): + completed_fixture(root,seed) + summary=summarize(root) + self.assertEqual(summary[0]['runs'],2) + self.assertEqual(read_json(root/'run-status.json')[2]['status'],'not_run') + self.assertIn('2/5',(root/'RESULTS.md').read_text(encoding='utf-8')) + self.assertIn('matrix: **incomplete**',(root/'RESULTS.md').read_text(encoding='utf-8')) + def test_modified_artifact_is_rejected_from_completed_count(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d); run_dir=completed_fixture(root,11) + summarize(root) + self.assertTrue((root/'summary.csv').exists()) + (run_dir/'metrics.json').write_text('[]\n',encoding='utf-8') + self.assertEqual(summarize(root),[]) + status=read_json(root/'run-status.json')[0] + self.assertEqual(status['status'],'invalid_artifact') + self.assertIn('hash mismatch',status['artifact_error']) + self.assertFalse((root/'summary.csv').exists()) + def test_unreadable_seed_does_not_hide_other_completed_runs(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);completed_fixture(root,11) + bad=root/'seed-23';bad.mkdir() + (bad/'status.json').write_text('{broken',encoding='utf-8') + summary=summarize(root) + self.assertEqual(summary[0]['runs'],1) + self.assertEqual(read_json(root/'run-status.json')[1]['status'],'invalid_artifact') + def test_provider_is_explicit_not_magic_fallback(self): + with patch.dict(os.environ,{},clear=True):self.assertIsNone(provider_model()) + def test_existing_output_refused(self): + with tempfile.TemporaryDirectory() as d: + with self.assertRaises(FileExistsError):execute(Path(d),11) + + def test_full_comparison_orchestration_with_test_doubles(self): + events=[] + class Backend: + def __init__(self,*args,**kwargs): self.identity={'model':'jev-1.13.0','test_double':True} + def close(self): pass + def predict(self,config,state): + events.append(state['phase']) + return {'model':'jev-1.13.0','choice':'a','probabilities':{'a':.7,'b':.3},'usage':{'input_tokens':2,'output_tokens':0}} + class Proposer: + def __init__(self,*args,**kwargs): self.identity={'test_double':True} + def propose(self,cfg,feedback,iteration,history): + return {'task':cfg.task,'instructions':cfg.instructions+' revised','criteria':cfg.criteria} + def rows(phase,n): + return [Example(phase+str(i),{'phase':phase,'i':i},'a' if i%2 else 'b',phase+str(i)) for i in range(n)] + splits={s:rows(s,6) for s in ('train','validation','calibration')} + from graph_synthesis.dspy_benchmark import run + with tempfile.TemporaryDirectory() as d, patch.dict(os.environ,{'TYPESAFE_API_KEY':'test-only','DSPY_PROPOSER_MODEL':'openai/test-only','OPENAI_API_KEY':'test-only'},clear=True), patch.object(run,'load_task',return_value=(CFG,splits,{'baseline_choice':{}},{'demonstrations':[]})), patch.object(run,'panels',return_value={'test':rows('test',6)}), patch.object(run,'TypeSafeBackend',Backend), patch.object(run,'LegacyBackend',Backend), patch.object(run,'LoggedProposer',Proposer): + output=Path(d)/'seed-11' + result=execute(output,11,iterations=1,workers=2) + self.assertEqual(read_json(output/'status.json')['status'],'completed') + self.assertEqual(len(result),2*3*7) + self.assertEqual({r['arm'] for r in result},{'dspy_accuracy','dspy_composite','without_dspy_baseline_choice'}) + self.assertTrue((output/'relation_support'/'calibration-freeze.json').exists()) + self.assertGreater(events.index('test'),events.index('calibration')) + +if __name__=='__main__':unittest.main()