diff --git a/.github/workflows/dspy-diagnostics.yml b/.github/workflows/dspy-diagnostics.yml new file mode 100644 index 0000000..54ad0e0 --- /dev/null +++ b/.github/workflows/dspy-diagnostics.yml @@ -0,0 +1,238 @@ +name: Complete and audit live DSPy benchmark comparison +on: + workflow_dispatch: +permissions: + contents: read + actions: read +concurrency: + group: dspy-complete-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false +jobs: + complete: + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 85 + env: + OPENBLAS_NUM_THREADS: '1' + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: {persist-credentials: false} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: {python-version: '3.12'} + - run: python -m pip install numpy==2.3.5 scipy==1.17.0 + - name: Test fixture accounting and full synthetic journal replay + run: python -B -m unittest graph_synthesis.dspy_completion.test_complete graph_synthesis.dspy_diagnostics.test_run graph_synthesis.dspy_diagnostics.test_roundtrip -v + - name: Wait for original primary captures to close; retain actual workflow status + env: + GH_TOKEN: ${{ github.token }} + run: | + python - <<'PY' + import json,os,pathlib,time,urllib.request + base='https://api.github.com/repos/CompleteDotTech/paper-package/actions/' + def get(path): + request=urllib.request.Request(base+path,headers={'Authorization':'Bearer '+os.environ['GH_TOKEN'],'Accept':'application/vnd.github+json'}) + with urllib.request.urlopen(request,timeout=30) as response:return json.load(response) + run=get('runs/35342763186') + assert run['head_sha']=='1f081b6be42bb454f241fb8765d597f4674da13f','Wrong original source' + for attempt in range(150): + jobs=[j for j in get('runs/35342763186/jobs?per_page=100')['jobs'] if j['name'].startswith('primary (')] + if len(jobs)==8 and all(j['status']=='completed' for j in jobs):break + time.sleep(30) + else:raise RuntimeError('Original capture window expired') + root=pathlib.Path(os.environ['RUNNER_TEMP'])/'completed';root.mkdir() + (root/'original-workflow-status.json').write_text(json.dumps({'run':run,'primary_jobs':jobs},indent=2)+'\n') + print('Original capture jobs closed. Full raw completeness must now be proved independently.') + PY + - name: Retrieve original live observations without repeating inference + env: + GH_TOKEN: ${{ github.token }} + run: gh run download 35342763186 --repo CompleteDotTech/paper-package --pattern 'dspy-v2-primary-*' --dir "$RUNNER_TEMP/original" + - name: Require every frozen search and raw final response, then complete fixture reporting + run: | + python -B - <<'PY' + import os,pathlib,shutil + from graph_synthesis.dspy_completion.complete import complete,install + from graph_synthesis.dspy_benchmark import study + from graph_synthesis.dspy_benchmark_analysis.report import load_capture + root=pathlib.Path(os.environ['RUNNER_TEMP']);paths=sorted(p.parent for p in (root/'original').rglob('freeze.json')) + assert len(paths)==8, 'A full comparison requires eight frozen formulation captures' + identities=set() + for source in paths: + destination=root/'completed'/'primary'/source.name + complete(source,destination) + capture=load_capture(destination) + identities.add((capture.task,capture.arm)) + print('Verified every raw final probability:',capture.task,capture.arm,flush=True) + assert identities=={(t,a) for t,arms in study.ARMS.items() for a in arms} + for i,p in enumerate((root/'original').rglob('environment.txt')): + target=root/'completed'/'environments'/f'primary-{i}.txt';target.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,target) + PY + - name: Preserve verified completed captures and original status + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: dspy-v2-completed-primary + path: ${{ runner.temp }}/completed/ + retention-days: 30 + if-no-files-found: warn + primary_analysis: + needs: complete + runs-on: ubuntu-latest + timeout-minutes: 35 + env: {OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1'} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: {persist-credentials: false} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: {python-version: '3.12'} + - run: python -m pip install numpy==2.3.5 scipy==1.17.0 matplotlib==3.10.8 + - env: {GH_TOKEN: '${{ github.token }}'} + run: gh run download '${{ github.run_id }}' --repo CompleteDotTech/paper-package --name dspy-v2-completed-primary --dir "$RUNNER_TEMP/input" + - name: Audit, bootstrap, compare calibration and replay graph compiler + run: | + python -B - <<'PY' + import os,pathlib,sys + from graph_synthesis.dspy_completion.complete import install,analysis_view + install() + from graph_synthesis.dspy_benchmark_analysis.report import load_capture,analyze + from graph_synthesis.dspy_benchmark import study + root=pathlib.Path(os.environ['RUNNER_TEMP']);paths=sorted(p.parent for p in (root/'input'/'primary').rglob('execution.json')) + captures=[load_capture(p) for p in paths];assert len(captures)==8 + output=root/'evidence'/'primary';analyze([analysis_view(c) for c in captures],output,graphs=True) + ambiguity=[r for c in captures for r in study.read(c.directory/'ambiguity-results.json')] + study.write_json(output/'ambiguity-results.json',ambiguity) + with (output/'REPORT.md').open('a') as f:f.write('\n\n## Ambiguous entity fixtures\n\nAll 100 original fixture inputs were captured. Binary classification uses the 88 labelled same/different fixtures; the 12 original uncertain fixtures have separate confidence/entropy diagnostics in ambiguity-results.json. The raw inventory includes every fixture. No main held-out score or prompt was changed by this reporting correction.\n') + study.write_json(output/'artifact-manifest.json',{'sha256':{p.relative_to(output).as_posix():study.sha(p) for p in sorted(output.rglob('*')) if p.is_file() and p.name!='artifact-manifest.json'}}) + from graph_synthesis.dspy_diagnostics.quality import main + sys.argv=['quality','--input',str(root/'input'/'primary'),'--output',str(root/'evidence'/'quality')];main() + PY + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: dspy-completed-primary-analysis + path: ${{ runner.temp }}/evidence/ + retention-days: 30 + if-no-files-found: warn + transfer: + needs: complete + strategy: + fail-fast: false + max-parallel: 8 + matrix: {shard: [0,1,2,3,4,5,6,7]} + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: {persist-credentials: false} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: {python-version: '3.12'} + - run: python -m pip install numpy==2.3.5 scipy==1.17.0 + - env: {GH_TOKEN: '${{ github.token }}'} + run: gh run download '${{ github.run_id }}' --repo CompleteDotTech/paper-package --name dspy-v2-completed-primary --dir "$RUNNER_TEMP/input" + - name: Reuse the exact completed transfer capture without inference + env: + GH_TOKEN: ${{ github.token }} + run: | + test "$(gh api repos/CompleteDotTech/paper-package/actions/runs/35345560822 --jq .head_sha)" = 'f197f9f24076bf85b7739916669dae9fdb428adf' + gh run download 35345560822 --repo CompleteDotTech/paper-package --name 'dspy-completed-transfer-${{ matrix.shard }}' --dir "$RUNNER_TEMP/transfer" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: dspy-completed-transfer-${{ matrix.shard }} + path: ${{ runner.temp }}/transfer/ + retention-days: 30 + if-no-files-found: warn + diagnostics: + needs: complete + strategy: + fail-fast: false + matrix: {task: [relation_support, entity_resolution]} + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: {persist-credentials: false} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: {python-version: '3.12'} + - run: python -m pip install numpy==2.3.5 scipy==1.17.0 + - env: {GH_TOKEN: '${{ github.token }}'} + run: gh run download '${{ github.run_id }}' --repo CompleteDotTech/paper-package --name dspy-v2-completed-primary --dir "$RUNNER_TEMP/input" + - name: Reuse the exact completed repeatability capture without inference + env: + GH_TOKEN: ${{ github.token }} + run: | + test "$(gh api repos/CompleteDotTech/paper-package/actions/runs/35345560822 --jq .head_sha)" = 'f197f9f24076bf85b7739916669dae9fdb428adf' + gh run download 35345560822 --repo CompleteDotTech/paper-package --name 'dspy-completed-diagnostic-${{ matrix.task }}' --dir "$RUNNER_TEMP/diagnostic" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: dspy-completed-diagnostic-${{ matrix.task }} + path: ${{ runner.temp }}/diagnostic/ + retention-days: 30 + if-no-files-found: warn + audit: + needs: [primary_analysis, transfer, diagnostics] + runs-on: ubuntu-latest + timeout-minutes: 35 + env: {OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1'} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: {persist-credentials: false} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: {python-version: '3.12'} + - run: python -m pip install numpy==2.3.5 scipy==1.17.0 matplotlib==3.10.8 + - name: Retrieve all completed evidence and pilot overhead + env: {GH_TOKEN: '${{ github.token }}'} + run: | + gh run download '${{ github.run_id }}' --repo CompleteDotTech/paper-package --name dspy-v2-completed-primary --dir "$RUNNER_TEMP/primary-input" + gh run download '${{ github.run_id }}' --repo CompleteDotTech/paper-package --name dspy-completed-primary-analysis --dir "$RUNNER_TEMP/evidence" + gh run download '${{ github.run_id }}' --repo CompleteDotTech/paper-package --pattern 'dspy-completed-transfer-*' --dir "$RUNNER_TEMP/transfer-input" + gh run download '${{ github.run_id }}' --repo CompleteDotTech/paper-package --pattern 'dspy-completed-diagnostic-*' --dir "$RUNNER_TEMP/diagnostic-input" + for RUN in 35339699653 35340617823; do + gh run download "$RUN" --repo CompleteDotTech/paper-package --pattern 'dspy-live-*' --dir "$RUNNER_TEMP/pilots/$RUN" + done + - name: Independently reconstruct every multi-call policy, diagnostic request and metric + run: | + python -B - <<'PY' + import os,pathlib + from graph_synthesis.dspy_benchmark_v2.multicall import install + install() + from graph_synthesis.dspy_benchmark_analysis.multicall_report import report + root=pathlib.Path(os.environ['RUNNER_TEMP']) + report(root/'transfer-input',root/'primary-input'/'primary'/'relation_support--fewshot_contract',root/'evidence'/'multicall') + PY + python -B -m graph_synthesis.dspy_diagnostics.report --input "$RUNNER_TEMP/diagnostic-input" --primary "$RUNNER_TEMP/primary-input/primary" --output "$RUNNER_TEMP/evidence/repeatability" + - name: Preserve raw captures, failed pilots, environments and original workflow status + run: | + python - <<'PY' + import gzip,json,os,pathlib,shutil + root=pathlib.Path(os.environ['RUNNER_TEMP']);out=root/'evidence' + for kind,source in [('primary','primary-input/primary'),('transfer','transfer-input'),('diagnostics','diagnostic-input')]: + for p in (root/source).rglob('execution.json'): + if (p.parent/'protocol.json').exists():shutil.copytree(p.parent,out/'captures'/kind/p.parent.name) + shutil.copytree(root/'primary-input'/'environments',out/'environments') + shutil.copyfile(root/'primary-input'/'original-workflow-status.json',out/'original-workflow-status.json') + pilots=[] + for p in (root/'pilots').rglob('calls.jsonl.gz'): + run=next(x for x in p.parts if x in ('35339699653','35340617823')) + shutil.copytree(p.parent,out/'pilots'/run/p.parent.name) + calls=[json.loads(line) for line in gzip.open(p,'rt')] + pilots.append({'run':run,'formulation':p.parent.name,'http_attempts':len(calls),'successful_responses':sum(c['error'] is None for c in calls),'reported_input_tokens':sum((c.get('response') or {}).get('usage',{}).get('input_tokens',0) for c in calls),'test_calls':sum(c['site'].startswith('test/') for c in calls),'note':'Nested-output pilot, not pooled with v2 searches.'}) + (out/'pilot-summary.json').write_text(json.dumps(pilots,indent=2)+'\n') + PY + git archive --format=tar HEAD > "$RUNNER_TEMP/evidence/repository-source.tar" + python -m pip freeze > "$RUNNER_TEMP/evidence/analysis-environment.txt" + python - <<'PY' + import hashlib,json,os,pathlib + root=pathlib.Path(os.environ['RUNNER_TEMP'])/'evidence' + files={p.relative_to(root).as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(root.rglob('*')) if p.is_file()} + (root/'package-manifest.json').write_text(json.dumps({'sha256':files},indent=2)+'\n') + PY + - name: Preserve complete audited comparison + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: dspy-final-complete-evidence + path: ${{ runner.temp }}/evidence/ + retention-days: 30 + if-no-files-found: warn diff --git a/experiments/dspy-calibration-20260918/FIXTURE_REPORTING.md b/experiments/dspy-calibration-20260918/FIXTURE_REPORTING.md new file mode 100644 index 0000000..9a2b371 --- /dev/null +++ b/experiments/dspy-calibration-20260918/FIXTURE_REPORTING.md @@ -0,0 +1,30 @@ +# Reporting correction for ambiguous entity fixtures + +The entity primary jobs completed all five real DSPy searches, all 390 calibration +inputs, all 413 held-out pairs, and all 100 fixture inputs. Reporting then rejected +the 12 fixture rows whose original gold label is `uncertain`, outside the task's +fixed binary `same`/`different` label schema. A CI failure at that point is not a +missing inference observation and does not justify paying to repeat completed calls. + +This correction restores the original research analyzer's semantic-eligibility +convention: classification metrics and paired comparisons use the 88 binary-labelled +fixtures; all 100 predictions remain in the raw and reconstructed evidence. The 12 +uncertain fixtures receive a separate descriptive report of confidence, entropy and +high-confidence decision rates. They are not relabelled, counted as correct or +silently removed from the inventory. No binary Brier, NLL or accuracy is defined for +an uncertain gold label. The complete original source analyzer already excludes +labels outside each task's label schema from semantic metrics and paired intervals. + +This is a reporting correction made after the first primary evaluation scores were +available, not a newly preregistered scoring rule. It changes no prompt, seed, +calibrator, search choice, source input or main held-out test metric. Original partial +files are preserved with hashes. Completion succeeds only when every required frozen +search and every raw calibration/test response is present and its request can be +reconstructed exactly. Missing evidence remains a failure; no synthetic prediction +or repeated paid inference substitutes for it. + +Completed capture manifests distinguish the original live observations from this +zero-inference report reconstruction. The original workflow status remains recorded +as failed where applicable. Downstream transfer and repeatability runs consume the +verified complete raw observations and frozen prompts, not a fabricated successful +workflow status. diff --git a/experiments/dspy-calibration-20260918/REPEATABILITY_PROTOCOL.md b/experiments/dspy-calibration-20260918/REPEATABILITY_PROTOCOL.md new file mode 100644 index 0000000..f11bf0e --- /dev/null +++ b/experiments/dspy-calibration-20260918/REPEATABILITY_PROTOCOL.md @@ -0,0 +1,40 @@ +# Post-freeze repeatability and original batching layout + +This diagnostic extension is fixed before reading amendment-v2 test scores. It makes +no new proposals and does not change any primary prompt, selection rule or calibrator. +Use every completed original formulation's baseline plus all five frozen accuracy +champions. Do not select a favorable seed. Native service variation is separate from +prompt-search variability and must not be credited as DSPy improvement when a prompt +is unchanged. + +For each task, rerun the original 20 evaluation `repeatability_ids` three times. +For the original 20 development `batching_ids`, compare a shared request containing +all non-few-shot formulations against individual formulation/seed requests. Few-shot +input retains its six demonstrations and is placed in a separate shared-state bank. +This preserves the original grouping rule while expanding it to baseline plus five +DSPy variants. Each row has 24 configurations overall: 18 ordinary, six few-shot. + +There are 120 repeated-evaluation requests, 40 batched development requests, and 480 +individual development requests per task: 640 successful requests per task, 1,280 +across both tasks, before retries. Requests are limited to two workers and minimum +0.8-second spacing per task, at most 2,000 attempts and the existing token limit. +Authentication failures and distributions outside the amendment-v2 interpretation +stop execution. Original probabilities and raw responses are retained. + +Evaluate raw, temperature, and temperature-plus-bias probabilities on the repeatability +panel using only the already-fitted, disjoint primary calibrators. Report seed-wise +means over three service replicates, between-seed dispersion, and within-seed service +dispersion separately. Twenty repeated inputs remain twenty inputs, not 300 new test +examples. These originally selected diagnostic panels are not population estimates. + +Record label/probability disagreement across repeats and between batched/individual +requests. Individual request ordering is shuffled deterministically within that phase; +batched and individual phases are not randomized across server load. Therefore +latency comparisons are descriptive, not a controlled hardware-throughput claim. +Report actual input usage per isolated baseline/DSPy prediction and packing savings +for identical configurations/examples. Optimizer overhead, local proposer CPU and CI +runtime are separate from deployment inference costs. + +No label-accuracy claim is made from the development batching panel. Every request, +probability, calibration transformation and metric is reconstructed in a network-free +audit. Partial task captures must never be represented as the complete comparison. diff --git a/graph_synthesis/dspy_benchmark_analysis/report.py b/graph_synthesis/dspy_benchmark_analysis/report.py index fe9b883..81a2ba7 100644 --- a/graph_synthesis/dspy_benchmark_analysis/report.py +++ b/graph_synthesis/dspy_benchmark_analysis/report.py @@ -17,6 +17,7 @@ 'correct_positive_edges', 'wrong_positive_edges') METHODS = ('raw', 'temperature', 'temperature_bias') BOOTSTRAPS = 2000 +NUMERICAL_TIE_GAP = 1e-14 def close(a, b, context=''): @@ -31,6 +32,44 @@ def close(a, b, context=''): elif a != b: raise ValueError('Mismatched value: '+context) +def aurc_tie_bounds(probabilities, rows, task): + """Bounds from permuting only confidence scores indistinguishable at 1e-14.""" + p = np.asarray(probabilities, dtype=float) + confidence = p.max(axis=1) + labels = LABELS[task] + wrong = p.argmax(axis=1) != np.asarray([labels.index(r['gold_label']) for r in rows]) + order = sorted(range(len(rows)), key=lambda i: -confidence[i]) + low, high = [], [] + start = 0 + while start < len(order): + end = start + 1 + while end < len(order) and confidence[order[start]] - confidence[order[end]] <= NUMERICAL_TIE_GAP: + end += 1 + errors = [int(wrong[i]) for i in order[start:end]] + low.extend(sorted(errors)) + high.extend(sorted(errors, reverse=True)) + start = end + ranks = np.arange(1, len(rows) + 1) + return (float(np.mean(np.cumsum(low) / ranks)), + float(np.mean(np.cumsum(high) / ranks))) + + +def numerical_temperature_ties(raw, transformed, rows): + """Identify rank changes solely within unresolved floating-point top ties.""" + before = np.asarray(raw) + after = np.asarray(transformed) + changed = np.flatnonzero(before.argmax(axis=1) != after.argmax(axis=1)) + tie_ids = [] + for i in changed: + raw_top = np.sort(before[i])[-2:] + calibrated_top = np.sort(after[i])[-2:] + if (raw_top[1] - raw_top[0] > NUMERICAL_TIE_GAP or + calibrated_top[1] - calibrated_top[0] > NUMERICAL_TIE_GAP): + raise ValueError('Scalar temperature changed a resolved argmax: ' + rows[i]['id']) + tie_ids.append(rows[i]['id']) + return tie_ids + + def clusters(rows): """Connected components of overlapping source/entity identities, not individual pairs.""" parent=list(range(len(rows))); seen={} @@ -120,7 +159,7 @@ def load_capture(directory): for search in frozen['payload']['searches']: if digest(read(directory/f"search-{search['seed']}.json"))!=search['ledger_sha256']: raise ValueError('Search ledger integrity failure') - cal=read(directory/'calibration.json');results=read(directory/'results.json');panels={};keys=set() + cal=read(directory/'calibration.json');results=read(directory/'results.json');panels={};keys=set();aurc_drift=[] for panel,rows in expected_panels.items(): saved=read(directory/(panel+'-predictions.json')) if [r['id'] for r in saved['rows']]!=[r['id'] for r in rows]:raise ValueError('Panel IDs changed') @@ -135,8 +174,18 @@ def load_capture(directory): for method in METHODS: matches=[r for r in results if r['panel']==panel and r['selection']==key[0] and r['seed']==key[1] and r['calibration']==method] if len(matches)!=1:raise ValueError('Duplicate or missing result') - recomputed=evaluate_metrics(calibrated(p[i],cal['fits'][i],method),rows,task) - close(recomputed,matches[0]['metrics'],f'{task}/{arm}/{panel}/{key}/{method}') + transformed=calibrated(p[i],cal['fits'][i],method) + recomputed=evaluate_metrics(transformed,rows,task) + recorded=matches[0]['metrics'] + context=f'{task}/{arm}/{panel}/{key}/{method}' + close({k:v for k,v in recomputed.items() if k!='aurc'}, + {k:v for k,v in recorded.items() if k!='aurc'},context) + if not np.isclose(recomputed['aurc'],recorded['aurc'],atol=2e-10,rtol=1e-9): + lower,upper=aurc_tie_bounds(transformed,rows,task) + if not lower-2e-10 <= recorded['aurc'] <= upper+2e-10: + raise ValueError('AURC exceeds near-tie permutation bounds: '+context) + aurc_drift.append({'context':context,'recorded':recorded['aurc'], + 'recomputed':recomputed['aurc'],'tie_bounds':[lower,upper]}) keys.add((panel,*key,method)) if len(keys)!=len(results):raise ValueError('Unaccounted results') calls={};n_calls=tokens=0;phase=Counter();latency=defaultdict(list) @@ -177,7 +226,7 @@ def load_capture(directory): for i,cfg in enumerate(configs): raw=probabilities({k:call['response']['answers'][f'v{i}__{k}'] for k in cfg},task,cfg) close(raw,saved['probabilities'][i][j],f'raw/{panel}/{i}/{j}') - execution={**execution,'attempts_by_phase':dict(phase), + execution={**execution,'numerical_aurc_tie_drift':aurc_drift,'attempts_by_phase':dict(phase), 'latency_seconds_by_phase':{k:{'median':float(np.median(v)),'p95':float(np.quantile(v,.95))} for k,v in latency.items()}} return Capture(directory,task,arm,protocol,cal,results,panels,execution,calls) @@ -231,7 +280,7 @@ def analyze(captures,out,graphs=False): flat=[{**{k:r[k] for k in ('task','arm','panel','selection','seed','calibration')},**{k:r['metrics'][k] for k in ('n',)+METRICS}} for r in results] write_csv(out/'all-results.csv',flat);write_json(out/'seed-summary.json',summary) write_json(out/'execution.json',[{'task':c.task,'arm':c.arm,**c.execution} for c in captures]) - intervals=[];changes=[];ledger_stats=[];calibration_effects=[] + intervals=[];changes=[];ledger_stats=[];calibration_effects=[];temperature_ties=[] for cap in captures: for seed in SEEDS: ledger=read(cap.directory/f'search-{seed}.json') @@ -258,11 +307,16 @@ def analyze(captures,out,graphs=False): for method in METHODS[1:]: transformed=calibrated(p[i],cap.calibration['fits'][i],method);cal=evaluate_metrics(transformed,rows,cap.task) changed=int(np.sum(p[i].argmax(axis=1)!=transformed.argmax(axis=1))) - if method=='temperature' and changed:raise ValueError('Scalar temperature changed argmax') + if method=='temperature': + ids=numerical_temperature_ties(p[i],transformed,rows) + if ids:temperature_ties.append({'task':cap.task,'arm':cap.arm,'panel':panel,**v,'ids':ids}) calibration_effects.append({'task':cap.task,'arm':cap.arm,'panel':panel,**v,'calibration':method,'changed_labels':changed, **{m+'_delta':cal[m]-raw[m] for m in METRICS}}) write_json(out/'paired-intervals.json',intervals);write_json(out/'prediction-changes.json',changes) write_json(out/'search-summary.json',ledger_stats);write_json(out/'calibration-effects.json',calibration_effects) + write_json(out/'numerical-ties.json',{'scalar_temperature_argmax_ties':temperature_ties, + 'confidence_rank_tolerance':NUMERICAL_TIE_GAP, + 'note':'Raw observations and fitted calibrators are unchanged; these are floating-point tie sensitivities, not new Jev calls.'}) if graphs: replay=[] for cap in captures: @@ -300,7 +354,7 @@ def render(out,summary,captures,graphs): '','## Main held-out panels','', '| Task / original formulation | Baseline accuracy | DSPy accuracy mean ± SD | DSPy raw NLL | DSPy temperature NLL | DSPy temperature + bias NLL |', '|---|---:|---:|---:|---:|---:|',*table,'', - 'Scalar temperature is a probability-quality intervention and leaves argmax accuracy unchanged. Temperature plus bias may change labels. Neither calibration method is guaranteed to improve held-out results.', + 'Scalar temperature is analytically rank preserving; one recorded near-tie may change its floating-point argmax after transformation. See numerical-ties.json. Temperature plus bias may change labels. Neither calibration method is guaranteed to improve held-out results.', '', '## Evidence and interpretation','', '- `all-results.csv` and `all-results.json`: every seed, formulation, panel, selection rule and calibration method, including confusion matrices in JSON.', '- `seed-summary.json`: means, sample standard deviations, minima and maxima; no best-test-seed selection.', diff --git a/graph_synthesis/dspy_benchmark_analysis/test_report.py b/graph_synthesis/dspy_benchmark_analysis/test_report.py index 40a3b97..becfd76 100644 --- a/graph_synthesis/dspy_benchmark_analysis/test_report.py +++ b/graph_synthesis/dspy_benchmark_analysis/test_report.py @@ -2,9 +2,26 @@ import unittest import numpy as np from graph_synthesis.dspy_benchmark.study import evaluate_metrics, LABELS -from .report import clusters, weighted_scores, paired_interval +from .report import (aurc_tie_bounds, clusters, numerical_temperature_ties, + paired_interval, weighted_scores) class TestReport(unittest.TestCase): + def test_temperature_near_tie_is_reported_but_resolved_flip_fails(self): + rows=[{'id':'tie'}] + self.assertEqual(numerical_temperature_ties( + np.array([[.45999999999999996,.46]]),np.array([[.4,.4]]),rows),['tie']) + with self.assertRaisesRegex(ValueError,'resolved argmax'): + numerical_temperature_ties(np.array([[.4,.6]]),np.array([[.6,.4]]),rows) + + def test_aurc_bounds_only_permute_near_equal_confidence(self): + rows=[{'id':'a','gold_label':'same'},{'id':'b','gold_label':'same'}, + {'id':'c','gold_label':'different'}] + p=np.array([[.8,.2],[.2,.8],[.3,.7]]) + lower,upper=aurc_tie_bounds(p,rows,'entity_resolution') + self.assertLess(lower,upper) + self.assertAlmostEqual(lower,(0+1/2+1/3)/3) + self.assertAlmostEqual(upper,(1+1/2+1/3)/3) + def test_overlapping_identity_units_share_component(self): result=clusters([{'groups':['a','b']},{'groups':['b','c']},{'groups':['d']},{'groups':['e','c']}]) self.assertEqual([x.tolist() for x in result],[[0,1,3],[2]]) diff --git a/graph_synthesis/dspy_completion/__init__.py b/graph_synthesis/dspy_completion/__init__.py new file mode 100644 index 0000000..daffb0f --- /dev/null +++ b/graph_synthesis/dspy_completion/__init__.py @@ -0,0 +1 @@ +"""Complete reporting from full raw captures without repeating paid model calls.""" diff --git a/graph_synthesis/dspy_completion/complete.py b/graph_synthesis/dspy_completion/complete.py new file mode 100644 index 0000000..27e1940 --- /dev/null +++ b/graph_synthesis/dspy_completion/complete.py @@ -0,0 +1,157 @@ +"""Reconstruct complete v2 observations; distinguish unknown-label diagnostic fixtures. + +This module makes no network calls. A failed CI status does not become a successful +capture unless every frozen search, calibration row and evaluation response exists. +""" +from __future__ import annotations +import argparse +from collections import Counter +from copy import copy +import gzip +import json +from pathlib import Path +import shutil +import numpy as np +from graph_synthesis.dspy_benchmark import study +from graph_synthesis.dspy_benchmark_v2.adapter import probabilities,distribution +from graph_synthesis.dspy_benchmark_analysis import report + +_original_metrics=study.evaluate_metrics + + +def eligible_metrics(p,rows,task): + labels=study.LABELS[task];unknown=[r for r in rows if r['gold_label'] not in labels] + if unknown and (task!='entity_resolution' or any(r['gold_label']!='uncertain' for r in unknown)): + raise ValueError('Unknown label is not an original ambiguous entity fixture') + indices=[i for i,r in enumerate(rows) if r['gold_label'] in labels] + if not indices:raise ValueError('No semantically eligible classification rows') + result=_original_metrics(np.asarray(p)[indices],[rows[i] for i in indices],task) + if unknown: + result['input_rows']=len(rows);result['ambiguous_fixture_rows']=len(unknown) + return result + + +def install(): + study.probabilities=probabilities + study.evaluate_metrics=eligible_metrics + report.evaluate_metrics=eligible_metrics + + +def expected_payload(task,arm,row,configs,demos): + state=study.state(task,row) + if arm=='fewshot_contract': + state={'labeled_examples':[{'input':study.state(task,d),'answer':d['gold_label']} for d in demos],'input':state} + return {'model':study.MODEL,'state':state, + 'questions':{f'v{i}__{key}':q for i,config in enumerate(configs) for key,q in config.items()}} + + +def complete(source,out): + install() + if out.exists():raise ValueError('Never overwrite original or completed evidence') + protocol=study.read(source/'protocol.json');task,arm=protocol['task'],protocol['arm'] + if protocol.get('revision')!=2 or protocol['seeds']!=study.SEEDS or protocol['model']!=study.MODEL:raise ValueError('Wrong experiment') + plan,parts,panels=study.prepare(task) + for name,want in protocol['data_hashes'].items(): + if study.sha(study.ROOT/name)!=want:raise ValueError('Data hash changed') + for name,rows in parts.items(): + if protocol['split_ids'][name]!=[r['id'] for r in rows]:raise ValueError('Split changed') + freeze=study.read(source/'freeze.json') + if study.digest(freeze['payload'])!=freeze['sha256'] or freeze['payload']['protocol_sha256']!=study.digest(protocol):raise ValueError('Frozen selection invalid') + searches=freeze['payload']['searches'] + if [s['seed'] for s in searches]!=study.SEEDS:raise ValueError('Missing frozen searches') + for search in searches: + ledger=study.read(source/f"search-{search['seed']}.json") + if len(ledger)!=4 or study.digest(ledger)!=search['ledger_sha256']:raise ValueError('Missing or changed search ledger') + variants=[('baseline',0,protocol['baseline'])]+[(selection,s['seed'],s[selection+'_questions']) for s in searches for selection in ('accuracy','nll')] + order=list(plan['question_specs'][task][arm]);configs=[{key:q[key] for key in order} for _,_,q in variants] + journal={};attempts=tokens=0;defects=[];choice_answers=0 + with gzip.open(source/'calls.jsonl.gz','rt',encoding='utf-8') as handle: + for line in handle: + call=json.loads(line);attempts+=1 + if call['request_sha256']!=study.digest(call['payload']) or call['payload']['model']!=study.MODEL:raise ValueError('Invalid request identity') + response=call.get('response') + if response: + if response['model']!=study.MODEL:raise ValueError('Response model drift') + tokens+=int(response.get('usage',{}).get('input_tokens',0)) + for key,answer in response['answers'].items(): + if answer['type']=='choice': + choice_answers+=1;p=list(answer['probabilities'].values());distribution(p) + if abs(sum(p)-1)>1e-6:defects.append({'site':call['site'],'question':key,'raw_mass':sum(p),'raw_probabilities':answer['probabilities']}) + if (call['site'].startswith('calibration/') or call['site'].startswith('test/')) and call['error'] is None: + if call['site'] in journal:raise ValueError('Duplicate final response') + journal[call['site']]=call + used=set() + def reconstruct(rows,phase): + output=[] + for row in rows: + site=phase+'/'+row['id'] + if site not in journal:raise ValueError('Missing live observation: '+site) + call=journal[site];payload=expected_payload(task,arm,row,configs,parts['demonstrations']) + if study.digest(payload)!=call['request_sha256']:raise ValueError('Reconstructed request changed') + if set(call['response']['answers'])!=set(payload['questions']):raise ValueError('Incomplete answer bank') + used.add(site) + output.append([probabilities({key:call['response']['answers'][f'v{i}__{key}'] for key in q},task,q) for i,q in enumerate(configs)]) + return np.asarray(output).transpose(1,0,2) + cal=study.read(source/'calibration.json');pc=reconstruct(parts['calibration'],'calibration') + report.close(pc.tolist(),cal['probabilities'],'calibration probabilities') + y=[study.LABELS[task].index(r['gold_label']) for r in parts['calibration']] + if cal['gold']!=y or cal['ids']!=[r['id'] for r in parts['calibration']]:raise ValueError('Calibration identity changed') + for i,p in enumerate(pc): + fitted=study.fit_calibration(p,y) + for method in ('raw','temperature','temperature_bias'): + if not np.allclose(study.calibrated(p,fitted,method),study.calibrated(p,cal['fits'][i],method),atol=2e-5,rtol=2e-5):raise ValueError('Calibrator cannot be reproduced') + predictions={};results=[];ambiguity=[] + for panel,rows in panels.items(): + p=reconstruct(rows,'test/'+panel) + saved={'rows':[{'id':r['id'],'gold_label':r['gold_label'],'groups':sorted(study.units(task,r)),'family':r.get('family')} for r in rows], + 'variants':[{'selection':s,'seed':seed} for s,seed,_ in variants],'probabilities':p.tolist()} + original=source/(panel+'-predictions.json') + if original.exists():report.close(saved,study.read(original),'saved panel') + predictions[panel]=saved + unknown=[i for i,r in enumerate(rows) if r['gold_label'] not in study.LABELS[task]] + for i,(selection,seed,cfg) in enumerate(variants): + for method in ('raw','temperature','temperature_bias'): + transformed=study.calibrated(p[i],cal['fits'][i],method) + results.append({'task':task,'arm':arm,'panel':panel,'selection':selection,'seed':seed,'calibration':method, + 'metrics':eligible_metrics(transformed,rows,task),'config_sha256':study.digest(cfg)}) + if unknown: + values=transformed[unknown];confidence=values.max(axis=1) + ambiguity.append({'task':task,'arm':arm,'panel':panel,'selection':selection,'seed':seed,'calibration':method, + 'n_ambiguous':len(unknown),'ids':[rows[j]['id'] for j in unknown], + 'mean_max_probability':float(confidence.mean()),'max_probability':float(confidence.max()), + 'fraction_confidence_at_least_0_9':float(np.mean(confidence>=.9)), + 'mean_entropy_nats':float(np.mean(-np.sum(values*np.log(np.clip(values,1e-15,1)),axis=1))), + 'note':'Gold uncertain is outside the binary schema. No binary accuracy, NLL or Brier score is defined for these rows.'}) + if used!=set(journal):raise ValueError('Unexpected unaccounted final responses') + shutil.copytree(source,out) + if (out/'results.json').exists():shutil.copyfile(out/'results.json',out/'original-results-before-completion.json') + for panel,value in predictions.items():study.write_json(out/(panel+'-predictions.json'),value) + study.write_json(out/'results.json',results);study.write_json(out/'ambiguity-results.json',ambiguity) + study.write_json(out/'probability-quality.json',{'choice_answers':choice_answers,'rounded_mass_defects':len(defects),'defects':defects, + 'note':'Bounded input normalization fixed before this capture, distinct from fitted calibration.'}) + study.write_json(out/'completion.json',{'source_files_sha256':{p.name:study.sha(p) for p in sorted(source.iterdir()) if p.is_file()}, + 'source_workflow_run':35342763186,'completion_source_sha256':study.sha(__file__),'new_inference_calls':0, + 'verified_final_response_count':len(used),'fixture_rule':'Binary classification metrics exclude original uncertain labels; all 100 fixture observations retained, 12 ambiguity rows reported separately.'}) + study.write_json(out/'execution.json',{'status':'completed','revision':2,'live':True,'http_attempts':attempts,'reported_input_tokens':tokens, + 'estimated_jev_usd':tokens*.042/1_000_000,'price_basis':'$0.042/M input; estimate, not invoice', + 'dspy_searches':5,'proposals_planned':15,'completion':'All required actual v2 responses verified; reporting completed offline with zero extra inference.'}) + study.write_json(out/'artifact-manifest.json',{'sha256':{p.name:study.sha(p) for p in sorted(out.iterdir()) if p.is_file() and p.name!='artifact-manifest.json'}}) + return out + + +def analysis_view(capture): + """The raw audit sees every fixture; classification statistics use eligible rows only.""" + view=copy(capture);view.panels=dict(capture.panels) + if capture.task=='entity_resolution': + saved=dict(view.panels['fixtures']);indices=[i for i,r in enumerate(saved['rows']) if r['gold_label'] in study.LABELS[capture.task]] + if len(indices)!=88:raise ValueError('Unexpected eligible entity fixture count') + saved['rows']=[saved['rows'][i] for i in indices] + saved['probabilities']=np.asarray(saved['probabilities'])[:,indices].tolist();view.panels['fixtures']=saved + return view + + +def main(): + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--input',type=Path,required=True);p.add_argument('--output',type=Path,required=True) + a=p.parse_args();complete(a.input,a.output) + +if __name__=='__main__':main() diff --git a/graph_synthesis/dspy_completion/test_complete.py b/graph_synthesis/dspy_completion/test_complete.py new file mode 100644 index 0000000..3fe6bbe --- /dev/null +++ b/graph_synthesis/dspy_completion/test_complete.py @@ -0,0 +1,29 @@ +import unittest +import numpy as np +from .complete import eligible_metrics,expected_payload,analysis_view +from graph_synthesis.dspy_benchmark import study + +class CompletionTests(unittest.TestCase): + def test_uncertain_is_not_invented_binary_ground_truth(self): + rows=[{'id':'a','gold_label':'same'},{'id':'b','gold_label':'different'},{'id':'c','gold_label':'uncertain'}] + result=eligible_metrics(np.array([[.8,.2],[.1,.9],[.99,.01]]),rows,'entity_resolution') + self.assertEqual(result['n'],2);self.assertEqual(result['input_rows'],3) + self.assertEqual(result['ambiguous_fixture_rows'],1);self.assertEqual(result['accuracy'],1.) + def test_other_unknown_labels_are_not_silently_dropped(self): + with self.assertRaises(ValueError):eligible_metrics([[.8,.2]],[{'id':'a','gold_label':'mistyped'}],'entity_resolution') + with self.assertRaises(ValueError):eligible_metrics([[.8,.1,.1]],[{'id':'a','gold_label':'uncertain'}],'relation_support') + def test_original_fixture_populations(self): + _,_,panels=study.prepare('entity_resolution');rows=panels['fixtures'] + self.assertEqual(len(rows),100) + self.assertEqual(sum(r['gold_label']=='uncertain' for r in rows),12) + result=eligible_metrics(np.tile([.8,.2],(100,1)),rows,'entity_resolution') + self.assertEqual(result['n'],88);self.assertEqual(result['input_rows'],100) + def test_fewshot_payload_keeps_fixed_examples_and_no_target_label(self): + plan,parts,_=study.prepare('relation_support') + q=plan['question_specs']['relation_support']['fewshot_contract'] + p=expected_payload('relation_support','fewshot_contract',parts['train'][0],[q]*11,parts['demonstrations']) + self.assertEqual(len(p['state']['labeled_examples']),6) + self.assertNotIn('gold_label',p['state']['input']) + self.assertEqual(len(p['questions']),11) + +if __name__=='__main__':unittest.main() diff --git a/graph_synthesis/dspy_diagnostics/__init__.py b/graph_synthesis/dspy_diagnostics/__init__.py new file mode 100644 index 0000000..a9e3dfb --- /dev/null +++ b/graph_synthesis/dspy_diagnostics/__init__.py @@ -0,0 +1 @@ +"""Post-freeze live repeatability, batching and measurement-quality diagnostics.""" diff --git a/graph_synthesis/dspy_diagnostics/quality.py b/graph_synthesis/dspy_diagnostics/quality.py new file mode 100644 index 0000000..1e87098 --- /dev/null +++ b/graph_synthesis/dspy_diagnostics/quality.py @@ -0,0 +1,132 @@ +"""Post-capture diagnostics; never used for prompt or calibrator selection.""" +from __future__ import annotations +import argparse +from collections import Counter,defaultdict +import gzip +import json +from pathlib import Path +import numpy as np +from graph_synthesis.dspy_benchmark import study +from graph_synthesis.dspy_benchmark_v2.adapter import distribution + + +def diagnose(directory): + protocol=study.read(directory/'protocol.json');task=protocol['task'];arm=protocol['arm'] + execution=study.read(directory/'execution.json') + if execution.get('status')!='completed' or execution.get('live') is not True:raise ValueError('Capture did not complete live') + for name,want in study.read(directory/'artifact-manifest.json')['sha256'].items(): + if not (directory/name).resolve().is_relative_to(directory.resolve()) or study.sha(directory/name)!=want:raise ValueError('Capture integrity mismatch') + frozen=study.read(directory/'freeze.json') + if study.digest(frozen['payload'])!=frozen['sha256'] or frozen['payload']['protocol_sha256']!=study.digest(protocol):raise ValueError('Freeze mismatch') + if protocol.get('revision')!=2:raise ValueError('Only amendment-v2 captures are comparable here') + freeze=frozen['payload'] + variants=[('baseline',0,protocol['baseline'])]+[(select,r['seed'],r[select+'_questions']) for r in freeze['searches'] for select in ('accuracy','nll')] + configs=[study.digest(v[2]) for v in variants] + grouped=defaultdict(list) + for i,key in enumerate(configs):grouped[key].append(i) + groups=[v for v in grouped.values() if len(v)>1] + flags=defaultdict(set);quality=Counter();mass_errors=[] + with gzip.open(directory/'calls.jsonl.gz','rt',encoding='utf-8') as handle: + for line in handle: + call=json.loads(line) + for key,answer in (call.get('response') or {}).get('answers',{}).items(): + if answer.get('type')!='choice':continue + p=answer['probabilities'];distribution(list(p.values()));quality['choice_answers']+=1 + mass=float(sum(p.values())) + if abs(mass-1)>1e-6: + quality['mass_defects']+=1;quality['mass_defects_'+call['site'].split('/')[0]]+=1 + mass_errors.append(abs(mass-1));flags[call['site']].add(key) + duplicate_diagnostics=[];sensitivity=[];fits=study.read(directory/'calibration.json')['fits'] + for panel in protocol['panels']: + saved=study.read(directory/(panel+'-predictions.json'));p=np.asarray(saved['probabilities']);rows=saved['rows'] + for indices in groups: + spread=np.max(p[indices],axis=0)-np.min(p[indices],axis=0) + predictions=np.argmax(p[indices],axis=2) + duplicate_diagnostics.append({'panel':panel,'identical_config_sha256':configs[indices[0]], + 'variant_indices':indices,'variant_roles':[{'selection':variants[i][0],'seed':variants[i][1]} for i in indices], + 'rows':len(rows),'rows_with_any_probability_difference':int(np.sum(np.any(spread>1e-12,axis=1))), + 'rows_with_label_disagreement':int(np.sum(np.any(predictions!=predictions[0],axis=0))), + 'max_probability_spread':float(np.max(spread)), + 'note':'Descriptive same-request, identical-prompt nuisance diagnostic; not additional independent test cases.'}) + # The 12 preserved entity fixtures labelled uncertain have no binary + # accuracy target. Keep them in raw/duplicate diagnostics, not metrics. + eligible=[i for i,row in enumerate(rows) if row['gold_label'] in study.LABELS[task]] + keep=[i for i in eligible if not flags['test/'+panel+'/'+rows[i]['id']]] + if not keep:continue + strict_rows=[rows[i] for i in keep] + for i,(selection,seed,_) in enumerate(variants): + for method in ('raw','temperature','temperature_bias'): + pp=study.calibrated(p[i],fits[i],method)[keep] + sensitivity.append({'task':task,'arm':arm,'panel':panel,'selection':selection,'seed':seed,'calibration':method, + 'common_subset_rows':len(keep),'excluded_mass_defect_rows':len(eligible)-len(keep), + 'excluded_nonbinary_rows':len(rows)-len(eligible), + 'metrics':study.evaluate_metrics(pp,strict_rows,task), + 'note':'Post-hoc data-quality sensitivity on the same rows for all variants; fitted calibrators unchanged. Not the primary estimand.'}) + return {'task':task,'arm':arm,'distinct_final_configurations':len(grouped), + 'accuracy_seeds_with_unchanged_baseline':[v[1] for i,v in enumerate(variants) if v[0]=='accuracy' and configs[i]==configs[0]], + 'nll_seeds_with_unchanged_baseline':[v[1] for i,v in enumerate(variants) if v[0]=='nll' and configs[i]==configs[0]], + 'mass_quality':dict(quality),'max_mass_defect':max(mass_errors,default=0.), + 'duplicate_prompt_diagnostics':duplicate_diagnostics},sensitivity + + +def charts(directory,out): + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + protocol=study.read(directory/'protocol.json');task=protocol['task'];arm=protocol['arm'] + figure_dir=out/'figures';figure_dir.mkdir(parents=True,exist_ok=True) + fig,ax=plt.subplots(figsize=(9,5)) + for seed in study.SEEDS: + ledger=study.read(directory/f'search-{seed}.json');score=ledger[0]['validation']['accuracy'];values=[score] + for entry in ledger[1:]: + if entry['status']=='accepted':score=entry['validation']['accuracy'] + values.append(score) + ax.plot(range(len(values)),values,marker='o',label=f'Seed {seed}') + ax.set_xlabel('Candidate round');ax.set_ylabel('Best validation accuracy');ax.set_xticks(range(study.ROUNDS+1)) + ax.set_title(f'{task} / {arm} — validation only');ax.legend();ax.grid(axis='y',alpha=.2);fig.tight_layout() + stem=f'{task}--{arm}--convergence';fig.savefig(figure_dir/(stem+'.png'),dpi=300);fig.savefig(figure_dir/(stem+'.svg'));plt.close(fig) + saved=study.read(directory/'evaluation-predictions.json');fits=study.read(directory/'calibration.json')['fits'] + p=np.asarray(saved['probabilities']);y=np.asarray([study.LABELS[task].index(r['gold_label']) for r in saved['rows']]) + indices=[i for i,v in enumerate(saved['variants']) if v['selection']=='accuracy'] + fig,ax=plt.subplots(figsize=(8,6));ax.plot([0,1],[0,1],linestyle='--',label='Perfect calibration reference') + for selection,method,label in [('baseline','raw','Baseline raw'),('baseline','temperature','Baseline temperature'),('accuracy','raw','DSPy raw (five searches)'),('accuracy','temperature','DSPy temperature (five searches)')]: + chosen=[0] if selection=='baseline' else indices + conf=[];correct=[] + for i in chosen: + pred=study.calibrated(p[i],fits[i],method) + conf.extend(np.max(pred,axis=1));correct.extend(pred.argmax(axis=1)==y) + conf=np.asarray(conf);correct=np.asarray(correct);bins=np.minimum(9,(conf*10).astype(int));x=[];v=[] + for j in range(10): + mask=bins==j + if mask.any():x.append(float(conf[mask].mean()));v.append(float(correct[mask].mean())) + ax.plot(x,v,marker='o',label=label) + ax.set_xlabel('Mean predicted confidence');ax.set_ylabel('Observed accuracy');ax.set_xlim(0,1);ax.set_ylim(0,1) + ax.set_title(f'{task} / {arm}\nDescriptive reliability; repeated searches are not new examples') + ax.legend(fontsize=8);fig.tight_layout();stem=f'{task}--{arm}--reliability' + fig.savefig(figure_dir/(stem+'.png'),dpi=300);fig.savefig(figure_dir/(stem+'.svg'));plt.close(fig) + + +def main(): + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--input',type=Path,required=True);p.add_argument('--output',type=Path,required=True) + a=p.parse_args();a.output.mkdir(parents=True,exist_ok=False);results=[];sensitivity=[] + paths=sorted(p.parent for p in a.input.rglob('execution.json') if (p.parent/'protocol.json').exists()) + if len(paths)!=8:raise ValueError('Expected all eight formulations') + identities=set() + for directory in paths: + item,extra=diagnose(directory);results.append(item);sensitivity.extend(extra);charts(directory,a.output) + identities.add((item['task'],item['arm'])) + if identities!={(t,arm) for t,arms in study.ARMS.items() for arm in arms}:raise ValueError('Duplicate/missing formulation') + study.write_json(a.output/'quality-diagnostics.json',results);study.write_json(a.output/'strict-mass-subset-results.json',sensitivity) + text=['# Data-quality and optimizer diagnostics','', + 'These descriptive analyses use completed captures only. They do not change prompts, selected seeds, calibration fits or primary results.', + '', '| Task / formulation | Distinct final configurations | Accuracy seeds retaining baseline | Choice answers with mass defect | Largest mass defect |', + '|---|---:|---|---:|---:|'] + for r in results: + text.append(f"| {r['task']} / {r['arm']} | {r['distinct_final_configurations']} | {r['accuracy_seeds_with_unchanged_baseline']} | {r['mass_quality'].get('mass_defects',0)} | {r['max_mass_defect']:.4g} |") + text+=['','Identical question payloads can yield slightly different probabilities within the same request. `quality-diagnostics.json` quantifies those differences and any label disagreements. A difference between two unchanged prompts must not be attributed to successful prompt optimization.', + '', 'The primary analysis uses the bounded normalization rule fixed in amendment v2. `strict-mass-subset-results.json` is a sensitivity analysis excluding a row for every variant when any variant returned a non-unit-sum Choice vector on that row. Calibrators are not refit. This common subset avoids comparing different subsets across methods, but remains a post-hoc selected data-quality subset.', + '', 'Convergence figures contain only validation scores. Reliability figures pool descriptive predictions across five searches but do not treat those repeated predictions as independent new examples or display inferential confidence intervals.',''] + (a.output/'REPORT.md').write_text('\n'.join(text),encoding='utf-8') + study.write_json(a.output/'artifact-manifest.json',{'sha256':{p.relative_to(a.output).as_posix():study.sha(p) for p in sorted(a.output.rglob('*')) if p.is_file() and p.name!='artifact-manifest.json'}}) + +if __name__=='__main__':main() diff --git a/graph_synthesis/dspy_diagnostics/report.py b/graph_synthesis/dspy_diagnostics/report.py new file mode 100644 index 0000000..b3a0171 --- /dev/null +++ b/graph_synthesis/dspy_diagnostics/report.py @@ -0,0 +1,148 @@ +"""Verify fresh repeatability/batching observations without inference.""" +from __future__ import annotations +import argparse +from collections import defaultdict +import gzip +import json +from pathlib import Path +import statistics +import numpy as np +from graph_synthesis.dspy_benchmark import study +from graph_synthesis.dspy_benchmark_analysis.report import close,write_csv +from .run import load,bank_call + + +def audit(directory,primary): + for name,want in study.read(directory/'artifact-manifest.json')['sha256'].items(): + if not (directory/name).resolve().is_relative_to(directory.resolve()) or study.sha(directory/name)!=want:raise ValueError('Capture hash mismatch') + protocol=study.read(directory/'protocol.json');execution=study.read(directory/'execution.json');task=protocol['task'] + if protocol['model']!=study.MODEL or execution['status']!='completed' or not execution['live']:raise ValueError('Incomplete or wrong-model capture') + plan,parts,_=study.prepare(task);original=plan['tasks'][task];lookup={r['id']:r for r in original['evaluation']+original['development']} + if protocol['repeatability_ids']!=original['repeatability_ids'] or protocol['batching_ids']!=original['batching_ids']:raise ValueError('Panels changed') + repeat=[lookup[i] for i in protocol['repeatability_ids']];batching=[lookup[i] for i in protocol['batching_ids']] + configs={};fits={} + for arm in study.ARMS[task]: + found=[p.parent for p in primary.rglob('protocol.json') if study.read(p).get('task')==task and study.read(p).get('arm')==arm] + if len(found)!=1:raise ValueError('Missing primary family') + config,fit,freeze=load(found[0],task,arm) + if freeze!=protocol['upstream_freeze_sha256'][arm] or study.sha(found[0]/'calibration.json')!=protocol['source_calibration_sha256'][arm]:raise ValueError('Upstream mismatch') + close(config,protocol['configs'][arm],'config');configs[arm]=config;fits[arm]=fit + journal={};attempts=tokens=0;phases=defaultdict(lambda:{'calls':0,'input_tokens':0,'latencies':[]}) + with gzip.open(directory/'calls.jsonl.gz','rt',encoding='utf-8') as handle: + for line in handle: + call=json.loads(line);attempts+=1 + if study.digest(call['payload'])!=call['request_sha256'] or call['payload']['model']!=study.MODEL:raise ValueError('Request mismatch') + if call['response'] and call['response']['model']!=study.MODEL:raise ValueError('Model drift') + usage=int((call.get('response') or {}).get('usage',{}).get('input_tokens',0));tokens+=usage + phase=call['site'].rsplit('/',1)[0];phases[phase]['calls']+=1;phases[phase]['input_tokens']+=usage;phases[phase]['latencies'].append(call['latency_seconds']) + if call['error'] is None: + if call['site'] in journal:raise ValueError('Duplicate successful call') + journal[call['site']]=call + if attempts!=execution['http_attempts'] or tokens!=execution['reported_input_tokens'] or len(journal)!=640:raise ValueError('Execution count mismatch') + used=set() + class Replay: + def send(self,payload,site): + if site in used or site not in journal or study.digest(payload)!=journal[site]['request_sha256']:raise ValueError('Reconstructed request mismatch') + used.add(site);return journal[site]['response'] + saved=study.read(directory/'predictions.json');k=len(study.LABELS[task]);demos=parts['demonstrations'] + arrays={kind:{a:np.asarray(p) for a,p in data.items()} for kind,data in saved.items()} + ordinary=[(a,i,c) for a in configs if a!='fewshot_contract' for i,c in enumerate(configs[a])] + fewshot=[('fewshot_contract',i,c) for i,c in enumerate(configs['fewshot_contract'])] + for r in range(3): + for name,bank in [('ordinary',ordinary),('fewshot',fewshot)]: + for j,row in enumerate(repeat): + for (a,i),p in bank_call(Replay(),task,row,bank,demos,f'repeat-{r}/{name}').items():close(p,arrays['repeatability'][a][r,i,j].tolist(),'repeat') + for name,bank in [('ordinary',ordinary),('fewshot',fewshot)]: + for j,row in enumerate(batching): + for (a,i),p in bank_call(Replay(),task,row,bank,demos,'batch/'+name).items():close(p,arrays['batched'][a][i,j].tolist(),'batched') + for a in configs: + for i,c in enumerate(configs[a]): + for j,row in enumerate(batching): + p=bank_call(Replay(),task,row,[(a,i,c)],demos,f'isolated/{a}/{i}')[(a,i)] + close(p,arrays['isolated'][a][i,j].tolist(),'isolated') + if used!=set(journal):raise ValueError('Unaccounted successful response') + results=study.read(directory/'repeat-metrics.json');expected=[];drift=[] + for a in configs: + if arrays['repeatability'][a].shape!=(3,6,20,k) or arrays['batched'][a].shape!=(6,20,k) or arrays['isolated'][a].shape!=(6,20,k):raise ValueError('Bad prediction shape') + for i in range(6): + selection='baseline' if i==0 else 'dspy';seed=0 if i==0 else study.SEEDS[i-1];fit_index=0 if i==0 else 2*i-1;p=arrays['repeatability'][a][:,i] + drift.append({'task':task,'arm':a,'selection':selection,'seed':seed, + 'repeat_rows_with_label_disagreement':int(np.sum(np.any(p.argmax(axis=2)!=p[0].argmax(axis=1),axis=0))), + 'repeat_max_probability_spread':float(np.max(np.max(p,axis=0)-np.min(p,axis=0))), + 'batch_vs_isolated_label_disagreements':int(np.sum(arrays['batched'][a][i].argmax(axis=1)!=arrays['isolated'][a][i].argmax(axis=1))), + 'batch_vs_isolated_max_probability_difference':float(np.max(np.abs(arrays['batched'][a][i]-arrays['isolated'][a][i])))}) + for method in ('raw','temperature','temperature_bias'): + for r in range(3): + value=study.calibrated(arrays['repeatability'][a][r,i],fits[a][fit_index],method) + expected.append({'task':task,'arm':a,'selection':selection,'seed':seed,'calibration':method,'repeat':r,'metrics':study.evaluate_metrics(value,repeat,task)}) + close(expected,results,'repeat metrics');close(drift,study.read(directory/'drift.json'),'drift') + stats={key:{'calls':v['calls'],'input_tokens':v['input_tokens'],'mean_request_seconds':float(np.mean(v['latencies'])), + 'median_request_seconds':float(np.median(v['latencies'])),'p95_request_seconds':float(np.quantile(v['latencies'],.95))} for key,v in phases.items()} + close(stats,study.read(directory/'performance.json'),'performance') + packing=[] + for name,armset in [('ordinary',[a for a in configs if a!='fewshot_contract']),('fewshot',['fewshot_contract'])]: + batch=stats['batch/'+name];keys=[f'isolated/{a}/{i}' for a in armset for i in range(6)] + separate_calls=sum(stats[x]['calls'] for x in keys);separate_tokens=sum(stats[x]['input_tokens'] for x in keys) + packing.append({'task':task,'input_group':name,'unique_inputs':20,'configurations':len(armset)*6, + 'batch_http_attempts':batch['calls'],'isolated_http_attempts':separate_calls, + 'batch_input_tokens':batch['input_tokens'],'isolated_input_tokens':separate_tokens, + 'input_token_saving_fraction':1-batch['input_tokens']/separate_tokens, + 'mean_batch_request_seconds':batch['mean_request_seconds'], + 'mean_isolated_request_seconds':statistics.mean(stats[x]['mean_request_seconds'] for x in keys), + 'note':'Request latency covers different numbers of predictions. Timing order/server load were not randomized between batch and isolated modes.'}) + costs=[] + for a in configs: + base=stats[f'isolated/{a}/0'];others=[stats[f'isolated/{a}/{i}'] for i in range(1,6)] + costs.append({'task':task,'arm':a,'unique_inputs':20,'baseline_input_tokens_per_prediction':base['input_tokens']/20, + 'dspy_mean_input_tokens_per_prediction':sum(r['input_tokens'] for r in others)/100, + 'baseline_mean_request_seconds':base['mean_request_seconds'], + 'dspy_mean_request_seconds':statistics.mean(r['mean_request_seconds'] for r in others), + 'note':'Separate one-configuration requests. Includes inference only, not prompt search or local proposer CPU time.'}) + return results,drift,packing,costs,{'task':task,**execution,'capture_manifest_sha256':study.sha(directory/'artifact-manifest.json')} + + +def main(): + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--input',type=Path,required=True);parser.add_argument('--primary',type=Path,required=True);parser.add_argument('--output',type=Path,required=True) + args=parser.parse_args();paths=sorted(p.parent for p in args.input.rglob('execution.json') if (p.parent/'protocol.json').exists()) + if len(paths)!=2:raise ValueError('Require both task captures') + results=[];drift=[];packing=[];costs=[];executions=[] + for path in paths: + r,d,p,c,e=audit(path,args.primary);results+=r;drift+=d;packing+=p;costs+=c;executions.append(e) + if {e['task'] for e in executions}!=set(study.ARMS):raise ValueError('Missing task') + summary=[] + for task,arms in study.ARMS.items(): + for arm in arms: + for method in ('raw','temperature','temperature_bias'): + baseline=[r for r in results if r['task']==task and r['arm']==arm and r['selection']=='baseline' and r['calibration']==method] + seeds=[[r for r in results if r['task']==task and r['arm']==arm and r['selection']=='dspy' and r['seed']==seed and r['calibration']==method] for seed in study.SEEDS] + if len(baseline)!=3 or any(len(s)!=3 for s in seeds):raise ValueError('Missing repetitions') + entry={'task':task,'arm':arm,'calibration':method,'unique_evaluation_examples':20,'search_seeds':5,'service_repetitions_per_seed':3} + for metric in ('accuracy','macro_f1','mcc','brier','log_loss','ece'): + b=[r['metrics'][metric] for r in baseline];s=[[r['metrics'][metric] for r in records] for records in seeds] + means=[statistics.mean(v) for v in s] + entry[metric]={'baseline_mean':statistics.mean(b),'baseline_service_std':statistics.stdev(b), + 'dspy_mean':statistics.mean(means),'between_search_std':statistics.stdev(means), + 'mean_within_search_service_std':statistics.mean(statistics.stdev(v) for v in s)} + summary.append(entry) + args.output.mkdir(parents=True,exist_ok=False) + for name,value in [('repeat-metrics',results),('drift',drift),('packing',packing),('isolated-costs',costs),('execution',executions),('replication-summary',summary)]:study.write_json(args.output/(name+'.json'),value) + flat=[{**{k:r[k] for k in ('task','arm','selection','seed','calibration','repeat')},**{k:r['metrics'][k] for k in ('n','accuracy','macro_f1','mcc','brier','log_loss','ece')}} for r in results] + write_csv(args.output/'repeat-metrics.csv',flat) + text=['# Fresh repeatability, batching and isolated inference costs','', + 'The original 20-row repeatability panels were evaluated three times per frozen configuration. The original 20-row batching panels are development examples, used for packing/performance diagnostics, not new held-out accuracy claims. All eight formulations, baseline plus five frozen accuracy-selected DSPy prompts, are included.', + '', 'Every recorded request and prediction was reconstructed from its raw response. The paired packing test retains the original grouping: all non-few-shot formulations share one state; few-shot input remains separate. Individual requests use exactly the same configurations and examples.', + '', '| Task / formulation | Baseline repeat accuracy | DSPy repeat accuracy | Between-search SD | Mean within-search service SD |', + '|---|---:|---:|---:|---:|'] + for r in summary: + if r['calibration']=='raw': + m=r['accuracy'];text.append(f"| {r['task']} / {r['arm']} | {m['baseline_mean']:.4f} | {m['dspy_mean']:.4f} | {m['between_search_std']:.4f} | {m['mean_within_search_service_std']:.4f} |") + text+=['','## Probability calibration and service variation','', + '`replication-summary.json` reports raw, temperature and temperature-plus-bias metrics. Calibrators are reused from the disjoint primary calibration split; no fits are made on these 20 examples. Means and standard deviations describe search and service repetition axes separately. Repeated observations are not new independent test examples.', + '', '`drift.json` counts label disagreements across service repeats and between batched and isolated calls, plus maximum probability differences. Do not interpret a score difference for an unchanged prompt as proof of optimization.', + '', '## Inference cost and packing','', + '`packing.json` compares actual reported input tokens for the same examples and configurations batched versus separate. `isolated-costs.json` measures baseline and DSPy token usage and request latency without cross-configuration batching. These exclude optimizer overhead, local model CPU time, CI time and external invoice verification. Batch request latency covers many predictions, so it is not directly equivalent to isolated one-configuration latency.', + '', 'The packing modes were executed in blocks, not randomized across server load; latency comparisons are descriptive. All raw calls, including retries, are retained. Input-token cost estimates use the documented Jev price and are not invoices.',''] + (args.output/'REPORT.md').write_text('\n'.join(text),encoding='utf-8') + study.write_json(args.output/'artifact-manifest.json',{'sha256':{p.relative_to(args.output).as_posix():study.sha(p) for p in sorted(args.output.rglob('*')) if p.is_file() and p.name!='artifact-manifest.json'}}) + +if __name__=='__main__':main() diff --git a/graph_synthesis/dspy_diagnostics/run.py b/graph_synthesis/dspy_diagnostics/run.py new file mode 100644 index 0000000..c2505da --- /dev/null +++ b/graph_synthesis/dspy_diagnostics/run.py @@ -0,0 +1,125 @@ +"""Fresh repeatability and original cross-formulation batching layout, after freezing.""" +from __future__ import annotations +import argparse +from concurrent.futures import ThreadPoolExecutor +from collections import defaultdict +import gzip +import json +from pathlib import Path +import random +import numpy as np +from graph_synthesis.dspy_benchmark import study +from graph_synthesis.dspy_benchmark_v2.adapter import probabilities,InvalidServiceResponse + + +def load(capture,task,arm): + protocol=study.read(capture/'protocol.json');freeze=study.read(capture/'freeze.json') + if protocol.get('revision')!=2 or (protocol['task'],protocol['arm'])!=(task,arm) or study.read(capture/'execution.json').get('status')!='completed': + raise ValueError('Require the matching complete v2 capture') + if study.digest(freeze['payload'])!=freeze['sha256'] or freeze['payload']['protocol_sha256']!=study.digest(protocol):raise ValueError('Freeze invalid') + plan,parts,_=study.prepare(task);order=list(plan['question_specs'][task][arm]) + configs=[protocol['baseline']] + for search in freeze['payload']['searches']: + if study.digest(study.read(capture/f"search-{search['seed']}.json"))!=search['ledger_sha256']:raise ValueError('Search changed') + configs.append(search['accuracy_questions']) + if [s['seed'] for s in freeze['payload']['searches']]!=study.SEEDS:raise ValueError('Incomplete searches') + configs=[{key:q[key] for key in order} for q in configs] + return configs,study.read(capture/'calibration.json')['fits'],freeze['sha256'] + + +def bank_call(live,task,row,bank,demos,phase): + fewshot={arm=='fewshot_contract' for arm,_,_ in bank} + if len(fewshot)!=1:raise ValueError('Do not mix incompatible input states') + value=study.state(task,row) + if True in fewshot: + value={'labeled_examples':[{'input':study.state(task,d),'answer':d['gold_label']} for d in demos],'input':value} + questions={f'{arm}__v{i}__{key}':q for arm,i,config in bank for key,q in config.items()} + response=live.send({'model':study.MODEL,'state':value,'questions':questions},phase+'/'+row['id']) + if set(response.get('answers',{}))!=set(questions):raise InvalidServiceResponse('Answer bank mismatch') + return {(arm,i):probabilities({key:response['answers'][f'{arm}__v{i}__{key}'] for key in config},task,config) for arm,i,config in bank} + + +def run(task,input_dir,out): + if out.exists():raise ValueError('Do not overwrite observations') + plan,parts,_=study.prepare(task);original=plan['tasks'][task] + lookup={r['id']:r for r in original['evaluation']+original['development']} + repeat=[lookup[i] for i in original['repeatability_ids']];batching=[lookup[i] for i in original['batching_ids']] + if len(repeat)!=20 or len(batching)!=20:raise ValueError('Unexpected original diagnostic panels') + captures={};fits={};freezes={};configs={} + for arm in study.ARMS[task]: + candidates=[p.parent for p in input_dir.rglob('protocol.json') if study.read(p).get('task')==task and study.read(p).get('arm')==arm] + if len(candidates)!=1:raise ValueError('Missing/duplicate primary capture') + captures[arm]=candidates[0];configs[arm],fits[arm],freezes[arm]=load(candidates[0],task,arm) + out.mkdir(parents=True) + study.write_json(out/'protocol.json',{'experiment_type':'repeatability_and_batching','task':task,'model':study.MODEL, + 'source_sha256':study.sha(__file__),'seeds':study.SEEDS,'repetitions':3,'upstream_freeze_sha256':freezes, + 'repeatability_ids':original['repeatability_ids'],'batching_ids':original['batching_ids'], + 'repeatability_origin':'Original evaluation subset','batching_origin':'Original development subset, used only for packing/performance diagnostics', + 'source_calibration_sha256':{arm:study.sha(captures[arm]/'calibration.json') for arm in captures}, + 'configs':configs,'expected_successful_calls':640}) + live=study.Live(out,max_calls=2000);demos=parts['demonstrations'];arms=list(configs) + ordinary=[(arm,i,config) for arm in arms if arm!='fewshot_contract' for i,config in enumerate(configs[arm])] + fewshot=[('fewshot_contract',i,config) for i,config in enumerate(configs['fewshot_contract'])] + banks=[('ordinary',ordinary),('fewshot',fewshot)] + k=len(study.LABELS[task]);repeated={arm:np.zeros((3,6,20,k)) for arm in arms} + batched={arm:np.zeros((6,20,k)) for arm in arms};isolated={arm:np.zeros((6,20,k)) for arm in arms} + for r in range(3): + jobs=[(name,bank,j,row) for name,bank in banks for j,row in enumerate(repeat)] + def one(item): + name,bank,j,row=item + return j,bank_call(live,task,row,bank,demos,f'repeat-{r}/{name}') + with ThreadPoolExecutor(max_workers=2) as pool: + for j,answers in pool.map(one,jobs): + for (arm,i),p in answers.items():repeated[arm][r,i,j]=p + jobs=[(name,bank,j,row) for name,bank in banks for j,row in enumerate(batching)] + def batch_one(item): + name,bank,j,row=item + return j,bank_call(live,task,row,bank,demos,'batch/'+name) + with ThreadPoolExecutor(max_workers=2) as pool: + for j,answers in pool.map(batch_one,jobs): + for (arm,i),p in answers.items():batched[arm][i,j]=p + jobs=[(arm,i,config,j,row) for arm in arms for i,config in enumerate(configs[arm]) for j,row in enumerate(batching)] + random.Random(20260918).shuffle(jobs) + def separate_one(item): + arm,i,config,j,row=item + return arm,i,j,bank_call(live,task,row,[(arm,i,config)],demos,f'isolated/{arm}/{i}')[(arm,i)] + with ThreadPoolExecutor(max_workers=2) as pool: + for arm,i,j,p in pool.map(separate_one,jobs):isolated[arm][i,j]=p + study.write_json(out/'predictions.json',{'repeatability':{a:p.tolist() for a,p in repeated.items()}, + 'batched':{a:p.tolist() for a,p in batched.items()},'isolated':{a:p.tolist() for a,p in isolated.items()}}) + metrics=[];drift=[] + for arm in arms: + for i in range(6): + selection='baseline' if i==0 else 'dspy';seed=0 if i==0 else study.SEEDS[i-1];fit_index=0 if i==0 else 2*i-1 + p=repeated[arm][:,i] + drift.append({'task':task,'arm':arm,'selection':selection,'seed':seed, + 'repeat_rows_with_label_disagreement':int(np.sum(np.any(p.argmax(axis=2)!=p[0].argmax(axis=1),axis=0))), + 'repeat_max_probability_spread':float(np.max(np.max(p,axis=0)-np.min(p,axis=0))), + 'batch_vs_isolated_label_disagreements':int(np.sum(batched[arm][i].argmax(axis=1)!=isolated[arm][i].argmax(axis=1))), + 'batch_vs_isolated_max_probability_difference':float(np.max(np.abs(batched[arm][i]-isolated[arm][i])))}) + for method in ('raw','temperature','temperature_bias'): + for r in range(3): + value=study.calibrated(repeated[arm][r,i],fits[arm][fit_index],method) + metrics.append({'task':task,'arm':arm,'selection':selection,'seed':seed,'calibration':method,'repeat':r, + 'metrics':study.evaluate_metrics(value,repeat,task)}) + study.write_json(out/'repeat-metrics.json',metrics);study.write_json(out/'drift.json',drift) + phases=defaultdict(lambda:{'calls':0,'input_tokens':0,'latencies':[]}) + with gzip.open(out/'calls.jsonl.gz','rt',encoding='utf-8') as f: + for line in f: + call=json.loads(line);phase=call['site'].rsplit('/',1)[0];entry=phases[phase] + entry['calls']+=1;entry['input_tokens']+=int((call.get('response') or {}).get('usage',{}).get('input_tokens',0));entry['latencies'].append(call['latency_seconds']) + stats={key:{'calls':v['calls'],'input_tokens':v['input_tokens'],'mean_request_seconds':float(np.mean(v['latencies'])), + 'median_request_seconds':float(np.median(v['latencies'])),'p95_request_seconds':float(np.quantile(v['latencies'],.95))} for key,v in phases.items()} + study.write_json(out/'performance.json',stats) + study.write_json(out/'execution.json',{'status':'completed','live':True,'task':task,'http_attempts':live.calls, + 'reported_input_tokens':live.tokens,'estimated_jev_usd':live.tokens*.042/1_000_000,'expected_successful_calls':640, + 'scope':'Three repeatability passes; all original non-few-shot formulations packed together; few-shot state isolated; matched individual calls.'}) + study.write_json(out/'artifact-manifest.json',{'sha256':{p.name:study.sha(p) for p in sorted(out.iterdir()) if p.is_file() and p.name!='artifact-manifest.json'}}) + + +def main(): + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--task',choices=list(study.ARMS),required=True) + p.add_argument('--input',type=Path,required=True);p.add_argument('--output',type=Path,required=True) + a=p.parse_args();run(a.task,a.input,a.output) + +if __name__=='__main__':main() diff --git a/graph_synthesis/dspy_diagnostics/test_roundtrip.py b/graph_synthesis/dspy_diagnostics/test_roundtrip.py new file mode 100644 index 0000000..8bf58e1 --- /dev/null +++ b/graph_synthesis/dspy_diagnostics/test_roundtrip.py @@ -0,0 +1,60 @@ +"""Synthetic journal replay regression, not live model accuracy evidence.""" +import gzip +from pathlib import Path +import tempfile +import threading +import unittest +from unittest.mock import patch +from graph_synthesis.dspy_benchmark import study +from .run import run +from .report import audit + + +class JournalBackend: + def __init__(self,out,max_calls=2000): + self.out=out;self.calls=0;self.tokens=0;self.lock=threading.Lock() + def send(self,payload,site): + answers={} + for key,question in payload['questions'].items(): + if question['type']=='noul':answers[key]={'type':'noul','noul':.8} + else: + keys=list(question['criteria']);vector={label:1/len(keys) for label in keys} + answers[key]={'type':'choice','choice':keys[0],'probabilities':vector} + response={'model':study.MODEL,'answers':answers,'usage':{'input_tokens':10}} + record={'site':site,'attempt':0,'payload':payload,'request_sha256':study.digest(payload), + 'response':response,'error':None,'http_status':200,'latency_seconds':.1} + with self.lock: + self.calls+=1;self.tokens+=10 + with gzip.open(self.out/'calls.jsonl.gz','at',encoding='utf-8') as f:f.write(study.canonical(record)+'\n') + return response + + +class ReplayTests(unittest.TestCase): + def test_entire_640_call_task_replays_exactly(self): + task='entity_resolution';original,_,_=study.prepare(task) + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);primary=root/'primary';primary.mkdir() + for arm in study.ARMS[task]: + out=primary/arm;out.mkdir();questions=original['question_specs'][task][arm] + protocol={'revision':2,'task':task,'arm':arm,'baseline':questions} + searches=[] + for seed in study.SEEDS: + ledger=[{'iteration':0,'status':'baseline'}] + study.write_json(out/f'search-{seed}.json',ledger) + searches.append({'seed':seed,'accuracy_questions':questions,'ledger_sha256':study.digest(ledger)}) + frozen={'protocol_sha256':study.digest(protocol),'searches':searches} + fit={'temperature':1.,'temperature_bias':{'temperature':1.,'bias':[0.,0.]}} + for name,value in [('protocol',protocol),('freeze',{'payload':frozen,'sha256':study.digest(frozen)}), + ('execution',{'status':'completed'}),('calibration',{'fits':[fit]*11})]:study.write_json(out/(name+'.json'),value) + output=root/'diagnostic' + with patch.object(study,'Live',JournalBackend):run(task,primary,output) + results,drift,packing,costs,execution=audit(output,primary) + self.assertEqual(execution['http_attempts'],640) + self.assertEqual(len(results),4*6*3*3) + self.assertEqual(len(drift),24);self.assertEqual(len(packing),2);self.assertEqual(len(costs),4) + self.assertTrue(all(r['repeat_rows_with_label_disagreement']==0 for r in drift)) + with (output/'predictions.json').open('a') as f:f.write(' ') + with self.assertRaises(ValueError):audit(output,primary) + + +if __name__=='__main__':unittest.main() diff --git a/graph_synthesis/dspy_diagnostics/test_run.py b/graph_synthesis/dspy_diagnostics/test_run.py new file mode 100644 index 0000000..a327805 --- /dev/null +++ b/graph_synthesis/dspy_diagnostics/test_run.py @@ -0,0 +1,35 @@ +import unittest +from graph_synthesis.dspy_benchmark import study +from .run import bank_call + +class FakeLive: + def __init__(self):self.payloads=[] + def send(self,payload,site): + self.payloads.append(payload);answers={} + for key,q in payload['questions'].items(): + if q['type']=='noul':answers[key]={'type':'noul','noul':.8} + else: + labels=list(q['criteria']);answers[key]={'type':'choice','choice':labels[0],'probabilities':{l:1/len(labels) for l in labels}} + return {'model':study.MODEL,'answers':answers} + +class DiagnosticsTests(unittest.TestCase): + def test_original_batch_panels_are_development(self): + for task in study.ARMS: + plan,parts,_=study.prepare(task);data=plan['tasks'][task] + self.assertTrue(set(data['batching_ids'])<={r['id'] for r in data['development']}) + self.assertTrue(set(data['repeatability_ids'])<={r['id'] for r in data['evaluation']}) + def test_cross_formulation_bank_retains_every_typed_question(self): + plan,parts,_=study.prepare('relation_support');specs=plan['question_specs']['relation_support'] + bank=[(arm,i,specs[arm]) for arm in study.ARMS['relation_support'] if arm!='fewshot_contract' for i in range(6)] + live=FakeLive();row=parts['train'][0] + results=bank_call(live,'relation_support',row,bank,parts['demonstrations'],'test-only') + self.assertEqual(len(results),18);self.assertEqual(len(live.payloads[0]['questions']),24) + self.assertNotIn('gold_label',live.payloads[0]['state']) + for p in results.values():self.assertAlmostEqual(sum(p),1) + def test_fewshot_state_cannot_be_mixed_with_plain_state(self): + plan,parts,_=study.prepare('relation_support');spec=plan['question_specs']['relation_support'] + with self.assertRaises(ValueError): + bank_call(FakeLive(),'relation_support',parts['train'][0], + [('baseline_choice',0,spec['baseline_choice']),('fewshot_contract',0,spec['fewshot_contract'])],parts['demonstrations'],'test-only') + +if __name__=='__main__':unittest.main()