From 20a9ecf8327703561a31d6f028323c068c00de97 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Fri, 23 Oct 2026 13:03:10 -0400 Subject: [PATCH] fix(eval): refuse contaminated independent L0 measurements --- docs/eval/l0-bounded-index-reuse.md | 9 ++ docs/eval/l0-outcome-contract.md | 146 ++++++++++++++++++++++++ eval/runner/cli.py | 4 +- eval/runner/l0.py | 43 +++++++- eval/tests/test_l0_outcome_contract.py | 147 +++++++++++++++++++++++++ eval/v2/PREREGISTRATION.md | 14 +++ 6 files changed, 358 insertions(+), 5 deletions(-) create mode 100644 docs/eval/l0-outcome-contract.md create mode 100644 eval/tests/test_l0_outcome_contract.py diff --git a/docs/eval/l0-bounded-index-reuse.md b/docs/eval/l0-bounded-index-reuse.md index 192c00ef..9fe34761 100644 --- a/docs/eval/l0-bounded-index-reuse.md +++ b/docs/eval/l0-bounded-index-reuse.md @@ -149,3 +149,12 @@ satisfy a missing derived-workspace file, even when both directories share a nam An unreadable candidate is an explicit error, not evidence of absence. Strict metadata reads refuse instead of treating an inaccessible alternative as a unique path match. + +The sealed driver now additionally records +`hook_outcomes=independent-hook-outcomes/v1`; this refuses residual session +contamination, unexplained silence and unsupported/contradictory hook outcomes +instead of admitting them as independent observations. Error cells have null +efficacy metrics and trigger the existing quarantine. Legacy rows remain +unchanged. The [outcome reconciliation and next-run gate](l0-outcome-contract.md) +separates delivered source controls from the still-required native enrollment, +measured aggregate-storage plan and separately accepted execution envelope. diff --git a/docs/eval/l0-outcome-contract.md b/docs/eval/l0-outcome-contract.md new file mode 100644 index 00000000..33077056 --- /dev/null +++ b/docs/eval/l0-outcome-contract.md @@ -0,0 +1,146 @@ +# L0 outcome reconciliation and the next-run gate + +This is source/instrument work for [#95](https://github.com/intigraph/bobbin/issues/95), +not a fresh evaluation or permission to start one. Original rows, their digests, +legacy scoring and the model/chunker gates remain unchanged. + +## Retained evidence and dispositions + +The frozen CSV still has 330/1,320 cells from ten of forty tasks: 213 injected, +83 skipped and 34 errors. Its SHA256 is +`54b0f5d8da2e4a583c5ffee2c94c2e0b51171b59b7c63d2994c43b97c44f7c85`. +The manifest binds the original stream separately. A public scalar projection +is not the hook's stdout/stderr archive. + +| Family | Count | Disposition | +|---|---:|---| +| Parsed injection | 213 | Retain exact old measurements and literal path convention; not automatically comparable to qualified scoring. | +| No parseable headers | 34 | Invalid measurements. Preserve errors, never replace with zero recall. Exact output/cause cannot be recovered from the scalar rows. | +| Previously injected | 23 | Invalid evidence for an independent no-dedup cell; retain old skip rows, refuse this condition in the new opt-in contract. | +| Operational intent | 42 | Explicit production non-injection; retain as a declared skip rather than retrieval failure. | +| Semantic gate | 18 | Explicit production non-injection; retain as a declared skip with the actual printed reason. | + +All 34 old error rows also contain numeric `0.0` file/hunk recall from the +legacy driver. Those bytes stay unchanged: the error status excludes them from +valid evidence. The new independent driver leaves these metrics null, making +that distinction explicit rather than silently treating an invalid measurement +as an observed miss. + +The 34 header-error rows occur in Django tasks. That association does not prove +which output form caused them. In particular, a synthetic headerless or JSON +complementary-files fixture is not a recovered historical output. Do not infer +that decoding arbitrary JSON repairs all 34 cells or that a headerless response +contains zero relevant code. + +The frozen native hook has two mechanisms: `--no-dedup` disables the fallback +session hash comparison, while the reducing ledger is separately enabled by +configuration and a nonempty session ID. It can filter chunks and emit the +observed "previously injected" reason even with that flag. This source path +explains the condition; exact historical ledger contents are not reconstructed. +The already-delivered sealed cell isolation archives/restores mutable state +before the next cell and rejects a baseline containing history. Reuse that +mechanism; do not add a second ledger reset or alter shared runtime behavior. + +## Versioned independent-cell contract + +The sealed opt-in driver records `hook_outcomes=independent-hook-outcomes/v1` +and passes the strict contract through to the real hook-result classifier. +The legacy driver still records `legacy` and retains its previous behavior. +`scoring_paths=workspace-qualified-or-literal/v1` is a separate existing +convention; changing outcome classification does not conceal a scoring change. + +Under the new contract: + +- A successful output containing supported standard chunk headers is injected. +- Exactly one explicit operational-intent or consistent semantic-gate reason + with no context is a skip. Printed scores are rounded: equal displayed + score/gate values may represent a valid strict comparison before rounding. +- A dedup skip, partial reducing message, unknown/multiple skip reasons, + contradictory context plus skip, unexplained silence, unsupported output, + nonzero exit or reported hook error is an error. +- Error cells carry no numeric file/hunk recall or density. The existing + isolation layer quarantines an error and stops subsequent cells; it does not + silently reset uncertain state and continue the matrix. + +This changes only the opt-in measurement path, not Rust retrieval, the CLI's +normal session behavior or historical outcomes. The preregistration amendment +records the interpretation before another campaign. Paired fixtures cover real +header syntax, headerless/complementary JSON refusal, explicit ordinary skips, +contamination and the actual task-driver propagation to null error metrics. +They are instrument controls, not retrieval-quality observations. + +## What a further run waits on + +The source work and the separately bounded execution envelope must be accepted +before another campaign. Do not silently resume the old output or repeat it to +fill missing cells. The following are concrete gates, not elapsed-time waits: + +1. Land reviewed, green outcome-contract source and freeze its harness, + preregistration, forty task YAMLs, eleven arms, budgets, config and scoring + hashes in a new output manifest. Preserve original data and pin the actual + executable independently of the harness revision. +2. Enroll the native executable intended for the current-source study. Existing + sealed reuse admits only its exact 0.28.0 runtime/model bytes. A released + 0.30.0 version string is not enrollment. Cold/derived-index equivalence, + repeated independent-cell positive controls and mutation refusals must bind + the proposed native binary and complete CPU model/tokenizer hashes before + it can be used. The old 0.28 runtime alone cannot establish today's-main + retrieval quality. +3. Accept a measured whole-run storage plan. The old 21 built/partial indexes + alone occupied 6,474,663,440 bytes, largest 450,579,646; they exclude full + clones, mirrors, models, journals and all missing tasks. The existing + 512-MiB per-index cap is not a total-run quota or a feasibility estimate. +4. Admit one fresh run under the unchanged CPU400%, memory6GiB, swap0, + no-GPU/no-agent/no-paid-model and 21,600-second whole-invocation limits. + This document does not raise a limit or enroll another execution lane. + +### Proposed bounded storage/execution plan (not yet accepted) + +Use one fresh private run root on the mounted fleet filesystem, a fresh score +stream and the delivered sealed full/derived indexes. Never use production +indexes or another run's mutable cache. Inventory all task/source/model inputs +read-only before admitting indexing. Reuse read-only Git objects/mirrors only +when the task's exact pre-image and checkout isolation are verified; do not +share mutable working trees or reuse an index across distinct commits. + +Propose a **32-GiB aggregate allocated-storage ceiling**, subject to acceptance +and a measured admission inventory. This is a proposed bound, not evidence that +all tasks fit or that the present host has room. It must cover clones/mirrors, +models, both index arms, results, journals and temporary files. External storage +must not be used to evade it; reflink apparent size alone is not allocated usage. +The existing per-index bound remains additive. If a reliable aggregate quota +and free-space floor cannot be enforced through the normal sanctioned lane, +the run remains blocked; a periodic `du` sample is not a hard quota. + +For admission, provide expected plus worst-case allocated storage, mount/quota +identity, source/model input sizes and a floor that protects other fleet writers. +Keep one task's mutable full/derived pair at a time only if the accepted +retention contract can archive its forensic outputs before owner-authorized +reclamation. This is a proposal; it does not authorize deleting the old campaign +or other owners' data. Measure cold-index, copy/seal, hook and journal costs +separately. Tiny-fixture native timings cannot predict full-corpus wall time. + +The actual supervisor must enforce the original wall/cgroup limits and stop on +quota/floor, seal, output-format, timeout or restoration failure. Retain raw +stdout/stderr for the failing cell privately under the accepted retention/byte +budget so another header failure is diagnosable; do not publish retrieved source +bodies. Preserve the terminal reason, immutable completed scores, quarantine and +explicit missing-cell coverage. Do not classify a cap hit as completion, auto- +retry a failed cell, increase the envelope or start a second campaign on this GO. + +A complete registered matrix is a prerequisite for the preregistered aggregate +analysis; successful source tests and the earlier 330 cells do not satisfy it. +The six-hour cap can still produce an honestly incomplete result. #96/#97 +remain gated on usable evidence and their own decisions. + +## Reproduce the bounded instrument checks + +`python3 -m pytest tests/test_l0_outcome_contract.py tests/test_l0.py tests/test_index_reuse.py tests/test_score_paths.py` +from `eval/` uses private fixtures and committed rows. It runs no corpus campaign, +model, GPU, shared index or provider. The normal full Python eval suite also +covers the surrounding driver. An inverse removing the strict driver forwarding +must fail the contamination/silence controls; removing the null-metric refusal +must fail the driver error controls. + +Rollback: revert this instrument change, retain both old and separately named +new outputs, and never reinterpret one contract's results as the other's. diff --git a/eval/runner/cli.py b/eval/runner/cli.py index 9b28b700..e812304d 100644 --- a/eval/runner/cli.py +++ b/eval/runner/cli.py @@ -1844,6 +1844,7 @@ def l0(tasks_dir, task_ids, arms, budgets, out_path, workdir, index_timeout, reu manifest["planned_cells"] = cells manifest["index_reuse"] = "sealed-no-coupling/v1" if reuse_model_dir else "legacy" manifest["scoring_paths"] = "workspace-qualified-or-literal/v1" if reuse_model_dir else "literal" + manifest["hook_outcomes"] = L0.INDEPENDENT_HOOK_CONTRACT if reuse_model_dir else "legacy" if reuse_limits is not None: manifest["resource_limits"] = reuse_limits manifest["wall_supervisor_required_seconds"] = 21600 @@ -1925,7 +1926,8 @@ def index(ws: Path, overrides=None): target, Path(bobbin), reuse_model_dir, overrides, journal, lambda a=arm, b=budget: L0.run_task( task, ws, [a], [b], bobbin, env, index, - nocoupling_workspace=fork, normalize_paths=True)) + nocoupling_workspace=fork, normalize_paths=True, + strict_hook_contract=True)) except Exception as exc: # Preserve the explicit failure and all prior cells; # remaining cells stay missing. No reset/resend follows. diff --git a/eval/runner/l0.py b/eval/runner/l0.py index c0825ea0..46f8918c 100644 --- a/eval/runner/l0.py +++ b/eval/runner/l0.py @@ -37,6 +37,7 @@ DEFAULT_BUDGET = 300 SEED = 20260924 +INDEPENDENT_HOOK_CONTRACT = "independent-hook-outcomes/v1" BOOTSTRAP_RESAMPLES = 10_000 # Shipped defaults every calibration-driven arm starts from (src/config.rs). @@ -145,16 +146,41 @@ def parse_injected_chunks(text: str) -> list[Chunk]: return chunks -def classify_hook_result(returncode: int, stdout: str, stderr: str) -> tuple[str, str]: +def classify_hook_result( + returncode: int, stdout: str, stderr: str, *, independent: bool = False +) -> tuple[str, str]: """(outcome, detail). See the module docstring for why stderr decides.""" for line in stderr.splitlines(): if line.startswith("bobbin inject-context"): return "error", line.strip() if returncode != 0: return "error", f"exit {returncode}: {stderr.strip()[:300]}" + reasons = [ln.strip() for ln in stderr.splitlines() if ln.startswith("bobbin: skipped")] + if independent: + # --no-dedup does not disable the frozen runtime's reducing ledger. + # Cell isolation should prevent this; observing it is invalid evidence. + if "previously injected" in stderr or any( + "session unchanged" in reason for reason in reasons + ): + return "error", "independent cell observed session-state contamination" + if reasons and stdout.strip(): + return "error", "independent cell produced both context and a skip reason" + if not stdout.strip(): + if len(reasons) != 1: + return "error", "independent cell has no unique explicit skip reason" + reason = reasons[0] + if reason == "bobbin: skipped (operational intent: Operational)": + return "skipped", reason + gate = re.fullmatch( + r"bobbin: skipped \(semantic=(-?\d+\.\d+) < gate=(-?\d+\.\d+)\)", reason + ) + # The runtime prints rounded scores: equality can still mean the + # unrounded score was below the gate. Never reject that rounding. + if gate and float(gate[1]) <= float(gate[2]): + return "skipped", reason + return "error", "independent cell has an unsupported or inconsistent skip reason" if parse_injected_chunks(stdout): return "injected", "" - reasons = [ln.strip() for ln in stderr.splitlines() if ln.startswith("bobbin: skipped")] if reasons: return "skipped", reasons[0] if stdout.strip(): @@ -304,6 +330,7 @@ def run_hook_arm( bobbin: str, *, timeout: int = 180, + independent: bool = False, ) -> tuple[str, str, list[Chunk], int]: spec = ABLATIONS[arm] write_calibration(workspace, spec.get("calibration", {})) @@ -334,7 +361,9 @@ def run_hook_arm( ) except subprocess.TimeoutExpired: return "error", f"timeout after {timeout}s", [], 0 - outcome, detail = classify_hook_result(r.returncode, r.stdout, r.stderr) + outcome, detail = classify_hook_result( + r.returncode, r.stdout, r.stderr, independent=independent + ) chunks = parse_injected_chunks(r.stdout) if outcome == "injected" else [] return outcome, detail, chunks, len(r.stdout) @@ -669,6 +698,7 @@ def run_task( *, nocoupling_workspace: Path | None = None, normalize_paths: bool = False, + strict_hook_contract: bool = False, ) -> list[ArmScore]: """Score every (arm, budget) for one task on an already-checked-out workspace. @@ -702,8 +732,13 @@ def run_task( ws = nocoupling_ws if arm == "-coupling" else workspace scoring_workspace = ws outcome, detail, chunks, chars = run_hook_arm( - ws, prompt, arm, budget, env, bobbin + ws, prompt, arm, budget, env, bobbin, + **({"independent": True} if strict_hook_contract else {}), ) + if strict_hook_contract and outcome == "error": + base.detail = detail + results.append(base) + continue else: if arm == "rg": chunks = baseline_rg(workspace, prompt, budget) diff --git a/eval/tests/test_l0_outcome_contract.py b/eval/tests/test_l0_outcome_contract.py new file mode 100644 index 00000000..c4c7f501 --- /dev/null +++ b/eval/tests/test_l0_outcome_contract.py @@ -0,0 +1,147 @@ +"""Independent-cell contract; fixtures are not new retrieval measurements.""" +import csv +import hashlib +import json +from collections import Counter +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from runner import l0 + +CONTEXT = "--- app.py:1-2 authenticate (function, score 0.91) ---\nx = 1\ny = 2\n" + + +@pytest.mark.parametrize("stdout,stderr,outcome", [ + (CONTEXT, "", "injected"), + ("", "bobbin: skipped (operational intent: Operational)", "skipped"), + ("", "bobbin: skipped (semantic=0.00 < gate=0.45)", "skipped"), + ("", "bobbin: skipped (semantic=0.45 < gate=0.45)", "skipped"), + ("", "bobbin: skipped (semantic=0.45 < gate=0.00)", "error"), + (CONTEXT, "bobbin: injecting 2 new chunks (1 previously injected, turn 2)", "error"), + ("", "bobbin: skipped (all 2 chunks previously injected, no complementary files)", "error"), + ("", "bobbin: skipped (session unchanged)", "error"), + ("", "", "error"), + ("", "bobbin: skipped (unknown future reason)", "error"), + (CONTEXT, "bobbin: skipped (semantic=0.00 < gate=0.45)", "error"), + ("banner without chunk headers", "", "error"), + (json.dumps({"hookSpecificOutput": {"additionalContext": "## Complementary Files\n- app.py"}}), + "", "error"), + ("", "bobbin inject-context: failed to read index", "error"), + ("", ("bobbin: skipped (semantic=0.00 < gate=0.45)\n" + "bobbin: skipped (operational intent: Operational)"), "error"), +]) +def test_independent_contract(stdout, stderr, outcome): + assert l0.classify_hook_result(0, stdout, stderr, independent=True)[0] == outcome + + +def test_legacy_contract_and_original_rows_are_not_reinterpreted(): + assert l0.classify_hook_result(0, "", "")[0] == "skipped" + reason = "bobbin: skipped (all 2 chunks previously injected, no complementary files)" + assert l0.classify_hook_result(0, "", reason) == ("skipped", reason) + root = Path(__file__).resolve().parents[2] / "docs/eval" + raw = (root / "l0-20261016-rows.csv").read_bytes() + manifest = json.loads((root / "l0-20261016-manifest.json").read_text()) + assert hashlib.sha256(raw).hexdigest() == manifest["rows_csv_sha256"] + rows = list(csv.DictReader(raw.decode().splitlines())) + assert Counter(row["outcome"] for row in rows) == {"injected": 213, "skipped": 83, "error": 34} + assert sum("previously injected" in row["detail"] for row in rows) == 23 + assert sum(row["detail"] == "injected text had no parseable chunk headers" for row in rows) == 34 + + +@pytest.mark.parametrize("stdout,stderr,outcome", [ + (CONTEXT, "", "injected"), + ("", "bobbin: skipped (semantic=0.00 < gate=0.45)", "skipped"), + ("", "bobbin: skipped (all 2 chunks previously injected, no complementary files)", "error"), + ("", "", "error"), + ("nonempty unsupported context", "", "error"), +]) +def test_driver_propagates_contract_without_numeric_errors(tmp_path, monkeypatch, stdout, stderr, outcome): + (tmp_path / "app.py").write_text("x = 1\ny = 2\n") + monkeypatch.setattr(l0, "gold_for_commit", lambda *args: {"app.py": [(1, 2)]}) + calls = [] + + def transport(cmd, **kwargs): + calls.append(cmd) + assert "--no-dedup" in cmd + return CompletedProcess(cmd, 0, stdout, stderr) + + monkeypatch.setattr(l0.subprocess, "run", transport) + row, = l0.run_task({"id": "fixture", "commit": "unused", "description": "authenticate"}, + tmp_path, ["full"], [300], "not-executed", {}, + lambda *args: None, strict_hook_contract=True) + assert len(calls) == 1 and row.outcome == outcome + if outcome == "error": + assert row.hunk_recall is None and row.file_recall is None and row.density is None + assert row.chunks == [] + elif outcome == "injected": + assert row.hunk_recall == 1 and row.density == 1 + else: + assert row.hunk_recall == 0 # A registered, explicit gate is a valid non-injection. + + +def test_nonzero_exit_cannot_be_overridden_by_valid_headers(): + assert l0.classify_hook_result(1, CONTEXT, "", independent=True)[0] == "error" + + +@pytest.mark.parametrize("contaminated", [False, True]) +def test_click_driver_selects_strict_contract_and_quarantines_error(tmp_path, monkeypatch, contaminated): + """Real CLI -> task -> hook -> classifier -> cell quarantine; fake indexing only.""" + import sys + + from click.testing import CliRunner + + from runner import bobbin_setup, cell_isolation, index_reuse, workspace + from runner.cli import cli + + binary = tmp_path / "fixture-bobbin" + output = "" if contaminated else CONTEXT + error = "bobbin: skipped (all 2 chunks previously injected, no complementary files)" if contaminated else "" + binary.write_text(f"#!{sys.executable}\nimport sys\n" + "if sys.argv[1] == '--version': print('bobbin private fixture')\n" + f"else:\n sys.stdin.read()\n sys.stdout.write({output!r})\n sys.stderr.write({error!r})\n") + binary.chmod(0o700) + model = tmp_path / "model" + model.mkdir() + task = {"id": "fixture", "repo": "fixture/source", "commit": "unused", "description": "authenticate"} + monkeypatch.setattr("runner.cli.load_task_by_id", lambda *a: task) + monkeypatch.setattr(bobbin_setup, "_find_bobbin", lambda: str(binary)) + + def clone(repo, parent): + ws = Path(parent) / "fixture--source" + ws.mkdir(parents=True) + (ws / "app.py").write_text("x = 1\ny = 2\n") + return ws + + def index(ws, **kwargs): + if "before_index" in kwargs: + kwargs["before_index"](ws) + (Path(ws) / ".bobbin").mkdir() + return {} + + empty_seal = {"payload": {"files": {}, "bytes": 0}} + monkeypatch.setattr(workspace, "clone_repo", clone) + monkeypatch.setattr(workspace, "checkout_parent", lambda *a: None) + monkeypatch.setattr(bobbin_setup, "setup_bobbin", index) + monkeypatch.setattr(index_reuse, "check_scope", lambda: {"fixture": "no campaign"}) + monkeypatch.setattr(index_reuse, "identity", lambda *a: {}) + monkeypatch.setattr(index_reuse, "seal", lambda *a: empty_seal) + monkeypatch.setattr(index_reuse, "verify", lambda *a: empty_seal) + monkeypatch.setattr(cell_isolation, "verify", lambda *a: empty_seal) + monkeypatch.setattr(l0, "gold_for_commit", lambda *a: {"app.py": [(1, 2)]}) + out = tmp_path / "scores.jsonl" + result = CliRunner().invoke(cli, ["l0", "--task", "fixture", "--arms", "full", + "--budgets", "300", "--out", str(out), + "--workdir", str(tmp_path / "work"), + "--reuse-model-dir", str(model)]) + assert result.exit_code == (1 if contaminated else 0), result.output + manifest, row = [json.loads(line) for line in out.read_text().splitlines()] + assert manifest["hook_outcomes"] == l0.INDEPENDENT_HOOK_CONTRACT + assert row["outcome"] == ("error" if contaminated else "injected") + if contaminated: + assert row["hunk_recall"] is None + journal = tmp_path / "work/cell-state/fixture/full/300/result.json" + assert json.loads(journal.read_text())["state"] == "quarantined" + else: + assert row["hunk_recall"] == 1 diff --git a/eval/v2/PREREGISTRATION.md b/eval/v2/PREREGISTRATION.md index a6f1fd1d..d88138b9 100644 --- a/eval/v2/PREREGISTRATION.md +++ b/eval/v2/PREREGISTRATION.md @@ -455,3 +455,17 @@ rows remain literal and untouched. Arms, budgets, tasks, gold exclusions, target metrics and statistical tests are unchanged. Any new campaign must retain this instrumentation identity and cannot silently pool its corrected rows with the prior partial run. + +### 2026-10-23: opt-in independent hook outcome contract + +Before any further campaign, sealed reuse records +`hook_outcomes=independent-hook-outcomes/v1`. It retains per-cell state isolation +and qualified-path scoring, and refuses dedup/reducing contamination, unsupported +or contradictory output and unexplained silence as errors with null efficacy +metrics. Exactly stated operational-intent and semantic-gate non-injections stay +skips; rounded printed gate values are not treated as exact latent scores. +Legacy outcomes and the original partial rows are unchanged. This is an +instrument amendment, not permission to run, a new result, a resource increase +or a changed model/chunker decision. See +[`l0-outcome-contract.md`](../../docs/eval/l0-outcome-contract.md) for the outcome +family dispositions and the native-enrollment/aggregate-storage acceptance gates.