diff --git a/docs/demos/l0-score-paths/README.md b/docs/demos/l0-score-paths/README.md new file mode 100644 index 00000000..3702b9e4 --- /dev/null +++ b/docs/demos/l0-score-paths/README.md @@ -0,0 +1,22 @@ +# Exact L0 scoring paths + +A synthetic header in the measured frozen native format names +`repos/fixture/app.py`; Git gold names `app.py`. The same two-line source scores +file recall 0 under legacy literal comparison and 1 through the opt-in exact +workspace mapping. A foreign-only path refuses, and original literal scoring +remains unchanged. This demonstrates the instrument, not retrieval quality. + +Run `python3 docs/demos/l0-score-paths/probe.py` from the repository root. +The source base is `9624689e5eca15f198910201fd89ebf3afd1af9e`; the candidate +adds the mapper and opt-in driver wiring. Exact candidate identity is in the PR. +No native executable, model, provider, graph or campaign is invoked. + +[Real terminal capture](output.log) and [timing](timing.log) can be downloaded +and replayed with: + +```sh +scriptreplay --log-timing timing.log --log-out output.log --maxdelay 0.1 +``` + +The private paired control also applies this mapper to retained actual native +hook chunks. Its outcome does not change or reinterpret the earlier partial run. diff --git a/docs/demos/l0-score-paths/output.log b/docs/demos/l0-score-paths/output.log new file mode 100644 index 00000000..6bb0eafa --- /dev/null +++ b/docs/demos/l0-score-paths/output.log @@ -0,0 +1,7 @@ +Script started on 2026-10-20 23:26:32-04:00 [COMMAND="stty -onlcr; python3 docs/demos/l0-score-paths/probe.py" ] +Before (legacy literal paths): file_recall=0.0 +After (opt-in exact workspace mapping): file_recall=1.0 +Foreign-only path: REFUSED (unknown or ambiguous native chunk path) +Original literal scoring unchanged; synthetic instrument only, no retrieval/campaign + +Script done on 2026-10-20 23:26:32-04:00 [COMMAND_EXIT_CODE="0"] diff --git a/docs/demos/l0-score-paths/probe.py b/docs/demos/l0-score-paths/probe.py new file mode 100644 index 00000000..55089e9e --- /dev/null +++ b/docs/demos/l0-score-paths/probe.py @@ -0,0 +1,29 @@ +"""Synthetic transport-path instrument demo; no retrieval, model or campaign.""" +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / 'eval')) +from runner.l0 import parse_injected_chunks, score_chunks +from runner.score_paths import workspace_chunks + +with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / 'fixture' + workspace.mkdir() + (workspace / 'app.py').write_text('def authenticate(token):\n return bool(token)\n') + native = parse_injected_chunks('--- repos/fixture/app.py:1-2 authenticate ---') + gold = {'app.py': [(1, 2)]} + before = score_chunks(native, gold)['file_recall'] + after = score_chunks(workspace_chunks(workspace, native), gold)['file_recall'] + assert before == 0 and after == 1 + print(f'Before (legacy literal paths): file_recall={before}') + print(f'After (opt-in exact workspace mapping): file_recall={after}') + try: + workspace_chunks(workspace, [('repos/foreign/app.py', 1, 2)]) + except ValueError as exc: + print(f'Foreign-only path: REFUSED ({exc})') + else: + raise AssertionError('foreign path accepted') + assert native == [('repos/fixture/app.py', 1, 2)] + assert score_chunks(native, gold)['file_recall'] == before + print('Original literal scoring unchanged; synthetic instrument only, no retrieval/campaign') diff --git a/docs/demos/l0-score-paths/timing.log b/docs/demos/l0-score-paths/timing.log new file mode 100644 index 00000000..dc74f660 --- /dev/null +++ b/docs/demos/l0-score-paths/timing.log @@ -0,0 +1,3 @@ +0.036814 103 +0.000012 68 +0.000012 85 diff --git a/docs/eval/l0-bounded-index-reuse.md b/docs/eval/l0-bounded-index-reuse.md index d5066a2f..192c00ef 100644 --- a/docs/eval/l0-bounded-index-reuse.md +++ b/docs/eval/l0-bounded-index-reuse.md @@ -124,10 +124,28 @@ Rollback: revert the source commit, retain all immutable prior/new results, and do not reinterpret a partial run as complete. No production indexing, release, resource increase, timer or service change is part of this source delivery. -A separate scoring caveat remains: the retained native hook emits repository- +A paired scoring probe found that the retained native hook emits repository- qualified paths such as `repos/fixture/app.py`, while the existing scorer matches -paths literally against Git-relative gold paths. This source change preserves -that scorer contract and the original scores; the private hook-equivalence -controls compare native chunks directly, not claimed recall improvements. Path -normalization and any resulting study interpretation require explicit paired -scorer controls before full-cohort quality acceptance. +paths literally against Git-relative gold paths. Legacy scoring and original +scores stay unchanged; the private hook-equivalence controls compare native +chunks directly, not claimed recall improvements. The opt-in correction below +is paired against those same chunks before any full-cohort interpretation. + +The opt-in scorer now resolves that path caveat with exact workspace ownership: +only a literal existing file or the exact current `repos//` +prefix can match. Exactly one nonlinked candidate is required. Ambiguity, +unknown/foreign-only paths and traversal refuse as an error. No gold-dependent +matching is used; normalization is applied before baseline character counting, +while actual hook character accounting is preserved. The manifest records +`workspace-qualified-or-literal/v1`; legacy scoring and old rows stay unchanged. +The paired private instrument probe checks the same retained actual native chunks, +and driver controls verify recall 0 before versus 1 with the opt-in mapping. +This repairs interpretation of paths, not a measured improvement in retrieval. + +Path qualification uses the workspace that produced the current arm’s chunks. +The no-coupling copy is checked independently: an original-workspace file cannot +satisfy a missing derived-workspace file, even when both directories share a name. + +An unreadable candidate is an explicit error, not evidence of absence. Strict +metadata reads refuse instead of treating an inaccessible alternative as a +unique path match. diff --git a/eval/runner/cli.py b/eval/runner/cli.py index 4acb4f5a..9b28b700 100644 --- a/eval/runner/cli.py +++ b/eval/runner/cli.py @@ -1843,6 +1843,7 @@ def l0(tasks_dir, task_ids, arms, budgets, out_path, workdir, index_timeout, reu cells = planned_cells(tasks, arm_list, budget_list) manifest["planned_cells"] = cells manifest["index_reuse"] = "sealed-no-coupling/v1" if reuse_model_dir else "legacy" + manifest["scoring_paths"] = "workspace-qualified-or-literal/v1" if reuse_model_dir else "literal" if reuse_limits is not None: manifest["resource_limits"] = reuse_limits manifest["wall_supervisor_required_seconds"] = 21600 @@ -1924,7 +1925,7 @@ def index(ws: Path, overrides=None): target, Path(bobbin), reuse_model_dir, overrides, journal, lambda a=arm, b=budget: L0.run_task( task, ws, [a], [b], bobbin, env, index, - nocoupling_workspace=fork)) + nocoupling_workspace=fork, normalize_paths=True)) except Exception as exc: # Preserve the explicit failure and all prior cells; # remaining cells stay missing. No reset/resend follows. diff --git a/eval/runner/l0.py b/eval/runner/l0.py index 5a90e69a..c0825ea0 100644 --- a/eval/runner/l0.py +++ b/eval/runner/l0.py @@ -668,6 +668,7 @@ def run_task( index_workspace: Callable[[Path, dict[str, str] | None], None], *, nocoupling_workspace: Path | None = None, + normalize_paths: bool = False, ) -> list[ArmScore]: """Score every (arm, budget) for one task on an already-checked-out workspace. @@ -696,8 +697,10 @@ def run_task( continue try: chars = 0 + scoring_workspace = workspace if arm in ABLATIONS: ws = nocoupling_ws if arm == "-coupling" else workspace + scoring_workspace = ws outcome, detail, chunks, chars = run_hook_arm( ws, prompt, arm, budget, env, bobbin ) @@ -715,7 +718,11 @@ def run_task( outcome, detail = ( ("injected", "") if chunks else ("skipped", "baseline found nothing") ) - chars = _chunk_chars(workspace, chunks) + if normalize_paths: + from runner.score_paths import workspace_chunks + chunks = workspace_chunks(scoring_workspace, chunks) + if arm not in ABLATIONS: + chars = _chunk_chars(workspace, chunks, strict=normalize_paths) except Exception as exc: # noqa: BLE001 -- recorded, never swallowed base.outcome, base.detail = "error", f"{type(exc).__name__}: {exc}"[:300] results.append(base) @@ -735,12 +742,14 @@ def run_task( return results -def _chunk_chars(workspace: Path, chunks: list[Chunk]) -> int: +def _chunk_chars(workspace: Path, chunks: list[Chunk], *, strict: bool = False) -> int: total = 0 for path, s, e in chunks: try: lines = (workspace / path).read_text(errors="replace").splitlines() - except OSError: + except OSError as exc: + if strict: + raise ValueError("unreadable native chunk path") from exc continue total += sum(len(x) + 1 for x in lines[s - 1 : e]) return total diff --git a/eval/runner/score_paths.py b/eval/runner/score_paths.py new file mode 100644 index 00000000..2d7ff885 --- /dev/null +++ b/eval/runner/score_paths.py @@ -0,0 +1,51 @@ +"""Resolve frozen native context paths without guessing another repository. + +Only the opt-in, identity-sealed L0 runner uses this mapping. Legacy rows and +literal scoring remain unchanged. An ambiguous or unknown path is an error, +never a zero-recall observation. +""" +import stat +from pathlib import Path, PurePosixPath + + +def workspace_chunks(workspace: Path, chunks: list[tuple[str, int, int]]) -> list[tuple[str, int, int]]: + """Map literal or exact repos// paths to existing files.""" + root = workspace.resolve(strict=True) + prefix = f"repos/{workspace.name}/" + result = [] + for path, start, end in chunks: + if (not isinstance(path, str) or not path or "\\" in path or "\0" in path + or PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts + or PurePosixPath(path).as_posix() != path): + raise ValueError("unsafe native chunk path") + names = {path} + if path.startswith(prefix): + names.add(path[len(prefix):]) + candidates = set() + for name in names: + if not name: + continue + target = root + metadata = None + for part in PurePosixPath(name).parts: + target = target / part + try: + metadata = target.lstat() + except (FileNotFoundError, NotADirectoryError): + metadata = None + break + except OSError as exc: + raise ValueError("unreadable native chunk path") from exc + if stat.S_ISLNK(metadata.st_mode): + raise ValueError("linked native chunk path") + if metadata is not None and stat.S_ISREG(metadata.st_mode): + try: + with target.open("rb") as contents: + contents.read(1) + except OSError as exc: + raise ValueError("unreadable native chunk path") from exc + candidates.add(name) + if len(candidates) != 1: + raise ValueError("unknown or ambiguous native chunk path") + result.append((candidates.pop(), start, end)) + return result diff --git a/eval/tests/test_score_paths.py b/eval/tests/test_score_paths.py new file mode 100644 index 00000000..54a9f21d --- /dev/null +++ b/eval/tests/test_score_paths.py @@ -0,0 +1,205 @@ +"""Private path instrument controls; no corpus or native/model invocation.""" + +import os + +import pytest + +from runner import l0 +from runner.score_paths import workspace_chunks + + +def fixture(tmp_path): + ws = tmp_path / 'fixture' + ws.mkdir() + (ws / 'app.py').write_text('one\ntwo\n') + return ws + + +def test_native_qualified_and_literal_paths_share_gold_without_changing_scorer(tmp_path): + ws = fixture(tmp_path) + gold = {'app.py': [(1, 2)]} + native = [('repos/fixture/app.py', 1, 2)] + assert l0.score_chunks(native, gold)['file_recall'] == 0 + normalized = workspace_chunks(ws, native) + assert normalized == [('app.py', 1, 2)] + assert workspace_chunks(ws, normalized) == normalized + assert l0.score_chunks(normalized, gold)['file_recall'] == 1 + assert native == [('repos/fixture/app.py', 1, 2)] + + +@pytest.mark.parametrize('path', [ + 'repos/other/app.py', 'repos/fixture/missing.py', '../app.py', + '/app.py', 'repos/fixture/../app.py', 'repos//fixture/app.py', + 'repos/fixture/./app.py', 'repos\\fixture\\app.py', '', 'app.py\0bad', +]) +def test_unknown_or_unsafe_path_is_not_a_zero_recall_observation(tmp_path, path): + with pytest.raises(ValueError): + workspace_chunks(fixture(tmp_path), [(path, 1, 2)]) + + +def test_ambiguous_qualified_path_refuses_instead_of_choosing_gold(tmp_path): + ws = fixture(tmp_path) + other = ws / 'repos/fixture' + other.mkdir(parents=True) + (other / 'app.py').write_text('different source\n') + with pytest.raises(ValueError, match='ambiguous'): + workspace_chunks(ws, [('repos/fixture/app.py', 1, 2)]) + # A real literal repos/ path stays valid when there is no stripped candidate. + (ws / 'app.py').unlink() + assert workspace_chunks(ws, [('repos/fixture/app.py', 1, 2)]) == [ + ('repos/fixture/app.py', 1, 2)] + + +def test_linked_source_path_refuses(tmp_path): + ws = fixture(tmp_path) + (ws / 'link.py').symlink_to(ws / 'app.py') + with pytest.raises(ValueError, match='linked'): + workspace_chunks(ws, [('repos/fixture/link.py', 1, 2)]) + + +def test_actual_task_driver_opt_in_maps_hook_paths_legacy_stays_literal(tmp_path, monkeypatch): + ws = fixture(tmp_path) + monkeypatch.setattr(l0, 'gold_for_commit', lambda *a: {'app.py': [(1, 2)]}) + monkeypatch.setattr(l0, 'run_hook_arm', lambda *a: ( + 'injected', '', [('repos/fixture/app.py', 1, 2)], 80)) + task = {'id': 'fixture', 'commit': 'unused', 'description': 'private'} + args = (task, ws, ['full'], [100], 'not-executed', {}, lambda *a: None) + legacy, = l0.run_task(*args) + actual, = l0.run_task(*args, normalize_paths=True) + assert legacy.file_recall == 0 and actual.file_recall == 1 + assert actual.injected_tokens == legacy.injected_tokens == 20 + assert actual.chunks == [('app.py', 1, 2)] + monkeypatch.setattr(l0, 'run_hook_arm', lambda *a: ( + 'injected', '', [('repos/foreign/app.py', 1, 2)], 80)) + refused, = l0.run_task(*args, normalize_paths=True) + assert refused.outcome == 'error' and refused.file_recall is None + + +def test_native_embed_baseline_normalizes_before_counting_characters(tmp_path, monkeypatch): + ws = fixture(tmp_path) + monkeypatch.setattr(l0, 'gold_for_commit', lambda *a: {'app.py': [(1, 2)]}) + monkeypatch.setattr(l0, 'baseline_embed', lambda *a: [('repos/fixture/app.py', 1, 2)]) + task = {'id': 'fixture', 'commit': 'unused', 'description': 'private'} + score, = l0.run_task(task, ws, ['embed'], [100], 'not-executed', {}, + lambda *a: None, normalize_paths=True) + assert score.file_recall == 1 and score.injected_tokens == len('one\ntwo\n') // 4 + + +@pytest.mark.parametrize('fork_name', ['fixture', 'separate-arm']) +def test_no_coupling_checks_its_own_source_not_the_original(tmp_path, monkeypatch, fork_name): + ws = fixture(tmp_path) + fork = tmp_path / 'derived' / fork_name + fork.mkdir(parents=True) + (fork / 'app.py').write_text('one\ntwo\n') + monkeypatch.setattr(l0, 'gold_for_commit', lambda *a: {'app.py': [(1, 2)]}) + monkeypatch.setattr(l0, 'run_hook_arm', lambda selected, *a: ( + 'injected', '', [(f'repos/{selected.name}/app.py', 1, 2)], 80)) + task = {'id': 'fixture', 'commit': 'unused', 'description': 'private'} + args = (task, ws, ['full', '-coupling'], [100], 'not-executed', {}, lambda *a: None) + full, derived = l0.run_task(*args, nocoupling_workspace=fork, normalize_paths=True) + assert full.file_recall == derived.file_recall == 1 + assert full.injected_tokens == derived.injected_tokens == 20 + (fork / 'app.py').unlink() + full, missing = l0.run_task(*args, nocoupling_workspace=fork, normalize_paths=True) + assert full.outcome == 'injected' and full.file_recall == 1 + assert missing.outcome == 'error' and missing.file_recall is None + + +@pytest.mark.skipif(getattr(os, 'geteuid', lambda: 0)() == 0, + reason='POSIX non-root traversal is required to prove permission refusal') +def test_unreadable_alternative_does_not_make_a_path_unique(tmp_path): + ws = fixture(tmp_path) + parent = ws / 'repos' + alternative = parent / 'fixture' / 'app.py' + alternative.parent.mkdir(parents=True) + alternative.write_text('other source\n') + try: + parent.chmod(0) + with pytest.raises(ValueError, match='unreadable'): + workspace_chunks(ws, [('repos/fixture/app.py', 1, 2)]) + finally: + parent.chmod(0o700) + with pytest.raises(ValueError, match='ambiguous'): + workspace_chunks(ws, [('repos/fixture/app.py', 1, 2)]) + alternative.unlink() + assert workspace_chunks(ws, [('repos/fixture/app.py', 1, 2)]) == [('app.py', 1, 2)] + + +@pytest.mark.skipif(getattr(os, 'geteuid', lambda: 0)() == 0, + reason='POSIX non-root reads are required to prove permission refusal') +@pytest.mark.parametrize('path', ['app.py', 'repos/fixture/app.py']) +def test_unreadable_file_contents_are_refused_with_readable_positive(tmp_path, path): + ws = fixture(tmp_path) + source = ws / 'app.py' + raw = [(path, 1, 2)] + assert workspace_chunks(ws, raw) == [('app.py', 1, 2)] + try: + source.chmod(0) + with pytest.raises(PermissionError): + source.read_text() + with pytest.raises(ValueError, match='unreadable'): + workspace_chunks(ws, raw) + finally: + source.chmod(0o600) + + +@pytest.mark.skipif(getattr(os, 'geteuid', lambda: 0)() == 0, + reason='POSIX non-root reads are required to prove permission refusal') +def test_unreadable_alternative_file_is_not_discarded(tmp_path): + ws = fixture(tmp_path) + other = ws / 'repos/fixture/app.py' + other.parent.mkdir(parents=True) + other.write_text('alternative source\n') + try: + other.chmod(0) + with pytest.raises(PermissionError): + other.read_text() + with pytest.raises(ValueError, match='unreadable'): + workspace_chunks(ws, [('repos/fixture/app.py', 1, 2)]) + finally: + other.chmod(0o600) + + +@pytest.mark.skipif(getattr(os, 'geteuid', lambda: 0)() == 0, + reason='POSIX non-root reads are required to prove permission refusal') +@pytest.mark.parametrize('late_failure', [False, True]) +def test_actual_driver_records_read_failure_as_error_only_in_opt_in( + tmp_path, monkeypatch, late_failure): + import runner.score_paths as paths + + ws = fixture(tmp_path) + source = ws / 'app.py' + monkeypatch.setattr(l0, 'gold_for_commit', lambda *a: {'app.py': [(1, 2)]}) + monkeypatch.setattr(l0, 'baseline_embed', lambda *a: [('repos/fixture/app.py', 1, 2)]) + task = {'id': 'fixture', 'commit': 'unused', 'description': 'fixture'} + args = (task, ws, ['embed'], [100], 'not-executed', {}, lambda *a: None) + positive, = l0.run_task(*args, normalize_paths=True) + assert positive.outcome == 'injected' and positive.file_recall == 1 + assert positive.injected_tokens == len('one\ntwo\n') // 4 + original = paths.workspace_chunks + + def revoke_after_mapping(*args): + mapped = original(*args) + source.chmod(0) + with pytest.raises(PermissionError): + source.read_text() + return mapped + + try: + if late_failure: + monkeypatch.setattr(paths, 'workspace_chunks', revoke_after_mapping) + else: + source.chmod(0) + with pytest.raises(PermissionError): + source.read_text() + rejected, = l0.run_task(*args, normalize_paths=True) + assert rejected.outcome == 'error' + assert rejected.file_recall is None and rejected.hunk_recall is None + assert 'unreadable' in rejected.detail + # Retain historical literal scoring and permissive character accounting. + monkeypatch.setattr(l0, 'baseline_embed', lambda *a: [('app.py', 1, 2)]) + legacy, = l0.run_task(*args) + assert legacy.outcome == 'injected' and legacy.file_recall == 1 + assert legacy.injected_tokens == 0 + finally: + source.chmod(0o600) diff --git a/eval/v2/PREREGISTRATION.md b/eval/v2/PREREGISTRATION.md index 7060a8d2..a6f1fd1d 100644 --- a/eval/v2/PREREGISTRATION.md +++ b/eval/v2/PREREGISTRATION.md @@ -438,3 +438,20 @@ a separate decision. **Cost bound.** At most 20 Jev calls per task per set: 40 tasks x 2 sets x 20 = 1,600 calls, measured at ~0.4 s and ~300 input tokens for a one-line passage (up to ~120 lines per passage here). + +## L0 path instrumentation correction (2026-10-21) + +The bounded-index opt-in runner records `scoring_paths: +workspace-qualified-or-literal/v1`. The frozen native hook can emit +`repos//`, while Git gold uses repository-relative paths. +Literal comparison reports zero overlap for the same source file. Opt-in scoring +resolves a chunk only when exactly one existing, nonlinked workspace file matches +its literal path or the exact current workspace-label prefix removed. Unknown, +ambiguous, foreign-only or unsafe paths produce an explicit error, not a zero +score. It never searches by basename, gold membership, suffix or another repo. +The mapping applies equally to native baseline results and hook results; raw +hook character/token accounting is unchanged. The legacy runner and all retained +rows remain literal and untouched. Arms, budgets, tasks, gold exclusions, target +metrics and statistical tests are unchanged. Any new campaign must retain this +instrumentation identity and cannot silently pool its corrected rows with the +prior partial run.