From 48f11b95175fcd39bfba0a7ae3fd2f744a7259d1 Mon Sep 17 00:00:00 2001 From: Sricharan Vagvala <127071883+theon199@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:40:59 -0800 Subject: [PATCH] Expose catalogue selection context without changing frozen pickers Reconcile catalogue assessment metadata with the selected record and retain existing source-bound picker replays. Includes controls and source pins. Prepared with substantial OpenAI Codex assistance. --- AGENTS.md | 7 +- docs/research/agent-procedure.md | 18 +- docs/research/priority-selection-context.md | 134 +++++++++++++++ scripts/choose_contextual_research_task.py | 89 ++++++++++ tests/test_research_coordination.py | 2 +- tests/test_research_priority_context.py | 179 ++++++++++++++++++++ 6 files changed, 420 insertions(+), 9 deletions(-) create mode 100644 docs/research/priority-selection-context.md create mode 100644 scripts/choose_contextual_research_task.py create mode 100644 tests/test_research_priority_context.py diff --git a/AGENTS.md b/AGENTS.md index fa006ceb6b..ff81117fca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,9 +7,10 @@ before using the task picker; its coverage fields identify prior checks and remaining obligations. Explicit user tasks take precedence over that default assignment; do not reroll an already assigned task. -For a fresh assignment, run `python3 -B scripts/choose_current_research_task.py`. -This entry point resolves generic baseline commands and retains each frozen -card beside the current command. The historical picker and root guide stay +For a fresh assignment, run `python3 -B scripts/choose_contextual_research_task.py`. +This entry point reports whether the catalogue's assessment context matches the +selected record, then retains the existing current-command routes and frozen +cards. The historical pickers and root guide stay unchanged because research packages pin their bytes. The current command examples are in [the standing procedure](docs/research/agent-procedure.md). diff --git a/docs/research/agent-procedure.md b/docs/research/agent-procedure.md index 06b35b9805..6afed3acaa 100644 --- a/docs/research/agent-procedure.md +++ b/docs/research/agent-procedure.md @@ -102,15 +102,15 @@ Run this standard-library helper: ```sh # Python 3.10 or later; no third-party dependencies. -python3 scripts/choose_current_research_task.py +python3 scripts/choose_contextual_research_task.py # To reproduce a selection, pass the emitted 32-hex-digit nonce: -python3 scripts/choose_current_research_task.py --seed YOUR_NONCE +python3 scripts/choose_contextual_research_task.py --seed YOUR_NONCE # Expertise or an explicit user preference may override the allocation: -python3 scripts/choose_current_research_task.py --track complex-mechanisms +python3 scripts/choose_contextual_research_task.py --track complex-mechanisms # Retrieve a specific card after assignment: -python3 scripts/choose_current_research_task.py --task connector-alphabet-change +python3 scripts/choose_contextual_research_task.py --task connector-alphabet-change # Optional: surface structured handoffs/claims from an existing overlap cache. -python3 scripts/choose_current_research_task.py --coordination-cache /tmp/overlap.json +python3 scripts/choose_contextual_research_task.py --coordination-cache /tmp/overlap.json ``` It reads [the task catalogue](agent-task-catalog.json), draws a @@ -121,6 +121,14 @@ that a task is open. A deterministic seed is a reproducibility aid, not novelty. Each card supplies a first-step command, expected evidence, a next experiment, its connection to the bottleneck, allowed variations and scope-change triggers. +The contextual entry point delegates selection and command routing to the +unchanged pickers, then adds `catalog_selection`: the original catalogue context +and its κ/source/review comparison with `selected`. A mismatch means the +assessment predates or differs from the selection; missing fields are unknown. +This does not assess whether a task remains useful or authenticate the scientific +closure. Use `selected` for the current review and replay. See the +[source-pinned discrepancy and controls](priority-selection-context.md). + **Resolve the generic orientation command before running a historical card.** If its `first_step.command` is `python3 -B scripts/research_baseline.py`, use the current orientation command in [current priorities](current-priorities.md). diff --git a/docs/research/priority-selection-context.md b/docs/research/priority-selection-context.md new file mode 100644 index 0000000000..1241a5069c --- /dev/null +++ b/docs/research/priority-selection-context.md @@ -0,0 +1,134 @@ +# Catalogue assessment versus the selected result + +The new contextual task CLI exposes the catalogue's original `selection_context` +as `catalog_selection.context` and compares its κ, source commit and review +pointer with the selected record. It delegates to the unchanged current picker. +This resolves an onboarding discrepancy, not a mathematical claim or a new +research-priority assessment. + +At base **`7c49fe32c55de02c2c873655c804b16e4c3fef0a`**, revision 8 of the +catalogue and [priority guide](current-priorities.md) name PR5599. The +[selected record](../../certificates/selected-result.json) and its +[application review](pr-intake-evidence/pr6030-audit/PROOF-REVIEW.md) select +PR6030. The old current CLI emitted the latter under `selected` but omitted the +catalogue's assessment context altogether, while linking the former guide. + +| Field | Catalogue assessment | Selected record | +| --- | --- | --- | +| κ | `62009/20000000` = .00310045 | `781451/250000000` = .003125804 | +| Source commit | `09dd0965711ca7eeb169057a420abab1c9052137` | `772dd10a5859a1d15f4851feb371519789213ad8` | +| Review directory | `pr5599-completion` | `pr6030-audit` | + +The source comparison uses the original `audited_source_head`, rather than the +separate portability-repair head. It does not infer a primary source from the +order of `source_prs`, nor from a larger number in an open PR title. + +## Contract and reproduction + +```sh +python3 -B scripts/choose_contextual_research_task.py --task scope-navigation --seed 00000000000000000000000000000063 +``` + +On the pinned inputs, expect `catalog_selection.matches_selected: false`, with +exact old/new values under `differences.kappa`, `differences.source_commit` and +`differences.review`, and no `uncompared_fields`. `selected.reproduce` remains +`make frontier6030-verify`. The 32-digit picker seed is 99 in hexadecimal; it +only makes this explicitly chosen card's output reproducible. + +Equal rational representations compare equal. A changed review or source at +the same κ still differs. Missing comparison fields produce `null` (unknown) +when no known difference exists; a known difference still produces `false`. +The exact historical context is retained, including its own dated scope text. + +Before comparison, both reread JSON byte strings must match the hashes in the +child picker's receipt. A change between the child and wrapper reads fails +without emitting a successful JSON receipt. This detects mixed snapshots at +that boundary; it is not an atomic filesystem snapshot or authentication of +the full selected construction. A stable checkout remains required. + +Both existing pickers, the catalogue, priority guide and selected record retain their +bytes. Existing task choices, methods, scope keys, coverage flags and command +routes are preserved. The contextual CLI adds only `catalog_selection`; existing +consumers can keep their original entry point and output contract. Matching +the three metadata fields establishes neither task novelty nor current coverage. + +## Verification and source pins + +The following command checks the actual discrepancy fixture and controls for +same-κ provenance changes, rational equivalence, missing context, malformed +metadata, changed-between-reads inputs and the real child/wrapper CLI: + +```sh +python3 -B -m unittest discover -s tests -p 'test_research_priority_context.py' -v +``` + +Expected: **10 tests, OK**. The adjacent command is +`python3 -B -m unittest discover -s tests -p 'test_research_current_assignment*.py' -v` +(15 tests). Both pass normally and with `-O -W error` on Linux, CPython 3.12.14. +The real original/contextual receipt for the command above is identical after removing +the added field. The existing controls preserve all live cards, frozen generic +and historical routes, help/error propagation and sparse-input distinctions. +The 27 onboarding and nine coordination tests also pass. The final replay +dependency plan selects no scientific packages; the new code has no scientific +runtime dependency. + +The unchanged #6179 conformance replay also passes with 22 live cards, two +historical route fixtures and seven rejection controls: +`python3 -I -W error -B research/current-picker-lazy-validation/verify.py`. +That replay reconstructs the current picker's exact predecessor source, which +is why this contribution uses a separate entry point. The historical timing +receipt is checked for conformance, not rebenchmarked; no speed claim is made +for the contextual CLI's extra subprocess. + +Immutable SHA-256 pins at the exact base above: + +| Repository path | SHA-256 | +| --- | --- | +| `docs/research/agent-task-catalog.json` | `ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81` | +| `docs/research/current-priorities.md` | `70f61c8d140ae6e892f2b8f065bba4eae3e76f7a8f55f632fd4e2f9f34039bdc` | +| `certificates/selected-result.json` | `9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa` | +| `scripts/choose_research_task.py` | `484f553c488c6f427ca81f19f3ee2720399dcc28ce61a3886d50bdf28ca72d12` | +| `scripts/choose_current_research_task.py` (unchanged) | `04e194b14a1f4157be5be83e8c13995d69c3efe8996773d918acba6bfcabb471` | + +Only Python's standard library is used. The selected full cold replay, native +maps, compiler, precision/tape and analytic recovery are outside this metadata +repair. The five selected assumptions and κ remain unchanged. + +## Assignment, overlap and open next question + +OS randomness (`secrets.randbelow(100)`) yielded assignment seed **99**: +research consolidation. Three candidates were considered: this catalogue/record +discrepancy; source-certificate pointer coverage (current inputs agree, with +the broader predecessor #1864); and ledger/status navigation (nearby work +#3695, #4138 and #6050). The chosen bounded experiment was to reproduce and +expose this exact discrepancy while preserving task selection. Stop after the +three-field comparison and its failure controls; no global survey or search for +a stronger bound is part of this result. + +The local claim was recorded at 2026-11-09 17:14:30 UTC, with an expected finish +by 17:47:48 UTC. GitHub reported no pinned issue. The start-here discussion, +last 20 handoff comments, 25 recently active discussions and relevant open and +closed PR searches were inspected before implementation. No matching repair +was found in that bounded coverage; this is not a proof of novelty. + +This extends the exact-output boundary in Ali Romman's #5378 and Mihir +Srivastava's #5520, preserving Nick Petalas's #6179 source-bound conformance +and benchmark. The omission of `selection_context` was already observed in +#4906's startup audit; that work repaired the generic command route. This +change adds source/claim reconciliation and unknown-context handling rather +than claiming discovery of the earlier omission. No pending construction is a +dependency. + +**Open next question:** should the next maintainer priority assessment revise +the catalogue's tasks as well as its selected context? The comparison now makes +the old assessment visible, but cannot establish whether its advice remains +applicable. First inspect each affected card's current source-specific coverage +and live handoffs; a substantive strategic reassessment and dependent-input +review are required before rewriting frozen catalogue bytes. + +Prepared by **Sricharan Vagvala (@theon199)** with substantial **OpenAI Codex** +assistance. Credit Douglas Colkitt, Ali Romman, Mihir Srivastava, Nick Petalas, +ToikY and the +retained picker/test contributors; DreamingOfClouds and the selected source's +[contributor map](community-p1046030-contributors.md) retain all construction +credit. Apache-2.0. No independent human or proof-kernel review is claimed. diff --git a/scripts/choose_contextual_research_task.py b/scripts/choose_contextual_research_task.py new file mode 100644 index 0000000000..1aad9d738c --- /dev/null +++ b/scripts/choose_contextual_research_task.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Show catalogue assessment drift alongside the current task-picker receipt. + +The frozen current and legacy pickers retain their source and behavior. +Sricharan Vagvala, with substantial OpenAI Codex assistance. Apache-2.0. +""" +from fractions import Fraction +from hashlib import sha256 +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +CURRENT = ROOT / "scripts/choose_current_research_task.py" + + +def catalog_selection_context(result, root): + """Compare declared context, without interpreting it as scientific coverage.""" + documents = [] + for name, digest_key in ( + ("docs/research/agent-task-catalog.json", "catalog_sha256"), + ("certificates/selected-result.json", "selected_record_sha256"), + ): + raw = (root / name).read_bytes() + if sha256(raw).hexdigest() != result[digest_key]: + raise ValueError("input changed since task selection: " + name) + documents.append(json.loads(raw)) + catalog, selected = documents + context = catalog.get("selection_context") + if context is not None and not isinstance(context, dict): + raise ValueError("catalogue selection_context must be an object") + differences, missing = {}, [] + for field, selected_field in ( + ("kappa", "kappa"), ("source_commit", "audited_source_head"), + ("review", "review"), + ): + before = (context or {}).get(field) + after = selected.get(selected_field) + if before is None or after is None: + missing.append(field) + continue + if not all(isinstance(value, str) and value.strip() for value in (before, after)): + raise ValueError("invalid selection context field: " + field) + if field == "kappa": + try: + equal = Fraction(before) == Fraction(after) + except (ValueError, ZeroDivisionError) as exc: + raise ValueError("invalid selection context kappa") from exc + else: + equal = before == after + if not equal: + differences[field] = {"catalog": before, "selected": after} + return { + "context": context, + "matches_selected": False if differences else (None if missing else True), + "differences": differences, + "uncompared_fields": missing, + "scope": ( + "Compare only kappa, source_commit/audited_source_head and review. " + "Missing fields are unknown. The catalogue context is historical; " + "use selected for the current result and replay. Matching metadata " + "does not establish current task coverage or scientific validation. " + "Input hashes bind this comparison to the picker receipt, not the " + "complete selected source closure." + ), + } + + +def main(): + child = subprocess.run([sys.executable, "-B", str(CURRENT), *sys.argv[1:]], + capture_output=True, text=True, encoding="utf-8") + if child.returncode or any(arg in ("-h", "--help") for arg in sys.argv[1:]): + sys.stdout.write(child.stdout) + sys.stderr.write(child.stderr) + return child.returncode + sys.stderr.write(child.stderr) + try: + result = json.loads(child.stdout) + result["catalog_selection"] = catalog_selection_context(result, ROOT) + except (OSError, ValueError, KeyError, TypeError) as exc: + sys.stderr.write("catalogue selection comparison failed: " + str(exc) + "\n") + return 2 + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_research_coordination.py b/tests/test_research_coordination.py index 95d7dda49a..f89523f248 100644 --- a/tests/test_research_coordination.py +++ b/tests/test_research_coordination.py @@ -121,7 +121,7 @@ def test_root_entry_and_priorities_links(self): self.assertIn('docs/research/current-priorities.md',root) self.assertIn('docs/research/agent-procedure.md',root) procedure=(ROOT/'docs/research/agent-procedure.md').read_text() - self.assertLess(procedure.index('**Before drawing a card**'),procedure.index('python3 scripts/choose_current_research_task.py')) + self.assertLess(procedure.index('**Before drawing a card**'),procedure.index('python3 scripts/choose_contextual_research_task.py')) if __name__=='__main__':unittest.main() diff --git a/tests/test_research_priority_context.py b/tests/test_research_priority_context.py new file mode 100644 index 0000000000..9cb05d9903 --- /dev/null +++ b/tests/test_research_priority_context.py @@ -0,0 +1,179 @@ +"""Keep catalogue assessment provenance distinct from the selected result. + +Sricharan Vagvala, with substantial OpenAI Codex assistance. Apache-2.0. +""" +import copy +from contextlib import contextmanager +from hashlib import sha256 +import importlib.util +import json +from pathlib import Path +import subprocess +import shutil +import sys +import tempfile +import unittest + +from test_research_current_assignment import ROOT, frozen_cli_tree + +SCRIPT = "choose_contextual_research_task.py" +SPEC = importlib.util.spec_from_file_location("contextual_picker", ROOT / "scripts" / SCRIPT) +current = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(current) + + +@contextmanager +def contextual_cli_tree(): + with frozen_cli_tree() as root: + shutil.copyfile(ROOT / "scripts" / SCRIPT, root / "scripts" / SCRIPT) + yield root + + +# The three differing fields at base 7c49fe32; these are metadata fixtures, +# not independently authenticated construction or admission evidence. +CATALOG_CONTEXT = { + "kappa": "62009/20000000", + "source_commit": "09dd0965711ca7eeb169057a420abab1c9052137", + "review": "docs/research/pr-intake-evidence/pr5599-completion/PROOF-REVIEW.md", +} +SELECTED = { + "kappa": "781451/250000000", + "audited_source_head": "772dd10a5859a1d15f4851feb371519789213ad8", + "review": "docs/research/pr-intake-evidence/pr6030-audit/PROOF-REVIEW.md", +} +NAMES = ("docs/research/agent-task-catalog.json", "certificates/selected-result.json") + + +def write_receipt(root, context, selected): + receipt = {} + for name, key, value in zip( + NAMES, ("catalog_sha256", "selected_record_sha256"), + ({"selection_context": context}, selected), + ): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + raw = json.dumps(value).encode("utf-8") + path.write_bytes(raw) + receipt[key] = sha256(raw).hexdigest() + return receipt + + +class ResearchPriorityContextTests(unittest.TestCase): + def compare(self, context=CATALOG_CONTEXT, selected=SELECTED): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + receipt = write_receipt(root, context, selected) + before = copy.deepcopy(receipt) + result = current.catalog_selection_context(receipt, root) + self.assertEqual(receipt, before) + return result + + def matching_context(self): + return {"kappa": SELECTED["kappa"], + "source_commit": SELECTED["audited_source_head"], + "review": SELECTED["review"]} + + def test_actual_checkpoint_discrepancy_exposes_all_three_fields(self): + result = self.compare() + self.assertFalse(result["matches_selected"]) + self.assertEqual(result["context"], CATALOG_CONTEXT) + self.assertEqual(set(result["differences"]), {"kappa", "source_commit", "review"}) + self.assertEqual(result["differences"]["kappa"], + {"catalog": "62009/20000000", "selected": "781451/250000000"}) + self.assertEqual(result["uncompared_fields"], []) + + def test_equal_rationals_are_not_a_false_discrepancy(self): + context = self.matching_context() + context["kappa"] = "0.003125804" + result = self.compare(context) + self.assertIs(result["matches_selected"], True) + self.assertEqual(result["differences"], {}) + + def test_same_kappa_does_not_hide_changed_source_or_review(self): + for field in ("source_commit", "review"): + with self.subTest(field=field): + context = self.matching_context() + context[field] = CATALOG_CONTEXT[field] + result = self.compare(context) + self.assertIs(result["matches_selected"], False) + self.assertEqual(set(result["differences"]), {field}) + + def test_absent_context_is_unknown(self): + result = self.compare(None) + self.assertIsNone(result["matches_selected"]) + self.assertIsNone(result["context"]) + self.assertEqual(set(result["uncompared_fields"]), set(CATALOG_CONTEXT)) + + def test_missing_source_head_is_unknown_even_when_other_fields_match(self): + for missing_side in ("catalog", "selected"): + with self.subTest(missing_side=missing_side): + context, selected = self.matching_context(), dict(SELECTED) + if missing_side == "catalog": + context.pop("source_commit") + else: + selected.pop("audited_source_head") + result = self.compare(context, selected) + self.assertIsNone(result["matches_selected"]) + self.assertEqual(result["uncompared_fields"], ["source_commit"]) + + def test_known_difference_survives_incomplete_context(self): + context = dict(CATALOG_CONTEXT) + context.pop("source_commit") + result = self.compare(context) + self.assertIs(result["matches_selected"], False) + self.assertEqual(result["uncompared_fields"], ["source_commit"]) + + def test_change_after_child_receipt_is_rejected_for_each_input(self): + for name in NAMES: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + receipt = write_receipt(root, CATALOG_CONTEXT, SELECTED) + path = root / name + # Even a semantically inert rewrite belongs to another snapshot. + path.write_bytes(path.read_bytes() + b"\n") + with self.assertRaisesRegex(ValueError, "input changed since task selection"): + current.catalog_selection_context(receipt, root) + + def test_malformed_context_is_rejected(self): + for context in ([], "old context", {"kappa": 0.003125804}, + {"kappa": "1/0"}, {"kappa": "not a rational"}): + with self.subTest(context=context), self.assertRaises(ValueError): + self.compare(context) + + def test_real_cli_preserves_choices_and_retains_original_context(self): + with contextual_cli_tree() as root: + path = root / NAMES[0] + catalog = json.loads(path.read_bytes()) + context = {**CATALOG_CONTEXT, "scope": "Historical assessment; retain verbatim."} + catalog["selection_context"] = context + path.write_text(json.dumps(catalog), encoding="utf-8") + (root / NAMES[1]).write_text(json.dumps({ + **SELECTED, "source_manifest": "fixture.json", "reproduce": "fixture replay", + }), encoding="utf-8") + args = ["--seed", "0" * 32, "--task", "portable-replay"] + def invoke(script): + return subprocess.run([sys.executable, "-B", str(root / "scripts" / script), *args], + capture_output=True, text=True, encoding="utf-8") + old = invoke("choose_current_research_task.py") + new = invoke(SCRIPT) + self.assertEqual(old.returncode, 0, old.stderr) + self.assertEqual(new.returncode, 0, new.stderr) + before, after = json.loads(old.stdout), json.loads(new.stdout) + report = after.pop("catalog_selection") + self.assertIs(report["matches_selected"], False) + self.assertEqual(report["context"], context) + # The existing current-command routes and all other output stay exact. + self.assertEqual(after, before) + self.assertEqual(invoke("choose_current_research_task.py").stdout, old.stdout) + + def test_real_cli_preserves_child_help_and_failure(self): + with contextual_cli_tree() as root: + for args in (["--help"], ["--unknown-option"], ["--seed", "bad"]): + with self.subTest(args=args): + def invoke(script): + return subprocess.run( + [sys.executable, "-B", str(root / "scripts" / script), *args], + capture_output=True, text=True, encoding="utf-8") + old, new = invoke("choose_current_research_task.py"), invoke(SCRIPT) + self.assertEqual((new.returncode, new.stdout, new.stderr), + (old.returncode, old.stdout, old.stderr))