diff --git a/adapters/candidate.py b/adapters/candidate.py index 65806b51..5e287bfe 100644 --- a/adapters/candidate.py +++ b/adapters/candidate.py @@ -163,8 +163,8 @@ def _run(transport, repo: str, commit: str): def _dispatches(transport, repo: str, commit: str): """Every dispatch of the eval job on `commit`, by its GitHub id, conclusion and attempt, oldest first, whatever it - concluded; None where the list is not read whole, holds a run of another commit, event or workflow, or holds a run - still going (issue 613). The list shows a run's latest attempt alone (P-27, decision 0111).""" + concluded; None where the list is not read whole, holds a run of another commit, event or workflow, holds a run + still going, or names one twice (issue 613). The list shows a run's latest attempt alone (P-27, decision 0111).""" try: answer = transport.send(request("GET", "/repos/%s/actions/workflows/%s/runs?head_sha=%s&event=workflow_dispatch" "&per_page=100" % (repo, EVAL_WORKFLOW, commit))) @@ -177,7 +177,8 @@ def _dispatches(transport, repo: str, commit: str): ours = [r for r in runs if isinstance(r, dict) and r.get("head_sha") == commit and r.get("status") == "completed" and r.get("event") == "workflow_dispatch" and r.get("path") == ".github/workflows/%s" % EVAL_WORKFLOW and type(r.get("id")) is int and r["id"] > 0 and type(r.get("run_number")) is int] - if len(ours) != len(runs): + # Each dispatch once, by its id: one listed twice is one dispatch's evidence, never two (Muse's read of #642). + if len(ours) != len(runs) or len({r["id"] for r in ours}) != len(ours): return None return [(r["id"], r.get("conclusion"), r.get("run_attempt")) for r in sorted(ours, key=lambda r: r["run_number"])] diff --git a/examples/invalid/eval-ledger.no-preflight.json b/examples/invalid/eval-ledger.no-preflight.json new file mode 100644 index 00000000..7c0a70e6 --- /dev/null +++ b/examples/invalid/eval-ledger.no-preflight.json @@ -0,0 +1,150 @@ +{ + "schema": "eval-ledger/1", + "salt": "8f14e45fceea167a5a36dedd4bea2543", + "release": "factory/v0.1.0", + "commit": "9e927b4000000000000000000000000000000000", + "role": "pr-review", + "intent": "review-change", + "set": { + "count": 2, + "sha256": "05b3abf2579a5eb66403cd78be557fd860633a1fe2103c7642030defe32c657f" + }, + "cases": [ + { + "id": "poc-0001", + "sha256": "754be4b69c49114336398cc52faaf3b06b709a5674ac584fcc9433cede863d5a", + "verdict": "pass", + "reasons": [], + "usage": { + "turns": 4, + "cost_usd": 0.21, + "seconds": 38.5 + }, + "natural": true, + "result": { + "schema": "result/1", + "run": "r_01JA6Z", + "release": "factory/v0.3.1", + "status": "completed", + "reason": null, + "dispositions": [], + "findings": [ + { + "ref": "n1", + "location": { + "path": "kernel/decide.py", + "start_line": 88, + "end_line": 94 + }, + "category": "functional", + "severity": "major", + "criterion": "PR-3", + "adopted_from": { + "lens": "coderabbit", + "surface": { + "type": "thread", + "key": "PRRT_kw1" + } + } + }, + { + "ref": "n2", + "location": { + "path": "tests/test_decide.py", + "start_line": 410, + "end_line": 410 + }, + "category": "delivery", + "severity": "minor", + "criterion": null + } + ], + "effects": { + "submit_review": [ + { + "id": "e1", + "comments": [ + { + "finding": "n1", + "body": "parse() reads the first item before it checks that the list has one; return early on an empty list." + } + ], + "body": null + } + ], + "create_check_run": [ + { + "id": "e2", + "annotations": [ + { + "finding": "n2", + "body": "The new test asserts that no error was raised, not the Decision." + } + ] + } + ] + }, + "coverage": [ + { + "criterion": "PR-1", + "verdict": "met", + "reason": null + }, + { + "criterion": "PR-2", + "verdict": "met", + "reason": null + }, + { + "criterion": "PR-3", + "verdict": "finding", + "reason": null + }, + { + "criterion": "PR-4", + "verdict": "met", + "reason": null + }, + { + "criterion": "PR-5", + "verdict": "not_applicable", + "reason": "no_rule_added" + } + ], + "usage": { + "engine": "claude-cli", + "engine_version": "2.1.302", + "model": "claude-opus", + "answering_model": "claude-opus", + "effort": "medium", + "session": "7f3a9c2e-5b1d-4e8a-9c6f-2d4b8a1e0f37", + "input_tokens": 2, + "cache_creation_input_tokens": 20475, + "cache_read_input_tokens": 0, + "output_tokens": 194, + "turns": 2, + "cost_usd": 0.167688, + "seconds": 2.714 + } + } + }, + { + "id": "poc-0002", + "sha256": "1b657e2ca5226c6d61636cf2d155df3aeef3cd95422f9ae09924410f72ed07d4", + "verdict": "unreadable", + "reasons": [ + "engine_timeout" + ], + "usage": { + "turns": null, + "cost_usd": null, + "seconds": 0 + }, + "natural": false, + "cause": { + "subtype": "error_max_turns", + "exit": 1 + } + } + ] +} diff --git a/harness/eval/__init__.py b/harness/eval/__init__.py index 442a80bf..7668666b 100644 --- a/harness/eval/__init__.py +++ b/harness/eval/__init__.py @@ -740,11 +740,13 @@ def ledger_of(kept: dict, found: dict, assessed: dict, causes: dict, alternates: """The `eval-ledger/1` of a gate set's run: each case as `report` would list it, whether it is natural, and the Result the review returned for it, under a salt drawn for this run alone, so that its hash in the report tells nothing of what it holds; and each case asked first the same way, but for whether it is natural, from `ahead`, `(assessed, - causes, usages, results)` of them (issue 613). It stays in the scratch for `seal`, and is never uploaded but sealed (issue 612).""" - asked, told, spent, answers = ahead or ({}, {}, {}, {}) + causes, usages, results, alternates, surplus)` of them (issue 613). It stays in the scratch for `seal`, and is never uploaded but sealed (issue 612).""" + # Each case asked first with its marks, at an alternate place alone or drawing gating Findings away (Muse's read of + # #646), as the set's are listed. + asked, told, spent, answers, beside, away = ahead or ({}, {}, {}, {}, set(), {}) first = [dict(case, **({"result": answers[case["id"]]} if case["id"] in answers else {})) for case in report(kept, dict(found, cases=kept.get("preflight", [])), asked, causes=told, - usages=spent)["cases"]] + alternates=beside, surplus=away, usages=spent)["cases"]] natural = {e["id"]: e.get("natural") is True for e in found["cases"]} cases = [dict(case, natural=natural[case["id"]], **({"result": results[case["id"]]} if case["id"] in results else {})) for case in report(kept, found, assessed, causes=causes, alternates=alternates, surplus=surplus, @@ -1003,19 +1005,19 @@ def gate_scored(kept: dict, found: dict, place: pathlib.Path, review, causes: di """`(assessed, report)` of a gate set's run: each case, with its Result, written to the ledger in the scratch, and the report of counts alone that binds it (issue 612). The cases its preflight names are asked first, scored apart; where `preflight_stops`, none of the set's own is asked, and each is `unanswered` (issue 613).""" - results, first, spent, answers = {}, dict(found, cases=kept["preflight"]), {}, {} + results, first, spent, answers, beside, away = {}, dict(found, cases=kept["preflight"]), {}, {}, set(), {} # What a preflight case's read raised, as `score` reads the set's own, so that the ledger says why it was not read. try: told = _read(place / PREFLIGHT, "fetch-causes.json") except Stop: told = {} told = dict(told) if isinstance(told, dict) else {} - ahead = assess(first, _read(place / PREFLIGHT, "fetched.json"), place / PREFLIGHT, review, told, usages=spent, - results=answers) + ahead = assess(first, _read(place / PREFLIGHT, "fetched.json"), place / PREFLIGHT, review, told, beside, away, + usages=spent, results=answers) why = {e["id"]: "unanswered" for e in found["cases"]} if preflight_stops(ahead) else _read(place, "fetched.json") assessed = assess(found, why, place, review, causes, alternates, surplus, usages, results) ledger = (json.dumps(ledger_of(kept, found, assessed, causes, alternates, results, surplus, usages, - (ahead, told, spent, answers)), indent=2, sort_keys=True) + "\n").encode("utf-8") + (ahead, told, spent, answers, beside, away)), indent=2, sort_keys=True) + "\n").encode("utf-8") with open(os.open(within(place, LEDGER), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as out: out.write(ledger) return assessed, aggregate(kept, found, assessed, place, results, ledger, usages, ahead) diff --git a/harness/eval/run.py b/harness/eval/run.py index 7bcda19b..0bda2a8a 100644 --- a/harness/eval/run.py +++ b/harness/eval/run.py @@ -10,8 +10,10 @@ when the time it has left is less than a case's tier allows, so that the job ends before its own timeout: a case not reached is unanswered, and the run fails. -Nothing of a case, a Task or a Result is printed or written: each Result is scored in this step's memory and never -kept, and only `eval-report.json`, of hashes, verdicts, codes and causes, is written in the workspace, by `score`. +Nothing of a case, a Task or a Result is printed: each Result is scored in this step's memory, and only the report, +`eval-report.json`, of hashes, verdicts, codes and causes, or of counts alone for a gate set, is written in the +workspace, by `score`. A gate set's Results are kept whole in the ledger, in the scratch alone, for `seal`, which +uploads nothing of them but their ciphertext (issue 612). The CLI's call is watched for a case's cause alone: its exit, its subtype, or that it timed out (decision 0131).""" from __future__ import annotations diff --git a/tests/test_evals.py b/tests/test_evals.py index 0a0c69a4..38897a08 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -2730,6 +2730,55 @@ def test_a_gate_set_marks_no_group_of_natural_cases_or_others_too_small_to_hide_ self.set(three, gate=True, natural=natural) self.assertEqual(self.step(job.plan), code, natural) self.clean() + # A group of exactly the least is enough (Muse's read of #628). + four = dict(three, **{"poc-four": three["poc-three"].replace(b"poc-three", b"poc-four")}) + self.set(four, gate=True, natural=tuple(four)[:2]) + self.assertEqual(self.step(job.plan), "planned 4 cases after 3 to preflight") + self.clean() + + def test_a_preflight_case_keeps_its_marks_in_the_ledger_as_the_sets_do(self): # Muse's read of #646 + # A case asked first is listed in the ledger as the set's are, but for whether it is natural: passed at an + # alternate place alone, or drawing gating Findings away from every place, it says so (decision 0170). + from harness import eval as job + self.set(self.cases, gate=True, natural=[]) + found = job.manifest(self.root, "pr-review-gate")[0] + kept = {"release": "factory/v0.1.0", "commit": "a" * 40, "set": {"count": 2, "sha256": "0" * 64}, + "preflight": job.preflight_of(self.root, found)} + first = [e["id"] for e in kept["preflight"]] + asked = {i: ("pass", []) for i in first} + ledger = job.ledger_of(kept, dict(found, cases=[]), {}, {}, set(), {}, + ahead=(asked, {}, {}, {}, {first[0]}, {first[1]: 2})) + marks = {c["id"]: (c.get("at_alternate"), c.get("surplus_gating")) for c in ledger["preflight"]} + self.assertEqual(marks, {first[0]: (True, None), first[1]: (None, 2), first[2]: (None, None)}) + + def test_a_gate_run_carries_its_preflights_marks_into_the_ledger(self): # the Claude read of the follow-on to #646 + # The run step asks the preflight with containers for its marks, and hands them to the ledger: a preflight case + # the scorer marks at an alternate place, or as drawing gating Findings away, is listed so. + from harness import eval as job + self.set(self.cases, gate=True, natural=[]) + self.step(job.plan) + self.step(the_fetch(), self.home(self.cases, gate=True)) + real, first = job.assess, sorted(self.pre) + + def marking(found, why, place, review, causes=None, alternates=None, surplus=None, usages=None, results=None): + said = real(found, why, place, review, causes, alternates, surplus, usages, results) + if alternates is not None and [e["id"] for e in found["cases"]] == list(self.pre): + alternates.add(first[0]) + surplus[first[1]] = 2 + return said + with mock.patch.object(job, "assess", marking): + self.step(job.score, lambda case: example_result()) + ledger = json.loads((self.temp / "heldout" / job.LEDGER).read_text()) + marks = {c["id"]: (c.get("at_alternate"), c.get("surplus_gating")) for c in ledger["preflight"]} + self.assertEqual(marks, {first[0]: (True, None), first[1]: (None, 2), first[2]: (None, None)}) + + def test_a_gate_sets_faults_are_the_codes_of_cases_not_read_alone(self): # Muse's read of #628 + # A case the reviewer failed is counted under its kind, never as a fault: only a case not read says its code, + # by how many (decision 0164). + from harness import eval as job + assessed = {"a": ("fail", ["finding_not_raised"]), "b": ("unreadable", ["not_found"]), "c": ("pass", []), + "d": ("unreadable", ["not_found", "too_large"])} + self.assertEqual(job.faults_of(assessed), [{"code": "not_found", "count": 2}, {"code": "too_large", "count": 1}]) def gate_run(self, natural=None): """A gate set of the two cases, planned, fetched and scored as passed: what `score` left in the scratch.""" @@ -3173,6 +3222,12 @@ def test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them(self self.assertEqual([p for p in check_examples.run(live) if p.startswith("evals/gate/")], ["evals/gate/cases.json: its preflight is refused where it is read: eval_set_unreadable"]) (live / "evals" / "pr-review" / "moved.json").rename(live / "evals" / "pr-review" / "cases.json") + # A gate set the schema lets leave out its preflight, or its bars, is held as the job reads it, never crashes. + for left in ("preflight", "bars"): + (live / "evals" / "gate" / "cases.json").write_text(json.dumps({k: v for k, v in found.items() if k != left})) + self.assertEqual([p for p in check_examples.run(live) if p.startswith("evals/gate/")], + ["evals/gate/cases.json: is refused where it is read: eval_set_unreadable"], left) + (live / "evals" / "gate" / "cases.json").write_text(json.dumps(found)) # And an example of one is held as the job reads it. (live / "examples").mkdir() (live / "examples" / "eval-set.gate.json").write_text(json.dumps(dict(found, kind_least=-1, bars={ @@ -3192,6 +3247,10 @@ def test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them(self ["its record does not say %r" % job.bars_said(dict(found, kind_least=21))]) self.assertEqual(check_examples._bars_stated(self.root, {k: v for k, v in found.items() if k != "record"}), ["names no record that states its bars"]) + # A gate set the schema lets leave out its bars or its least is a problem named, never a crash (Muse's read of #642). + for left in ("bars", "kind_least"): + self.assertEqual(check_examples._bars_stated(self.root, {k: v for k, v in found.items() if k != left}), + ["gives no bars and least of a kind that can be stated"], left) def test_the_gate_pools_every_dispatch_on_the_commit_and_one_faulted_closes_it(self): # issue 613 from harness import eval as job diff --git a/tests/test_release.py b/tests/test_release.py index 456a3da4..0e8bf111 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -19,7 +19,7 @@ from adapters import github, tags from harness import eval as eval_gate from release import current, cut, move -from tools import check_release +from tools import check_release, check_rulesets ROOT = pathlib.Path(__file__).resolve().parent.parent SKILL = ROOT / ".claude" / "skills" / "cut-a-release" / "SKILL.md" @@ -361,30 +361,39 @@ def test_the_first_cut_moved_next_and_its_release_is_neither_moved_nor_cut_again with self.subTest(tag=tag): ref = self.answered(made, "/repos/%s/git/ref/tags/%s" % (REPO, tag)) self.assertEqual((ref["status"], ref["body"]["object"]["sha"]), (200, commit)) - deployed = self.answered(made, "/repos/%s/deployments?environment=factory-next" % REPO)["body"] + # Each read is GitHub's answer, 200 (the Claude read of the follow-on to #627). + body = lambda item, path: (lambda answer: self.assertEqual(answer["status"], 200, path) or answer["body"])( + self.answered(item, path)) + deployed = body(made, "/repos/%s/deployments?environment=factory-next" % REPO) self.assertEqual([(d["id"], d["sha"], d["description"]) for d in deployed], [(made["printed"]["deployments"][0], commit, fixture["version"])]) - statuses = self.answered(made, "/repos/%s/deployments/%d/statuses" % (REPO, deployed[0]["id"]))["body"] + statuses = body(made, "/repos/%s/deployments/%d/statuses" % (REPO, deployed[0]["id"])) self.assertEqual([s["state"] for s in statuses], ["success"]) again = self.item(fixture, "refused_move_of_the_release") self.assertEqual((again["conclusion"], again["printed"]), ("failure", {"writes": [], "deployments": [], "refused": cut.ALREADY_RELEASED})) - self.assertEqual(len(self.answered(again, "/repos/%s/deployments?environment=factory-next" % REPO)["body"]), 1) + self.assertEqual(len(body(again, "/repos/%s/deployments?environment=factory-next" % REPO)), 1) calls = [e["request"]["call"].split(" ")[:2] for e in again["exchanges"]] forcing = again["exchanges"][calls.index(["PATCH", "/repos/%s/git/refs/tags/%s" % (REPO, fixture["version"])])] asked, forced = forcing["request"]["body"], forcing["response"] self.assertTrue(asked["force"] is True and asked["sha"] != commit, asked) self.assertEqual((forced["status"], forced["body"]["message"].split("\n")[0]), (422, "Repository rule violations found")) + # Made by the Maintainer, whose admin role the ruleset gives no bypass, and refused as a protected ref. + maintainer = json.loads((ROOT / "config" / "hooks.json").read_text())["maintainer"]["login"] + self.assertEqual(forcing["request"]["as"], {"kind": "person", "name": maintainer}) + self.assertIn("the admin role", forcing["request"]["call"]) + self.assertIn("Cannot update this protected ref.", forced["body"]["message"].split("\n")) # The exchanges are kept in the order they were made: the reads after the PATCH were made after it. made_at = [email.utils.parsedate_to_datetime(e["response"]["headers"]["date"]) for e in again["exchanges"]] self.assertEqual(made_at, sorted(made_at)) read = "/repos/%s/git/ref/tags/%s" % (REPO, fixture["version"]) self.assertLess(again["exchanges"].index(forcing), calls.index(["GET", read])) - self.assertEqual(self.answered(again, read)["body"]["object"]["sha"], commit) - rules = self.answered(again, "/repos/%s/rulesets/25996980" % REPO)["body"] + self.assertEqual(body(again, read)["object"]["sha"], commit) + rules = body(again, "/repos/%s/rulesets/25996980" % REPO) self.assertEqual((rules["target"], rules["enforcement"], rules["bypass_actors"]), ("tag", "active", [])) - self.assertLessEqual({"update", "deletion"}, {rule["type"] for rule in rules["rules"]}) + # The rules the repo holds this ruleset to, force-pushes among them (tools/check_rulesets.py). + self.assertLessEqual(check_rulesets.TAG_RULESETS["factory releases are immutable"], {rule["type"] for rule in rules["rules"]}) self.assertIn("refs/tags/factory/v*.*.*", rules["conditions"]["ref_name"]["include"]) self.assertEqual(rules["conditions"]["ref_name"]["exclude"], []) @@ -408,16 +417,21 @@ def test_a_rollback_with_no_earlier_release_was_refused_and_moved_nothing(self): self.assertEqual(refs, {fixture["version"]: commit, channel: commit}) for tag in (fixture["version"], channel): with self.subTest(tag=tag): - ref = self.answered(refused, "/repos/%s/git/ref/tags/%s" % (REPO, tag))["body"] - self.assertEqual(ref["object"]["sha"], commit) + ref = self.answered(refused, "/repos/%s/git/ref/tags/%s" % (REPO, tag)) + self.assertEqual((ref["status"], ref["body"]["object"]["sha"]), (200, commit)) step = move.asked({"RELEASE_" + k.upper(): v for k, v in refused["asked"].items()}, channel) self.assertEqual(cut.plan(step, refs, {}, move.channels(rings)), ([], cut.NO_EARLIER_RELEASE)) run = self.answered(refused, "/repos/%s/actions/runs/%d" % (REPO, refused["run"]))["body"] made = self.item(json.loads(self.CUT.read_text()), "cut_and_move_of_next") - deployed = self.answered(refused, "/repos/%s/deployments?environment=factory-next" % REPO)["body"] + # Each read is GitHub's answer, 200: a read it refused would show nothing of what the run left. + listing = self.answered(refused, "/repos/%s/deployments?environment=factory-next" % REPO) + self.assertEqual(listing["status"], 200) + deployed = listing["body"] self.assertEqual([(d["id"], d["sha"], d["description"]) for d in deployed], [(made["printed"]["deployments"][0], commit, fixture["version"])]) - statuses = self.answered(refused, "/repos/%s/deployments/%d/statuses" % (REPO, deployed[0]["id"]))["body"] + answered = self.answered(refused, "/repos/%s/deployments/%d/statuses" % (REPO, deployed[0]["id"])) + self.assertEqual(answered["status"], 200) + statuses = answered["body"] self.assertEqual([s["state"] for s in statuses], ["success"]) for made_at in [deployed[0]["created_at"]] + [s["created_at"] for s in statuses]: self.assertLess(made_at, run["created_at"]) diff --git a/tests/test_release_step.py b/tests/test_release_step.py index 625f7677..3a5e2f74 100644 --- a/tests/test_release_step.py +++ b/tests/test_release_step.py @@ -1,7 +1,7 @@ """The release workflow's step (#197's rulings 4′, 4″ and 5′): the commit's facts read by `adapters/candidate.py`, the tags written by `adapters/tags.py`, the Deployment by `adapters/deployments.py`, and `release.move step`, which sends a plan's writes and reads each back. No test writes to GitHub: every answer is a replay, in the shapes GitHub's REST -reference gives, since no release step has run yet; the first cut's fixture is a later pull request's.""" +reference gives; what the release step's runs answered is in `release/fixtures/`, which `tests/test_release.py` reads.""" from __future__ import annotations import base64 @@ -398,6 +398,19 @@ def judged(reports, runs=None, head=None): runs[2].update(change) self.assertEqual(judged([counts(), counts(), counts()], runs)[0], "eval_not_run") self.assertEqual(judged([counts()] * 3, runs=[])[0], "eval_pool_short") + # A list not read whole, or not answered 200, leaves the commit unjudged; a dispatch listed three times is one + # dispatch's evidence, never a pool (Muse's read of #642). + runs = [dict(run_of(id=100 + i, run_number=i), status="completed", run_attempt=1) for i in range(3)] + listing = "/repos/%s/actions/workflows/eval.yml/runs?head_sha=%s&event=workflow_dispatch&per_page=100" % (REPO, D) + # Each with the reports a pool would read after it, so that a list read as whole would pass. + for n, listed in enumerate((ex("GET", listing, 200, {"total_count": 4, "workflow_runs": runs}), + ex("GET", listing, 500, {"total_count": 3, "workflow_runs": runs}), + dispatches([runs[0]] * 3))): + ran = [100] * 3 if n == 2 else [100, 101, 102] + reads = [named, gate_manifest(), listed] + [r for i, run in enumerate(ran) + for r in report_download(counts(), ident=500 + i, run=run)] + with self.subTest(n=n): + self.assertEqual(candidate.gate(github.Replay(reads), REPO, D, VERSION), "eval_not_run") # The set named must be a gate set, by a name of one part; a configuration not read names nothing. development = (json.dumps({k: v for k, v in gate_set.items() if k not in eval_gate.OWN + ("gate",)}, indent=2) + "\n").encode() diff --git a/tools/check_examples.py b/tools/check_examples.py index b3c77e53..4342162d 100644 --- a/tools/check_examples.py +++ b/tools/check_examples.py @@ -274,8 +274,12 @@ def _bars_stated(root: pathlib.Path, found: dict) -> list: path = root / found.get("record", "") if "record" not in found or not path.is_file(): return ["names no record that states its bars"] - if bars_said(found) not in path.read_text(encoding="utf-8"): - return ["its record does not say %r" % bars_said(found)] + try: + said = bars_said(found) + except (KeyError, TypeError, ValueError, IndexError): # bars or a least the schema leaves out (Muse's read of #642) + return ["gives no bars and least of a kind that can be stated"] + if said not in path.read_text(encoding="utf-8"): + return ["its record does not say %r" % said] return [] @@ -370,7 +374,9 @@ def load(path: pathlib.Path): if kind == "feed" and not errors and "decision" in schemas: errors = _table(value, schemas["decision"]) if kind == "eval-set" and not errors and value.get("gate") is True: - errors = _bars_stated(root, value) + _preflight_held(root, value) + # Held first as the job reads it, so that a gate set the schema lets leave out what it gives of its own is + # a problem named, never a crash further on (the Claude read of the follow-on to #642). + errors = _held(value) or _bars_stated(root, value) + _preflight_held(root, value) for err in errors: problems.append("%s: %s" % (rel, err)) for folder in ("config", "lifecycle", "evals"): diff --git a/tools/mutants.py b/tools/mutants.py index 2e555c39..8fa34335 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -4076,6 +4076,47 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: '13:07:04 GMT', '23:07:04 GMT', CUT_FIXTURE), ('the tag ruleset exempts the Release', 'release/fixtures/cut.v0-1-0.json', '"exclude": []', '"exclude": ["refs/tags/factory/v0.1.0"]', CUT_FIXTURE), + # Muse's read of #646: a preflight case's marks in the ledger, and the ledger's preflight required. + ('the run step hands the ledger no preflight mark', 'harness/eval/__init__.py', + '(ahead, told, spent, answers, beside, away)), indent=2', + '(ahead, told, spent, answers, set(), {})), indent=2', ('tests.test_evals.EvalJob.test_a_gate_run_carries_its_preflights_marks_into_the_ledger',)), + ('the preflight is asked with no containers for its marks', 'harness/eval/__init__.py', + 'review, told, beside, away,\n usages=spent', + 'review, told,\n usages=spent', ('tests.test_evals.EvalJob.test_a_gate_run_carries_its_preflights_marks_into_the_ledger',)), + ('a preflight case keeps no mark in the ledger', 'harness/eval/__init__.py', + ' alternates=beside, surplus=away, usages=spent)["cases"]]', + ' usages=spent)["cases"]]', ('tests.test_evals.EvalJob.test_a_preflight_case_keeps_its_marks_in_the_ledger_as_the_sets_do',)), + ('a ledger need hold no preflight', 'schemas/eval-ledger.schema.json', + ' "cases",\n "preflight"\n', + ' "cases"\n', ('tests.test_evals.TheSet.test_a_case_dropped_under_evals_in_any_file_is_caught',)), + # Muse's read of #642: the pool's list read whole, each dispatch once, and a gate set with no bars named, not raised. + ('the pool reads a list not read whole', 'adapters/candidate.py', + ' if not isinstance(runs, list) or body.get("total_count") != len(runs):\n return None\n ours', + ' if not isinstance(runs, list):\n return None\n ours', ('tests.test_release_step.Candidate.test_a_gate_set_is_judged_by_every_dispatch_on_the_commit',)), + ('the pool counts a dispatch listed twice', 'adapters/candidate.py', + ' or len({r["id"] for r in ours}) != len(ours):', + ':', ('tests.test_release_step.Candidate.test_a_gate_set_is_judged_by_every_dispatch_on_the_commit',)), + ('a gate set with no bars crashes the check', 'tools/check_examples.py', + ' except (KeyError, TypeError, ValueError, IndexError): # bars or a least', + ' except ZeroDivisionError: # bars or a least', ('tests.test_evals.EvalJob.test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them',)), + # Muse's read of #628: a group of exactly the least, and a gate set's faults of cases not read alone. + ('a group of exactly the least is refused', 'harness/eval/__init__.py', + 'any(0 < group < ORIGIN_LEAST for group', + 'any(0 < group <= ORIGIN_LEAST for group', ('tests.test_evals.EvalJob.test_a_gate_set_marks_no_group_of_natural_cases_or_others_too_small_to_hide_one',)), + ('a case the reviewer failed is counted a fault', 'harness/eval/__init__.py', + 'for verdict_of, reasons in assessed.values() if verdict_of == "unreadable" for code in reasons]', + 'for verdict_of, reasons in assessed.values() for code in reasons]', ('tests.test_evals.EvalJob.test_a_gate_sets_faults_are_the_codes_of_cases_not_read_alone',)), + # Muse's read of #627: the refused rollback's evidence is read only where GitHub answered it. + ('the refused rollback\'s deployments are read from a refused answer', 'release/fixtures/rollback.refused.json', + 'deployments?environment=factory-next (gh api --include)",\n "headers": {\n "accept": "application/vnd.github+json"\n },\n "body": null\n },\n "response": {\n "status": 200,', + 'deployments?environment=factory-next (gh api --include)",\n "headers": {\n "accept": "application/vnd.github+json"\n },\n "body": null\n },\n "response": {\n "status": 500,', ('tests.test_release.Fixture.test_a_rollback_with_no_earlier_release_was_refused_and_moved_nothing',)), + # Muse's read of #617: the PATCH is the Maintainer's admin role's, refused as a protected ref, and the ruleset holds + # the rules tools/check_rulesets.py holds it to. + ('the refused PATCH is no admin role\'s', 'release/fixtures/cut.v0-1-0.json', ', the admin role)', ')', CUT_FIXTURE), + ('the PATCH is refused for another reason', 'release/fixtures/cut.v0-1-0.json', + 'Cannot update this protected ref.', 'Cannot do that.', CUT_FIXTURE), + ('the tag ruleset need not refuse a force-push', 'release/fixtures/cut.v0-1-0.json', + '"type": "non_fast_forward"', '"type": "creation"', CUT_FIXTURE), # The refused rollback's fixture: the rules refuse it for every tag read after it, and it moved and recorded nothing. ('a rollback with no earlier Release moved next', 'release/fixtures/rollback.refused.json', '"ref": "refs/tags/factory/v0-next",\n "object": {\n "sha": "499835d5', @@ -9195,7 +9236,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if kind == "eval-set" and not errors and value.get("gate") is True:\n', ' if False:\n', ('tests.test_evals.EvalJob.test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them',)), ('a record that does not say the bars is enough', 'tools/check_examples.py', - ' if bars_said(found) not in path.read_text(encoding="utf-8"):\n', + ' if said not in path.read_text(encoding="utf-8"):\n', ' if False:\n', ('tests.test_evals.EvalJob.test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them',)), ('a gate set named is judged by its newest successful run', 'adapters/candidate.py', ' if name:\n return pooled_gate(', @@ -9298,8 +9339,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' or kept.get("preflight") != preflight_of(root, found):\n', ':\n', ('tests.test_evals.EvalJob.test_a_gate_sets_preflight_is_of_the_development_set_and_no_gate_sets_case',)), ("the check holds no live gate set's preflight", 'tools/check_examples.py', - '_bars_stated(root, value) + _preflight_held(root, value)', - '_bars_stated(root, value)', ('tests.test_evals.EvalJob.test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them',)), + '_held(value) or _bars_stated(root, value) + _preflight_held(root, value)', + '_held(value) or _bars_stated(root, value)', ('tests.test_evals.EvalJob.test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them',)), ('the preflight is never fetched', 'harness/eval/__init__.py', ' if kept["preflight"]:\n ahead = _fetched(', ' if False:\n ahead = _fetched(', ('tests.test_evals.EvalJob.test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it',)),