What Muse's owed reads of #617, #627, #628, #642 and #646 found

adapters/candidate.py

@@ -163,8 +163,8 @@ def _run(transport, repo: str, commit: str):
 
 def _dispatches(transport, repo: str, commit: str):
     """Every dispatch of the eval job on `commit`, by its GitHub id, conclusion and attempt, oldest first, whatever it
-    concluded; None where the list is not read whole, holds a run of another commit, event or workflow, or holds a run
-    still going (issue 613). The list shows a run's latest attempt alone (P-27, decision 0111)."""
+    concluded; None where the list is not read whole, holds a run of another commit, event or workflow, holds a run
+    still going, or names one twice (issue 613). The list shows a run's latest attempt alone (P-27, decision 0111)."""
     try:
         answer = transport.send(request("GET", "/repos/%s/actions/workflows/%s/runs?head_sha=%s&event=workflow_dispatch"
                                                "&per_page=100" % (repo, EVAL_WORKFLOW, commit)))
@@ -177,7 +177,8 @@ def _dispatches(transport, repo: str, commit: str):
     ours = [r for r in runs if isinstance(r, dict) and r.get("head_sha") == commit and r.get("status") == "completed"
             and r.get("event") == "workflow_dispatch" and r.get("path") == ".github/workflows/%s" % EVAL_WORKFLOW
             and type(r.get("id")) is int and r["id"] > 0 and type(r.get("run_number")) is int]
-    if len(ours) != len(runs):
+    # Each dispatch once, by its id: one listed twice is one dispatch's evidence, never two (Muse's read of #642).
+    if len(ours) != len(runs) or len({r["id"] for r in ours}) != len(ours):
         return None
     return [(r["id"], r.get("conclusion"), r.get("run_attempt")) for r in sorted(ours, key=lambda r: r["run_number"])]
 

examples/invalid/eval-ledger.no-preflight.json

@@ -0,0 +1,150 @@
+{
+  "schema": "eval-ledger/1",
+  "salt": "8f14e45fceea167a5a36dedd4bea2543",
+  "release": "factory/v0.1.0",
+  "commit": "9e927b4000000000000000000000000000000000",
+  "role": "pr-review",
+  "intent": "review-change",
+  "set": {
+    "count": 2,
+    "sha256": "05b3abf2579a5eb66403cd78be557fd860633a1fe2103c7642030defe32c657f"
+  },
+  "cases": [
+    {
+      "id": "poc-0001",
+      "sha256": "754be4b69c49114336398cc52faaf3b06b709a5674ac584fcc9433cede863d5a",
+      "verdict": "pass",
+      "reasons": [],
+      "usage": {
+        "turns": 4,
+        "cost_usd": 0.21,
+        "seconds": 38.5
+      },
+      "natural": true,
+      "result": {
+        "schema": "result/1",
+        "run": "r_01JA6Z",
+        "release": "factory/v0.3.1",
+        "status": "completed",
+        "reason": null,
+        "dispositions": [],
+        "findings": [
+          {
+            "ref": "n1",
+            "location": {
+              "path": "kernel/decide.py",
+              "start_line": 88,
+              "end_line": 94
+            },
+            "category": "functional",
+            "severity": "major",
+            "criterion": "PR-3",
+            "adopted_from": {
+              "lens": "coderabbit",
+              "surface": {
+                "type": "thread",
+                "key": "PRRT_kw1"
+              }
+            }
+          },
+          {
+            "ref": "n2",
+            "location": {
+              "path": "tests/test_decide.py",
+              "start_line": 410,
+              "end_line": 410
+            },
+            "category": "delivery",
+            "severity": "minor",
+            "criterion": null
+          }
+        ],
+        "effects": {
+          "submit_review": [
+            {
+              "id": "e1",
+              "comments": [
+                {
+                  "finding": "n1",
+                  "body": "parse() reads the first item before it checks that the list has one; return early on an empty list."
+                }
+              ],
+              "body": null
+            }
+          ],
+          "create_check_run": [
+            {
+              "id": "e2",
+              "annotations": [
+                {
+                  "finding": "n2",
+                  "body": "The new test asserts that no error was raised, not the Decision."
+                }
+              ]
+            }
+          ]
+        },
+        "coverage": [
+          {
+            "criterion": "PR-1",
+            "verdict": "met",
+            "reason": null
+          },
+          {
+            "criterion": "PR-2",
+            "verdict": "met",
+            "reason": null
+          },
+          {
+            "criterion": "PR-3",
+            "verdict": "finding",
+            "reason": null
+          },
+          {
+            "criterion": "PR-4",
+            "verdict": "met",
+            "reason": null
+          },
+          {
+            "criterion": "PR-5",
+            "verdict": "not_applicable",
+            "reason": "no_rule_added"
+          }
+        ],
+        "usage": {
+          "engine": "claude-cli",
+          "engine_version": "2.1.302",
+          "model": "claude-opus",
+          "answering_model": "claude-opus",
+          "effort": "medium",
+          "session": "7f3a9c2e-5b1d-4e8a-9c6f-2d4b8a1e0f37",
+          "input_tokens": 2,
+          "cache_creation_input_tokens": 20475,
+          "cache_read_input_tokens": 0,
+          "output_tokens": 194,
+          "turns": 2,
+          "cost_usd": 0.167688,
+          "seconds": 2.714
+        }
+      }
+    },
+    {
+      "id": "poc-0002",
+      "sha256": "1b657e2ca5226c6d61636cf2d155df3aeef3cd95422f9ae09924410f72ed07d4",
+      "verdict": "unreadable",
+      "reasons": [
+        "engine_timeout"
+      ],
+      "usage": {
+        "turns": null,
+        "cost_usd": null,
+        "seconds": 0
+      },
+      "natural": false,
+      "cause": {
+        "subtype": "error_max_turns",
+        "exit": 1
+      }
+    }
+  ]
+}

harness/eval/__init__.py

@@ -740,11 +740,13 @@ def ledger_of(kept: dict, found: dict, assessed: dict, causes: dict, alternates:
     """The `eval-ledger/1` of a gate set's run: each case as `report` would list it, whether it is natural, and the Result
     the review returned for it, under a salt drawn for this run alone, so that its hash in the report tells nothing of
     what it holds; and each case asked first the same way, but for whether it is natural, from `ahead`, `(assessed,
-    causes, usages, results)` of them (issue 613). It stays in the scratch for `seal`, and is never uploaded but sealed (issue 612)."""
-    asked, told, spent, answers = ahead or ({}, {}, {}, {})
+    causes, usages, results, alternates, surplus)` of them (issue 613). It stays in the scratch for `seal`, and is never uploaded but sealed (issue 612)."""
+    # Each case asked first with its marks, at an alternate place alone or drawing gating Findings away (Muse's read of
+    # #646), as the set's are listed.
+    asked, told, spent, answers, beside, away = ahead or ({}, {}, {}, {}, set(), {})
     first = [dict(case, **({"result": answers[case["id"]]} if case["id"] in answers else {}))
              for case in report(kept, dict(found, cases=kept.get("preflight", [])), asked, causes=told,
-                                usages=spent)["cases"]]
+                                alternates=beside, surplus=away, usages=spent)["cases"]]
     natural = {e["id"]: e.get("natural") is True for e in found["cases"]}
     cases = [dict(case, natural=natural[case["id"]], **({"result": results[case["id"]]} if case["id"] in results else {}))
              for case in report(kept, found, assessed, causes=causes, alternates=alternates, surplus=surplus,
@@ -1003,19 +1005,19 @@ def gate_scored(kept: dict, found: dict, place: pathlib.Path, review, causes: di
     """`(assessed, report)` of a gate set's run: each case, with its Result, written to the ledger in the scratch, and the
     report of counts alone that binds it (issue 612). The cases its preflight names are asked first, scored apart; where
     `preflight_stops`, none of the set's own is asked, and each is `unanswered` (issue 613)."""
-    results, first, spent, answers = {}, dict(found, cases=kept["preflight"]), {}, {}
+    results, first, spent, answers, beside, away = {}, dict(found, cases=kept["preflight"]), {}, {}, set(), {}
     # What a preflight case's read raised, as `score` reads the set's own, so that the ledger says why it was not read.
     try:
         told = _read(place / PREFLIGHT, "fetch-causes.json")
     except Stop:
         told = {}
     told = dict(told) if isinstance(told, dict) else {}
-    ahead = assess(first, _read(place / PREFLIGHT, "fetched.json"), place / PREFLIGHT, review, told, usages=spent,
-                   results=answers)
+    ahead = assess(first, _read(place / PREFLIGHT, "fetched.json"), place / PREFLIGHT, review, told, beside, away,
+                   usages=spent, results=answers)
     why = {e["id"]: "unanswered" for e in found["cases"]} if preflight_stops(ahead) else _read(place, "fetched.json")
     assessed = assess(found, why, place, review, causes, alternates, surplus, usages, results)
     ledger = (json.dumps(ledger_of(kept, found, assessed, causes, alternates, results, surplus, usages,
-                                   (ahead, told, spent, answers)), indent=2, sort_keys=True) + "\n").encode("utf-8")
+                                   (ahead, told, spent, answers, beside, away)), indent=2, sort_keys=True) + "\n").encode("utf-8")
     with open(os.open(within(place, LEDGER), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as out:
         out.write(ledger)
     return assessed, aggregate(kept, found, assessed, place, results, ledger, usages, ahead)

harness/eval/run.py

@@ -10,8 +10,10 @@
 when the time it has left is less than a case's tier allows, so that the job ends before its own timeout: a case not
 reached is unanswered, and the run fails.
 
-Nothing of a case, a Task or a Result is printed or written: each Result is scored in this step's memory and never
-kept, and only `eval-report.json`, of hashes, verdicts, codes and causes, is written in the workspace, by `score`.
+Nothing of a case, a Task or a Result is printed: each Result is scored in this step's memory, and only the report,
+`eval-report.json`, of hashes, verdicts, codes and causes, or of counts alone for a gate set, is written in the
+workspace, by `score`. A gate set's Results are kept whole in the ledger, in the scratch alone, for `seal`, which
+uploads nothing of them but their ciphertext (issue 612).
 The CLI's call is watched for a case's cause alone: its exit, its subtype, or that it timed out (decision 0131)."""
 from __future__ import annotations
 

tests/test_evals.py

@@ -2730,6 +2730,55 @@ def test_a_gate_set_marks_no_group_of_natural_cases_or_others_too_small_to_hide_
                 self.set(three, gate=True, natural=natural)
                 self.assertEqual(self.step(job.plan), code, natural)
                 self.clean()
+            # A group of exactly the least is enough (Muse's read of #628).
+            four = dict(three, **{"poc-four": three["poc-three"].replace(b"poc-three", b"poc-four")})
+            self.set(four, gate=True, natural=tuple(four)[:2])
+            self.assertEqual(self.step(job.plan), "planned 4 cases after 3 to preflight")
+            self.clean()
+
+    def test_a_preflight_case_keeps_its_marks_in_the_ledger_as_the_sets_do(self):  # Muse's read of #646
+        # A case asked first is listed in the ledger as the set's are, but for whether it is natural: passed at an
+        # alternate place alone, or drawing gating Findings away from every place, it says so (decision 0170).
+        from harness import eval as job
+        self.set(self.cases, gate=True, natural=[])
+        found = job.manifest(self.root, "pr-review-gate")[0]
+        kept = {"release": "factory/v0.1.0", "commit": "a" * 40, "set": {"count": 2, "sha256": "0" * 64},
+                "preflight": job.preflight_of(self.root, found)}
+        first = [e["id"] for e in kept["preflight"]]
+        asked = {i: ("pass", []) for i in first}
+        ledger = job.ledger_of(kept, dict(found, cases=[]), {}, {}, set(), {},
+                               ahead=(asked, {}, {}, {}, {first[0]}, {first[1]: 2}))
+        marks = {c["id"]: (c.get("at_alternate"), c.get("surplus_gating")) for c in ledger["preflight"]}
+        self.assertEqual(marks, {first[0]: (True, None), first[1]: (None, 2), first[2]: (None, None)})
+
+    def test_a_gate_run_carries_its_preflights_marks_into_the_ledger(self):  # the Claude read of the follow-on to #646
+        # The run step asks the preflight with containers for its marks, and hands them to the ledger: a preflight case
+        # the scorer marks at an alternate place, or as drawing gating Findings away, is listed so.
+        from harness import eval as job
+        self.set(self.cases, gate=True, natural=[])
+        self.step(job.plan)
+        self.step(the_fetch(), self.home(self.cases, gate=True))
+        real, first = job.assess, sorted(self.pre)
+
+        def marking(found, why, place, review, causes=None, alternates=None, surplus=None, usages=None, results=None):
+            said = real(found, why, place, review, causes, alternates, surplus, usages, results)
+            if alternates is not None and [e["id"] for e in found["cases"]] == list(self.pre):
+                alternates.add(first[0])
+                surplus[first[1]] = 2
+            return said
+        with mock.patch.object(job, "assess", marking):
+            self.step(job.score, lambda case: example_result())
+        ledger = json.loads((self.temp / "heldout" / job.LEDGER).read_text())
+        marks = {c["id"]: (c.get("at_alternate"), c.get("surplus_gating")) for c in ledger["preflight"]}
+        self.assertEqual(marks, {first[0]: (True, None), first[1]: (None, 2), first[2]: (None, None)})
+
+    def test_a_gate_sets_faults_are_the_codes_of_cases_not_read_alone(self):  # Muse's read of #628
+        # A case the reviewer failed is counted under its kind, never as a fault: only a case not read says its code,
+        # by how many (decision 0164).
+        from harness import eval as job
+        assessed = {"a": ("fail", ["finding_not_raised"]), "b": ("unreadable", ["not_found"]), "c": ("pass", []),
+                    "d": ("unreadable", ["not_found", "too_large"])}
+        self.assertEqual(job.faults_of(assessed), [{"code": "not_found", "count": 2}, {"code": "too_large", "count": 1}])
 
     def gate_run(self, natural=None):
         """A gate set of the two cases, planned, fetched and scored as passed: what `score` left in the scratch."""
@@ -3173,6 +3222,12 @@ def test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them(self
             self.assertEqual([p for p in check_examples.run(live) if p.startswith("evals/gate/")],
                              ["evals/gate/cases.json: its preflight is refused where it is read: eval_set_unreadable"])
             (live / "evals" / "pr-review" / "moved.json").rename(live / "evals" / "pr-review" / "cases.json")
+            # A gate set the schema lets leave out its preflight, or its bars, is held as the job reads it, never crashes.
+            for left in ("preflight", "bars"):
+                (live / "evals" / "gate" / "cases.json").write_text(json.dumps({k: v for k, v in found.items() if k != left}))
+                self.assertEqual([p for p in check_examples.run(live) if p.startswith("evals/gate/")],
+                                 ["evals/gate/cases.json: is refused where it is read: eval_set_unreadable"], left)
+            (live / "evals" / "gate" / "cases.json").write_text(json.dumps(found))
             # And an example of one is held as the job reads it.
             (live / "examples").mkdir()
             (live / "examples" / "eval-set.gate.json").write_text(json.dumps(dict(found, kind_least=-1, bars={
@@ -3192,6 +3247,10 @@ def test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them(self
                          ["its record does not say %r" % job.bars_said(dict(found, kind_least=21))])
         self.assertEqual(check_examples._bars_stated(self.root, {k: v for k, v in found.items() if k != "record"}),
                          ["names no record that states its bars"])
+        # A gate set the schema lets leave out its bars or its least is a problem named, never a crash (Muse's read of #642).
+        for left in ("bars", "kind_least"):
+            self.assertEqual(check_examples._bars_stated(self.root, {k: v for k, v in found.items() if k != left}),
+                             ["gives no bars and least of a kind that can be stated"], left)
 
     def test_the_gate_pools_every_dispatch_on_the_commit_and_one_faulted_closes_it(self):  # issue 613
         from harness import eval as job