diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index a74b5fb4..5cea9424 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -1,14 +1,17 @@ # The eval job, `factory / eval` (design §4.9 and §3.14, issue 309; decision 0053): the review Intent's held-out set, run # on the commit it judges, main's own head, for the Release the dispatch names. Dispatched alone, by the Maintainer -# before a cut; the Release's gate reads its report (#197's ruling 4″). Its steps are harness.eval's modules and pinned -# actions, in the order tools/check_workflows.py holds: -# the set planned before any credential is used; the CLI installed at the Engine's pin by its checksum, with the package -# the scrub needs, as root and before anything is here; each case fetched with the Maintainer's read-only token for the -# home, held by that step alone, into a scratch directory outside the workspace; each case given to the review Intent with the model's credential alone, as a live Run is; for a -# gate set, its ledger sealed to the Maintainer's certificates and the ciphertext uploaded; the report, of hashes, -# verdicts and codes alone, or of counts alone for a gate set, uploaded whatever the verdicts; and the run judged by the -# gate's own test. The dispatch names the set, the review Intent's where it names none (issue 612). -# Nothing of a case, a Result or the home reaches the log: each step prints counts and codes. +# before a cut; the Release's gate reads its report (#197's ruling 4″). The dispatch names the set, the review Intent's +# where it names none (issue 612). The run is split into parts, each a job of its own (issue 611): the preflight's job, +# which asks a gate set's preflight and fails where it stops the dispatch, so no shard runs; a matrix of shards behind +# it, each with its own time limit, which asks its share of the set; and the merge job, which holds no credential and +# writes the one report from what each part uploaded, whatever became of them. Each job's steps are harness.eval's +# modules and pinned actions, in the order tools/check_workflows.py holds: the part planned before any credential is +# used; the CLI installed at the Engine's pin by its checksum, with the package the scrub needs, as root and before +# anything is here; each case, and its head, fetched with the Maintainer's read-only token for the home, held by that +# step alone, into a scratch directory outside the workspace; each case given to the review Intent with the model's +# credential alone, as a live Run is; for a gate set, the part's ledger sealed to the Maintainer's certificates and the +# ciphertext uploaded; and what the part found uploaded, counts alone for a gate set. Nothing of a case, a Result or +# the home reaches the log: each step prints counts and codes. name: factory on: workflow_dispatch: @@ -22,14 +25,76 @@ on: default: pr-review permissions: {} jobs: + preflight: + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: read + environment: factory-eval + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: python3 -m harness.eval.plan + timeout-minutes: 1 + env: + RELEASE: ${{ inputs.release }} + EVAL_SET: ${{ inputs.set }} + EVAL_PART: preflight + - env: + CLI_VERSION: "2.1.306" + CLI_CHECKSUM: "520bc6c1f40af466bd03a278b5ae1af1a2938bc7de5b9e5cb15526da20d71cfe" + timeout-minutes: 3 + run: | + url="https://downloads.claude.ai/claude-code-releases/${CLI_VERSION}/linux-x64/claude" + curl -sSfL --proto '=https' --proto-redir '=https' -o /tmp/claude "${url}" + echo "${CLI_CHECKSUM} /tmp/claude" | sha256sum -c + chmod 0755 /tmp/claude + sudo mv /tmp/claude /usr/local/bin/claude + sudo apt-get update -q + sudo apt-get install -y --no-install-recommends bubblewrap + - run: python3 -m harness.eval.fetch + timeout-minutes: 3 + env: + HELDOUT_TOKEN: ${{ secrets.HELDOUT_TOKEN }} + HELDOUT_OWNER: ${{ secrets.HELDOUT_OWNER }} + HELDOUT_REPO: ${{ secrets.HELDOUT_REPO }} + - run: python3 -m harness.eval.run + timeout-minutes: 109 + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CLI: /usr/local/bin/claude + - run: python3 -m harness.eval.seal + timeout-minutes: 1 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-ledger-preflight + path: eval-ledger.cms + if-no-files-found: ignore + retention-days: 90 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-part-preflight + path: eval-part.json + if-no-files-found: error + retention-days: 90 + - run: python3 -m harness.eval.judge + timeout-minutes: 1 eval: + needs: preflight + strategy: + fail-fast: false + matrix: + shard: + - 0 + - 1 + - 2 + - 3 runs-on: ubuntu-24.04 timeout-minutes: 120 permissions: contents: read environment: factory-eval - outputs: - ledger_sha256: ${{ steps.seal.outputs.ledger_sha256 }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -39,6 +104,7 @@ jobs: env: RELEASE: ${{ inputs.release }} EVAL_SET: ${{ inputs.set }} + EVAL_PART: ${{ matrix.shard }} - env: CLI_VERSION: "2.1.306" CLI_CHECKSUM: "520bc6c1f40af466bd03a278b5ae1af1a2938bc7de5b9e5cb15526da20d71cfe" @@ -63,14 +129,47 @@ jobs: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} CLAUDE_CLI: /usr/local/bin/claude - run: python3 -m harness.eval.seal - id: seal timeout-minutes: 1 - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: eval-ledger + name: eval-ledger-${{ matrix.shard }} path: eval-ledger.cms if-no-files-found: ignore retention-days: 90 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-part-${{ matrix.shard }} + path: eval-part.json + if-no-files-found: error + retention-days: 90 + merge: + needs: + - preflight + - eval + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + outputs: + ledger_sha256: ${{ steps.merge.outputs.ledger_sha256 }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: python3 -m harness.eval.plan + timeout-minutes: 1 + env: + RELEASE: ${{ inputs.release }} + EVAL_SET: ${{ inputs.set }} + EVAL_PART: merge + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: eval-part-* + path: parts + - run: python3 -m harness.eval.merge + id: merge + timeout-minutes: 1 - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: eval-report diff --git a/adapters/heldout.py b/adapters/heldout.py index 3e817e62..cd511a89 100644 --- a/adapters/heldout.py +++ b/adapters/heldout.py @@ -79,11 +79,37 @@ def beside(transport, owner: str, name: str, role: str, ident: str, kind: str, s if not (isinstance(role, str) and PART.fullmatch(role) and isinstance(ident, str) and PART.fullmatch(ident) and kind in BESIDE and isinstance(sha256, str) and SHA256.fullmatch(sha256)): return None, "not_this_set" + data, code = _raw(transport, owner, name, "%s/%s.%s" % (role, ident, kind)) + if code is not None: + return None, code + if len(data) > BESIDE[kind]: + return None, "too_large" + if hashlib.sha256(data).hexdigest() != sha256: + return None, "hash_differs" + return data, None + + + +def salt(transport, owner: str, name: str, role: str, set_name: str): + """The salt a gate set's shards are drawn by, `{role}/{set_name}.salt` in the home, read raw with the home's token, + 16 to 256 bytes; or None where there is none to read whole (issue 611). It is no case and pins nothing: it keeps a + shard's cases from being told by their place in the public manifest, so a shard's counts say nothing of which + case is which kind.""" + if not (isinstance(role, str) and PART.fullmatch(role) and isinstance(set_name, str) and PART.fullmatch(set_name)): + return None + data, _ = _raw(transport, owner, name, "%s/%s.salt" % (role, set_name)) + return data if data is not None and 16 <= len(data) <= 256 else None + + +def _raw(transport, owner: str, name: str, path: str) -> tuple: + """`(bytes, None)` of the file at `path` in the home, read whole with the raw media type and the home's token, or + `(None, code)`: `unanswered` for a home that could name another repo or an answer not of that type, and + `not_found`. The raw media type answers a file of more than 1 MiB whole, in one answer and no redirect + (trials/P-62.json).""" if not home(owner, name): return None, "unanswered" try: - answer = transport.send(request("GET", "/repos/%s/%s/contents/%s/%s.%s" % (owner, name, role, ident, kind), - accept=RAW)) + answer = transport.send(request("GET", "/repos/%s/%s/contents/%s" % (owner, name, path), accept=RAW)) except Unanswered: return None, "unanswered" if answer["status"] == 404: @@ -92,10 +118,4 @@ def beside(transport, owner: str, name: str, role: str, ident: str, kind: str, s if answer["status"] != 200 or not isinstance(body, str) \ or not str(answer["headers"].get("content-type", "")).startswith(RAW + ";"): return None, "unanswered" - data = raw(body) - if len(data) > BESIDE[kind]: - return None, "too_large" - if hashlib.sha256(data).hexdigest() != sha256: - return None, "hash_differs" - return data, None - + return raw(body), None diff --git a/config/harness.json b/config/harness.json index d5810d8a..e205f749 100644 --- a/config/harness.json +++ b/config/harness.json @@ -26,5 +26,6 @@ }, "advisory": [ "coderabbit" - ] + ], + "eval_shards": 4 } diff --git a/docs/decisions/NNNN-a-dispatchs-run-is-split-into-a-preflight-job-shards-and-a-merge-and-a-gate-sets-shards-are-drawn-by-a-salt.md b/docs/decisions/NNNN-a-dispatchs-run-is-split-into-a-preflight-job-shards-and-a-merge-and-a-gate-sets-shards-are-drawn-by-a-salt.md new file mode 100644 index 00000000..34915b1e --- /dev/null +++ b/docs/decisions/NNNN-a-dispatchs-run-is-split-into-a-preflight-job-shards-and-a-merge-and-a-gate-sets-shards-are-drawn-by-a-salt.md @@ -0,0 +1,33 @@ +# NNNN. A dispatch's run is split into a preflight job, shards and a merge, and a gate set's shards are drawn by a salt + +Status: accepted. The Factory Builder's rulings of 2026-11-08 (e) and of 2026-11-09 (a) to (d) on issue 611, "A case's head is read from the private home, pinned by its hash, and the run is sharded": this is the second of its two pull requests, the run sharded; the first read each case's head from the home. It amends decision 0170's row "When", where it says a gate set's preflight is asked in the run step before the set's cases, which a job of its own now does; decision 0164's row "The seal", where it seals one ledger and binds it in the report and names the job's one output, `ledger_sha256`, which the merge job now sets; and its row "The upload", where the ciphertext is the one artifact `eval-ledger`, which is now each part's, `eval-ledger-`. Nothing else in either changes. + +## Decision + +| What | Decided | Because | +|---|---|---| +| The parts | `.github/workflows/eval.yml` holds three jobs. `preflight` asks a gate set's preflight, and nothing for a development set, which has none. `eval`, a matrix of the shards `config/harness.json` names, `eval_shards`, four, needs it, and each shard asks its share of the set, with a time limit of its own and `fail-fast: false`. `merge` needs both and runs whatever became of them, `if: always()`, holds neither credential and no environment, its own token reading contents alone, and writes the one report from what each part uploaded. Each job plans as its part, `EVAL_PART`; a part the configuration does not name is `part_unreadable` | The issue's: one time limit for every case, serial, could not hold a set of forty or more. The Factory Builder's ruling (e): four shards behind one preflight, since four preflights would raise false stops to about 1.2% a run, and about 3.6% across a pool of three | +| Which cases a shard asks | A development set's case is the shard of its place in the manifest, modulo the shards, so each shard's mix of kinds is even. A gate set's is the shard of the sha256 of a salt the home keeps, `/.salt`, read with the home's token at the fetch, the run's id and attempt, and the case's id, modulo the shards. A gate set with no salt, or one not 16 to 256 bytes, stops the fetch, `no_shard_salt`, a code its seating must satisfy | The Factory Builder's ruling (a): a part's counts by kind, over a partition anyone can compute from the public manifest, would tell which cases are of which kind, which decisions 0094 and 0099 keep from the public. There is no private channel between jobs, so the partition is made unknowable instead | +| What a part writes | Each part's `run` step writes `eval-part/1` where the upload finds it, `eval-part-`: how many it was given and passed, of each kind how many and how many passed, of its blocking cases two words, whether there is one and whether every one passed, never how many (decision 0120), its natural and constructed counts, the attempt a gate set's shard was drawn for, each code of a case not read, its usage and its models, and for the preflight its counts apart. A development set's part lists its cases as its report does; a gate set's lists none, and its cases with their Results go to a ledger of the part, under a salt of its own, sealed by that part's `seal` and uploaded as `eval-ledger-`. No step's line, and no gate set's part, names a case | Every word the report says of its kinds is said from summed counts, as `split_ok`, `kinds_pass` and `blocking_pass` say it of the whole set, so the merge needs no case and no credential | +| The merge | The merge downloads every `eval-part-*`, reads each as `eval-part/1` of this run's Release, commit and set, and writes the one report, uploaded as `eval-report` as before, so the Release's gate reads it unchanged. A part not there, not this run's, of another part, a gate set's holding any ledger but its own, or a gate set's shard drawn for another attempt than the merge's is none; its cases are `unanswered`, a fault of the run, and then no kind can be read, nor the split. A preflight that stopped leaves no shard to run, and the report says so, `preflight_failed`. The judge then judges the report as before | A shard that failed faults the run, as the issue's done-when asks | +| The preflight's job | It judges what it wrote: a preflight that `preflight_stops` fails the job, by its counts and codes, so no shard is asked | Decision 0170's stop, kept before any held-out case is asked | +| The ledger and the output | A gate set's report holds `ledger` as a list, each part's salted and sealed sha256, the preflight's first; a whole run's holds one, `whole`. The gate refuses a part named twice, or a whole run's beside a part's. The run's one output, `ledger_sha256`, is set by the merge step alone: the sha256 of the canonical JSON list of the parts' sealed hashes, in the report's order | The Factory Builder's ruling (c): one string output for every reader, the detail in the report. A matrix's outputs are its last writer's, so no shard sets it | +| The download | `actions/download-artifact` by its commit, the one already pinned in this repo's workflows, the merge job's alone | The Factory Builder's ruling (b) | +| A run not split | Where no part is named, each step runs the whole set and writes the report itself, as before | So a run, or a test, of one job reads as the merge of its parts does | +| Residual: what a part still publishes | Each gate set part publishes, of a subset of the set no reader can name, its counts by kind, its natural and constructed counts, its blocking words and its usage. Across runs and a pool of three these are of different unknown subsets, and from many of them a reader could estimate how many natural cases are of each kind, a pooled count, never which case is which: `ORIGIN_LEAST` keeps a group from naming one. The blocking words, too, tell over many runs about how many blocking cases there are | The Factory Builder's ruling (a), stated as a residual; and the security read of this change | +| Residual: the salt | The partition is unknowable only while the salt is random and secret. The seating makes it `os.urandom(32)` (issue 653, "The ledger's recipient: the Maintainer's certificate under config/ledger/ and its pin in .github/ledger-recipients"); a salt guessed, or leaked later, tells every past run's partition for as long as its artifacts are kept, ninety days, and is rotated by writing a new one in the home | The security read | +| Residual: a part's timing | A shard that runs out of time leaves nothing, and its cases are `unanswered`: the run is faulted, never read as the reviewer's. A gate set's shard run again, by GitHub's re-run of failed jobs, draws its cases anew for its attempt, and the merge, run again with it, reads no shard of the first attempt: the run stays faulted, and a new dispatch is the way on | A shard's limit is its own; a slow one fails closed | + +## Why + +A dispatch over a set of forty or more cases ran them one after another, under one time limit, in one job: a set larger than the limit, or a slow Engine, faulted the run however the reviewer did. Split into shards behind one preflight, each shard has its own limit and they run side by side; the merge, holding no credential, makes the one report the gate reads. A gate set must not tell which of its cases is which kind, so its shards are drawn by a salt no reader has, and each part of it publishes counts alone. + +## Boundaries + +| Boundary | Check | +|---|---| +| A development set's case is its place's shard, and the merge of the shards' parts is the report of the whole; a shard not there, or not this run's part, leaves its cases unanswered and the run faulted; a part the configuration does not name is refused; no step's line names a case | `tests/test_evals.py::test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole`, `harness/eval/__init__.py`, mutant: "a shard prints a case id", mutant: "a development set's shard asks every case", mutant: "a shard missing is read as whole", mutant: "a part of another part is merged", mutant: "a part of another commit is merged", mutant: "the merge's split needs no blocking case", mutant: "a development set's part that holds a ledger is merged", mutant: "a shard the configuration does not name is planned" | +| A gate set's case is the shard its salt, the run's id and attempt and its id draw, never its place; no salt stops the fetch; each part writes counts alone and seals its own ledger; the merge holds each part's ledger, the preflight's first, and the run's output hashes their list; a preflight that stops fails its own job and the merge says so | `tests/test_evals.py::test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone`, `harness/eval/__init__.py`, mutant: "a gate set's shard is chosen by its place in the manifest", mutant: "a gate set's shard artifact names a case", mutant: "a gate set's part that lists a case is merged", mutant: "a gate set with no salt is sharded by its place", mutant: "a gate set's shards are drawn by no run", mutant: "a gate set's cases no part answered are not counted", mutant: "a gate set's shard of another attempt is merged", mutant: "a gate set's part with no ledger of its own is merged", mutant: "a preflight that stops passes its own job", mutant: "the merge names no ledger as the run's output", mutant: "the run's output hashes the last part's ledger alone", mutant: "the one part found where it was put is not read" | +| The salt is read raw from the home, 16 to 256 bytes, or not at all | `tests/test_adapters.py::test_a_gate_sets_salt_is_read_raw_from_the_home_and_none_short_of_it`, `adapters/heldout.py`, mutant: "a salt of any length is read" | +| The gate refuses a part's ledger named twice, a whole run's beside a part's, or none | `tests/test_evals.py::test_the_gate_passes_a_good_aggregate_report_and_refuses_a_bad_one`, `harness/eval/__init__.py`, mutant: "a part's ledger named twice is read", mutant: "a whole run's ledger is read beside a part's", mutant: "a report with no ledger passes" | +| eval.yml holds the preflight, the shards the configuration names and the merge, each part's steps and names, the merge needing both and run always, with no credential, and its one output | `tests/test_evals.py::test_the_eval_workflow_names_each_credential_in_one_step_alone`, `tools/check_workflows.py`, mutant: "the shards' matrix is not the configuration's", mutant: "the merge may be skipped on a failure", mutant: "a part may need nothing", mutant: "the merge downloads every artifact" | diff --git a/examples/eval-aggregate.json b/examples/eval-aggregate.json index 0ff1e32d..bead06b9 100644 --- a/examples/eval-aggregate.json +++ b/examples/eval-aggregate.json @@ -80,8 +80,11 @@ "met": true }, "blocking_pass": false, - "ledger": { - "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", - "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" - } + "ledger": [ + { + "part": "whole", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] } diff --git a/examples/eval-part.json b/examples/eval-part.json new file mode 100644 index 00000000..05952f9e --- /dev/null +++ b/examples/eval-part.json @@ -0,0 +1,66 @@ +{ + "schema": "eval-part/1", + "part": "2", + "release": "factory/v0.1.0", + "commit": "9e927b4000000000000000000000000000000000", + "role": "pr-review", + "intent": "review-change", + "set": { + "count": 44, + "sha256": "05b3abf2579a5eb66403cd78be557fd860633a1fe2103c7642030defe32c657f" + }, + "attempt": "1", + "asked": 10, + "passed": 8, + "kinds": { + "finding": { + "cases": 5, + "passed": 4 + }, + "met": { + "cases": 5, + "passed": 4 + } + }, + "blocking": { + "some": true, + "passed": true + }, + "origin": { + "natural": { + "cases": 2, + "passed": 2 + }, + "constructed": { + "cases": 8, + "passed": 6 + } + }, + "faults": [], + "preflight": { + "cases": 0, + "passed": 0, + "faults": [] + }, + "usage": { + "answered": 44, + "input_tokens": 1320000, + "cache_creation_input_tokens": 88000, + "cache_read_input_tokens": 410000, + "output_tokens": 61000, + "turns": 210, + "cost_usd": 38.412, + "seconds": 5120.5 + }, + "models": [ + "a-model" + ], + "cases": [], + "ledger": [ + { + "part": "2", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] +} diff --git a/examples/invalid/eval-aggregate.a-fault-in-words.json b/examples/invalid/eval-aggregate.a-fault-in-words.json index f94eb292..a7df11a2 100644 --- a/examples/invalid/eval-aggregate.a-fault-in-words.json +++ b/examples/invalid/eval-aggregate.a-fault-in-words.json @@ -85,8 +85,11 @@ "met": true }, "blocking_pass": false, - "ledger": { - "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", - "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" - } + "ledger": [ + { + "part": "whole", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] } diff --git a/examples/invalid/eval-aggregate.a-fault-no-case-gave.json b/examples/invalid/eval-aggregate.a-fault-no-case-gave.json index 2fa45050..376c202f 100644 --- a/examples/invalid/eval-aggregate.a-fault-no-case-gave.json +++ b/examples/invalid/eval-aggregate.a-fault-no-case-gave.json @@ -85,8 +85,11 @@ "met": true }, "blocking_pass": false, - "ledger": { - "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", - "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" - } + "ledger": [ + { + "part": "whole", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] } diff --git a/examples/invalid/eval-aggregate.carries-its-cases.json b/examples/invalid/eval-aggregate.carries-its-cases.json index 81003a05..9254b6eb 100644 --- a/examples/invalid/eval-aggregate.carries-its-cases.json +++ b/examples/invalid/eval-aggregate.carries-its-cases.json @@ -80,10 +80,13 @@ "met": true }, "blocking_pass": false, - "ledger": { - "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", - "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" - }, + "ledger": [ + { + "part": "whole", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ], "cases": [ { "id": "poc-0001", diff --git a/examples/invalid/eval-aggregate.names-a-natural-case.json b/examples/invalid/eval-aggregate.names-a-natural-case.json index 246974aa..fc8224a5 100644 --- a/examples/invalid/eval-aggregate.names-a-natural-case.json +++ b/examples/invalid/eval-aggregate.names-a-natural-case.json @@ -83,8 +83,11 @@ "met": true }, "blocking_pass": false, - "ledger": { - "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", - "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" - } + "ledger": [ + { + "part": "whole", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] } diff --git a/examples/invalid/eval-aggregate.no-models.json b/examples/invalid/eval-aggregate.no-models.json index fe1980f9..fbe40e76 100644 --- a/examples/invalid/eval-aggregate.no-models.json +++ b/examples/invalid/eval-aggregate.no-models.json @@ -77,8 +77,11 @@ "met": true }, "blocking_pass": false, - "ledger": { - "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", - "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" - } + "ledger": [ + { + "part": "whole", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] } diff --git a/examples/invalid/eval-aggregate.no-preflight.json b/examples/invalid/eval-aggregate.no-preflight.json index c3caed93..d3e2de1f 100644 --- a/examples/invalid/eval-aggregate.no-preflight.json +++ b/examples/invalid/eval-aggregate.no-preflight.json @@ -75,8 +75,11 @@ "met": true }, "blocking_pass": false, - "ledger": { - "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", - "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" - } + "ledger": [ + { + "part": "whole", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] } diff --git a/examples/invalid/eval-aggregate.not-sealed.json b/examples/invalid/eval-aggregate.not-sealed.json index 3c6840d7..8bd6c22e 100644 --- a/examples/invalid/eval-aggregate.not-sealed.json +++ b/examples/invalid/eval-aggregate.not-sealed.json @@ -80,7 +80,10 @@ "met": true }, "blocking_pass": false, - "ledger": { - "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5" - } + "ledger": [ + { + "part": "whole", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5" + } + ] } diff --git a/examples/invalid/eval-part.a-shard-past-sixteen.json b/examples/invalid/eval-part.a-shard-past-sixteen.json new file mode 100644 index 00000000..866c0d2c --- /dev/null +++ b/examples/invalid/eval-part.a-shard-past-sixteen.json @@ -0,0 +1,66 @@ +{ + "schema": "eval-part/1", + "part": "16", + "release": "factory/v0.1.0", + "commit": "9e927b4000000000000000000000000000000000", + "role": "pr-review", + "intent": "review-change", + "set": { + "count": 44, + "sha256": "05b3abf2579a5eb66403cd78be557fd860633a1fe2103c7642030defe32c657f" + }, + "attempt": "1", + "asked": 10, + "passed": 8, + "kinds": { + "finding": { + "cases": 5, + "passed": 4 + }, + "met": { + "cases": 5, + "passed": 4 + } + }, + "blocking": { + "some": true, + "passed": true + }, + "origin": { + "natural": { + "cases": 2, + "passed": 2 + }, + "constructed": { + "cases": 8, + "passed": 6 + } + }, + "faults": [], + "preflight": { + "cases": 0, + "passed": 0, + "faults": [] + }, + "usage": { + "answered": 44, + "input_tokens": 1320000, + "cache_creation_input_tokens": 88000, + "cache_read_input_tokens": 410000, + "output_tokens": 61000, + "turns": 210, + "cost_usd": 38.412, + "seconds": 5120.5 + }, + "models": [ + "a-model" + ], + "cases": [], + "ledger": [ + { + "part": "2", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] +} diff --git a/examples/invalid/eval-part.two-ledgers.json b/examples/invalid/eval-part.two-ledgers.json new file mode 100644 index 00000000..6241df1c --- /dev/null +++ b/examples/invalid/eval-part.two-ledgers.json @@ -0,0 +1,71 @@ +{ + "schema": "eval-part/1", + "part": "2", + "release": "factory/v0.1.0", + "commit": "9e927b4000000000000000000000000000000000", + "role": "pr-review", + "intent": "review-change", + "set": { + "count": 44, + "sha256": "05b3abf2579a5eb66403cd78be557fd860633a1fe2103c7642030defe32c657f" + }, + "attempt": "1", + "asked": 10, + "passed": 8, + "kinds": { + "finding": { + "cases": 5, + "passed": 4 + }, + "met": { + "cases": 5, + "passed": 4 + } + }, + "blocking": { + "some": true, + "passed": true + }, + "origin": { + "natural": { + "cases": 2, + "passed": 2 + }, + "constructed": { + "cases": 8, + "passed": 6 + } + }, + "faults": [], + "preflight": { + "cases": 0, + "passed": 0, + "faults": [] + }, + "usage": { + "answered": 44, + "input_tokens": 1320000, + "cache_creation_input_tokens": 88000, + "cache_read_input_tokens": 410000, + "output_tokens": 61000, + "turns": 210, + "cost_usd": 38.412, + "seconds": 5120.5 + }, + "models": [ + "a-model" + ], + "cases": [], + "ledger": [ + { + "part": "2", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + }, + { + "part": "2", + "salted_sha256": "3f1d0c1e2a7b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5", + "sealed_sha256": "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" + } + ] +} diff --git a/harness/eval/AGENTS.md b/harness/eval/AGENTS.md index 5b834c5e..1cfe1a6a 100644 --- a/harness/eval/AGENTS.md +++ b/harness/eval/AGENTS.md @@ -4,13 +4,15 @@ The eval job's steps, a module each, in the order `__init__.py` gives. Their rul issue 612, "The eval job reads a named set, and a gate set's report is aggregates only", issue 613, "The gate pools k dispatches on one commit and reads all of them", and issue 611, "A case's head is read from the private home, pinned by its hash, and the run is sharded". A dispatch names its set. A set whose manifest says it is a gate set is reported by counts alone, `eval-aggregate/1`, and each case's outcome with its Result goes to a ledger, `eval-ledger/1`, kept in the scratch. -`seal.py` encrypts the ledger with `openssl cms` to each of the Maintainer's certificates that its pins, in a protected path under `.github/`, name by sha256 (`seal.RECIPIENTS`). The ciphertext alone is uploaded, as `eval-ledger`. +`seal.py` encrypts the ledger with `openssl cms` to each of the Maintainer's certificates that its pins, in a protected path under `.github/`, name by sha256 (`seal.RECIPIENTS`). The ciphertext alone is uploaded, +as `eval-ledger`, or `eval-ledger-` for a part. No key that opens a ledger is in the job or the repo, and nothing automated reads one. Until the Maintainer commits his certificates, a gate set's run fails at the seal, `no_recipient`. -The ciphertext lives only as a run artifact under its retention until a storage decision is made; its hash is the job's output, `ledger_sha256`. Where ledgers are kept is not built yet. +The ciphertext lives only as a run artifact under its retention until a storage decision is made; the run's one output, `ledger_sha256`, is its hash, or in a split run the merge's hash of every part's. Where ledgers are kept is not built yet. A gate set gives its own bars and least of a kind. The Release's gate pools every dispatch on a commit of the gate set `config/harness.json` names, `gate_set`. Until one is named, the gate reads the review Intent's set by its newest successful dispatch, which a re-dispatch can still make the best of several. A gate set's dispatch asks its preflight first, a few cases of the review Intent's set scored apart, so that a broken pipeline stops it before any held-out case is asked. A case's head is read from the home beside the case, its tarball and its tree each by the sha256 the case pins, and held to that tree (issue 611); nothing in the job reads with no credential. +The run is split into the preflight's job, a matrix of shards and a merge job with no credential, which writes the one report from each part's counts; a gate set's shards are drawn by a salt the home keeps. | Rule | Held by | |---|---| @@ -23,11 +25,13 @@ A case's head is read from the home beside the case, its tarball and its tree ea | No set but a gate set names a gate set's case, by its id or its hash | `tests/test_evals.py::test_no_set_but_a_gate_set_names_a_case_of_one` | | A gate set marks no group of natural cases, or of others, under ten but none | `tests/test_evals.py::test_a_gate_set_marks_no_group_of_natural_cases_or_others_too_small_to_hide_one` | | A gate set's judge prints counts and codes, never a case | `tests/test_evals.py::test_a_gate_sets_judge_says_counts_and_codes_and_never_a_case` | -| The seal and the ledger's upload come after the run and before the report's; the seal is given nothing and sets the job's one output | `tests/test_evals.py::test_the_eval_workflow_names_each_credential_in_one_step_alone` | +| The seal and the ledger's upload come after the run and before the report's; the seal is given nothing, and the merge step alone sets the run's one output | `tests/test_evals.py::test_the_eval_workflow_names_each_credential_in_one_step_alone` | | A gate set gives its own bars, least of a kind and the record that states them word for word, and its job and gate hold them; no other set gives any | `tests/test_evals.py::test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them` | | The pool is three runs at least, every one, each kind summed under the set's bars; one faulted closes it | `tests/test_evals.py::test_the_gate_pools_every_dispatch_on_the_commit_and_one_faulted_closes_it` | | A gate set's preflight is three to five of the review Intent's cases, none a gate set's by its id or its hash | `tests/test_evals.py::test_a_gate_sets_preflight_is_of_the_development_set_and_no_gate_sets_case` | | The preflight is asked first; a fault in it, or no pass, stops the dispatch before a case of the set is asked, and the gate reads it as faulted | `tests/test_evals.py::test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it` | | Each case's head is read from the home by its pins, unpacked and held to its tree; one not the one pinned, or not read whole, closes the gate | `tests/test_evals.py::test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate` | +| A split run's merge is the report of the whole; a part missing leaves its cases unanswered; no step names a case | `tests/test_evals.py::test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole` | +| A gate set's shards are drawn by the home's salt, never by place, and its parts write counts alone | `tests/test_evals.py::test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone` | | The Release's gate judges a gate set named in the configuration by every completed dispatch on the commit | `tests/test_release_step.py::test_a_gate_set_is_judged_by_every_dispatch_on_the_commit` | | Each of these rules has a mutant that a test kills | `tools/mutants.py` | diff --git a/harness/eval/__init__.py b/harness/eval/__init__.py index f2bc9c12..20a82c30 100644 --- a/harness/eval/__init__.py +++ b/harness/eval/__init__.py @@ -12,8 +12,11 @@ (`harness/task.from_admission`, `harness/run.answer`), and scores its Result with `tools/eval_result.py`; for a gate set, the development cases its manifest names as its preflight first, scored apart (issue 613); - `seal`, for a gate set alone, encrypts the ledger `run` left in the scratch to the Maintainer's certificates (issue 612); +- `merge`, in the merge job of a run split into parts, writes the one report from what the preflight's job and each + shard's wrote (issue 611); - `judge` fails the run unless the report passes `verdict`, the gate's own test, which the Release's gate reads too: each - kind's bar of decision 0150, and no fault of the Engine or the run (decision 0120). + kind's bar of decision 0150, and no fault of the Engine or the run (decision 0120). The preflight's job judges what it + wrote, and fails where the preflight stops the dispatch. Nothing a step prints is of a case, a Result or the home: counts and codes alone. A step that fails ends on one code and exits 1, the judge's count of the report's codes before it, and no traceback, which could quote a case, reaches the @@ -74,6 +77,10 @@ OWN = ("bars", "kind_least", "record", "preflight") # what a gate set's manifest gives of its own, and no other set's does # The scratch's folder of the development cases a gate set's dispatch asks first, apart from the set's own (issue 613). PREFLIGHT = "preflight" +# The parts a dispatch's run is split into (issue 611): the preflight's job, each shard's of the matrix, and the merge +# job that writes the one report from what each part wrote, `PART`, of counts alone for a gate set. +PART, PART_SCHEMA, MERGE = "eval-part.json", "eval-part/1", "merge" +RUN_NUMBER = re.compile(r"[1-9][0-9]{0,19}") # What a case's `cause` may say, each held to a shape that holds no words: an exception's class, the last line of this # repo's own code it passed, a code, and the CLI's subtype and exit. Never a message, which could quote a case. CLASS = re.compile(r"[A-Z][A-Za-z0-9_]{0,63}") @@ -252,10 +259,35 @@ def _write(directory, name: str, document) -> None: f.write(json.dumps(document, indent=2, sort_keys=True) + "\n") +def shards_of(root: pathlib.Path): + """How many shards the run is split into, `config/harness.json`'s `eval_shards`, or None where it names none.""" + try: + shards = json.loads((root / "config" / "harness.json").read_text(encoding="utf-8")).get("eval_shards") + except (OSError, ValueError, AttributeError): + return None + return shards if type(shards) is int and 1 <= shards <= 16 else None + + +def part_named(env: dict, root: pathlib.Path): + """`(part, shards)`: the part of the run this job is, `EVAL_PART` (the preflight, a shard by its number from 0, or the + merge), and how many shards there are; `(None, None)` where no part is named, and the job runs the whole set and + writes the report itself. A part not one of these is `part_unreadable` (issue 611).""" + part = env.get("EVAL_PART") + if part is None: + return None, None + shards = shards_of(root) + if shards is None: + raise Stop("part_unreadable") + if part not in (PREFLIGHT, MERGE) and part not in [str(i) for i in range(shards)]: + raise Stop("part_unreadable") + return part, shards + + def plan(env: dict, root: pathlib.Path, workspace: pathlib.Path) -> str: """What the run judges, kept for the steps after it: the Release the dispatch names, the commit checked out, and - the set it names, `EVAL_SET`, the review Intent's where none is given, by its name, count and hash. A set of no case - is `eval_set_empty`, and the run ends before anything is read.""" + the set it names, `EVAL_SET`, the review Intent's where none is given, by its name, count and hash; and the part + of the run this job is, `part_named`'s. A set of no case is `eval_set_empty`, and the run ends before anything is + read.""" release, commit, name = env.get("RELEASE"), env.get("GITHUB_SHA"), env.get("EVAL_SET", DEFAULT_SET) if not (isinstance(release, str) and RELEASE.fullmatch(release)): raise Stop("release_unreadable") @@ -266,6 +298,7 @@ def plan(env: dict, root: pathlib.Path, workspace: pathlib.Path) -> str: raise Stop("eval_set_empty") held_out(root, name, found) ahead = preflight_of(root, found) + part, shards = part_named(env, root) # The heads beside the scratch, made new as it is: a folder an earlier job left is no head of this run. Both are # placed first, and the heads made first, so that a heads folder left behind stops the plan before any scratch is # made; a scratch left behind stops it with the new heads beside it, and either stops the next (issue 626). @@ -283,8 +316,9 @@ def plan(env: dict, root: pathlib.Path, workspace: pathlib.Path) -> str: # that the fetch and the install spend it too (the run step's BUDGET). _write(place, "plan.json", {"release": release, "commit": commit, "role": found["role"], "intent": found["intent"], "name": name, "set": {"count": found["count"], "sha256": digest}, "preflight": ahead, - "planned_at": now()}) - return "planned %d cases" % found["count"] + (" after %d to preflight" % len(ahead) if ahead else "") + "part": part, "shards": shards, "planned_at": now()}) + return "planned %d cases" % found["count"] + (" after %d to preflight" % len(ahead) if ahead else "") \ + + ("" if part is None else ", as the part %s of %d shards" % (part, shards)) def held_out(root: pathlib.Path, name: str, found: dict) -> None: @@ -404,11 +438,39 @@ def fetch(env: dict, root: pathlib.Path, workspace: pathlib.Path, transport) -> first, the same way, into the scratch's own folder of it, PREFLIGHT; its heads beside the set's, by their ids, which `preflight_of` holds apart (issue 613).""" kept, found, place = _planned(env, root, workspace) - said = "" - if kept["preflight"]: + part, said = kept.get("part"), "" + if part == MERGE: + raise Stop("part_unreadable") + if kept["preflight"] and part in (None, PREFLIGHT): ahead = _fetched(env, workspace, transport, dict(found, cases=kept["preflight"]), place / PREFLIGHT) said = ", %d of %d to preflight" % (ahead, len(kept["preflight"])) - return "fetched %d of %d cases" % (_fetched(env, workspace, transport, found, place), found["count"]) + said + mine = [] if part == PREFLIGHT else shard_cases(env, kept, found, transport) + _write(place, "part.json", [e["id"] for e in mine]) + of = "" if part in (None, PREFLIGHT) else " of shard %s" % part + return "fetched %d of %d cases%s" % (_fetched(env, workspace, transport, dict(found, cases=mine), place), len(mine), of) + said + + +def shard_cases(env: dict, kept: dict, found: dict, transport) -> list: + """The cases of the set this job's part asks, in the manifest's order (issue 611): every case where no part is + named; for a development set, the shard of each case's place in the manifest, its place modulo the shards; and for + a gate set, the shard drawn by the sha256 of the set's salt in the home, this run's id and attempt and the case's + id, so that no reader of the public manifest can tell which cases a shard holds, nor so which kinds its counts + are of. A gate set whose salt cannot be read stops the fetch, `no_shard_salt`; one whose run cannot be named, + `part_unreadable`.""" + from adapters import heldout + part, shards = kept.get("part"), kept.get("shards") + if part is None: + return list(found["cases"]) + if not gate_set(found): + return [e for place, e in enumerate(found["cases"]) if place % shards == int(part)] + salt = heldout.salt(transport, env.get("HELDOUT_OWNER"), env.get("HELDOUT_REPO"), found["role"], kept["name"]) + if salt is None: + raise Stop("no_shard_salt") + run, attempt = env.get("GITHUB_RUN_ID"), env.get("GITHUB_RUN_ATTEMPT") + if not all(isinstance(n, str) and RUN_NUMBER.fullmatch(n) for n in (run, attempt)): + raise Stop("part_unreadable") + drawn = lambda e: int(hashlib.sha256(b"\0".join([salt, run.encode(), attempt.encode(), e["id"].encode()])).hexdigest(), 16) + return [e for e in found["cases"] if drawn(e) % shards == int(part)] def _fetched(env: dict, workspace: pathlib.Path, transport, found: dict, place: pathlib.Path) -> int: @@ -806,6 +868,21 @@ def faults_of(assessed: dict) -> list: return [{"code": code, "count": codes.count(code)} for code in sorted(set(codes))] +def preflight_counts(ahead: dict) -> dict: + """The cases a gate set's dispatch asked first, counted apart: how many, how many passed, and each code of one not + read, by how many (issue 613).""" + return {"cases": len(ahead), "passed": len([s for s in ahead.values() if s == (PASS, [])]), + "faults": faults_of(ahead)} + + +def kept_ledger(place: pathlib.Path, document: dict) -> bytes: + """`document`, a ledger, written to the scratch alone, readable by the runner's user alone, for `seal`; its bytes.""" + data = (json.dumps(document, indent=2, sort_keys=True) + "\n").encode("utf-8") + with open(os.open(within(place, LEDGER), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as out: + out.write(data) + return data + + def preflight_stops(ahead: dict) -> bool: """Whether the cases a gate set's dispatch asked first stop it before any of the set's own is asked: one of them stopped by the Engine or the run, or none passed, as a scorer that fails every case would have it (issue 613). One @@ -834,13 +911,12 @@ def aggregate(kept: dict, found: dict, assessed: dict, place: pathlib.Path, resu "intent": found["intent"], "set": dict(kept["set"]), "passed": len(passed), "kinds": counted, "origin": {name: {"cases": len(ids), "passed": len(ids & passed)} for name, ids in origin.items()}, "faults": faults_of(assessed), - "preflight": {"cases": len(ahead), "passed": len([s for s in ahead.values() if s == (PASS, [])]), - "faults": faults_of(ahead)}, + "preflight": preflight_counts(ahead), "usage": usage_of(results), "engine": engine_of(found["role"]), "models": models(usages or {}), "bars": {kind: list(bars_of(found)[kind]) for kind in KINDS}, "kind_least": kind_least_of(found), "split_ok": split_ok(found, place), "kinds_pass": kinds_pass(found, place, assessed), "blocking_pass": blocking_pass(found, place, assessed), - "ledger": {"salted_sha256": hashlib.sha256(ledger).hexdigest()}} + "ledger": [{"part": "whole", "salted_sha256": hashlib.sha256(ledger).hexdigest()}]} def _judged(document: dict, found: dict, digest: str, commit: str, release: str): @@ -872,6 +948,10 @@ def aggregate_verdict(document, found: dict, digest: str, commit: str, release: first = [fault["code"] for fault in ahead["faults"]] if any(len(set(said)) != len(said) or not set(said) <= eval_result_codes() for said in (codes, first)): return "count_differs" + # Each part's ledger once, and a whole run's alone (issue 611). + parts = [entry["part"] for entry in document["ledger"]] + if len(set(parts)) != len(parts) or ("whole" in parts and parts != ["whole"]): + return "count_differs" if ahead["cases"] != len(found["preflight"]) or ahead["passed"] + sum(f["count"] for f in ahead["faults"]) > ahead["cases"]: return "count_differs" if set(first) & set(ENGINE_FAULTS + RUN_FAULTS) or ahead["passed"] == 0: @@ -880,6 +960,9 @@ def aggregate_verdict(document, found: dict, digest: str, commit: str, release: return "engine_fault" if set(codes) & set(RUN_FAULTS): return "run_fault" + # A report that passes is sealed: a ledger of every part it read (issues 612 and 611). + if not document["ledger"]: + return "count_differs" # Each count is the job's: the natural cases are the manifest's, the passes are the report's own, and no case is # both passed and not read. A report whose counts do not add up is not the job's. natural, origin, passed = len([e for e in found["cases"] if e.get("natural") is True]), document["origin"], document["passed"] @@ -999,6 +1082,10 @@ def score(env: dict, root: pathlib.Path, workspace: pathlib.Path, review) -> str causes = {} causes = dict(causes) if isinstance(causes, dict) else {} alternates, surplus, usages = set(), {}, {} + if kept.get("part") == MERGE: + raise Stop("part_unreadable") + if kept.get("part") is not None: + return part_scored(kept, found, place, workspace, review, causes, env.get("GITHUB_RUN_ATTEMPT")) if gate_set(found): return written_out(workspace, found, *gate_scored(kept, found, place, review, causes, alternates, surplus, usages)) @@ -1020,22 +1107,225 @@ def gate_scored(kept: dict, found: dict, place: pathlib.Path, review, causes: di """`(assessed, report)` of a gate set's run: each case, with its Result, written to the ledger in the scratch, and the report of counts alone that binds it (issue 612). The cases its preflight names are asked first, scored apart; where `preflight_stops`, none of the set's own is asked, and each is `unanswered` (issue 613).""" - results, first, spent, answers = {}, dict(found, cases=kept["preflight"]), {}, {} + results = {} + ahead, told, spent, answers = preflight_asked(kept, found, place, review) + why = {e["id"]: "unanswered" for e in found["cases"]} if preflight_stops(ahead) else _read(place, "fetched.json") + assessed = assess(found, why, place, review, causes, alternates, surplus, usages, results) + ledger = kept_ledger(place, ledger_of(kept, found, assessed, causes, alternates, results, surplus, usages, + (ahead, told, spent, answers))) + return assessed, aggregate(kept, found, assessed, place, results, ledger, usages, ahead) + + +def preflight_asked(kept: dict, found: dict, place: pathlib.Path, review) -> tuple: + """`(assessed, causes, usages, results)` of the cases a gate set's dispatch asks first, as `assess` gives them, each + read where `fetch` put them (issue 613).""" + spent, answers = {}, {} # What a preflight case's read raised, as `score` reads the set's own, so that the ledger says why it was not read. try: told = _read(place / PREFLIGHT, "fetch-causes.json") except Stop: told = {} told = dict(told) if isinstance(told, dict) else {} - ahead = assess(first, _read(place / PREFLIGHT, "fetched.json"), place / PREFLIGHT, review, told, usages=spent, - results=answers) - why = {e["id"]: "unanswered" for e in found["cases"]} if preflight_stops(ahead) else _read(place, "fetched.json") - assessed = assess(found, why, place, review, causes, alternates, surplus, usages, results) - ledger = (json.dumps(ledger_of(kept, found, assessed, causes, alternates, results, surplus, usages, - (ahead, told, spent, answers)), indent=2, sort_keys=True) + "\n").encode("utf-8") - with open(os.open(within(place, LEDGER), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as out: - out.write(ledger) - return assessed, aggregate(kept, found, assessed, place, results, ledger, usages, ahead) + ahead = assess(dict(found, cases=kept["preflight"]), _read(place / PREFLIGHT, "fetched.json"), place / PREFLIGHT, + review, told, usages=spent, results=answers) + return ahead, told, spent, answers + + +def part_cases(kept: dict, found: dict, place: pathlib.Path) -> dict: + """`found`, but for the cases this job's part asks, as `fetch` listed them in the scratch, in the manifest's order.""" + ids = _read(place, "part.json") + if not isinstance(ids, list): + raise Stop("scratch_unreadable") + return dict(found, cases=[e for e in found["cases"] if e["id"] in set(ids)]) + + +def counts_of(found: dict, place: pathlib.Path, assessed: dict) -> tuple: + """`(kinds, blocking)` of `found`'s cases: of each kind, how many and how many passed, which the merge sums; and of + those that expect a blocking Finding, two words, whether there is one and whether every one passed, never how many + (decision 0120); `(None, None)` where `_expected` can say nothing.""" + expected = _expected(found, place) + if expected is None: + return None, None + passed = {case for case, said in assessed.items() if said == (PASS, [])} + kinds = {kind: {"cases": len([c for c, e in expected.items() if e["verdict"] == kind]), + "passed": len([c for c, e in expected.items() if e["verdict"] == kind and c in passed])} for kind in KINDS} + blocked = blocking_cases(found, expected) + return kinds, {"some": bool(blocked), "passed": all(c in passed for c in blocked)} + + +def part_scored(kept: dict, found: dict, place: pathlib.Path, workspace: pathlib.Path, review, causes: dict, + attempt=None) -> str: + """The `run` step of one part of a split run (issue 611): the preflight's cases, or a shard's, assessed, and what + the part found written as `PART` where the upload finds it, `eval-part/1`. A development set's part lists its + cases as the report does; a gate set's holds counts alone, and its cases, with their Results, go to a ledger of the + part in the scratch for `seal`, under a salt of its own (issue 612).""" + gate, alternates, surplus, usages, results = gate_set(found), set(), {}, {}, {} + ahead, told, spent, answers = {}, {}, {}, {} + if kept["part"] == PREFLIGHT: + mine, assessed = dict(found, cases=[]), {} + if gate: + ahead, told, spent, answers = preflight_asked(kept, found, place, review) + else: + mine = part_cases(kept, found, place) + assessed = assess(mine, _read(place, "fetched.json"), place, review, causes, alternates, surplus, usages, results) + ledger = [] + if gate: + data = kept_ledger(place, ledger_of(dict(kept, preflight=kept["preflight"] if kept["part"] == PREFLIGHT else []), + mine, assessed, causes, alternates, results, surplus, usages, + (ahead, told, spent, answers))) + ledger = [{"part": kept["part"], "salted_sha256": hashlib.sha256(data).hexdigest()}] + kinds, blocking = counts_of(mine, place, assessed) + passed = {case for case, said in assessed.items() if said == (PASS, [])} + natural = {e["id"] for e in mine["cases"] if e.get("natural") is True} + origin = {"natural": natural, "constructed": {e["id"] for e in mine["cases"]} - natural} + written = {"schema": PART_SCHEMA, "part": kept["part"], "release": kept["release"], "commit": kept["commit"], + "role": found["role"], "intent": found["intent"], "set": dict(kept["set"]), + # The attempt a gate set's shard was drawn for: the merge reads no shard of another (issue 611). + "attempt": attempt if gate and kept["part"] != PREFLIGHT else None, "asked": len(mine["cases"]), + "passed": len(passed), "kinds": kinds, "blocking": blocking, + "origin": {name: {"cases": len(ids), "passed": len(ids & passed)} for name, ids in origin.items()}, + "faults": faults_of(assessed), + "preflight": preflight_counts(ahead), + # The preflight's cases are counted apart, and nothing of them in the set's usage (decision 0170). + "usage": usage_of(results), "models": models(usages), + "cases": [] if gate else report(kept, mine, assessed, causes=causes, alternates=alternates, + surplus=surplus, usages=usages)["cases"], + "ledger": ledger} + _write(workspace, PART, written) + return "scored %d cases of the part %s, %d passed" % (len(mine["cases"]) + len(ahead), kept["part"], len(passed)) + + +def part_names(kept: dict) -> list: + """The parts of a split run, the preflight and each shard by its number, as many shards as the plan kept, which + the configuration bounds at sixteen: a count the scratch holds otherwise is no plan of this run's.""" + shards = next((n for n in range(1, 17) if type(kept.get("shards")) is int and n == kept["shards"]), None) + if shards is None: + raise Stop("scratch_unreadable") + return [PREFLIGHT] + [str(i) for i in range(shards)] + + +def _part_file(path: str): + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError, RecursionError): + return None + + +def parts_of(kept: dict, found: dict, folder: pathlib.Path, attempt=None) -> dict: + """`{part: what it wrote, or None}` for the preflight and each shard, read from `folder` as the merge job's download + puts them, `eval-part-/eval-part.json`, or the one part it found at `eval-part.json` itself. One not there, + not `eval-part/1`, of another part, Release, commit or set than this run's, a gate set's that lists a case or holds + no ledger but its own, a development set's that holds one, or a gate set's shard drawn for another attempt than + `attempt`, the merge's, is None, and its cases are not answered: a shard run again draws its cases anew.""" + names, gate = part_names(kept), gate_set(found) + read = {} + for part in names: + try: + read[part] = _part_file(within(within(folder, "eval-part-%s" % part), PART)) + except Stop: + read[part] = None + flat = _part_file(os.path.join(os.path.realpath(str(folder)), PART)) + if isinstance(flat, dict) and flat.get("part") in names and read[flat["part"]] is None and not any(read.values()): + read[flat["part"]] = flat + for part, document in read.items(): + if document is None: + continue + if _errors(document, "eval-part") or document["part"] != part or _judged( + document, found, kept["set"]["sha256"], kept["commit"], kept["release"]) \ + or (document["cases"] or [e["part"] for e in document["ledger"]] != [part] if gate else document["ledger"]) \ + or (gate and part != PREFLIGHT and document["attempt"] != attempt): + read[part] = None + return read + + +def _summed(counts: list): + """`{cases, passed}` summed over `counts`; None where any is None.""" + if any(c is None for c in counts): + return None + return {key: sum(c[key] for c in counts) for key in ("cases", "passed")} + + +def merged(kept: dict, found: dict, read: dict, root: pathlib.Path) -> dict: + """The one report of a split run, from what each part wrote (issue 611): a development set's `eval-report/1`, its + cases from the shards in the manifest's order, and a gate set's `eval-aggregate/1`, of the parts' counts summed, + its preflight the preflight's, its ledger each part's. A case no part answered is `unanswered`, a fault of the run; + then no kind can be read, nor the split. Every word the report says of its kinds is said from the summed counts, + as `split_ok`, `kinds_pass` and `blocking_pass` would say it of the whole set.""" + shards = [read[part] for part in part_names(kept)[1:]] + present = [d for d in shards if d is not None] + whole = sum(d["asked"] for d in present) == found["count"] + kinds = {k: _summed([None if d["kinds"] is None else d["kinds"][k] for d in present]) for k in KINDS} if whole else None + kinds = None if kinds is None or None in kinds.values() else kinds + blocked = None if kinds is None or any(d["blocking"] is None for d in present) else { + "some": any(d["blocking"]["some"] for d in present), "passed": all(d["blocking"]["passed"] for d in present)} + least, bars = kind_least_of(found), bars_of(found) + # A part says whether it has a blocking case, never how many, so the merge can hold a least of one at most. + held = None if blocked is None or BLOCKING_LEAST > 1 else \ + min(kinds[k]["cases"] for k in KINDS) >= least and (blocked["some"] or BLOCKING_LEAST == 0) + words = None if kinds is None else {k: _clears(kinds[k]["passed"], kinds[k]["cases"], bars[k]) for k in KINDS} + clear = None if blocked is None else blocked["passed"] + unanswered = found["count"] - sum(d["asked"] for d in present) + answering = sorted({m for d in present for m in d["models"]}) + if not gate_set(found): + rows = {row["id"]: row for d in present for row in d["cases"]} + cases = [rows.get(e["id"]) or {"id": e["id"], "sha256": e["sha256"], "verdict": "unreadable", + "reasons": ["unanswered"], "usage": dict(NOT_ASKED)} for e in found["cases"]] + named = identity(root, found["role"]) + return {"schema": "eval-report/1", "release": kept["release"], "commit": kept["commit"], "role": found["role"], + "intent": found["intent"], "set": dict(kept["set"]), "models": answering, + "prompt_sha256": named["prompt_sha256"], "rubric_sha256": named["rubric_sha256"], "cases": cases, + "passed": len([c for c in cases if c["verdict"] == PASS]), "split_ok": held, "kinds_pass": words, + "blocking_pass": clear} + faults = {} + for d in present: + for fault in d["faults"]: + faults[fault["code"]] = faults.get(fault["code"], 0) + fault["count"] + if unanswered: + faults["unanswered"] = faults.get("unanswered", 0) + unanswered + natural = len([e for e in found["cases"] if e.get("natural") is True]) + origin = {"natural": {"cases": natural, "passed": sum(d["origin"]["natural"]["passed"] for d in present)}, + "constructed": {"cases": found["count"] - natural, + "passed": sum(d["origin"]["constructed"]["passed"] for d in present)}} + first = read[PREFLIGHT] + ahead = first["preflight"] if first is not None else { + "cases": len(kept["preflight"]), "passed": 0, "faults": [{"code": "unanswered", "count": len(kept["preflight"])}]} + usage = dict({field: sum(d["usage"][field] for d in present) for field in SUMMED + ("answered",)}, + cost_usd=round(sum(d["usage"]["cost_usd"] for d in present), 6), + seconds=round(sum(d["usage"]["seconds"] for d in present), 3)) + return {"schema": AGGREGATE, "release": kept["release"], "commit": kept["commit"], "role": found["role"], + "intent": found["intent"], "set": dict(kept["set"]), "passed": sum(d["passed"] for d in present), + "kinds": None if kinds is None else {k: dict(kinds[k], interval=wilson(kinds[k]["passed"], kinds[k]["cases"])) + for k in KINDS}, + "origin": origin, "faults": [{"code": c, "count": n} for c, n in sorted(faults.items())], + "preflight": ahead, "usage": usage, "engine": engine_of(found["role"]), "models": answering, + "bars": {kind: list(bars[kind]) for kind in KINDS}, "kind_least": least, "split_ok": held, + "kinds_pass": words, "blocking_pass": clear, + "ledger": [entry for part in part_names(kept) if read[part] is not None + for entry in read[part]["ledger"]]} + + +def ledger_digest(ledger: list) -> str: + """The job's one output, `ledger_sha256`: the sha256 of the parts' sealed hashes as a canonical JSON list, in the + report's order, the preflight's first (the Factory Builder's ruling (c) on issue 611). The report holds each.""" + return hashlib.sha256(json.dumps([entry.get("sealed_sha256") for entry in ledger], separators=(",", ":")).encode()).hexdigest() + + +def merge(env: dict, root: pathlib.Path, workspace: pathlib.Path) -> str: + """The merge job's step (issue 611): the one report, `merged`'s, written where the upload finds it, from each part + the download put under `parts/` in the workspace; and for a gate set, `ledger_digest` as the job's one output.""" + kept, found, _ = _planned(env, root, workspace) + if kept.get("part") != MERGE: + raise Stop("part_unreadable") + read = parts_of(kept, found, pathlib.Path(within(workspace, "parts")), env.get("GITHUB_RUN_ATTEMPT")) + written = merged(kept, found, read, root) + _write(workspace, REPORT, written) + output = env.get("GITHUB_OUTPUT") + if gate_set(found) and isinstance(output, str) and os.path.isabs(output): + with open(output, "a", encoding="utf-8") as out: + out.write("ledger_sha256=%s\n" % ledger_digest(written["ledger"])) + return "merged %d of %d parts, %d passed" % (len([d for d in read.values() if d is not None]), len(read), + written["passed"]) def closed_of(document: dict) -> list: @@ -1096,8 +1386,13 @@ def configured_model(root: pathlib.Path): def judge(env: dict, root: pathlib.Path, workspace: pathlib.Path) -> str: - """The run's verdict on its own report: `verdict`'s, which must be `pass`.""" + """The run's verdict on its own report: `verdict`'s, which must be `pass`. The preflight's job of a split run judges + what it wrote instead: a preflight that `preflight_stops` fails it, so no shard is asked (issue 611).""" kept, found, _ = _planned(env, root, workspace) + if kept.get("part") == PREFLIGHT: + return judged_preflight(workspace, found) + if kept.get("part") not in (None, MERGE): + raise Stop("part_unreadable") try: with open(within(workspace, REPORT), encoding="utf-8") as f: written = json.load(f) @@ -1113,6 +1408,25 @@ def judge(env: dict, root: pathlib.Path, workspace: pathlib.Path) -> str: return tally(written, configured_model(root)) +def judged_preflight(workspace: pathlib.Path, found: dict) -> str: + """The preflight job's verdict on what it wrote: `preflight_failed` where its cases `preflight_stops`, by its counts + and codes as the gate reads them; a development set has none, and passes.""" + try: + with open(within(workspace, PART), encoding="utf-8") as f: + written = json.load(f) + except (OSError, ValueError, RecursionError): + raise Stop("report_unreadable") from None + if _errors(written, "eval-part") or written["part"] != PREFLIGHT: + raise Stop("report_unreadable") + ahead, codes = written["preflight"], {f["code"] for f in written["preflight"]["faults"]} + said = "preflight %d of %d passed" % (ahead["passed"], ahead["cases"]) + if gate_set(found) and (ahead["cases"] != len(found["preflight"]) or ahead["passed"] == 0 + or codes & set(ENGINE_FAULTS + RUN_FAULTS)): + print(", ".join([said] + ["%s %d" % (f["code"], f["count"]) for f in ahead["faults"]])) + raise Stop("preflight_failed") + return said + + def main(step, *given) -> int: """One step of the job, run in the workspace: what it says on success, or its one code on failure, and nothing else of its own. Whatever it raised is never printed, since its message could quote a case.""" diff --git a/harness/eval/merge.py b/harness/eval/merge.py new file mode 100644 index 00000000..ecf721a6 --- /dev/null +++ b/harness/eval/merge.py @@ -0,0 +1,5 @@ +"""The merge job's step of a split run (issue 611): the one report, from what the preflight and each shard wrote.""" +from harness.eval import exit_with, merge + +if __name__ == "__main__": + exit_with(merge) diff --git a/harness/eval/seal.py b/harness/eval/seal.py index 7b7fa7c5..d36370f7 100644 --- a/harness/eval/seal.py +++ b/harness/eval/seal.py @@ -8,7 +8,9 @@ Maintainer's certificates, each pinned by its sha256 in a protected path under `.github/`, so that a session's edit of who can read the ledgers is asked about, and a merge of one is his. Nothing here is given a credential. The ciphertext lives only as the run's artifact, under its retention, until where ledgers are kept is decided (the Maintainer's ruling -on issue 612); its sha256 is the job's output, `ledger_sha256`, and the report's. +on issue 612); its sha256 is the report's. A run split into parts seals each part's ledger in that part's job, bound +in what the part wrote, `eval-part.json`, and the merge job names the run's one output, `ledger_sha256`, from every +part's (issue 611); a run not split names it here. A development set has no ledger, its report being its cases': the step seals nothing. A gate set with no certificate pinned, or one that is not the one pinned, fails the run by a code, and nothing is uploaded but what was before. A @@ -23,7 +25,8 @@ import re import subprocess -from harness.eval import AGGREGATE, LEDGER, REPORT, Stop, _planned, _write, exit_with, gate_set, within +from harness.eval import AGGREGATE, LEDGER, PART, PART_SCHEMA, REPORT, Stop, _planned, _write, exit_with, gate_set, \ + ledger_digest, within # Each certificate the ledger is sealed to, a line of its sha256, two spaces and its path, as `sha256sum` writes one. RECIPIENTS = pathlib.Path(".github") / "ledger-recipients" @@ -101,7 +104,7 @@ def seal(plain: str, certs: list, out: str, call=subprocess.run) -> bytes: def step(env, root, workspace, call=subprocess.run): - _, found, place = _planned(env, root, workspace) + kept, found, place = _planned(env, root, workspace) # Nothing but what this step seals is uploaded as the ledger: a file or a link already there, from the checkout or # a step before, is refused, whatever the set. target = os.path.join(os.path.realpath(str(workspace)), SEALED) @@ -109,8 +112,10 @@ def step(env, root, workspace, call=subprocess.run): raise Stop("ledger_in_workspace") if not gate_set(found): return "sealed nothing: a development set has no ledger" + # A part of a split run binds its ledger in what it wrote, `PART`, as a whole run does in its report (issue 611). + named, schema = (REPORT, AGGREGATE) if kept.get("part") is None else (PART, PART_SCHEMA) try: - with open(within(workspace, REPORT), encoding="utf-8") as f: + with open(within(workspace, named), encoding="utf-8") as f: written = json.load(f) except (OSError, ValueError, RecursionError): raise Stop("report_unreadable") from None @@ -120,8 +125,9 @@ def step(env, root, workspace, call=subprocess.run): except OSError: raise Stop("ledger_unreadable") from None # The ledger sealed is the one the report hashed: the report binds it, and the ciphertext's hash binds the seal. - if not isinstance(written, dict) or written.get("schema") != AGGREGATE or not isinstance(written.get("ledger"), dict) \ - or written["ledger"].get("salted_sha256") != hashlib.sha256(data).hexdigest(): + entries = written.get("ledger") if isinstance(written, dict) and written.get("schema") == schema else None + if not (isinstance(entries, list) and len(entries) == 1 and isinstance(entries[0], dict) + and entries[0].get("salted_sha256") == hashlib.sha256(data).hexdigest()): raise Stop("ledger_not_the_reports") certs = recipients(root, place) # openssl reads a copy of the bytes the report hashed and writes into the scratch; only what reads back sealed whole @@ -132,13 +138,14 @@ def step(env, root, workspace, call=subprocess.run): sealed = seal(checked, certs, sealing, call) with open(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "wb") as out: out.write(sealed) - written["ledger"]["sealed_sha256"] = hashlib.sha256(sealed).hexdigest() - _write(workspace, REPORT, written) - # And as the job's output, `ledger_sha256`, where the runner names its file of outputs: a hash, which says nothing. + entries[0]["sealed_sha256"] = hashlib.sha256(sealed).hexdigest() + _write(workspace, named, written) + # And, for a whole run, as the job's output, `ledger_sha256`, where the runner names its file of outputs: a hash, + # which says nothing. A split run's is the merge job's, of every part (issue 611). output = env.get("GITHUB_OUTPUT") - if isinstance(output, str) and os.path.isabs(output): + if kept.get("part") is None and isinstance(output, str) and os.path.isabs(output): with open(output, "a", encoding="utf-8") as out: - out.write("ledger_sha256=%s\n" % written["ledger"]["sealed_sha256"]) + out.write("ledger_sha256=%s\n" % ledger_digest(entries)) for done in (within(place, LEDGER), checked, sealing): os.remove(done) return "sealed the ledger to %d recipients" % len(certs) diff --git a/schemas/eval-aggregate.schema.json b/schemas/eval-aggregate.schema.json index 98253a4f..bf8f4795 100644 --- a/schemas/eval-aggregate.schema.json +++ b/schemas/eval-aggregate.schema.json @@ -525,21 +525,29 @@ ] }, "ledger": { - "description": "The sha256 of the ledger, under its salt, and of its ciphertext, which the seal step writes: a report without both is not a sealed run's.", - "type": "object", - "additionalProperties": false, - "required": [ - "salted_sha256", - "sealed_sha256" - ], - "properties": { - "salted_sha256": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" - }, - "sealed_sha256": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" + "description": "Each part's ledger, the whole run's or, for a run split into parts, the preflight's and each shard's, in that order: the sha256 of the ledger under its salt, and of its ciphertext, which the seal step writes (issues 612 and 611). A report without both of each is not a sealed run's, and one with none passes no gate: a split run none of whose parts was read has none.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "part", + "salted_sha256", + "sealed_sha256" + ], + "properties": { + "part": { + "type": "string", + "pattern": "^(whole|preflight|[0-9]|1[0-5])$" + }, + "salted_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "sealed_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } } } } diff --git a/schemas/eval-part.schema.json b/schemas/eval-part.schema.json new file mode 100644 index 00000000..8063ea98 --- /dev/null +++ b/schemas/eval-part.schema.json @@ -0,0 +1,592 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "What one part of a split eval run wrote, for the merge job to make the one report of (issue 611)", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "part", + "release", + "commit", + "role", + "intent", + "set", + "attempt", + "asked", + "passed", + "kinds", + "blocking", + "origin", + "faults", + "preflight", + "usage", + "models", + "cases", + "ledger" + ], + "properties": { + "schema": { + "const": "eval-part/1" + }, + "part": { + "description": "The preflight, or a shard by its number from 0.", + "type": "string", + "pattern": "^(preflight|[0-9]|1[0-5])$" + }, + "release": { + "type": "string", + "pattern": "^factory/v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "role": { + "type": "string", + "pattern": "^[a-z][a-z-]*[a-z]$" + }, + "intent": { + "type": "string", + "pattern": "^[a-z][a-z-]*[a-z]$" + }, + "set": { + "type": "object", + "additionalProperties": false, + "required": [ + "count", + "sha256" + ], + "properties": { + "count": { + "type": "integer", + "minimum": 0 + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + "attempt": { + "description": "The attempt of the run a gate set's shard was drawn for, by which the merge refuses a shard of another; null for the preflight and a development set's.", + "type": [ + "string", + "null" + ], + "pattern": "^[1-9][0-9]{0,19}$" + }, + "asked": { + "description": "How many of the set's cases the part was given.", + "type": "integer", + "minimum": 0 + }, + "passed": { + "type": "integer", + "minimum": 0 + }, + "kinds": { + "description": "Of each kind, the part's cases and how many passed: counts alone, which the merge sums. Null where a case of the part was not read or its answer is no kind.", + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "finding", + "met" + ], + "properties": { + "finding": { + "type": "object", + "additionalProperties": false, + "required": [ + "cases", + "passed" + ], + "properties": { + "cases": { + "type": "integer", + "minimum": 0 + }, + "passed": { + "type": "integer", + "minimum": 0 + } + } + }, + "met": { + "type": "object", + "additionalProperties": false, + "required": [ + "cases", + "passed" + ], + "properties": { + "cases": { + "type": "integer", + "minimum": 0 + }, + "passed": { + "type": "integer", + "minimum": 0 + } + } + } + } + }, + "blocking": { + "description": "Of the part's cases that expect a blocking Finding, whether there is one and whether every one passed: two words, never how many (decision 0120). Null where `kinds` is.", + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "some", + "passed" + ], + "properties": { + "some": { + "type": "boolean" + }, + "passed": { + "type": "boolean" + } + } + }, + "origin": { + "description": "The cases the manifest marks natural and those it does not, each counted with how many passed.", + "type": "object", + "additionalProperties": false, + "required": [ + "natural", + "constructed" + ], + "properties": { + "natural": { + "type": "object", + "additionalProperties": false, + "required": [ + "cases", + "passed" + ], + "properties": { + "cases": { + "type": "integer", + "minimum": 0 + }, + "passed": { + "type": "integer", + "minimum": 0 + } + } + }, + "constructed": { + "type": "object", + "additionalProperties": false, + "required": [ + "cases", + "passed" + ], + "properties": { + "cases": { + "type": "integer", + "minimum": 0 + }, + "passed": { + "type": "integer", + "minimum": 0 + } + } + } + } + }, + "faults": { + "description": "Each code of a case not read, with how many cases gave it: never which.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "count" + ], + "properties": { + "code": { + "enum": [ + "schema", + "not_found", + "hash_differs", + "too_large", + "unanswered", + "not_this_set", + "facts_not_current", + "facts_read_short", + "diff_unreadable", + "diff_not_content", + "pusher_unnamed", + "diff_not_whole", + "issue_not_whole", + "issue_unreadable", + "credential_missing", + "engine_version", + "engine_model", + "engine_failed", + "no_answer", + "engine_timeout", + "task_too_large", + "rubric_unreadable", + "head_unreadable", + "answer_off_rubric", + "result_not_completed", + "finding_not_raised", + "finding_uncited", + "finding_cites_another", + "finding_elsewhere", + "category_differs", + "severity_differs", + "location_differs", + "coverage_disagrees", + "finding_raised_wrongly", + "finding_not_in_hunk", + "surplus_at_place", + "surplus_gating_elsewhere", + "criterion_not_met", + "answer_disagrees_with_verdict", + "location_unreadable", + "criterion_unknown" + ] + }, + "count": { + "type": "integer", + "minimum": 1 + } + } + } + }, + "preflight": { + "description": "The development cases the dispatch asked first, its set's `preflight`, scored apart and counted in nothing else here: how many there were, how many passed, and each code of one not read (issue 613). One stopped by the Engine or the run, or none passed, stopped the dispatch before any of the set's cases was asked, and each of those is then `unanswered`.", + "type": "object", + "additionalProperties": false, + "required": [ + "cases", + "passed", + "faults" + ], + "properties": { + "cases": { + "type": "integer", + "minimum": 0 + }, + "passed": { + "type": "integer", + "minimum": 0 + }, + "faults": { + "description": "Each code of a preflight case not read, with how many cases gave it: never which.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "count" + ], + "properties": { + "code": { + "enum": [ + "schema", + "not_found", + "hash_differs", + "too_large", + "unanswered", + "not_this_set", + "facts_not_current", + "facts_read_short", + "diff_unreadable", + "diff_not_content", + "pusher_unnamed", + "diff_not_whole", + "issue_not_whole", + "issue_unreadable", + "credential_missing", + "engine_version", + "engine_model", + "engine_failed", + "no_answer", + "engine_timeout", + "task_too_large", + "rubric_unreadable", + "head_unreadable", + "answer_off_rubric", + "result_not_completed", + "finding_not_raised", + "finding_uncited", + "finding_cites_another", + "finding_elsewhere", + "category_differs", + "severity_differs", + "location_differs", + "coverage_disagrees", + "finding_raised_wrongly", + "finding_not_in_hunk", + "surplus_at_place", + "surplus_gating_elsewhere", + "criterion_not_met", + "answer_disagrees_with_verdict", + "location_unreadable", + "criterion_unknown" + ] + }, + "count": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + }, + "usage": { + "description": "What the Results spent, summed over those whose usage was whole, and how many those were.", + "type": "object", + "additionalProperties": false, + "required": [ + "answered", + "input_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "output_tokens", + "turns", + "cost_usd", + "seconds" + ], + "properties": { + "answered": { + "type": "integer", + "minimum": 0 + }, + "input_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_creation_input_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_read_input_tokens": { + "type": "integer", + "minimum": 0 + }, + "output_tokens": { + "type": "integer", + "minimum": 0 + }, + "turns": { + "type": "integer", + "minimum": 0 + }, + "cost_usd": { + "type": "number", + "minimum": 0 + }, + "seconds": { + "type": "number", + "minimum": 0 + } + } + }, + "models": { + "description": "Each model the CLI said answered a case, by the id it gave, not the configured name (issue 616): one where the run is whole.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z0-9.-]{1,64}$" + } + }, + "cases": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "sha256", + "verdict", + "reasons", + "usage" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "verdict": { + "enum": [ + "pass", + "fail", + "unreadable" + ] + }, + "reasons": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "schema", + "not_found", + "hash_differs", + "too_large", + "unanswered", + "not_this_set", + "facts_not_current", + "facts_read_short", + "diff_unreadable", + "diff_not_content", + "pusher_unnamed", + "diff_not_whole", + "issue_not_whole", + "issue_unreadable", + "credential_missing", + "engine_version", + "engine_model", + "engine_failed", + "no_answer", + "engine_timeout", + "task_too_large", + "rubric_unreadable", + "head_unreadable", + "answer_off_rubric", + "result_not_completed", + "finding_not_raised", + "finding_uncited", + "finding_cites_another", + "finding_elsewhere", + "category_differs", + "severity_differs", + "location_differs", + "coverage_disagrees", + "finding_raised_wrongly", + "finding_not_in_hunk", + "surplus_at_place", + "surplus_gating_elsewhere", + "criterion_not_met", + "answer_disagrees_with_verdict", + "location_unreadable", + "criterion_unknown" + ] + } + }, + "usage": { + "description": "What the case used, from the CLI's own answer: turns and cost in USD, null where the CLI reported none or was never asked, and seconds, 0 where it was never asked (issue 616). Numbers alone.", + "type": "object", + "additionalProperties": false, + "required": [ + "turns", + "cost_usd", + "seconds" + ], + "properties": { + "turns": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "cost_usd": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "seconds": { + "type": "number", + "minimum": 0 + } + } + }, + "at_alternate": { + "description": "A pass whose Finding was at a place of the case's also_at alone, not its location (decision 0143): a mark the report can count, saying no place.", + "type": "boolean", + "const": true + }, + "cause": { + "description": "Why a case was unanswered or stopped by the Engine, where the job found it: an exception's class and the last line of this repo's code it passed, a step's code, the CLI's subtype and exit, or that it timed out. Never a message.", + "type": "object", + "additionalProperties": false, + "properties": { + "exception": { + "type": "string", + "pattern": "^[A-Z][A-Za-z0-9_]{0,63}$" + }, + "at": { + "type": "string", + "pattern": "^(adapters|controller|harness|kernel|release|tools)/[a-z0-9_/]+\\.py:[1-9][0-9]{0,5}$" + }, + "stop": { + "type": "string", + "pattern": "^[a-z][a-z_]{0,63}$" + }, + "subtype": { + "type": "string", + "pattern": "^[a-z][a-z_]{0,63}$" + }, + "exit": { + "type": "integer", + "minimum": -255, + "maximum": 255 + }, + "timed_out": { + "const": true + } + } + }, + "surplus_gating": { + "description": "On a case that expects a Finding, how many gating Findings the Result put away from every place the case gives (issue 610): a count the report can sum, saying no place.", + "type": "integer", + "minimum": 1 + } + } + }, + "description": "A development set's cases of the part, as its report lists them; none for a gate set, whose part holds counts alone." + }, + "ledger": { + "description": "A gate set's part's ledger: the sha256 of it under its salt, and of its ciphertext, which the seal step writes. None for a development set.", + "type": "array", + "maxItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "part", + "salted_sha256", + "sealed_sha256" + ], + "properties": { + "part": { + "type": "string", + "pattern": "^(preflight|[0-9]|1[0-5])$" + }, + "salted_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "sealed_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + } + } +} diff --git a/schemas/harness.schema.json b/schemas/harness.schema.json index 30882490..224ae194 100644 --- a/schemas/harness.schema.json +++ b/schemas/harness.schema.json @@ -143,6 +143,12 @@ "pattern": "^[a-z0-9][a-z0-9-]*$" } }, + "eval_shards": { + "description": "How many shards the eval job's run is split into, each a job of its own with its own time limit, behind one preflight job (issue 611).", + "type": "integer", + "minimum": 1, + "maximum": 16 + }, "gate_set": { "description": "The gate set the Release's eval gate reads, by its name under evals/: every dispatch of the eval job on the commit is pooled (issue 613). Absent, the gate reads the review Intent's set, evals/pr-review, by its newest successful dispatch.", "type": "string", diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 7a2012ce..e6b29bb0 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -3565,6 +3565,20 @@ def send(self, req): self.assertEqual(heldout.case(Anything(), "o", "home", "pr-review", entry), (self.ONE, None)) self.assertEqual(asked, ["/repos/o/home/contents/pr-review/c-1.json"]) + def test_a_gate_sets_salt_is_read_raw_from_the_home_and_none_short_of_it(self): # issue 611 + # The salt a gate set's shards are drawn by is `/.salt` in the home, read raw, 16 to 256 bytes. + salt = b"s" * 16 + read = lambda found, role="pr-review", name="pr-review-gate": heldout.salt(github.Replay(found), "o", "home", role, name) + self.assertEqual(read([home_beside("pr-review-gate", "salt", salt)]), salt) + self.assertEqual(read([home_beside("pr-review-gate", "salt", b"s" * 256)]), b"s" * 256) + for found in ([home_beside("pr-review-gate", "salt", b"s" * 15)], [home_beside("pr-review-gate", "salt", b"s" * 257)], + [home_beside("pr-review-gate", "salt", salt, status=404)], [], + [home_beside("pr-review-gate", "salt", salt, **{"content-type": "application/json"})]): + self.assertIsNone(read(found)) + for role, name in (("pr-review/..", "pr-review-gate"), ("pr-review", "../x"), (None, "pr-review-gate")): + self.assertIsNone(read([home_beside("pr-review-gate", "salt", salt)], role, name), (role, name)) + self.assertIsNone(heldout.salt(github.Replay([]), "o", "..", "pr-review", "pr-review-gate")) + def test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins(self): # issue 611 # A head's tarball and its tree, beside the case, are asked with the raw media type, whose answer is the file's # bytes whole however large (trials/P-62.json), and handed back only as the bytes the case pinned. diff --git a/tests/test_evals.py b/tests/test_evals.py index d23f21e9..6906147b 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -734,7 +734,7 @@ def test_the_eval_jobs_name_is_reserved(self): problems = check_workflows.run(clone({".github/workflows/other.yml": by_name})) self.assertTrue(caught(problems, ".github/workflows/other.yml: the job eval is reserved for eval.yml")) problems = check_workflows.run(clone({".github/workflows/eval.yml": GOOD})) - self.assertTrue(caught(problems, ".github/workflows/eval.yml: holds the job eval and no other")) + self.assertTrue(caught(problems, ".github/workflows/eval.yml: holds the jobs preflight, eval, merge and no other")) def test_the_eval_workflow_names_each_credential_in_one_step_alone(self): # Issue 309, ruling 10: dispatched alone with the Release it judges; one job on factory-eval, granted @@ -751,26 +751,26 @@ def test_the_eval_workflow_names_each_credential_in_one_step_alone(self): EVAL_WORKFLOW.index(" - run: python3 -m harness.eval.fetch\n")] model = " CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}\n" broken = { - "the token beside the model": (EVAL_WORKFLOW.replace(token, token + model), "its fetch step is given under env"), - "the model beside the token": (EVAL_WORKFLOW.replace(model, model + token), "its run step is given under env"), + "the token beside the model": (EVAL_WORKFLOW.replace(token, token + model), "\'s fetch step is given under env"), + "the model beside the token": (EVAL_WORKFLOW.replace(model, model + token), "\'s run step is given under env"), "the token handed to the plan": (EVAL_WORKFLOW.replace(" RELEASE: ${{ inputs.release }}\n", " RELEASE: ${{ inputs.release }}\n" + token), - "its plan step is given under env"), + "\'s plan step is given under env"), "the token given to the upload": (EVAL_WORKFLOW.replace(" retention-days: 90\n", " retention-days: 90\n token: ${{ secrets.HELDOUT_TOKEN }}\n"), - "its upload step is given under with"), + "\'s upload step is given under with"), "the token under another name": (EVAL_WORKFLOW.replace("${{ secrets.HELDOUT_TOKEN }}", "${{ secrets.HELDOUT_PAT }}"), - "its fetch step is given under env"), + "\'s fetch step is given under env"), "a token minted beside it": (EVAL_WORKFLOW.replace(" - run: python3 -m harness.eval.fetch\n", " - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0\n" " - run: python3 -m harness.eval.fetch\n"), "in that order"), "the token on the job": (EVAL_WORKFLOW.replace(" environment: factory-eval\n", " environment: factory-eval\n env:\n" + token[4:]), - "its job has exactly"), + "has exactly"), "a shorter timeout": (EVAL_WORKFLOW.replace("timeout-minutes: 120", "timeout-minutes: 20"), "with a timeout of 120 minutes"), "another runner": (EVAL_WORKFLOW.replace("runs-on: ubuntu-24.04", "runs-on: ubuntu-24.04-big"), - "its job runs on ubuntu-24.04"), + "runs on ubuntu-24.04"), "another environment": (EVAL_WORKFLOW.replace("environment: factory-eval", "environment: factory-run"), "runs on the environment factory-eval"), "a write": (EVAL_WORKFLOW.replace(" contents: read\n", " contents: read\n actions: read\n"), @@ -790,29 +790,51 @@ def test_the_eval_workflow_names_each_credential_in_one_step_alone(self): "no set": (EVAL_WORKFLOW.replace(" set:\n required: false\n type: string\n" " default: pr-review\n", ""), "with the inputs release"), "the set kept from the plan": (EVAL_WORKFLOW.replace(" EVAL_SET: ${{ inputs.set }}\n", ""), - "its plan step is given under env"), + "\'s plan step is given under env"), # The seal, given nothing, after the run and before the uploads; the ledger's upload, its ciphertext alone. - "no seal": (EVAL_WORKFLOW.replace(" - run: python3 -m harness.eval.seal\n id: seal\n timeout-minutes: 1\n", ""), + "no seal": (EVAL_WORKFLOW.replace(" - run: python3 -m harness.eval.seal\n timeout-minutes: 1\n", ""), "in that order"), "the seal given the model's credential": (EVAL_WORKFLOW.replace( " - run: python3 -m harness.eval.seal\n", " - run: python3 -m harness.eval.seal\n env:\n" + model), - "its seal step is given under env"), + "\'s seal step is given under env"), "the scratch uploaded as the ledger": (EVAL_WORKFLOW.replace("path: eval-ledger.cms", "path: ${{ runner.temp }}"), - "its ledger step is given under with"), + "\'s ledger step is given under with"), "a development set's run failed for no ledger": (EVAL_WORKFLOW.replace("if-no-files-found: ignore", "if-no-files-found: error"), - "its ledger step is given under with"), + "\'s ledger step is given under with"), # The job's one output, the sealed ledger's hash, which the seal step alone sets, under the one id a step may take. - "no output": (EVAL_WORKFLOW.replace(" outputs:\n ledger_sha256: ${{ steps.seal.outputs.ledger_sha256 }}\n", ""), - "its job has exactly"), - "an output of the fetch": (EVAL_WORKFLOW.replace("steps.seal.outputs.ledger_sha256", "steps.home.outputs.token"), - "its job has one output"), - "a second output": (EVAL_WORKFLOW.replace(" ledger_sha256: ${{ steps.seal.outputs.ledger_sha256 }}\n", - " ledger_sha256: ${{ steps.seal.outputs.ledger_sha256 }}\n" - " token: ${{ secrets.HELDOUT_TOKEN }}\n"), "its job has one output"), - "the seal under another id": (EVAL_WORKFLOW.replace(" id: seal\n", " id: sealing\n"), - "its seal step has the id seal"), - "the seal with no id": (EVAL_WORKFLOW.replace(" id: seal\n", ""), "its seal step has the id seal"), + "no output": (EVAL_WORKFLOW.replace(" outputs:\n ledger_sha256: ${{ steps.merge.outputs.ledger_sha256 }}\n", ""), + "has exactly"), + "an output of the fetch": (EVAL_WORKFLOW.replace("steps.merge.outputs.ledger_sha256", "steps.home.outputs.token"), + "has one output"), + "a second output": (EVAL_WORKFLOW.replace(" ledger_sha256: ${{ steps.merge.outputs.ledger_sha256 }}\n", + " ledger_sha256: ${{ steps.merge.outputs.ledger_sha256 }}\n" + " token: ${{ secrets.HELDOUT_TOKEN }}\n"), "has one output"), + "the merge under another id": (EVAL_WORKFLOW.replace(" id: merge\n", " id: merging\n"), + "'s merge step has the id merge"), + "the merge with no id": (EVAL_WORKFLOW.replace(" id: merge\n", ""), "'s merge step has the id merge"), + # The parts (issue 611): the shards need the preflight, the merge needs both and runs whatever became of + # them, the shards are the configuration's, and the merge holds no credential. + "shards that need no preflight": (EVAL_WORKFLOW.replace(" needs: preflight\n", ""), "has exactly"), + "a merge skipped on a failure": (EVAL_WORKFLOW.replace(" if: always()\n", " if: success()\n"), "if: always()"), + "a merge that needs no shard": (EVAL_WORKFLOW.replace(" - preflight\n - eval\n", " - preflight\n"), + "its job merge needs preflight, eval"), + "a shard the configuration does not name": (EVAL_WORKFLOW.replace(" - 3\n", " - 3\n - 4\n"), + "a matrix of the shards"), + "a shard's failure that cancels the rest": (EVAL_WORKFLOW.replace("fail-fast: false", "fail-fast: true"), + "a matrix of the shards"), + "the merge holding the home's token": (EVAL_WORKFLOW.replace(" EVAL_PART: merge\n", " EVAL_PART: merge\n" + token), + "'s plan step is given under env"), + "the merge on the eval environment": (EVAL_WORKFLOW.replace(" outputs:\n", " environment: factory-eval\n outputs:\n"), + "its job merge has exactly"), + "a shard planned as the preflight": (EVAL_WORKFLOW.replace(" EVAL_PART: ${{ matrix.shard }}\n", " EVAL_PART: preflight\n"), + "'s plan step is given under env"), + "one shard's part named for all": (EVAL_WORKFLOW.replace("name: eval-part-${{ matrix.shard }}", "name: eval-part"), + "'s upload step is given under with"), + "the merge's own limit": (EVAL_WORKFLOW.replace(" timeout-minutes: 10\n", " timeout-minutes: 120\n"), + "its job merge runs on ubuntu-24.04, with a timeout of 10 minutes"), + "a download of every artifact": (EVAL_WORKFLOW.replace("pattern: eval-part-*", "pattern: eval-*"), + "'s download step is given under with"), "the report under the ledger's name": (EVAL_WORKFLOW.replace("name: eval-report", "name: eval-ledger"), "in that order"), "the fetch first": (EVAL_WORKFLOW.replace(" - run: python3 -m harness.eval.plan\n env:\n" @@ -827,20 +849,20 @@ def test_the_eval_workflow_names_each_credential_in_one_step_alone(self): " - run: make verify\n"), "in that order"), "a step skipped": (EVAL_WORKFLOW.replace(" - run: python3 -m harness.eval.judge\n", " - run: python3 -m harness.eval.judge\n if: false\n"), - "its judge step has a key it may not"), + "\'s judge step has a key it may not"), "an id on fetch": (EVAL_WORKFLOW.replace(" - run: python3 -m harness.eval.fetch\n", " - run: python3 -m harness.eval.fetch\n id: home\n"), - "its fetch step has a key it may not"), + "\'s fetch step has a key it may not"), "the whole workspace uploaded": (EVAL_WORKFLOW.replace("path: eval-report.json", "path: ."), - "its upload step is given under with"), + "\'s upload step is given under with"), "a credential left on disk": (EVAL_WORKFLOW.replace("persist-credentials: false", "persist-credentials: true"), - "its checkout step is given under with"), + "\'s checkout step is given under with"), "an upload of another commit": (EVAL_WORKFLOW.replace("upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9", "upload-artifact@" + "0" * 40), "in that order"), "an upload of its own": (EVAL_WORKFLOW.replace("uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9" " # v7.0.2", "run: python3 -m harness.eval.upload"), "in that order"), "the home as a variable": (EVAL_WORKFLOW.replace("${{ secrets.HELDOUT_OWNER }}", "${{ vars.HELDOUT_OWNER }}"), - "its fetch step is given under env"), + "\'s fetch step is given under env"), "a credential for every step": (EVAL_WORKFLOW.replace("permissions: {}\n", "permissions: {}\nenv:\n" " KEY: ${{ secrets.HELDOUT_TOKEN }}\n"), "nothing else above its job"), @@ -849,21 +871,21 @@ def test_the_eval_workflow_names_each_credential_in_one_step_alone(self): "another name": (EVAL_WORKFLOW.replace("name: factory\n", "name: eval\n"), "nothing else above its job"), "a grant above the job": (EVAL_WORKFLOW.replace("permissions: {}\n", "permissions:\n contents: read\n"), "nothing else above its job"), - "an empty env on the judge": (EVAL_WORKFLOW + " env:\n", "its judge step has a env: it is not given"), + "an empty env on the judge": (EVAL_WORKFLOW + " env:\n", "\'s judge step has a env: it is not given"), # The CLI's install, before the fetch, at the Engine's pin by its checksum, word for # word as the Run's jobs install it (issue 309, amendment 1). # Bubblewrap is in the block, and the block comes before the fetch: it runs as root, before the cases are here. "no bubblewrap": (EVAL_WORKFLOW.replace(" sudo apt-get install -y --no-install-recommends bubblewrap\n", ""), - "its steps are"), + "'s steps are"), "the install after the fetch": (EVAL_WORKFLOW.replace(FETCH_STEP, "").replace( - " - env:\n CLI_VERSION", FETCH_STEP + " - env:\n CLI_VERSION"), "its steps are"), + " - env:\n CLI_VERSION", FETCH_STEP + " - env:\n CLI_VERSION"), "'s steps are"), "the CLI at another version": (EVAL_WORKFLOW.replace('CLI_VERSION: "2.1.306"', 'CLI_VERSION: "2.1.305"'), - "its install step is given under env"), + "\'s install step is given under env"), "the CLI by another checksum": (EVAL_WORKFLOW.replace('CLI_CHECKSUM: "520bc6', 'CLI_CHECKSUM: "000000'), - "its install step is given under env"), + "\'s install step is given under env"), "the model's credential at the install": (EVAL_WORKFLOW.replace(' CLI_VERSION: "2.1.306"\n', ' CLI_VERSION: "2.1.306"\n' + model), - "its install step is given under env"), + "\'s install step is given under env"), "the CLI's install a module of its own": (EVAL_WORKFLOW.replace( install[install.index(" run: |"):], " run: python3 -m harness.eval.install\n"), "in that order"), "the CLI unchecked": (EVAL_WORKFLOW.replace(' echo "${CLI_CHECKSUM} /tmp/claude" | sha256sum -c\n', ""), @@ -873,10 +895,10 @@ def test_the_eval_workflow_names_each_credential_in_one_step_alone(self): "the CLI after the model": (EVAL_WORKFLOW.replace(install, "").replace( " - uses: actions/upload-artifact", install + " - uses: actions/upload-artifact"), "in that order"), "the CLI from another path": (EVAL_WORKFLOW.replace("CLAUDE_CLI: /usr/local/bin/claude", "CLAUDE_CLI: /tmp/claude"), - "its run step is given under env"), + "\'s run step is given under env"), "a second job": (EVAL_WORKFLOW + " other:\n runs-on: ubuntu-24.04\n timeout-minutes: 5\n" " environment: factory-poller\n steps:\n - run: make verify\n", - "it holds the job eval and no other"), + "it holds the jobs preflight, eval, merge"), } for why, (text, words) in broken.items(): self.assertNotEqual(text, EVAL_WORKFLOW, why) @@ -904,7 +926,7 @@ def test_the_eval_workflow_names_each_credential_in_one_step_alone(self): self.assertTrue(caught(problems, "names the model's credential, which only"), problems) problems = check_workflows.run(clone({".github/workflows/eval.yml": EVAL_WORKFLOW.replace( " - run: python3 -m harness.eval.fetch\n", " - run: python3 -m harness.eval.fetch\n id: x\n")})) - self.assertTrue(caught(problems, "eval.yml: its fetch step"), problems) + self.assertTrue(caught(problems, "eval.yml: its job preflight's fetch step"), problems) # The Maintainer's token, in another workflow under its one name, in any case, is the eval job's alone. other = GOOD.replace(" - run: make verify\n", " - run: make verify\n env:\n" " T: ${{ secrets.Heldout_Token }}\n", 1) @@ -2964,7 +2986,7 @@ def test_a_gate_sets_report_holds_counts_alone_and_its_ledger_every_case(self): self.assertEqual([(c["id"], c["natural"], c["verdict"], c["result"], c["usage"]) for c in ledger["cases"]], [(list(self.cases)[0], True, "pass", result, job.used(result["usage"])), ("poc-two", False, "pass", result, job.used(result["usage"]))]) - self.assertEqual(written["ledger"]["salted_sha256"], hashlib.sha256(kept.read_bytes()).hexdigest()) + self.assertEqual(written["ledger"], [{"part": "whole", "salted_sha256": hashlib.sha256(kept.read_bytes()).hexdigest()}]) found = job.manifest(self.root, "pr-review-gate")[0] salts = {job.ledger_of(dict(ledger, preflight=[]), found, {c["id"]: ("pass", []) for c in found["cases"]}, {}, set(), {})["salt"] for _ in range(2)} | {ledger["salt"]} @@ -3024,9 +3046,11 @@ def test_the_seal_encrypts_the_ledger_to_each_pinned_certificate_alone(self): # # The plaintext is gone, and the report binds the ciphertext, whole, for the gate. self.assertEqual([p.name for p in (self.temp / "heldout").iterdir() if "ledger" in p.name], []) written = json.loads((self.work / job.REPORT).read_text()) - self.assertEqual(written["ledger"]["sealed_sha256"], hashlib.sha256(sealed.read_bytes()).hexdigest()) + self.assertEqual(written["ledger"][0]["sealed_sha256"], hashlib.sha256(sealed.read_bytes()).hexdigest()) # The job's output is the same hash, after what the runner's file held. - self.assertEqual(outputs.read_text(), "earlier=1\nledger_sha256=%s\n" % written["ledger"]["sealed_sha256"]) + self.assertEqual(outputs.read_text(), "earlier=1\nledger_sha256=%s\n" % job.ledger_digest(written["ledger"])) + self.assertEqual(job.ledger_digest(written["ledger"]), + hashlib.sha256(('["%s"]' % written["ledger"][0]["sealed_sha256"]).encode()).hexdigest()) self.assertEqual(validate(written, load("schemas/eval-aggregate.schema.json"), schemas=eval_result.schemas()), []) self.assertEqual(self.step(job.judge), "2 of 2 passed; no code; preflight 3 of 3 passed") # A development set has no ledger, and the step seals nothing; the gate set of its cases is gone, as none may share one. @@ -3106,7 +3130,7 @@ def call(args, **given): self.assertTrue((place / job.LEDGER).exists()) # The ledger sealed is the one the report hashed, of a gate set's report. written = json.loads((self.work / job.REPORT).read_text()) - for report in (dict(written, ledger={"salted_sha256": "0" * 64}), dict(written, schema="eval-report/1"), []): + for report in (dict(written, ledger=[{"part": "whole", "salted_sha256": "0" * 64}]), dict(written, schema="eval-report/1"), []): (self.work / job.REPORT).write_text(json.dumps(report)) self.assertEqual(sealed_with(), "ledger_not_the_reports") (self.work / job.REPORT).write_text(json.dumps(written)) @@ -3153,7 +3177,7 @@ def aggregate_of(self, finding=(20, 6), met=(20, 18), natural=(30, 18), faults=( "engine": job.engine_of("pr-review"), "models": ["claude-opus"], "bars": {"finding": [3, 10], "met": [9, 10]}, "kind_least": 20, "split_ok": True, "kinds_pass": {"finding": job._clears(finding[1], finding[0], (3, 10)), "met": job._clears(met[1], met[0], (9, 10))}, - "blocking_pass": True, "ledger": {"salted_sha256": "e" * 64, "sealed_sha256": "f" * 64}} + "blocking_pass": True, "ledger": [{"part": "whole", "salted_sha256": "e" * 64, "sealed_sha256": "f" * 64}]} return found, "d" * 64, report def test_the_gate_passes_a_good_aggregate_report_and_refuses_a_bad_one(self): # issue 612 @@ -3187,7 +3211,12 @@ def changed(path, value): for report, code in ( # A gate set's report is of counts alone, and a development set's is per case: neither stands for the other. (loaded("examples/eval-report.json"), "report_of_another_kind"), - (changed(["ledger", "sealed_sha256"], KeyError), "report_unreadable"), + (changed(["ledger", 0, "sealed_sha256"], KeyError), "report_unreadable"), + (changed(["ledger"], []), "count_differs"), + # A part's ledger named twice, or a whole run's beside a part's, is no run's (issue 611). + (changed(["ledger"], good["ledger"] * 2), "count_differs"), + (changed(["ledger"], [dict(good["ledger"][0], part="0"), dict(good["ledger"][0], part="0")]), "count_differs"), + (changed(["ledger"], good["ledger"] + [dict(good["ledger"][0], part="0")]), "count_differs"), (changed(["models"], KeyError), "report_unreadable"), (changed(["cases"], [{"id": "c-0", "verdict": "pass"}]), "report_unreadable"), (changed(["faults"], [{"code": "case c-0 timed out", "count": 1}]), "report_unreadable"), @@ -3498,6 +3527,193 @@ def test_a_gate_sets_preflight_is_of_the_development_set_and_no_gate_sets_case(s self.assertEqual(check_examples._preflight_held(self.root, manifest), ["its preflight is refused where it is read: eval_set_overlaps_a_gate_set"]) + def part_run(self, part: str, home, review, run=("4242", "1"), certs: bool = False) -> tuple: + """One job of a split run, on a runner of its own: its scratch, its heads and its workspace, planned, fetched and + scored as the part `part`, and sealed where `certs`; `(what each step said, its workspace)` (issue 611).""" + from harness import eval as job + from harness.eval import seal + base = pathlib.Path(self.dir.name) / ("job-%s-%s" % (part, run[1])) + shutil.rmtree(str(base), ignore_errors=True) + (base / "temp").mkdir(parents=True) + (base / "work").mkdir() + env = dict(self.env, RUNNER_TEMP=str(base / "temp"), EVAL_PART=part, GITHUB_RUN_ID=run[0], GITHUB_RUN_ATTEMPT=run[1]) + said = [] + for step, given in ((job.plan, ()), (the_fetch(), (home,)), (job.score, (review,))) + (((seal.step, ()),) if certs else ()): + try: + said.append(step(env, self.root, base / "work", *given)) + except job.Stop as stop: + said.append(stop.code) + break + return said, base / "work" + + def merged_run(self, parts: dict, attempt: str = "1", flat: bool = False) -> tuple: + """The merge job of a split run, given each part's workspace by its name: `(what it said, the report, its outputs)`.""" + from harness import eval as job + base = pathlib.Path(self.dir.name) / "job-merge" + shutil.rmtree(str(base), ignore_errors=True) + (base / "temp").mkdir(parents=True) + (base / "work" / "parts").mkdir(parents=True) + for name, work in parts.items(): + # The download puts each part in a folder of its own name, or one alone where it is put. + folder = base / "work" / "parts" if flat else base / "work" / "parts" / ("eval-part-%s" % name) + folder.mkdir(exist_ok=True) + shutil.copy(str(work / job.PART), str(folder / job.PART)) + env = dict(self.env, RUNNER_TEMP=str(base / "temp"), EVAL_PART="merge", GITHUB_OUTPUT=str(base / "outputs"), + GITHUB_RUN_ATTEMPT=attempt) + (base / "outputs").write_text("") + job.plan(env, self.root, base / "work") + said = job.merge(env, self.root, base / "work") + return said, json.loads((base / "work" / job.REPORT).read_text()), (base / "outputs").read_text() + + def shards(self, count: int) -> None: + (self.root / "config").mkdir(exist_ok=True) + (self.root / "config" / "harness.json").write_text(json.dumps({"eval_shards": count})) + + def test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole(self): # issue 611 + # A development set's case k is the shard k modulo the shards; the preflight's job asks nothing, as the set has + # no preflight; and the merge job's report is the one the whole run would write, its words said from the + # shards' counts. No step's line names a case. + from adapters import github + from harness import eval as job + cases = {"poc-k%d" % i: (json.dumps(dict(json.loads(next(iter(self.cases.values()))), id="poc-k%d" % i), indent=2) + + "\n").encode() for i in range(5)} + self.set(cases) + self.shards(2) + result = example_result() + review = lambda case: result + ids = list(cases) + parts = {} + for part, mine in (("preflight", []), ("0", ids[0::2]), ("1", ids[1::2])): + said, parts[part] = self.part_run(part, github.Replay(home_of({i: cases[i] for i in mine})), review) + self.assertEqual(said[1], "fetched %d of %d cases%s" % (len(mine), len(mine), "" if part == "preflight" else " of shard " + part)) + for line in said: + for ident in ids: + self.assertNotIn(ident, line, (part, line)) + said, written, outputs = self.merged_run(parts) + self.assertEqual(said, "merged 3 of 3 parts, 5 passed") + self.assertEqual(outputs, "", "a development set has no ledger to name") + # The merge holds the split to a blocking case at least, by the parts' words, as `split_ok` holds the whole set. + kept = json.loads((pathlib.Path(self.dir.name) / "job-merge" / "temp" / "heldout" / "plan.json").read_text()) + read = {p: json.loads((w / job.PART).read_text()) for p, w in parts.items()} + none = {p: dict(d, blocking={"some": False, "passed": True}) for p, d in read.items()} + one = dict(none, **{"1": dict(none["1"], blocking={"some": True, "passed": False})}) + with mock.patch.object(job, "BLOCKING_LEAST", 1): + self.assertEqual([job.merged(kept, job.manifest(self.root)[0], r, self.root)["split_ok"] for r in (none, one)], [False, True]) + self.assertEqual(job.merged(kept, job.manifest(self.root)[0], one, self.root)["blocking_pass"], False) + # The whole run's report, from one job. + self.step(job.plan) + self.step(the_fetch(), self.home(cases)) + self.step(job.score, review) + whole = json.loads((self.work / job.REPORT).read_text()) + self.assertEqual(written, whole) + self.assertEqual(job.verdict(written, job.manifest(self.root)[0], json.loads(self.path().read_text()) and + job.manifest(self.root)[1], self.env["GITHUB_SHA"], self.env["RELEASE"]), "pass") + # A shard that left nothing leaves its cases unanswered, a fault of the run, and no kind can be read. + said, short, _ = self.merged_run({"preflight": parts["preflight"], "0": parts["0"]}) + self.assertEqual(said, "merged 2 of 3 parts, 3 passed") + self.assertEqual({c["id"]: c["reasons"] for c in short["cases"]}, + {i: [] if i in ids[0::2] else ["unanswered"] for i in ids}) + self.assertEqual((short["split_ok"], short["kinds_pass"], short["blocking_pass"]), (None, None, None)) + self.assertEqual(job.verdict(short, *job.manifest(self.root), self.env["GITHUB_SHA"], self.env["RELEASE"]), "run_fault") + # So does one that is not this run's part, of another commit, or one that holds a ledger. + kept = json.loads((parts["1"] / job.PART).read_text()) + for change in ({"part": "0"}, {"commit": "b" * 40}, + {"ledger": [{"part": "1", "salted_sha256": "e" * 64, "sealed_sha256": "f" * 64}]}): + (parts["1"] / job.PART).write_text(json.dumps(dict(kept, **change))) + self.assertEqual(self.merged_run(parts)[0], "merged 2 of 3 parts, 3 passed", change) + # The parts are the plan's, its shards bounded as the configuration bounds them: a count past it is no plan's. + self.assertEqual(job.part_names({"shards": 2}), ["preflight", "0", "1"]) + for shards in (0, 17, "2", True, None): + with self.assertRaises(job.Stop): + job.part_names({"shards": shards}) + # A part the configuration does not name, or the merge's own part asked to fetch, is refused. + for part in ("2", "-1", "", "whole"): + self.assertEqual(self.part_run(part, github.Replay([]), review)[0], ["part_unreadable"], part) + self.assertEqual(self.part_run("merge", github.Replay([]), review)[0][1], "part_unreadable") + + def test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone(self): # issue 611 + # A gate set's case goes to the shard the sha256 of the home's salt, the run's id and attempt and its id give, + # never by its place in the public manifest; the preflight's job asks the preflight alone and the shards need + # it; each part writes counts alone and seals a ledger of its own; the merge's report holds each part's ledger, + # the preflight's first, and the job's one output hashes their list. No part's line or file names a case. + from adapters import github + from harness import eval as job + from tests.test_adapters import home_beside + keys = pathlib.Path(self.dir.name) / "keys" + keys.mkdir() + (first, _), = [recipient(keys, "first")] + pin(self.root, {"first": first}) + cases = {"poc-g%d" % i: (json.dumps(dict(json.loads(next(iter(self.cases.values()))), id="poc-g%d" % i), indent=2) + + "\n").encode() for i in range(6)} + self.set(cases, gate=True, natural=[]) + self.shards(2) + ids = list(cases) + drawn = lambda salt, run, attempt: { + i for i in ids if int(hashlib.sha256(b"\0".join([salt, run.encode(), attempt.encode(), i.encode()])).hexdigest(), 16) % 2 == 0} + # A salt whose shard 0 is not the cases of even place, so the place in the manifest could not have chosen it. + salt = next(s for s in (b"salt-%02d-of-the-gate-set" % n for n in range(99)) + if drawn(s, "4242", "1") not in (set(ids[0::2]), set(ids[1::2]), set(), set(ids))) + result = example_result() + review = lambda case: result + salted = home_beside("pr-review-gate", "salt", salt) + parts = {} + said, parts["preflight"] = self.part_run("preflight", github.Replay(home_of(self.pre)), review, certs=True) + self.assertEqual(said[1:3], ["fetched 0 of 0 cases, 3 of 3 to preflight", "scored 3 cases of the part preflight, 0 passed"]) + for part in ("0", "1"): + mine = drawn(salt, "4242", "1") if part == "0" else set(ids) - drawn(salt, "4242", "1") + home = github.Replay([salted] + home_of({i: cases[i] for i in ids if i in mine})) + said, parts[part] = self.part_run(part, home, review, certs=True) + self.assertEqual(said[1], "fetched %d of %d cases of shard %s" % (len(mine), len(mine), part)) + self.assertEqual(said[3], "sealed the ledger to 1 recipients") + self.assertEqual(set(json.loads((pathlib.Path(self.dir.name) / ("job-%s-1" % part) / "temp" / "heldout" / "part.json").read_text())), mine) + for part, work in parts.items(): + text = (work / job.PART).read_text() + self.assertEqual(json.loads(text)["cases"], []) + for ident in ids + list(self.pre): + self.assertNotIn(ident, text, part) + said, written, outputs = self.merged_run(parts) + self.assertEqual(said, "merged 3 of 3 parts, 6 passed") + self.assertEqual([e["part"] for e in written["ledger"]], ["preflight", "0", "1"]) + self.assertEqual(outputs, "ledger_sha256=%s\n" % hashlib.sha256( + json.dumps([e["sealed_sha256"] for e in written["ledger"]], separators=(",", ":")).encode()).hexdigest()) + self.assertEqual(written["preflight"], {"cases": 3, "passed": 3, "faults": []}) + found, digest = job.manifest(self.root, "pr-review-gate") + self.assertEqual(job.verdict(written, found, digest, self.env["GITHUB_SHA"], self.env["RELEASE"]), "pass") + # A part says of its blocking cases two words, never how many (decision 0120). + self.assertEqual(json.loads((parts["0"] / job.PART).read_text())["blocking"], {"some": False, "passed": True}) + # Another attempt of the run draws its shards anew, so the merge of another attempt reads no shard of this one: + # a shard run again is no part of a run whose others were drawn before it. + self.assertTrue(any(drawn(salt, "4242", str(a)) != drawn(salt, "4242", "1") for a in range(2, 9))) + self.assertEqual(self.merged_run(parts, attempt="2")[0], "merged 1 of 3 parts, 0 passed") + again = drawn(salt, "4242", "2") + home = github.Replay([salted] + home_of({i: cases[i] for i in ids if i in again})) + said, _ = self.part_run("0", home, review, run=("4242", "2")) + self.assertEqual(set(json.loads((pathlib.Path(self.dir.name) / "job-0-2" / "temp" / "heldout" / "part.json").read_text())), again) + # A gate set's part that holds no ledger but its own is no part of the run. + kept = json.loads((parts["1"] / job.PART).read_text()) + row = {"id": "c-0", "sha256": "0" * 64, "verdict": "pass", "reasons": [], "usage": dict(job.NOT_ASKED)} + for change in ({"ledger": []}, {"ledger": [dict(kept["ledger"][0], part="0")]}, {"cases": [row]}): + (parts["1"] / job.PART).write_text(json.dumps(dict(kept, **change))) + self.assertEqual(self.merged_run(parts)[0], "merged 2 of 3 parts, %d passed" % len(drawn(salt, "4242", "1")), change) + (parts["1"] / job.PART).write_text(json.dumps(kept)) + # The one part the download found, put where it was put, is read by what it says it is. + self.assertEqual(self.merged_run({"preflight": parts["preflight"]}, flat=True)[0], "merged 1 of 3 parts, 0 passed") + # A gate set with no salt in the home stops the fetch. + self.assertEqual(self.part_run("0", github.Replay([home_beside("pr-review-gate", "salt", b"", status=404)]), review)[0][1], + "no_shard_salt") + # A preflight that stops fails its own job, and the merge, with no shard, says so. + failing = lambda case: dict(result, status="failed") + said, stopped = self.part_run("preflight", github.Replay(home_of(self.pre)), failing, certs=True) + from harness.eval import judge as judging + env = dict(self.env, RUNNER_TEMP=str(pathlib.Path(self.dir.name) / "job-preflight-1" / "temp"), EVAL_PART="preflight") + with mock.patch("sys.stdout", new_callable=io.StringIO): + with self.assertRaises(job.Stop) as stop: + job.judge(env, self.root, stopped) + self.assertEqual(stop.exception.code, "preflight_failed") + said, alone, _ = self.merged_run({"preflight": stopped}) + self.assertEqual(job.verdict(alone, found, digest, self.env["GITHUB_SHA"], self.env["RELEASE"]), "preflight_failed") + self.assertEqual(dict(alone["faults"][0]), {"code": "unanswered", "count": 6}) + def test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it(self): # issue 613 from harness import eval as job missed = dict(example_result(), findings=[]) @@ -3540,7 +3756,7 @@ def run(review, home=None): self.assertEqual(written["preflight"], preflight) self.assertFalse(set(asked) & set(self.cases)) self.assertEqual((written["passed"], written["faults"]), (0, [{"code": "unanswered", "count": 2}])) - sealed = dict(written, ledger=dict(written["ledger"], sealed_sha256="f" * 64)) + sealed = dict(written, ledger=[dict(written["ledger"][0], sealed_sha256="f" * 64)]) self.assertEqual(job.verdict(sealed, found, digest, "a" * 40, "factory/v0.1.0"), "preflight_failed") ledger = json.loads((self.temp / "heldout" / job.LEDGER).read_text()) self.assertEqual(validate(ledger, load("schemas/eval-ledger.schema.json"), schemas=eval_result.schemas()), []) diff --git a/tests/test_release_step.py b/tests/test_release_step.py index 625f7677..3fcdc147 100644 --- a/tests/test_release_step.py +++ b/tests/test_release_step.py @@ -325,7 +325,7 @@ def counts(finding=6, met=18, **change): "engine": eval_gate.engine_of("pr-review"), "models": ["claude-opus"], "bars": {"finding": [3, 10], "met": [9, 10]}, "kind_least": 20, "split_ok": True, "kinds_pass": {"finding": finding * 10 >= 60, "met": met * 10 >= 180}, "blocking_pass": True, - "ledger": {"salted_sha256": "e" * 64, "sealed_sha256": "f" * 64}} + "ledger": [{"part": "whole", "salted_sha256": "e" * 64, "sealed_sha256": "f" * 64}]} document.update(change) return document diff --git a/tests/test_tools.py b/tests/test_tools.py index 9c000f6a..155ec832 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -2560,9 +2560,12 @@ def test_each_step_limit_holds_what_its_step_needs(self): # issue 388 # least; the Engine's step calls the CLI for its tier's seconds at most, with two reads of its version. from harness.eval import run as eval_run live = ROOT / ".github" / "workflows" - limit = re.search(r" - run: python3 -m harness\.eval\.run\n timeout-minutes: (\d+)\n", - (live / "eval.yml").read_text()) - self.assertGreaterEqual(int(limit.group(1)) * 60, eval_run.BUDGET) + # The preflight's job and the shards' each run it (issue 611): each limit holds the budget. + limits = re.findall(r" - run: python3 -m harness\.eval\.run\n timeout-minutes: (\d+)\n", + (live / "eval.yml").read_text()) + self.assertEqual(len(limits), 2) + for limit in limits: + self.assertGreaterEqual(int(limit) * 60, eval_run.BUDGET) engine = re.search(r" timeout-minutes: (\d+)\n run: python3 -m harness\.run\n", (live / "run-jobs.yml").read_text()) tiers = json.loads((ROOT / "config" / "harness.json").read_text()) diff --git a/tools/check_examples.py b/tools/check_examples.py index b3c77e53..1e371fde 100644 --- a/tools/check_examples.py +++ b/tools/check_examples.py @@ -32,6 +32,7 @@ "written": ("document", "pass"), "view-meta": ("document", "pass"), "view-items": ("document", "pass"), "health": ("document", "pass"), "eval-report": ("document", "commit"), "eval-aggregate": ("document", "commit"), "eval-ledger": ("document", "commit"), + "eval-part": ("document", "commit"), "facts": "record", "issue-facts": "record", "seen": "record", "decision": "record", "issue-decision": "record", "finding": "record", "effect": "part", "disposition": "part", "coverage": "part", "run-name": "part", "run-record": "part", "view-item": "part", @@ -47,7 +48,7 @@ # is held by that kind's own rules, so the existing records are not swept. CODES_ONLY = ("task", "result", "effect", "disposition", "run-name", "coverage", "run-record", "pass", "writes", "written", "applied", "outcome", "view-meta", "view-items", "view-item", "health", "role", "feed", - "eval-set", "eval-report", "eval-aggregate", "eval-ledger", "rings") + "eval-set", "eval-report", "eval-aggregate", "eval-ledger", "eval-part", "rings") TEXT_FIELD = "body" UNTRUSTED_FIELD = "untrusted" UNTRUSTED_KIND = "view-item" diff --git a/tools/check_workflows.py b/tools/check_workflows.py index 8dac04c3..991e71bd 100644 --- a/tools/check_workflows.py +++ b/tools/check_workflows.py @@ -89,8 +89,9 @@ # act's, whose wake approve_job_problems holds the same way (trial P-55), and the queue act's, which queue_job_problems # holds so (decision 0128). The release job # moves the Release every Run calls, so it cannot be one: it runs main's file, and release_problems holds every line of -# it (issue 197's ruling 4′). -SHAPED = {"eval.yml": ("shim", "install"), "run-jobs.yml": ("install",), "approve-job.yml": ("install",), +# it (issue 197's ruling 4′). The eval job's shards name what each uploads by the matrix's shard, an expression under +# with:, which eval_problems holds word for word, so it is spared that rule too (issue 611). +SHAPED = {"eval.yml": ("shim", "install", "expression"), "run-jobs.yml": ("install",), "approve-job.yml": ("install",), "queue-job.yml": ("install",), "release.yml": ("shim",)} # The one workflow that wakes on another's end, and the Run's, which it does not name (trial P-55). WAKES, RUN_WORKFLOW = "poller.yml", "sf-run" @@ -704,8 +705,18 @@ # report: an action in the one job that holds both credentials can read them both (decision 0053), so a bump of it edits # this too. UPLOAD = ("actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9", "# v7.0.2") -# A gate set's ledger is sealed after the run and its ciphertext uploaded before the report (issue 612). -EVAL_ORDER = ("checkout", "plan", "install", "fetch", "run", "seal", "ledger", "upload", "judge") +# The one download action, by its commit, the merge job's alone: what each part of the run uploaded (issue 611). +DOWNLOAD = ("actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333", "# v8.0.2") +# The run's parts, each a job (issue 611): the preflight's, the shards' matrix behind it, and the merge, which holds no +# credential and writes the one report whatever became of the rest. A gate set's ledger is sealed after the run, its +# ciphertext uploaded before what the part found (issue 612). +EVAL_JOBS = ("preflight", "eval", "merge") +EVAL_ORDERS = {"preflight": ("checkout", "plan", "install", "fetch", "run", "seal", "ledger", "upload", "judge"), + "eval": ("checkout", "plan", "install", "fetch", "run", "seal", "ledger", "upload"), + "merge": ("checkout", "plan", "download", "merge", "upload", "judge")} +EVAL_ORDER = EVAL_ORDERS["preflight"] +EVAL_NEEDS = {"preflight": None, "eval": ["preflight"], "merge": ["preflight", "eval"]} +SHARD = "${{ matrix.shard }}" # The CLI's install, the one the Run's jobs hold word for word in RUN_JOBS_FILE: the eval job installs the CLI a live # Run does, at the Engine's pin and by the vendor's checksum (issue 309, amendment 1). Its version and checksum are given # under env:, as RUN_JOBS_FILE gives them. Bubblewrap is the one package: the CLI refuses to run with the scrub on and @@ -718,18 +729,23 @@ "sudo mv /tmp/claude /usr/local/bin/claude", "sudo apt-get update -q", "sudo apt-get install -y --no-install-recommends bubblewrap"] -EVAL_JOB_KEYS = {"runs-on", "timeout-minutes", "permissions", "environment", "outputs", "steps"} -# The job's one output, the sealed ledger's sha256, which the seal step alone sets, under the one id a step may take: the -# Maintainer's ruling on issue 612 keeps it for whatever stores the ciphertext, once that is decided. -EVAL_OUTPUTS = {"ledger_sha256": "${{ steps.seal.outputs.ledger_sha256 }}"} -EVAL_IDS = {"seal": "seal"} -# Its runner, the one the CLI's linux-x64 install is for, and its timeout, which the run step's budget is held inside -# (harness/eval/run.py, BUDGET): a shorter one would end the job before the report is uploaded (issue 309, ruling 9). +EVAL_JOB_KEYS = {"preflight": {"runs-on", "timeout-minutes", "permissions", "environment", "steps"}, + "eval": {"needs", "strategy", "runs-on", "timeout-minutes", "permissions", "environment", "steps"}, + "merge": {"needs", "if", "runs-on", "timeout-minutes", "permissions", "outputs", "steps"}} +# The run's one output, the hash of every part's sealed ledger's, which the merge step alone sets, under the one id a +# step may take: the Maintainer's ruling on issue 612 keeps it for whatever stores the ciphertexts, and the Factory +# Builder's ruling (c) on issue 611 makes it the hash of their list. +EVAL_OUTPUTS = {"ledger_sha256": "${{ steps.merge.outputs.ledger_sha256 }}"} +EVAL_IDS = {"merge": "merge"} +# The runner, the one the CLI's linux-x64 install is for, and each part's timeout, which the run step's budget is held +# inside (harness/eval/run.py, BUDGET): a shorter one would end the job before what it found is uploaded (issue 309, +# ruling 9). The merge asks nothing, and is quick. EVAL_RUNNER = {"runs-on": "ubuntu-24.04", "timeout-minutes": "120"} +EVAL_RUNNERS = {"preflight": EVAL_RUNNER, "eval": EVAL_RUNNER, "merge": {"runs-on": "ubuntu-24.04", "timeout-minutes": "10"}} # What each step may be given, under with: or env:, exactly: a step's own keys beside these are refused. EVAL_GIVEN = { "checkout": {"with": {"persist-credentials": "false"}}, - "plan": {"env": {"RELEASE": "${{ inputs.release }}", "EVAL_SET": "${{ inputs.set }}"}}, + "plan": {"env": {"RELEASE": "${{ inputs.release }}", "EVAL_SET": "${{ inputs.set }}", "EVAL_PART": None}}, # The one personal token a workflow here may name, the Maintainer's, read-only on the home alone (decision 0053), # and the home's coordinates, secrets and never variables: a step's log shows what it is given, and GitHub masks a # secret's value there, and nowhere a variable's. @@ -741,13 +757,22 @@ "CLAUDE_CLI": "/usr/local/bin/claude"}}, # The seal holds no credential: what it seals to is in the checkout. "seal": {}, - # The ciphertext alone, and nothing where a development set has no ledger to seal. - "ledger": {"with": {"name": "eval-ledger", "path": "eval-ledger.cms", "if-no-files-found": "ignore", - "retention-days": "90"}}, - "upload": {"with": {"name": "eval-report", "path": "eval-report.json", "if-no-files-found": "error", - "retention-days": "90"}}, + # The ciphertext alone, and nothing where a development set has no ledger to seal; each part's by its own name. + "ledger": {"with": {"name": None, "path": "eval-ledger.cms", "if-no-files-found": "ignore", "retention-days": "90"}}, + "upload": {"with": {"name": None, "path": None, "if-no-files-found": "error", "retention-days": "90"}}, + "download": {"with": {"pattern": "eval-part-*", "path": "parts"}}, + "merge": {}, "judge": {}, } +# What differs from one job to another: the part each plans as, and the names of what it uploads. +EVAL_PARTS = {"preflight": {"plan": {"env": {"EVAL_PART": "preflight"}}, + "ledger": {"with": {"name": "eval-ledger-preflight"}}, + "upload": {"with": {"name": "eval-part-preflight", "path": "eval-part.json"}}}, + "eval": {"plan": {"env": {"EVAL_PART": SHARD}}, + "ledger": {"with": {"name": "eval-ledger-" + SHARD}}, + "upload": {"with": {"name": "eval-part-" + SHARD, "path": "eval-part.json"}}}, + "merge": {"plan": {"env": {"EVAL_PART": "merge"}}, + "upload": {"with": {"name": "eval-report", "path": "eval-report.json"}}}} EVAL_STEP_KEYS = {"uses", "run", "with", "env", "timeout-minutes"} # The release job and its workflow (issue 197's ruling 4′): dispatched by the Maintainer, with the command and its # version or target, or on a schedule that cuts only when the repo's FACTORY_RELEASE_AUTO is true and no Pause is set; @@ -801,7 +826,7 @@ # What names the release job's environment, its key or its token, in a workflow that is not release.yml. RELEASE_ONLY = re.compile(r"factory-release|RELEASE_APP_|RELEASE_TOKEN|release\.move", re.I) EVAL_TOP = {"name": "factory", "on": "", "permissions": "{}", "jobs": ""} -EVAL_ACTIONS = {"checkout", "upload", "ledger"} +EVAL_ACTIONS = {"checkout", "upload", "ledger", "download"} # The dispatch's inputs, each by what it is given: the Release it judges, and the set, the review Intent's by default. EVAL_INPUTS = {"release": [("required", "true"), ("type", "string")], "set": [("default", "pr-review"), ("required", "false"), ("type", "string")]} @@ -1398,14 +1423,17 @@ def _kind(step: dict, entries: list = ()) -> str: if uses is not None and "run" not in step: if (uses, step["uses"]["comment"]) == CHECKOUT: return "checkout" + if (uses, step["uses"]["comment"]) == DOWNLOAD: + return "download" named = [e["value"] for e in entries if e["step"] == step["uses"]["step"] and e["path"][-1:] == ["with"] and e["key"] == "name"] - return None if (uses, step["uses"]["comment"]) != UPLOAD else "ledger" if named == ["eval-ledger"] else "upload" + return None if (uses, step["uses"]["comment"]) != UPLOAD else \ + "ledger" if len(named) == 1 and named[0].startswith("eval-ledger") else "upload" command = _lines(step.get("run")) if "uses" not in step and step.get("run", {}).get("value") == "|" and command == CLI_INSTALL_RUN: return "install" # The install is its block alone: a module of that name is no install, and is refused as a step of no kind. - for name in EVAL_ORDER: + for name in EVAL_ORDERS["preflight"] + EVAL_ORDERS["merge"]: if name not in EVAL_ACTIONS | {"install"} and command == [EVAL_STEP % name] and "uses" not in step: return name return None @@ -1413,10 +1441,11 @@ def _kind(step: dict, entries: list = ()) -> str: def eval_problems(text: str, root: pathlib.Path = ROOT) -> list: """What is wrong with eval.yml's shape (issue 309, ruling 10), each a sentence. It runs on `workflow_dispatch` alone, - with two inputs, the Release it judges and the set, and one job, `eval`, on the environment `factory-eval`, granted - `contents: read` alone and given no env: of its own. Its steps are EVAL_ORDER, each given exactly what EVAL_GIVEN - says and nothing else: the Maintainer's token for the held-out home by `fetch` alone, the model's - credential by `run` alone, so no step holds both, the seal given nothing, and each upload takes its one file. Every other spelling is + with two inputs, the Release it judges and the set, and its jobs EVAL_JOBS (issue 611), each granted `contents: read` + alone and given no env: of its own: the preflight's and the shards', on the environment `factory-eval`, and the + merge, on none. Each job's steps are EVAL_ORDERS's, each given exactly what EVAL_GIVEN and EVAL_PARTS say and nothing + else: the Maintainer's token for the held-out home by `fetch` alone, the model's credential by `run` alone, so no + step holds both, and the merge neither; the seal given nothing, and each upload takes its one file. Every other spelling is refused: a check reads what it names, never what it can guess. The CLI's install is CLI_INSTALL_RUN, word for word, given the Engine's pin in config/harness.json and its checksum in CLI_CHECKSUMS, as the Run's jobs' is.""" entries, bad = read(text) @@ -1440,39 +1469,89 @@ def eval_problems(text: str, root: pathlib.Path = ROOT) -> list: for name, given in EVAL_INPUTS.items()): problems.append("runs on workflow_dispatch alone, with the inputs release, a required string, and set, an optional " "string whose default is pr-review") - if [e["key"] for e in entries if e["path"] == ["jobs"]] != [EVAL_JOB]: - problems.append("it holds the job %s and no other" % EVAL_JOB) - inside = {e["key"]: e for e in entries if e["path"] == ["jobs", EVAL_JOB]} - if set(inside) != EVAL_JOB_KEYS or inside["environment"]["value"] != EVAL_ENVIRONMENT: - problems.append("its job has exactly %s, and runs on the environment %s" - % (", ".join(sorted(EVAL_JOB_KEYS)), EVAL_ENVIRONMENT)) - if {k: inside[k]["value"] for k in EVAL_RUNNER if k in inside} != EVAL_RUNNER: - problems.append("its job runs on %(runs-on)s, with a timeout of %(timeout-minutes)s minutes" % EVAL_RUNNER) - if {e["key"]: e["value"] for e in entries if e["path"] == ["jobs", EVAL_JOB, "outputs"]} != EVAL_OUTPUTS: - problems.append("its job has one output, ledger_sha256, the seal step's") - granted = sorted((e["key"], e["value"]) for e in entries if e["path"] == ["jobs", EVAL_JOB, "permissions"]) + if [e["key"] for e in entries if e["path"] == ["jobs"]] != list(EVAL_JOBS): + problems.append("it holds the jobs %s, in that order, and no other" % ", ".join(EVAL_JOBS)) + shards = shards_named(root) + for job in EVAL_JOBS: + problems += _eval_job_problems(text, entries, job, given_of, shards) + return problems + + +def shards_named(root: pathlib.Path): + """How many shards `config/harness.json` names, `eval_shards`, or None (issue 611).""" + try: + shards = json.loads((root / "config" / "harness.json").read_text()).get("eval_shards") + except (OSError, ValueError, AttributeError): + return None + return shards if type(shards) is int and 1 <= shards <= 16 else None + + +def _items(text: str, entry: dict) -> list: + """The items of the block list under `entry`, a key with no value, as written: none where it has a value.""" + if entry["value"]: + return [] + values = [] + for raw in text.split("\n")[entry["line"]:]: + if not raw.strip() or raw.lstrip().startswith("#"): + continue + item = ITEM.match(raw.rstrip()) + if item is None or len(item.group(1)) < entry["indent"]: + break + values.append(item.group(2).strip()) + return values + + +def _eval_job_problems(text: str, entries: list, job: str, given_of: dict, shards) -> list: + """What is wrong with one job of eval.yml (issues 309, 612 and 611): its keys, its environment where it holds a + credential, its runner, what it needs, the shards' matrix as the configuration names them, the merge's one output + and `if: always()`, its grant, and its steps, EVAL_ORDERS's, each given exactly what EVAL_GIVEN and EVAL_PARTS say.""" + problems = [] + inside = {e["key"]: e for e in entries if e["path"] == ["jobs", job]} + if set(inside) != EVAL_JOB_KEYS[job] or ("environment" in inside and inside["environment"]["value"] != EVAL_ENVIRONMENT): + problems.append("its job %s has exactly %s%s" % (job, ", ".join(sorted(EVAL_JOB_KEYS[job])), + ", and runs on the environment %s" % EVAL_ENVIRONMENT if "environment" in EVAL_JOB_KEYS[job] else "")) + if {k: inside[k]["value"] for k in EVAL_RUNNERS[job] if k in inside} != EVAL_RUNNERS[job]: + problems.append("its job %s runs on %s, with a timeout of %s minutes" % (job, EVAL_RUNNERS[job]["runs-on"], + EVAL_RUNNERS[job]["timeout-minutes"])) + if EVAL_NEEDS[job] is not None and ("needs" not in inside or _listed(text, inside["needs"]) != EVAL_NEEDS[job]): + problems.append("its job %s needs %s" % (job, ", ".join(EVAL_NEEDS[job]))) + if job == "merge" and inside.get("if", {}).get("value") != "always()": + problems.append("its job merge runs whatever became of the rest: if: always()") + if job == "eval": + strategy = {e["key"]: e for e in entries if e["path"] == ["jobs", job, "strategy"]} + matrix = {e["key"]: e for e in entries if e["path"] == ["jobs", job, "strategy", "matrix"]} + if set(strategy) != {"fail-fast", "matrix"} or strategy["fail-fast"]["value"] != "false" or set(matrix) != {"shard"} \ + or shards is None or _items(text, matrix["shard"]) != [str(i) for i in range(shards)]: + problems.append("its job eval is a matrix of the shards config/harness.json names, shard 0 to the last, " + "and one shard's failure cancels no other: fail-fast: false") + outputs = {e["key"]: e["value"] for e in entries if e["path"] == ["jobs", job, "outputs"]} + if outputs != (EVAL_OUTPUTS if job == "merge" else {}): + problems.append("its job merge has one output, ledger_sha256, the merge step's, and no other job any") + granted = sorted((e["key"], e["value"]) for e in entries if e["path"] == ["jobs", job, "permissions"]) if granted != [("contents", "read")]: - problems.append("its job is granted contents: read and nothing else") - steps = _steps(entries, EVAL_JOB) + problems.append("its job %s is granted contents: read and nothing else" % job) + steps = _steps(entries, job) kinds = [_kind(step, entries) for step in steps] - if tuple(kinds) != EVAL_ORDER: - problems.append("its steps are %s, in that order, and no other" % ", ".join(EVAL_ORDER)) + if tuple(kinds) != EVAL_ORDERS[job] or _scalar_items(text, entries, job, steps_only=True): + problems.append("its job %s's steps are %s, in that order, and no other" % (job, ", ".join(EVAL_ORDERS[job]))) for step, kind in zip(steps, kinds): if kind is None: continue number = next(iter(step.values()))["step"] - own = set(step) - EVAL_STEP_KEYS - ({"id"} if kind in EVAL_IDS else set()) + ids = EVAL_IDS if job == "merge" else {} + own = set(step) - EVAL_STEP_KEYS - ({"id"} if kind in ids else set()) if own: - problems.append("its %s step has a key it may not: %s" % (kind, ", ".join(sorted(own)))) - if kind in EVAL_IDS and step.get("id", {}).get("value") != EVAL_IDS[kind]: - problems.append("its %s step has the id %s" % (kind, EVAL_IDS[kind])) + problems.append("its job %s's %s step has a key it may not: %s" % (job, kind, ", ".join(sorted(own)))) + if kind in ids and step.get("id", {}).get("value") != ids[kind]: + problems.append("its job %s's %s step has the id %s" % (job, kind, ids[kind])) for under in ("with", "env"): given = {e["key"]: e["value"] for e in entries if e["step"] == number - and e["path"] == ["jobs", EVAL_JOB, "steps", under]} - if given != given_of[kind].get(under, {}): - problems.append("its %s step is given under %s: %s, exactly" % (kind, under, given_of[kind].get(under, {}))) + and e["path"] == ["jobs", job, "steps", under]} + wanted = dict(given_of[kind].get(under, {}), **EVAL_PARTS[job].get(kind, {}).get(under, {})) + if given != wanted or None in wanted.values(): + problems.append("its job %s's %s step is given under %s: %s, exactly" % (job, kind, under, wanted)) if under in step and (step[under]["value"] or under not in given_of[kind]): - problems.append("its %s step has a %s: it is not given" % (kind, under)) + problems.append("its job %s's %s step has a %s: it is not given" % (job, kind, under)) return problems @@ -1923,9 +2002,9 @@ def run(root: pathlib.Path = ROOT) -> list: if EVAL_JOB in jobs and path.name != EVAL_WORKFLOW: problems.append("%s: the job %s is reserved for %s, the one job that reads the held-out home" % (path.relative_to(root), EVAL_JOB, EVAL_WORKFLOW)) - if path.name == EVAL_WORKFLOW and jobs != [EVAL_JOB]: - problems.append("%s: holds the job %s and no other, or it is not the eval job's workflow" - % (path.relative_to(root), EVAL_JOB)) + if path.name == EVAL_WORKFLOW and jobs != list(EVAL_JOBS): + problems.append("%s: holds the jobs %s and no other, or it is not the eval job's workflow" + % (path.relative_to(root), ", ".join(EVAL_JOBS))) if path.name == EVAL_WORKFLOW: problems += ["%s: %s" % (path.relative_to(root), p) for p in eval_problems(raw(path), root)] else: diff --git a/tools/mutants.py b/tools/mutants.py index 5a194866..0da97337 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -2454,7 +2454,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("a limit of none, or of another script's digits, is a limit", "tools/check_workflows.py", 'MINUTES = re.compile(r"^[1-9][0-9]*$")', 'MINUTES = re.compile(r"^[0-9]+$")', LIMITED), ("the eval job's run step is cut before its budget", ".github/workflows/eval.yml", - " timeout-minutes: 109\n", " timeout-minutes: 30\n", NEEDED), + " timeout-minutes: 109\n env:\n CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}\n CLAUDE_CLI: /usr/local/bin/claude\n - run: python3 -m harness.eval.seal\n timeout-minutes: 1\n - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2\n with:\n name: eval-ledger-${{ matrix.shard }}", + " timeout-minutes: 30\n env:\n CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}\n CLAUDE_CLI: /usr/local/bin/claude\n - run: python3 -m harness.eval.seal\n timeout-minutes: 1\n - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2\n with:\n name: eval-ledger-${{ matrix.shard }}", NEEDED), ("the Engine's step is cut before its call can end", ".github/workflows/run-jobs.yml", " timeout-minutes: 20\n", " timeout-minutes: 15\n", NEEDED), ("a uses: step must have a limit of its own", "tools/check_workflows.py", @@ -8037,20 +8038,20 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' or [e["key"] for e in inputs if len(e["path"]) == 3] != list(EVAL_INPUTS)', '', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ('the eval job runs on any environment', 'tools/check_workflows.py', - ' or inside["environment"]["value"] != EVAL_ENVIRONMENT:', + ' or ("environment" in inside and inside["environment"]["value"] != EVAL_ENVIRONMENT):', ':', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ('the eval job may have an env of its own', 'tools/check_workflows.py', - ' if set(inside) != EVAL_JOB_KEYS or', - ' if not EVAL_JOB_KEYS <= set(inside) or', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), + ' if set(inside) != EVAL_JOB_KEYS[job] or', + ' if not EVAL_JOB_KEYS[job] <= set(inside) or', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ('the eval job is granted more', 'tools/check_workflows.py', ' if granted != [("contents", "read")]:\n', ' if ("contents", "read") not in granted:\n', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ("the eval job's steps are in any order", 'tools/check_workflows.py', - ' if tuple(kinds) != EVAL_ORDER:\n', - ' if set(kinds) != set(EVAL_ORDER):\n', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), + ' if tuple(kinds) != EVAL_ORDERS[job] or', + ' if set(kinds) != set(EVAL_ORDERS[job]) or', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ('an eval step is given more than it needs', 'tools/check_workflows.py', - ' if given != given_of[kind].get(under, {}):\n', - ' if not set(given_of[kind].get(under, {}).items()) <= set(given.items()):\n', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), + ' if given != wanted or None in wanted.values():\n', + ' if not set(wanted.items()) <= set(given.items()) or None in wanted.values():\n', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ("another workflow may name the eval job's environment", 'tools/check_workflows.py', ' if EVAL_ONLY.search(line):\n', ' if False:\n', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), @@ -8181,7 +8182,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' sudo apt-get update -q\n sudo apt-get install -y --no-install-recommends bubblewrap\n', ' sudo apt-get update -q\n', ("tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone",)), ('the eval job installs after its fetch', 'tools/check_workflows.py', - '"plan", "install", "fetch", "run"', '"plan", "fetch", "install", "run"', ("tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone",)), + '"preflight": ("checkout", "plan", "install", "fetch", "run"', '"preflight": ("checkout", "plan", "fetch", "install", "run"', ("tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone",)), ('the canary installs a block of its own', 'harness/eval/canary.py', '"\\n".join(check_workflows.CLI_INSTALL_RUN)', '"\\n".join(check_workflows.CLI_INSTALL_RUN[:-1])', ("tests.test_evals.EvalJob.test_the_canary_installs_the_block_the_jobs_hold",)), ('the canary installs at no pin', 'harness/eval/canary.py', @@ -9261,7 +9262,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ('the eval blob is asked for as raw+json', 'adapters/heldout.py', 'RAW = "application/vnd.github.raw"', 'RAW = "application/vnd.github.raw+json"', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), ('what the home keeps beside a case is asked as JSON', 'adapters/heldout.py', - ' accept=RAW))', ' ))', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + 'path), accept=RAW))', 'path)))', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), ('a head of any size is read', 'adapters/heldout.py', ' if len(data) > BESIDE[kind]:\n', ' if False:\n', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), ('a head answered under another type is read', 'adapters/heldout.py', @@ -9280,8 +9281,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ('a file beside a case of any name is asked', 'adapters/heldout.py', ' and isinstance(ident, str) and PART.fullmatch(ident)\n', '\n', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), ('a home of any name is asked beside a case', 'adapters/heldout.py', - ' return None, "not_this_set"\n if not home(owner, name):\n return None, "unanswered"\n try:', - ' return None, "not_this_set"\n if False:\n return None, "unanswered"\n try:', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ' if not home(owner, name):\n return None, "unanswered"\n try:\n answer = transport.send(request("GET", "/repos/%s/%s/contents/%s" % (owner, name, path)', + ' if False:\n return None, "unanswered"\n try:\n answer = transport.send(request("GET", "/repos/%s/%s/contents/%s" % (owner, name, path)', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), ('a head is unpacked and never held to its tree', 'harness/eval/__init__.py', ' head.held(into, tree)\n except', ' except', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), ('a head not the one pinned is read as unreadable', 'harness/eval/__init__.py', @@ -9334,6 +9335,94 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' except Exception: # by its code alone', ' except ZeroDivisionError: # by its code alone', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), ("the Maintainer's tool leaves a file half written", 'tools/heads_into_home.py', ' with contextlib.suppress(OSError):\n os.unlink(temporary)\n', ' pass\n', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + # Issue 611, PR 2: the run split into the preflight's job, a matrix of shards and the merge. + ("a gate set's shard is chosen by its place in the manifest", 'harness/eval/__init__.py', + ' return [e for e in found["cases"] if drawn(e) % shards == int(part)]', + ' return [e for place, e in enumerate(found["cases"]) if place % shards == int(part)]', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ('a shard prints a case id', 'harness/eval/__init__.py', + 'of = "" if part in (None, PREFLIGHT) else " of shard %s" % part', + 'of = "" if part in (None, PREFLIGHT) else " of shard %s" % ",".join(e["id"] for e in mine)', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ("a gate set's shard artifact names a case", 'harness/eval/__init__.py', + '"cases": [] if gate else report(', + '"cases": report(', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ('a gate set with no salt is sharded by its place', 'harness/eval/__init__.py', + ' raise Stop("no_shard_salt")', + ' return [e for place, e in enumerate(found["cases"]) if place % shards == int(part)]', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ("a gate set's shards are drawn by no run", 'harness/eval/__init__.py', + 'b"\\0".join([salt, run.encode(), attempt.encode(), e["id"].encode()])', + 'b"\\0".join([salt, e["id"].encode()])', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ("a development set's shard asks every case", 'harness/eval/__init__.py', + ' return [e for place, e in enumerate(found["cases"]) if place % shards == int(part)]\n salt', + ' return list(found["cases"])\n salt', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ('a shard missing is read as whole', 'harness/eval/__init__.py', + ' whole = sum(d["asked"] for d in present) == found["count"]', + ' whole = True', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ("a gate set's cases no part answered are not counted", 'harness/eval/__init__.py', + ' if unanswered:\n faults["unanswered"]', + ' if False:\n faults["unanswered"]', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ("a gate set's shard of another attempt is merged", 'harness/eval/__init__.py', + '\n or (gate and part != PREFLIGHT and document["attempt"] != attempt):', + ':', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ("a gate set's part with no ledger of its own is merged", 'harness/eval/__init__.py', + '(document["cases"] or [e["part"] for e in document["ledger"]] != [part] if gate else document["ledger"])', + '(document["cases"] if gate else document["ledger"])', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ("the one part found where it was put is not read", 'harness/eval/__init__.py', + ' read[flat["part"]] = flat\n', + ' pass\n', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ('a part of another commit is merged', 'harness/eval/__init__.py', + ' or _judged(\n document, found, kept["set"]["sha256"], kept["commit"], kept["release"])', + '', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ("a gate set's part that lists a case is merged", 'harness/eval/__init__.py', + '(document["cases"] or [e["part"]', + '([e["part"]', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ("a development set's part that holds a ledger is merged", 'harness/eval/__init__.py', + ' if gate else document["ledger"])', + ' if gate else False)', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ("the merge's split needs no blocking case", 'harness/eval/__init__.py', + ' and (blocked["some"] or BLOCKING_LEAST == 0)', + '', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ('a report with no ledger passes', 'harness/eval/__init__.py', + ' if not document["ledger"]:\n return "count_differs"\n', + '', ('tests.test_evals.EvalJob.test_the_gate_passes_a_good_aggregate_report_and_refuses_a_bad_one',)), + ('the merge reads as many shards as the scratch says', 'harness/eval/__init__.py', + 'for n in range(1, 17) if type(kept.get("shards")) is int', + 'for n in range(0, 1000) if type(kept.get("shards")) is int', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ('a part of another part is merged', 'harness/eval/__init__.py', + ' or document["part"] != part', + '', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ('a preflight that stops passes its own job', 'harness/eval/__init__.py', + ' raise Stop("preflight_failed")\n return said', + ' return said\n return said', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ("the merge names no ledger as the run's output", 'harness/eval/__init__.py', + ' if gate_set(found) and isinstance(output, str) and os.path.isabs(output):', + ' if False:', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ("the run's output hashes the last part's ledger alone", 'harness/eval/__init__.py', + 'for entry in ledger], separators', + 'for entry in ledger[-1:]], separators', ('tests.test_evals.EvalJob.test_a_gate_sets_shards_are_drawn_by_the_homes_salt_and_merged_by_their_counts_alone',)), + ('a shard the configuration does not name is planned', 'harness/eval/__init__.py', + ' and part not in [str(i) for i in range(shards)]:', + ' and part not in [str(i) for i in range(16)]:', ('tests.test_evals.EvalJob.test_a_development_sets_run_split_into_shards_is_merged_into_the_report_of_the_whole',)), + ("a part's ledger named twice is read", 'harness/eval/__init__.py', + ' if len(set(parts)) != len(parts) or ("whole" in parts and parts != ["whole"]):', + ' if "whole" in parts and parts != ["whole"]:', ('tests.test_evals.EvalJob.test_the_gate_passes_a_good_aggregate_report_and_refuses_a_bad_one',)), + ("a whole run's ledger is read beside a part's", 'harness/eval/__init__.py', + ' if len(set(parts)) != len(parts) or ("whole" in parts and parts != ["whole"]):', + ' if len(set(parts)) != len(parts):', ('tests.test_evals.EvalJob.test_the_gate_passes_a_good_aggregate_report_and_refuses_a_bad_one',)), + ("the shards' matrix is not the configuration's", 'tools/check_workflows.py', + ' or _items(text, matrix["shard"]) != [str(i) for i in range(shards)]:', + ':', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), + ('the merge may be skipped on a failure', 'tools/check_workflows.py', + ' if job == "merge" and inside.get("if", {}).get("value") != "always()":', + ' if False:', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), + ('a part may need nothing', 'tools/check_workflows.py', + ' if EVAL_NEEDS[job] is not None and ("needs" not in inside or _listed(text, inside["needs"]) != EVAL_NEEDS[job]):', + ' if False:', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), + ('the merge downloads every artifact', 'tools/check_workflows.py', + '"download": {"with": {"pattern": "eval-part-*",', + '"download": {"with": {"pattern": "*",', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), + ('a salt of any length is read', 'adapters/heldout.py', + '16 <= len(data) <= 256', + 'len(data) > 0', ('tests.test_adapters.Heldout.test_a_gate_sets_salt_is_read_raw_from_the_home_and_none_short_of_it',)), # Issue 613, the preflight: each dispatch of a gate set asks a few development cases first, and a broken pipeline stops it. ('a gate set need name no preflight', 'harness/eval/__init__.py', 'OWN = ("bars", "kind_least", "record", "preflight")', @@ -9378,7 +9467,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: '_bars_stated(root, value) + _preflight_held(root, value)', '_bars_stated(root, value)', ('tests.test_evals.EvalJob.test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them',)), ('the preflight is never fetched', 'harness/eval/__init__.py', - ' if kept["preflight"]:\n ahead = _fetched(', + ' if kept["preflight"] and part in (None, PREFLIGHT):\n ahead = _fetched(', ' if False:\n ahead = _fetched(', ('tests.test_evals.EvalJob.test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it',)), ("a broken preflight asks the set's cases anyway", 'harness/eval/__init__.py', ' why = {e["id"]: "unanswered" for e in found["cases"]} if preflight_stops(ahead) else _read(place, "fetched.json")\n', @@ -9396,11 +9485,11 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' or not any(said == (PASS, []) for said in ahead.values())\n', ' or not all(said == (PASS, []) for said in ahead.values())\n', ('tests.test_evals.EvalJob.test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it',)), ('the report counts every preflight case passed', 'harness/eval/__init__.py', - '"preflight": {"cases": len(ahead), "passed": len([s for s in ahead.values() if s == (PASS, [])]),', - '"preflight": {"cases": len(ahead), "passed": len(ahead),', ('tests.test_evals.EvalJob.test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it',)), + ' return {"cases": len(ahead), "passed": len([s for s in ahead.values() if s == (PASS, [])]),', + ' return {"cases": len(ahead), "passed": len(ahead),', ('tests.test_evals.EvalJob.test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it',)), ('the report says no code of the preflight', 'harness/eval/__init__.py', - ' "faults": faults_of(ahead)},', - ' "faults": []},', ('tests.test_evals.EvalJob.test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it',)), + ' "faults": faults_of(ahead)}', + ' "faults": []}', ('tests.test_evals.EvalJob.test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it',)), ('the ledger lists no preflight case', 'harness/eval/__init__.py', '"cases": cases, "preflight": first}', '"cases": cases, "preflight": []}', ('tests.test_evals.EvalJob.test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it',)), @@ -9436,13 +9525,13 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: 'if said in ("engine_fault", "run_fault", "case_failed"):', ('tests.test_evals.EvalJob.test_a_gate_sets_judge_says_counts_and_codes_and_never_a_case',)), # Issue 612: a dispatch names its set; a gate set's report is of counts alone, its ledger sealed to pinned certificates. ("the sealed ledger's hash is not the job's output", 'harness/eval/seal.py', - ' out.write("ledger_sha256=%s\\n" % written["ledger"]["sealed_sha256"])\n', + ' out.write("ledger_sha256=%s\\n" % ledger_digest(entries))\n', ' pass\n', ('tests.test_evals.EvalJob.test_the_seal_encrypts_the_ledger_to_each_pinned_certificate_alone',)), ('the eval job may have any output', 'tools/check_workflows.py', - ' if {e["key"]: e["value"] for e in entries if e["path"] == ["jobs", EVAL_JOB, "outputs"]} != EVAL_OUTPUTS:\n', + ' if outputs != (EVAL_OUTPUTS if job == "merge" else {}):\n', ' if False:\n', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ('the seal may take any id', 'tools/check_workflows.py', - ' if kind in EVAL_IDS and step.get("id", {}).get("value") != EVAL_IDS[kind]:\n', + ' if kind in ids and step.get("id", {}).get("value") != ids[kind]:\n', ' if False:\n', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ("a development set may name a gate set's case", 'harness/eval/__init__.py', ' raise Stop("eval_set_overlaps_a_gate_set")\n', @@ -9486,7 +9575,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("a gate set's report names no model that answered", 'harness/eval/__init__.py', '"models": models(usages or {}),', '"models": [],', ('tests.test_evals.EvalJob.test_a_gate_sets_report_holds_counts_alone_and_its_ledger_every_case',)), ("a gate set's run keeps no usage", 'harness/eval/__init__.py', - 'review, causes, alternates, surplus, usages, results)', 'review, causes, alternates, surplus, {}, results)', ('tests.test_evals.EvalJob.test_a_gate_sets_report_holds_counts_alone_and_its_ledger_every_case',)), + 'assess(found, why, place, review, causes, alternates, surplus, usages, results)', 'assess(found, why, place, review, causes, alternates, surplus, {}, results)', ('tests.test_evals.EvalJob.test_a_gate_sets_report_holds_counts_alone_and_its_ledger_every_case',)), ("a gate set's ledger says no case's usage", 'harness/eval/__init__.py', ' usages=usages)["cases"]]', ' usages=None)["cases"]]', ('tests.test_evals.EvalJob.test_a_gate_sets_report_holds_counts_alone_and_its_ledger_every_case',)), ("a gate set's line says not who answered", 'harness/eval/__init__.py', @@ -9590,13 +9679,13 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' for done in (within(place, LEDGER), checked, sealing):\n os.remove(done)\n', '', ('tests.test_evals.EvalJob.test_the_seal_encrypts_the_ledger_to_each_pinned_certificate_alone',)), ("a ledger not the report's is sealed", 'harness/eval/seal.py', - ' \\\n or written["ledger"].get("salted_sha256") != hashlib.sha256(data).hexdigest():', - ':', ('tests.test_evals.EvalJob.test_the_seal_refuses_a_certificate_not_pinned_and_a_ledger_not_sealed_whole',)), + '\n and entries[0].get("salted_sha256") == hashlib.sha256(data).hexdigest()):', + '):', ('tests.test_evals.EvalJob.test_the_seal_refuses_a_certificate_not_pinned_and_a_ledger_not_sealed_whole',)), ("the report is not given the ciphertext's hash", 'harness/eval/seal.py', - ' written["ledger"]["sealed_sha256"] = hashlib.sha256(sealed).hexdigest()\n', + ' entries[0]["sealed_sha256"] = hashlib.sha256(sealed).hexdigest()\n', '', ('tests.test_evals.EvalJob.test_the_seal_encrypts_the_ledger_to_each_pinned_certificate_alone',)), ("the eval job's ledger upload is read as the report's", 'tools/check_workflows.py', - '"ledger" if named == ["eval-ledger"] else "upload"', + '"ledger" if len(named) == 1 and named[0].startswith("eval-ledger") else "upload"', '"upload"', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ("the eval job's set may be given any default", 'tools/check_workflows.py', ' for name, given in EVAL_INPUTS.items()):', @@ -10167,7 +10256,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' and e["key"] == "run-name" for e in entries) and name != "run.yml":', ('tests.test_tools.Workflows.test_an_acts_shim_is_the_runs_but_for_its_name_and_the_file_it_calls',)), ('the eval workflow may hold another job', 'tools/check_workflows.py', - ' if [e["key"] for e in entries if e["path"] == ["jobs"]] != [EVAL_JOB]:\n problems.append("it holds the job %s and no other" % EVAL_JOB)\n', + ' if [e["key"] for e in entries if e["path"] == ["jobs"]] != list(EVAL_JOBS):\n problems.append("it holds the jobs %s, in that order, and no other" % ", ".join(EVAL_JOBS))\n', '', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ("a shim's events as one scalar escape its hash", 'tools/check_workflows.py', ' events |= {e["value"] for e in entries if not e["path"] and e["key"] == "on" and e["value"]}\n', @@ -12548,8 +12637,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' "CLAUDE_CLI": "/usr/local/bin/claude"}},', ' "CLAUDE_CLI": "/tmp/claude"}},', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ("eval.yml's install is refused as any install is", 'tools/check_workflows.py', - 'SHAPED = {"eval.yml": ("shim", "install"),', - 'SHAPED = {"eval.yml": ("shim",),', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), + 'SHAPED = {"eval.yml": ("shim", "install", "expression"),', + 'SHAPED = {"eval.yml": ("shim", "expression"),', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ("an interrupt's report is printed", 'harness/eval/__init__.py', ' except BaseException: # a message', ' except Exception: # a message', ('tests.test_evals.EvalJob.test_a_step_says_counts_and_codes_and_never_what_it_raised',)), @@ -12566,7 +12655,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: '(tasks.Refused, KeyError, UnicodeEncodeError)', '(tasks.Refused, KeyError)', ('tests.test_evals.EvalJob.test_a_case_that_names_other_content_or_comes_too_late_is_not_reviewed',)), ('the eval job may run anywhere for any time', 'tools/check_workflows.py', - ' if {k: inside[k]["value"] for k in EVAL_RUNNER if k in inside} != EVAL_RUNNER:\n', + ' if {k: inside[k]["value"] for k in EVAL_RUNNERS[job] if k in inside} != EVAL_RUNNERS[job]:\n', ' if False:\n', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)), ("a failed case's report hides what the scorer said", 'harness/eval/__init__.py', 'assessed[entry["id"]] = (said["verdict"], said["reasons"])', @@ -12626,8 +12715,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if not isinstance(kind, str) or kind not in KINDS:\n return None\n', ' if not isinstance(kind, str) or kind not in KINDS:\n continue\n', ('tests.test_evals.EvalJob.test_a_set_short_of_a_kind_fails_the_run_and_closes_the_gate_saying_no_count',)), ('the plan keeps no time', 'harness/eval/__init__.py', - '\n "planned_at": now()}', - '}', ('tests.test_evals.EvalJob.test_the_step_gives_each_case_the_release_dispatched_and_a_fresh_run',)), + '\n "part": part, "shards": shards, "planned_at": now()}', + '\n "part": part, "shards": shards}', ('tests.test_evals.EvalJob.test_the_step_gives_each_case_the_release_dispatched_and_a_fresh_run',)), ('a module named install is the install', 'tools/check_workflows.py', 'EVAL_ACTIONS | {"install"}', 'EVAL_ACTIONS', ('tests.test_evals.TheSet.test_the_eval_workflow_names_each_credential_in_one_step_alone',)),