diff --git a/adapters/AGENTS.md b/adapters/AGENTS.md index 92699cea..09f62930 100644 --- a/adapters/AGENTS.md +++ b/adapters/AGENTS.md @@ -33,7 +33,7 @@ Not built yet, by the stories that hold their rules: contract.py, binding.py, is | An issue asks for a contract only by an Intent that asks or a person's `factory:contract` label; if none asks, `acceptance` is null and whole | `tests/test_adapters.py::test_an_issue_asks_for_a_contract_only_by_a_persons_label` | | Closing references read short, of another repo or named twice, are not whole | `tests/test_adapters.py::test_closing_references_are_whole_only_of_this_repo_and_every_one_counted` | | A pull request to another base than the default is not whole on its contract | `tests/test_adapters.py::test_a_pull_request_to_another_base_than_the_default_is_not_read_whole_on_its_contract` | -| A held-out case is the bytes its manifest hashed, or none and a code; one that could name another file is not asked | `tests/test_adapters.py::test_a_held_out_case_is_its_hashed_bytes`, `tests/test_adapters.py::test_a_held_out_case_not_read_whole_says_why` | +| A held-out case and its head are the bytes pinned, or none and a code | `tests/test_adapters.py::test_a_held_out_case_not_read_whole_says_why`, `tests/test_adapters.py::test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins` | | The title, and the head the snapshot was held to, are handed beside the facts, never in them | `tests/test_adapters.py::test_the_pull_requests_own_facts_and_the_title_beside_them`, `tests/test_adapters.py::test_a_snapshot_hands_back_its_head` | | A plan of writes not its pass record's sends nothing; a dispatch is sent once, never retried | `tests/test_effects.py::test_a_write_list_that_is_not_the_records_is_refused_whole`, `tests/test_effects.py::test_a_dispatch_is_sent_once_and_never_retried` | | Each write goes through its one credential, which sends only its own requests by path and body | `tests/test_effects.py::test_each_write_goes_through_its_one_credential_and_its_allowlist` | diff --git a/adapters/heldout.py b/adapters/heldout.py index d11d6f46..3e817e62 100644 --- a/adapters/heldout.py +++ b/adapters/heldout.py @@ -1,6 +1,7 @@ """The held-out home's cases, read for the eval job alone (design §3.13 and §4.9, issue 309): each case a manifest names, `GET /repos/{owner}/{repo}/contents/{role}/{id}.json`, through the one transport, with the Maintainer's read-only token for the home and nothing else (decision 0053). A case is handed back as the bytes its manifest hashed, or not at all, with a code that says why. +So is what the home keeps beside it, its head's tarball and its commit's tree, each by the sha256 the case pins (issue 611). The home's owner and name are the job's environment's, never this repo's, and no refusal, code or message here names them, a case's path or a byte of a case: what the job prints is counts and codes. A case is read from GitHub's JSON @@ -11,9 +12,13 @@ import hashlib import re -from adapters.github import Unanswered, request +from adapters.github import TARRED, Unanswered, raw, request LARGEST = 1024 * 1024 # bytes a case may be: a facts snapshot and a diff; GitHub inlines a file's content to 1 MiB +RAW = "application/vnd.github.raw" # a file's bytes as they are, whatever its size (P-62) +# What the home keeps beside a case, by its extension, and the bytes each may be: the head's tarball as the transport +# bounds one, and its tree, which GitHub truncates past 7 MB, as `harness/head.tree` refuses one truncated. +BESIDE = {"tar.gz": TARRED, "tree.json": 8 * 1024 * 1024} OWNER = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?") NAME = re.compile(r"[A-Za-z0-9_.-]{1,100}") PART = re.compile(r"[a-z0-9][a-z0-9-]*") # a Role or a case id, as eval-set/1 holds them: no dot, no slash @@ -64,3 +69,33 @@ def case(transport, owner: str, name: str, role: str, entry) -> tuple: return None, "hash_differs" return data, None + +def beside(transport, owner: str, name: str, role: str, ident: str, kind: str, sha256) -> tuple: + """`(bytes, None)` for the file the home keeps beside a case, `{role}/{ident}.{kind}`, its head's tarball (`tar.gz`) + or its commit's tree (`tree.json`), read whole with the raw media type and hashing to the sha256 the case pins; + `(None, code)` otherwise, as `case` says them, and `too_large` over the kind's bound (issue 611). The raw media type + answers a file of more than 1 MiB whole, in one answer and no redirect (trials/P-62.json), which the JSON answer + `case` reads does not: it holds a file's content to 1 MiB alone.""" + if not (isinstance(role, str) and PART.fullmatch(role) and isinstance(ident, str) and PART.fullmatch(ident) + and kind in BESIDE and isinstance(sha256, str) and SHA256.fullmatch(sha256)): + return None, "not_this_set" + if not home(owner, name): + return None, "unanswered" + try: + answer = transport.send(request("GET", "/repos/%s/%s/contents/%s/%s.%s" % (owner, name, role, ident, kind), + accept=RAW)) + except Unanswered: + return None, "unanswered" + if answer["status"] == 404: + return None, "not_found" + body = answer["body"] + if answer["status"] != 200 or not isinstance(body, str) \ + or not str(answer["headers"].get("content-type", "")).startswith(RAW + ";"): + return None, "unanswered" + data = raw(body) + if len(data) > BESIDE[kind]: + return None, "too_large" + if hashlib.sha256(data).hexdigest() != sha256: + return None, "hash_differs" + return data, None + diff --git a/docs/decisions/0138-the-review-reads-the-pull-requests-head-with-the-read-tools-alone-held-to-it-by-restricted.md b/docs/decisions/0138-the-review-reads-the-pull-requests-head-with-the-read-tools-alone-held-to-it-by-restricted.md index 5888da93..8658f675 100644 --- a/docs/decisions/0138-the-review-reads-the-pull-requests-head-with-the-read-tools-alone-held-to-it-by-restricted.md +++ b/docs/decisions/0138-the-review-reads-the-pull-requests-head-with-the-read-tools-alone-held-to-it-by-restricted.md @@ -41,7 +41,7 @@ What is owed: what codeload's address holds for a private repo read with an App' | Each archive it refuses, it refuses whole | `tests/test_harness.py::test_each_archive_the_unpacker_refuses_it_refuses_whole` | | It holds its caps, writes into a new folder, and nothing outside it | `tests/test_harness.py::test_the_unpacker_holds_its_caps_and_writes_nothing_outside_its_folder` | | An eval case requires its head, as 40 lowercase hex characters | `tools/check_examples.py`, `examples/invalid/eval-case.no-head.json`, `examples/invalid/eval-case.a-head-as-an-object.json`, `examples/invalid/eval-case.a-short-head.json` | -| Each case's head is read with no credential and removed once asked; one not read closes the gate | `tests/test_evals.py::test_each_cases_head_is_read_with_no_credential_and_one_not_read_closes_the_gate` | +| Each case's head is read with no credential and removed once asked; one not read closes the gate | `tests/test_evals.py::test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate`, `harness/eval/__init__.py`, mutant: "the eval fetch reads no case's head" | | A case is reviewed with the read tools in its head, as a live Run is | `tests/test_evals.py::test_a_case_is_reviewed_by_the_task_and_the_engine_a_live_run_is` | | The read tools are held to the folder at the pin, and by which flag | `trials/P-59.json`, `tests/test_tools.py::test_the_read_tools_on_a_linux_runner_are_held_to_their_folder_only_with_both_flags` | | The tarball is one redirect to codeload, which redirects no further over HTTPS | `trials/P-60.json`, `tests/test_tools.py::test_a_commits_tarball_is_one_redirect_to_codeload_which_never_redirects_over_https` | diff --git a/docs/decisions/NNNN-a-cases-head-is-read-from-the-held-out-home-pinned-by-the-case-and-held-to-its-tree.md b/docs/decisions/NNNN-a-cases-head-is-read-from-the-held-out-home-pinned-by-the-case-and-held-to-its-tree.md new file mode 100644 index 00000000..d04f9afb --- /dev/null +++ b/docs/decisions/NNNN-a-cases-head-is-read-from-the-held-out-home-pinned-by-the-case-and-held-to-its-tree.md @@ -0,0 +1,29 @@ +# NNNN. A case's head is read from the held-out home, pinned by the case, and held to its tree + +Status: accepted. The Factory Builder's rulings of 2026-11-08 on issue 611, "A case's head is read from the private home, pinned by its hash, and the run is sharded", points (a) to (d): this is the first of its two pull requests, the heads; the second shards the run. It amends decision 0138's rows "A transport with no token" and "The eval case" where they say the eval job reads a case's head with no credential, and its Boundaries row "Each case's head is read with no credential and removed once asked", whose check is now the test of the read from the home; and decision 0142's row "An eval case's head", which this closes. Nothing else in either changes. + +## Decision + +| What | Decided | Because | +|---|---|---| +| Where a head is kept | Beside each case in the held-out home, `/.json`, the home keeps the case's head as two files: `.tar.gz`, the commit's tarball as GitHub archived it, and `.tree.json`, the commit's tree, every level of it, as the API answered it. `eval-case/1` requires `head_sha256` and `tree_sha256`, the sha256 of each, beside `head` | The issue's: a head read anonymously shares the anonymous cap, 60 an hour, so a pool of three runs, or a run again within the hour, can fault on `head_unreadable`; and a head that is not on GitHub cannot be read at all. Pinned by the case, which the manifest pins, the head is the bytes the Maintainer chose | +| How it is read | `fetch` reads both files with the home's token, through the transport the case is read with, by `beside` in `adapters/heldout.py`: each with the raw media type, `RAW` in `adapters/heldout.py`, whose answer is a file's bytes whole however large, in one answer and no redirect (`trials/P-62.json`). An answer of another type, `+json` among them, which the transport would read as JSON, is no file. A file over its bound (the transport's tarball bound, `TARRED`, or 8 MiB for a tree) is `too_large`, and one that does not hash to its pin `hash_differs`. Nothing in the eval job reads with no credential, and the home's token is still sent nowhere else | The Factory Builder's ruling (c): no transition, and no case without its pins, since an anonymous read kept for one would keep the read the issue removes. Ruling (d): the raw media type, grounded by its trial, the git blobs API its fallback; and the ruling on P-62's verdict, that the raw media type is GitHub's `vnd.github.raw`, which the transport must ask for | +| How it is held | The tarball is unpacked as before (decision 0138), and then held to the tree file by file, by git's blob hash, by the check a live head is held by (decision 0142): `tree_of` in `harness/head.py` reads the tree file with the checks `tree` reads GitHub's answer with, and `held` holds the head to it. A head not the one pinned is `hash_differs`, a fault of the run; one not read, unpacked or held whole is `head_unreadable`, the Engine's or the Factory's. Either closes the gate | The Factory Builder's ruling (b): pin the tree as well as the tarball, and hold every eval head to it. 0142 left an eval head unheld for want of room under the anonymous cap; read with the home's token, there is room. An eval case is now graded on the files a live review would be given | +| Who puts them there | The Maintainer, at his terminal, in his checkout of the home: `GH_TOKEN="$(gh auth token)" python3 -m tools.heads_into_home `. For each case of the folder, it reads the head's tarball and tree from GitHub through the one transport, read-only, holds the one to the other, writes both beside the case, the tree compact as GitHub's answer held it, and writes the case anew with their sha256 after its `head`; a case it cannot pin, or whose head is over what the job reads, it leaves as it was. Each file is written whole and moved over its name, so a link there is replaced, never written through, and a case or a folder that is a link is not read. It prints counts, then each pinned case's id and the sha256 of its new bytes, never a byte of a case. It takes a role folder and touches nothing else, so a gate set made later is pinned as the review Intent's set is | The Factory Builder's ruling (c): the tool serves the coming gate set too, and prints counts and hashes alone. A session never sees a case | +| The set changes with it | Pinning changes each case's bytes, so the set's manifest under `evals/` takes the hashes the tool prints, in the pull request that brings this change. It merges only once the home holds the files, and once `trials/P-62.json` is on main | The Factory Builder's ruling (c) and (d) | +| The canary | `harness/eval/canary.py` still reads its case's head with no credential: its case is this repo's own example, of this public repo, never one of the home's, and pins no file the home keeps | The canary is no step of the eval job (decision 0106) | +| Residual: a head only in the home | The tool reads a head from GitHub. A head that is not there, one built for a case, is written beside its case by hand, its tarball as `git archive` writes one with the commit in its pax header and one folder, and its tree in the API's shape; nothing builds that yet | No case of the set needs one yet. The eval reads it as it reads any other | +| Residual: the home's answer | P-62 asked a public repo. Should the API answer a private repo's file otherwise, a redirect among them, the read fails closed: the transport follows no redirect for a file, and a head not read whole is `head_unreadable` | Nothing the eval reads can be given in its place; the first dispatch after the Maintainer's run shows it | + +## Why + +A case's head was read with no credential, from the public repo its change was made in, which ties every run of the set to an hourly cap it shares with nothing else of the Factory's, and to GitHub keeping the commit. Kept in the home and pinned by the case, the head is read with the token every case is already read with, and is the same bytes on every run. The tree beside it lets the eval hold the head to the commit as the live review does, so a case can no longer be graded on files the live review would refuse. + +## Boundaries + +| Boundary | Check | +|---|---| +| What the home keeps beside a case is asked with the raw media type and the home's token, and handed back only as the bytes the case pins: an answer of another type, `+json` among them, one not found, refused, over its bound or of another hash is none, and a name, kind, pin or home that could name another file is not asked | `tests/test_adapters.py::test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins`, `trials/P-62.json`, `adapters/heldout.py`, mutant: "the eval blob is asked for as raw+json", mutant: "a head beside a case of another hash is read", mutant: "what the home keeps beside a case is asked as JSON", mutant: "a head of any size is read", mutant: "a head answered under another type is read", mutant: "a head answered as raw JSON is read", mutant: "a head GitHub did not find is unanswered", mutant: "a head GitHub refused is read", mutant: "a file of any kind is asked beside a case", mutant: "a pin of any shape is asked by", mutant: "a file beside a case of any name is asked", mutant: "a home of any name is asked beside a case" | +| The fetch reads each case's head from the home by its pins, with no other transport, unpacks it and holds it to its tree; a head not the one pinned is `hash_differs`, one not read, unpacked or held whole `head_unreadable` and removed, a pin that is no sha256 reads nothing and the case is `not_this_set`, and the gate closes | `tests/test_evals.py::test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate`, `harness/eval/__init__.py`, `harness/eval/fetch.py`, `harness/head.py`, mutant: "the eval fetch reads no case's head", mutant: "a head is unpacked and never held to its tree", mutant: "a head not the one pinned is read as unreadable", mutant: "the tree's pin is never read", mutant: "a head is read beside another Role", mutant: "a tree that cannot be read stops the fetch", mutant: "a head not held is left in the heads folder", mutant: "a case whose pin is no sha256 has its head read" | +| An eval case requires both pins, each a whole sha256 | `tests/test_evals.py::test_a_case_dropped_under_evals_in_any_file_is_caught`, `tools/check_examples.py`, `schemas/eval-case.schema.json`, `examples/invalid/eval-case.no-head-pin.json`, `examples/invalid/eval-case.no-tree-pin.json`, `examples/invalid/eval-case.a-head-pin-cut-short.json`, `examples/invalid/eval-case.a-tree-pin-cut-short.json`, mutant: "a case need pin no head", mutant: "a case need pin no tree", mutant: "a head pin may be cut short", mutant: "a tree pin may be cut short" | +| The Maintainer's tool writes a head beside its case only once it is held to its tree and within what the job reads, pins what it wrote, writes each file whole and through no link, reads no case or folder that is a link, leaves a case it cannot pin as it was, reads no tree file as a case, prints counts and hashes alone, and says so in its exit; what it writes is what the fetch reads | `tests/test_evals.py::test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone`, `tools/heads_into_home.py`, mutant: "the Maintainer's tool writes a head not held to its tree", mutant: "the Maintainer's tool pins a case of another id", mutant: "the Maintainer's tool reads a tree answered otherwise", mutant: "the Maintainer's tool says every case was pinned", mutant: "the Maintainer's tool reads a tree it wrote as a case", mutant: "the Maintainer's tool pins a tree it did not write", mutant: "the Maintainer's tool reads with no token", mutant: "the Maintainer's tool writes through a link", mutant: "the Maintainer's tool reads a case that is a link", mutant: "the Maintainer's tool pins a head the job will refuse", mutant: "the Maintainer's tool reads a folder that is a link", mutant: "a write that fails stops the Maintainer's tool", mutant: "the Maintainer's tool leaves a file half written", mutant: "the Maintainer's tool writes a head beside a case it cannot pin" | diff --git a/examples/eval-case.also-at.json b/examples/eval-case.also-at.json index 677b63da..41a60425 100644 --- a/examples/eval-case.also-at.json +++ b/examples/eval-case.also-at.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/eval-case.json b/examples/eval-case.json index 71f4c29d..c5b29a6f 100644 --- a/examples/eval-case.json +++ b/examples/eval-case.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/eval-case.met.json b/examples/eval-case.met.json index c1988926..3359b0de 100644 --- a/examples/eval-case.met.json +++ b/examples/eval-case.met.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "met", "criterion": "PR-1", diff --git a/examples/eval-case.with-issue.json b/examples/eval-case.with-issue.json index b90c8556..ee635006 100644 --- a/examples/eval-case.with-issue.json +++ b/examples/eval-case.with-issue.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,3 +86,5 @@\n def parse(items):\n+ if not items:\n+ return []\n first = items[0]\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "met", "criterion": "PR-1", diff --git a/examples/invalid/eval-case.a-category-nobody-owns.json b/examples/invalid/eval-case.a-category-nobody-owns.json index 4921afac..c49ee422 100644 --- a/examples/invalid/eval-case.a-category-nobody-owns.json +++ b/examples/invalid/eval-case.a-category-nobody-owns.json @@ -237,6 +237,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/invalid/eval-case.a-head-as-an-object.json b/examples/invalid/eval-case.a-head-as-an-object.json index 51aa0301..a94a35dd 100644 --- a/examples/invalid/eval-case.a-head-as-an-object.json +++ b/examples/invalid/eval-case.a-head-as-an-object.json @@ -240,6 +240,8 @@ "head": { "sha": "c514ff021d93934192c9dbb14b6c31c2b421e6c7" }, + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/invalid/eval-case.a-head-pin-cut-short.json b/examples/invalid/eval-case.a-head-pin-cut-short.json new file mode 100644 index 00000000..afff1335 --- /dev/null +++ b/examples/invalid/eval-case.a-head-pin-cut-short.json @@ -0,0 +1,256 @@ +{ + "schema": "eval-case/1", + "id": "synthetic-early-return", + "role": "pr-review", + "intent": "review-change", + "facts": { + "schema": "facts/1", + "item": { + "repo": "combsmith/software-factory", + "type": "pull_request", + "number": 42 + }, + "state": "open", + "is_draft": false, + "class": "maintainer", + "author": "don-petry", + "last_pusher": "don-petry", + "targets_protected": true, + "mergeable": true, + "content_id": "sha256:5d0c", + "diff": { + "files": 3, + "lines": 120, + "truncated": false, + "characters": 4800 + }, + "changed_paths": [ + { + "path": "kernel/derive.py", + "previous": "kernel/derive_old.py" + }, + { + "path": "tests/test_decide.py", + "previous": null + }, + { + "path": "docs/glossary.md", + "previous": null + } + ], + "label_events": [ + { + "label": "factory:hold", + "action": "labeled", + "by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "at": "2026-10-11T11:40:00Z" + }, + { + "label": "factory:hold", + "action": "unlabeled", + "by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "at": "2026-10-11T11:45:00Z" + } + ], + "last_push_at": "2026-10-11T12:00:00Z", + "last_reopened_at": null, + "description_edited_at": null, + "expired_reviews": [], + "last_writer_action_at": "2026-10-11T12:00:00Z", + "reviews": [ + { + "by": { + "kind": "machine_user", + "name": "donpetry-bot", + "can_write": true + }, + "state": "dismissed", + "commit": "4b1a9c0", + "content_id": "sha256:4b1a", + "submitted_at": "2026-10-11T11:50:00Z", + "dismissed_by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "dismissed_from": "approved" + }, + { + "by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "state": "changes_requested", + "commit": "5d0c7e2", + "content_id": "sha256:5d0c", + "submitted_at": "2026-10-11T12:18:00Z", + "dismissed_by": null, + "dismissed_from": null + } + ], + "threads": [ + { + "id": "PRRT_kw7", + "resolved": false, + "opened_by": { + "kind": "lens", + "name": "pr-review", + "can_write": true + }, + "category": "functional", + "severity": "major", + "resolved_by": null, + "opened_at": "2026-10-11T12:15:00Z", + "ruling": null + }, + { + "id": "PRRT_kw1", + "resolved": false, + "opened_by": { + "kind": "lens", + "name": "coderabbit", + "can_write": false + }, + "category": null, + "severity": null, + "resolved_by": null, + "opened_at": "2026-10-11T12:15:00Z", + "ruling": null + }, + { + "id": "PRRT_kw3", + "resolved": true, + "opened_by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "category": "human", + "severity": null, + "resolved_by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "opened_at": "2026-10-11T12:16:00Z", + "ruling": null + } + ], + "checks": [ + { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml", + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + }, + { + "name": "Secret scan (gitleaks)", + "app_id": 15368, + "workflow": ".github/workflows/ci.yml", + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + }, + { + "name": "CodeQL", + "app_id": 57789, + "workflow": null, + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + } + ], + "queue": false, + "acts_since_writer": 2, + "acceptance": { + "approved": { + "digest": "sha256:c0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0de", + "commit": "1a2b3c4", + "by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "criteria": [ + { + "id": "AC-1", + "kind": "test" + }, + { + "id": "AC-2", + "kind": "test" + } + ] + }, + "head": "sha256:c0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0de", + "results": [ + { + "criterion": "AC-1", + "content_id": "sha256:4b1a", + "result": "failing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + }, + { + "criterion": "AC-1", + "content_id": "sha256:5d0c", + "result": "passing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + }, + { + "criterion": "AC-2", + "content_id": "sha256:5d0c", + "result": "passing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + } + ] + }, + "complete": { + "acceptance": true, + "reviews": true, + "threads": true, + "checks": true, + "diff": true, + "changed_paths": true, + "runs": true, + "timeline": true + }, + "read_at": "2026-10-11T12:20:00Z" + }, + "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", + "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", + "expected": { + "verdict": "finding", + "criterion": "PR-3", + "finding": { + "category": "functional", + "severity": "major", + "location": { + "path": "kernel/decide.py", + "start_line": 86, + "end_line": 94 + } + } + } +} diff --git a/examples/invalid/eval-case.a-place-in-also-at-with-no-lines.json b/examples/invalid/eval-case.a-place-in-also-at-with-no-lines.json index 0882bbc6..4c755c45 100644 --- a/examples/invalid/eval-case.a-place-in-also-at-with-no-lines.json +++ b/examples/invalid/eval-case.a-place-in-also-at-with-no-lines.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/invalid/eval-case.a-place-in-also-at-with-no-path.json b/examples/invalid/eval-case.a-place-in-also-at-with-no-path.json index 4d839688..d7d7c46b 100644 --- a/examples/invalid/eval-case.a-place-in-also-at-with-no-path.json +++ b/examples/invalid/eval-case.a-place-in-also-at-with-no-path.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/invalid/eval-case.a-short-head.json b/examples/invalid/eval-case.a-short-head.json index 4f7ea3fa..8e4aa0ba 100644 --- a/examples/invalid/eval-case.a-short-head.json +++ b/examples/invalid/eval-case.a-short-head.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff0", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/invalid/eval-case.a-tree-pin-cut-short.json b/examples/invalid/eval-case.a-tree-pin-cut-short.json new file mode 100644 index 00000000..2a74e6b8 --- /dev/null +++ b/examples/invalid/eval-case.a-tree-pin-cut-short.json @@ -0,0 +1,256 @@ +{ + "schema": "eval-case/1", + "id": "synthetic-early-return", + "role": "pr-review", + "intent": "review-change", + "facts": { + "schema": "facts/1", + "item": { + "repo": "combsmith/software-factory", + "type": "pull_request", + "number": 42 + }, + "state": "open", + "is_draft": false, + "class": "maintainer", + "author": "don-petry", + "last_pusher": "don-petry", + "targets_protected": true, + "mergeable": true, + "content_id": "sha256:5d0c", + "diff": { + "files": 3, + "lines": 120, + "truncated": false, + "characters": 4800 + }, + "changed_paths": [ + { + "path": "kernel/derive.py", + "previous": "kernel/derive_old.py" + }, + { + "path": "tests/test_decide.py", + "previous": null + }, + { + "path": "docs/glossary.md", + "previous": null + } + ], + "label_events": [ + { + "label": "factory:hold", + "action": "labeled", + "by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "at": "2026-10-11T11:40:00Z" + }, + { + "label": "factory:hold", + "action": "unlabeled", + "by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "at": "2026-10-11T11:45:00Z" + } + ], + "last_push_at": "2026-10-11T12:00:00Z", + "last_reopened_at": null, + "description_edited_at": null, + "expired_reviews": [], + "last_writer_action_at": "2026-10-11T12:00:00Z", + "reviews": [ + { + "by": { + "kind": "machine_user", + "name": "donpetry-bot", + "can_write": true + }, + "state": "dismissed", + "commit": "4b1a9c0", + "content_id": "sha256:4b1a", + "submitted_at": "2026-10-11T11:50:00Z", + "dismissed_by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "dismissed_from": "approved" + }, + { + "by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "state": "changes_requested", + "commit": "5d0c7e2", + "content_id": "sha256:5d0c", + "submitted_at": "2026-10-11T12:18:00Z", + "dismissed_by": null, + "dismissed_from": null + } + ], + "threads": [ + { + "id": "PRRT_kw7", + "resolved": false, + "opened_by": { + "kind": "lens", + "name": "pr-review", + "can_write": true + }, + "category": "functional", + "severity": "major", + "resolved_by": null, + "opened_at": "2026-10-11T12:15:00Z", + "ruling": null + }, + { + "id": "PRRT_kw1", + "resolved": false, + "opened_by": { + "kind": "lens", + "name": "coderabbit", + "can_write": false + }, + "category": null, + "severity": null, + "resolved_by": null, + "opened_at": "2026-10-11T12:15:00Z", + "ruling": null + }, + { + "id": "PRRT_kw3", + "resolved": true, + "opened_by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "category": "human", + "severity": null, + "resolved_by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "opened_at": "2026-10-11T12:16:00Z", + "ruling": null + } + ], + "checks": [ + { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml", + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + }, + { + "name": "Secret scan (gitleaks)", + "app_id": 15368, + "workflow": ".github/workflows/ci.yml", + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + }, + { + "name": "CodeQL", + "app_id": 57789, + "workflow": null, + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + } + ], + "queue": false, + "acts_since_writer": 2, + "acceptance": { + "approved": { + "digest": "sha256:c0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0de", + "commit": "1a2b3c4", + "by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "criteria": [ + { + "id": "AC-1", + "kind": "test" + }, + { + "id": "AC-2", + "kind": "test" + } + ] + }, + "head": "sha256:c0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0de", + "results": [ + { + "criterion": "AC-1", + "content_id": "sha256:4b1a", + "result": "failing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + }, + { + "criterion": "AC-1", + "content_id": "sha256:5d0c", + "result": "passing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + }, + { + "criterion": "AC-2", + "content_id": "sha256:5d0c", + "result": "passing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + } + ] + }, + "complete": { + "acceptance": true, + "reviews": true, + "threads": true, + "checks": true, + "diff": true, + "changed_paths": true, + "runs": true, + "timeline": true + }, + "read_at": "2026-10-11T12:20:00Z" + }, + "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", + "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d242901", + "expected": { + "verdict": "finding", + "criterion": "PR-3", + "finding": { + "category": "functional", + "severity": "major", + "location": { + "path": "kernel/decide.py", + "start_line": 86, + "end_line": 94 + } + } + } +} diff --git a/examples/invalid/eval-case.a-verdict-in-words.json b/examples/invalid/eval-case.a-verdict-in-words.json index 699ba814..4d7a417d 100644 --- a/examples/invalid/eval-case.a-verdict-in-words.json +++ b/examples/invalid/eval-case.a-verdict-in-words.json @@ -237,6 +237,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "the reviewer should object", "criterion": "PR-3", diff --git a/examples/invalid/eval-case.an-issue-with-no-body.json b/examples/invalid/eval-case.an-issue-with-no-body.json index 448315d3..8fd4df4c 100644 --- a/examples/invalid/eval-case.an-issue-with-no-body.json +++ b/examples/invalid/eval-case.an-issue-with-no-body.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,3 +86,5 @@\n def parse(items):\n+ if not items:\n+ return []\n first = items[0]\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "met", "criterion": "PR-1", diff --git a/examples/invalid/eval-case.four-places-in-also-at.json b/examples/invalid/eval-case.four-places-in-also-at.json index 6043f5c5..eabffbb9 100644 --- a/examples/invalid/eval-case.four-places-in-also-at.json +++ b/examples/invalid/eval-case.four-places-in-also-at.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/invalid/eval-case.no-head-pin.json b/examples/invalid/eval-case.no-head-pin.json new file mode 100644 index 00000000..f23aa9d8 --- /dev/null +++ b/examples/invalid/eval-case.no-head-pin.json @@ -0,0 +1,255 @@ +{ + "schema": "eval-case/1", + "id": "synthetic-early-return", + "role": "pr-review", + "intent": "review-change", + "facts": { + "schema": "facts/1", + "item": { + "repo": "combsmith/software-factory", + "type": "pull_request", + "number": 42 + }, + "state": "open", + "is_draft": false, + "class": "maintainer", + "author": "don-petry", + "last_pusher": "don-petry", + "targets_protected": true, + "mergeable": true, + "content_id": "sha256:5d0c", + "diff": { + "files": 3, + "lines": 120, + "truncated": false, + "characters": 4800 + }, + "changed_paths": [ + { + "path": "kernel/derive.py", + "previous": "kernel/derive_old.py" + }, + { + "path": "tests/test_decide.py", + "previous": null + }, + { + "path": "docs/glossary.md", + "previous": null + } + ], + "label_events": [ + { + "label": "factory:hold", + "action": "labeled", + "by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "at": "2026-10-11T11:40:00Z" + }, + { + "label": "factory:hold", + "action": "unlabeled", + "by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "at": "2026-10-11T11:45:00Z" + } + ], + "last_push_at": "2026-10-11T12:00:00Z", + "last_reopened_at": null, + "description_edited_at": null, + "expired_reviews": [], + "last_writer_action_at": "2026-10-11T12:00:00Z", + "reviews": [ + { + "by": { + "kind": "machine_user", + "name": "donpetry-bot", + "can_write": true + }, + "state": "dismissed", + "commit": "4b1a9c0", + "content_id": "sha256:4b1a", + "submitted_at": "2026-10-11T11:50:00Z", + "dismissed_by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "dismissed_from": "approved" + }, + { + "by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "state": "changes_requested", + "commit": "5d0c7e2", + "content_id": "sha256:5d0c", + "submitted_at": "2026-10-11T12:18:00Z", + "dismissed_by": null, + "dismissed_from": null + } + ], + "threads": [ + { + "id": "PRRT_kw7", + "resolved": false, + "opened_by": { + "kind": "lens", + "name": "pr-review", + "can_write": true + }, + "category": "functional", + "severity": "major", + "resolved_by": null, + "opened_at": "2026-10-11T12:15:00Z", + "ruling": null + }, + { + "id": "PRRT_kw1", + "resolved": false, + "opened_by": { + "kind": "lens", + "name": "coderabbit", + "can_write": false + }, + "category": null, + "severity": null, + "resolved_by": null, + "opened_at": "2026-10-11T12:15:00Z", + "ruling": null + }, + { + "id": "PRRT_kw3", + "resolved": true, + "opened_by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "category": "human", + "severity": null, + "resolved_by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "opened_at": "2026-10-11T12:16:00Z", + "ruling": null + } + ], + "checks": [ + { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml", + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + }, + { + "name": "Secret scan (gitleaks)", + "app_id": 15368, + "workflow": ".github/workflows/ci.yml", + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + }, + { + "name": "CodeQL", + "app_id": 57789, + "workflow": null, + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + } + ], + "queue": false, + "acts_since_writer": 2, + "acceptance": { + "approved": { + "digest": "sha256:c0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0de", + "commit": "1a2b3c4", + "by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "criteria": [ + { + "id": "AC-1", + "kind": "test" + }, + { + "id": "AC-2", + "kind": "test" + } + ] + }, + "head": "sha256:c0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0de", + "results": [ + { + "criterion": "AC-1", + "content_id": "sha256:4b1a", + "result": "failing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + }, + { + "criterion": "AC-1", + "content_id": "sha256:5d0c", + "result": "passing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + }, + { + "criterion": "AC-2", + "content_id": "sha256:5d0c", + "result": "passing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + } + ] + }, + "complete": { + "acceptance": true, + "reviews": true, + "threads": true, + "checks": true, + "diff": true, + "changed_paths": true, + "runs": true, + "timeline": true + }, + "read_at": "2026-10-11T12:20:00Z" + }, + "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", + "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", + "expected": { + "verdict": "finding", + "criterion": "PR-3", + "finding": { + "category": "functional", + "severity": "major", + "location": { + "path": "kernel/decide.py", + "start_line": 86, + "end_line": 94 + } + } + } +} diff --git a/examples/invalid/eval-case.no-head.json b/examples/invalid/eval-case.no-head.json index 1078d64e..e1d6afd8 100644 --- a/examples/invalid/eval-case.no-head.json +++ b/examples/invalid/eval-case.no-head.json @@ -249,5 +249,7 @@ "end_line": 94 } } - } + }, + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010" } diff --git a/examples/invalid/eval-case.no-place-in-also-at.json b/examples/invalid/eval-case.no-place-in-also-at.json index 2b505196..7dfe2724 100644 --- a/examples/invalid/eval-case.no-place-in-also-at.json +++ b/examples/invalid/eval-case.no-place-in-also-at.json @@ -238,6 +238,8 @@ }, "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "tree_sha256": "c8037a8b9d2fca258427534558b219fa94635816faf136ab59684a97d2429010", "expected": { "verdict": "finding", "criterion": "PR-3", diff --git a/examples/invalid/eval-case.no-tree-pin.json b/examples/invalid/eval-case.no-tree-pin.json new file mode 100644 index 00000000..b06e7f3c --- /dev/null +++ b/examples/invalid/eval-case.no-tree-pin.json @@ -0,0 +1,255 @@ +{ + "schema": "eval-case/1", + "id": "synthetic-early-return", + "role": "pr-review", + "intent": "review-change", + "facts": { + "schema": "facts/1", + "item": { + "repo": "combsmith/software-factory", + "type": "pull_request", + "number": 42 + }, + "state": "open", + "is_draft": false, + "class": "maintainer", + "author": "don-petry", + "last_pusher": "don-petry", + "targets_protected": true, + "mergeable": true, + "content_id": "sha256:5d0c", + "diff": { + "files": 3, + "lines": 120, + "truncated": false, + "characters": 4800 + }, + "changed_paths": [ + { + "path": "kernel/derive.py", + "previous": "kernel/derive_old.py" + }, + { + "path": "tests/test_decide.py", + "previous": null + }, + { + "path": "docs/glossary.md", + "previous": null + } + ], + "label_events": [ + { + "label": "factory:hold", + "action": "labeled", + "by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "at": "2026-10-11T11:40:00Z" + }, + { + "label": "factory:hold", + "action": "unlabeled", + "by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "at": "2026-10-11T11:45:00Z" + } + ], + "last_push_at": "2026-10-11T12:00:00Z", + "last_reopened_at": null, + "description_edited_at": null, + "expired_reviews": [], + "last_writer_action_at": "2026-10-11T12:00:00Z", + "reviews": [ + { + "by": { + "kind": "machine_user", + "name": "donpetry-bot", + "can_write": true + }, + "state": "dismissed", + "commit": "4b1a9c0", + "content_id": "sha256:4b1a", + "submitted_at": "2026-10-11T11:50:00Z", + "dismissed_by": { + "kind": "human", + "name": "don-petry", + "can_write": true + }, + "dismissed_from": "approved" + }, + { + "by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "state": "changes_requested", + "commit": "5d0c7e2", + "content_id": "sha256:5d0c", + "submitted_at": "2026-10-11T12:18:00Z", + "dismissed_by": null, + "dismissed_from": null + } + ], + "threads": [ + { + "id": "PRRT_kw7", + "resolved": false, + "opened_by": { + "kind": "lens", + "name": "pr-review", + "can_write": true + }, + "category": "functional", + "severity": "major", + "resolved_by": null, + "opened_at": "2026-10-11T12:15:00Z", + "ruling": null + }, + { + "id": "PRRT_kw1", + "resolved": false, + "opened_by": { + "kind": "lens", + "name": "coderabbit", + "can_write": false + }, + "category": null, + "severity": null, + "resolved_by": null, + "opened_at": "2026-10-11T12:15:00Z", + "ruling": null + }, + { + "id": "PRRT_kw3", + "resolved": true, + "opened_by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "category": "human", + "severity": null, + "resolved_by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "opened_at": "2026-10-11T12:16:00Z", + "ruling": null + } + ], + "checks": [ + { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml", + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + }, + { + "name": "Secret scan (gitleaks)", + "app_id": 15368, + "workflow": ".github/workflows/ci.yml", + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + }, + { + "name": "CodeQL", + "app_id": 57789, + "workflow": null, + "conclusion": "success", + "completed_at": "2026-10-11T12:06:00Z" + } + ], + "queue": false, + "acts_since_writer": 2, + "acceptance": { + "approved": { + "digest": "sha256:c0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0de", + "commit": "1a2b3c4", + "by": { + "kind": "human", + "name": "a-colleague", + "can_write": true + }, + "criteria": [ + { + "id": "AC-1", + "kind": "test" + }, + { + "id": "AC-2", + "kind": "test" + } + ] + }, + "head": "sha256:c0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0dec0de", + "results": [ + { + "criterion": "AC-1", + "content_id": "sha256:4b1a", + "result": "failing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + }, + { + "criterion": "AC-1", + "content_id": "sha256:5d0c", + "result": "passing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + }, + { + "criterion": "AC-2", + "content_id": "sha256:5d0c", + "result": "passing", + "check": { + "name": "verify", + "app_id": 15368, + "workflow": ".github/workflows/verify.yml" + } + } + ] + }, + "complete": { + "acceptance": true, + "reviews": true, + "threads": true, + "checks": true, + "diff": true, + "changed_paths": true, + "runs": true, + "timeline": true + }, + "read_at": "2026-10-11T12:20:00Z" + }, + "diff": "--- a/kernel/decide.py\n+++ b/kernel/decide.py\n@@ -86,9 +86,9 @@\n- if not items:\n- return []\n first = items[0]\n+ if not items:\n+ return []\n return parse(first, items[1:])\n", + "head": "c514ff021d93934192c9dbb14b6c31c2b421e6c7", + "head_sha256": "6bbe1f08b91c69085d33a368e265ba4f7ef7438228aa0047546ab018b61eeec2", + "expected": { + "verdict": "finding", + "criterion": "PR-3", + "finding": { + "category": "functional", + "severity": "major", + "location": { + "path": "kernel/decide.py", + "start_line": 86, + "end_line": 94 + } + } + } +} diff --git a/harness/eval/AGENTS.md b/harness/eval/AGENTS.md index 44ec90a3..5b834c5e 100644 --- a/harness/eval/AGENTS.md +++ b/harness/eval/AGENTS.md @@ -1,7 +1,8 @@ # harness/eval/ -The eval job's steps, a module each, in the order `__init__.py` gives. Their rules are `evals/AGENTS.md`'s, beside the sets they run, but for what two issues add, which are here: -issue 612, "The eval job reads a named set, and a gate set's report is aggregates only", and issue 613, "The gate pools k dispatches on one commit and reads all of them". +The eval job's steps, a module each, in the order `__init__.py` gives. Their rules are `evals/AGENTS.md`'s, beside the sets they run, but for what three issues add, which are here: +issue 612, "The eval job reads a named set, and a gate set's report is aggregates only", issue 613, "The gate pools k dispatches on one commit and reads all of them", +and issue 611, "A case's head is read from the private home, pinned by its hash, and the run is sharded". A dispatch names its set. A set whose manifest says it is a gate set is reported by counts alone, `eval-aggregate/1`, and each case's outcome with its Result goes to a ledger, `eval-ledger/1`, kept in the scratch. `seal.py` encrypts the ledger with `openssl cms` to each of the Maintainer's certificates that its pins, in a protected path under `.github/`, name by sha256 (`seal.RECIPIENTS`). The ciphertext alone is uploaded, as `eval-ledger`. No key that opens a ledger is in the job or the repo, and nothing automated reads one. Until the Maintainer commits his certificates, a gate set's run fails at the seal, `no_recipient`. @@ -9,6 +10,7 @@ The ciphertext lives only as a run artifact under its retention until a storage A gate set gives its own bars and least of a kind. The Release's gate pools every dispatch on a commit of the gate set `config/harness.json` names, `gate_set`. Until one is named, the gate reads the review Intent's set by its newest successful dispatch, which a re-dispatch can still make the best of several. A gate set's dispatch asks its preflight first, a few cases of the review Intent's set scored apart, so that a broken pipeline stops it before any held-out case is asked. +A case's head is read from the home beside the case, its tarball and its tree each by the sha256 the case pins, and held to that tree (issue 611); nothing in the job reads with no credential. | Rule | Held by | |---|---| @@ -26,5 +28,6 @@ A gate set's dispatch asks its preflight first, a few cases of the review Intent | The pool is three runs at least, every one, each kind summed under the set's bars; one faulted closes it | `tests/test_evals.py::test_the_gate_pools_every_dispatch_on_the_commit_and_one_faulted_closes_it` | | A gate set's preflight is three to five of the review Intent's cases, none a gate set's by its id or its hash | `tests/test_evals.py::test_a_gate_sets_preflight_is_of_the_development_set_and_no_gate_sets_case` | | The preflight is asked first; a fault in it, or no pass, stops the dispatch before a case of the set is asked, and the gate reads it as faulted | `tests/test_evals.py::test_a_gate_sets_dispatch_asks_its_preflight_first_and_a_broken_one_stops_it` | +| Each case's head is read from the home by its pins, unpacked and held to its tree; one not the one pinned, or not read whole, closes the gate | `tests/test_evals.py::test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate` | | The Release's gate judges a gate set named in the configuration by every completed dispatch on the commit | `tests/test_release_step.py::test_a_gate_set_is_judged_by_every_dispatch_on_the_commit` | | Each of these rules has a mutant that a test kills | `tools/mutants.py` | diff --git a/harness/eval/__init__.py b/harness/eval/__init__.py index 442a80bf..f2bc9c12 100644 --- a/harness/eval/__init__.py +++ b/harness/eval/__init__.py @@ -6,7 +6,8 @@ names; a set of no case ends the run here, `eval_set_empty`, before any credential is used (ruling 8); - `fetch` reads each case from the private home through `adapters/heldout.py`, with the Maintainer's read-only token, into a scratch directory outside the workspace, on a runner thrown away when the job ends, and each case's head, the - commit its diff was taken at, as its tarball with no credential, unpacked by hand beside it (decision 0138); + commit its diff was taken at, from the home too: its tarball and its commit's tree, kept beside the case and each + pinned by its sha256 there, the tarball unpacked by hand and held to the tree (decision 0138, issue 611); - `run` gives each case to the review Intent, its Task built and its Engine called by the functions a live Run's are (`harness/task.from_admission`, `harness/run.answer`), and scores its Result with `tools/eval_result.py`; for a gate set, the development cases its manifest names as its preflight first, scored apart (issue 613); @@ -358,43 +359,59 @@ def seen_of(review) -> dict: return said -def head_of(data: bytes, folder: pathlib.Path, ident: str, transport): - """The head a case names, its tarball read through `transport` and unpacked into `folder` under its id, or the code - `head_unreadable` when it cannot be read or unpacked whole; None, and nothing read, for a case with no head to read, - which its schema refuses after (decision 0138). Nothing of the head is printed or kept but its files there.""" +def head_of(env: dict, data: bytes, folder: pathlib.Path, role: str, ident: str, transport): + """The head a case names, read from the home through `transport` beside the case, its tarball and its commit's tree + each by the sha256 the case pins, unpacked into `folder` under its id and held to that tree file by file (issue 611; + decision 0142); or the code that says why not: `hash_differs` for a file not the one pinned, and `head_unreadable` + for one that cannot be read, unpacked or held whole. None, and nothing read, for a case with no head to read, which + its schema refuses after (decision 0138). Nothing of the head is printed or kept but its files there.""" + from adapters import heldout from harness import head try: case = json.loads(data) - repo, sha = case["facts"]["item"]["repo"], case["head"] + sha, pins = case["head"], (case["head_sha256"], case["tree_sha256"]) except (ValueError, TypeError, KeyError, RecursionError): return None - if not (isinstance(sha, str) and COMMIT.fullmatch(sha) and isinstance(repo, str)): + # A case whose head or pins are not whole is no case of this set, which its schema says after: nothing is read. + if not (isinstance(sha, str) and COMMIT.fullmatch(sha) and all(isinstance(p, str) and SHA256.fullmatch(p) for p in pins)): return None + read = {} + for kind, pin in zip(("tar.gz", "tree.json"), pins): + read[kind], code = heldout.beside(transport, env.get("HELDOUT_OWNER"), env.get("HELDOUT_REPO"), role, ident, + kind, pin) + if code is not None: + return "hash_differs" if code == "hash_differs" else "head_unreadable" folder.mkdir(mode=0o700, exist_ok=True) + into = None try: - head.unpack(head.fetch(transport, repo, sha), sha, within(folder, ident)) - except (head.Refused, OSError, Stop): - shutil.rmtree(str(folder / ident), ignore_errors=True) + tree = head.tree_of(json.loads(read["tree.json"])) + into = within(folder, ident) + head.unpack(read["tar.gz"], sha, into) + head.held(into, tree) + except (head.Refused, OSError, Stop, ValueError, RecursionError): + if into is not None: + shutil.rmtree(into, ignore_errors=True) return "head_unreadable" return None -def fetch(env: dict, root: pathlib.Path, workspace: pathlib.Path, transport, heads=None) -> str: +def fetch(env: dict, root: pathlib.Path, workspace: pathlib.Path, transport) -> str: """Each case the set names, read from the home into the scratch directory as the bytes its manifest hashed, and why each that is not there is not: `fetched.json`, by id, a code or null, and `fetch-causes.json`, by id, what a read - that raised raised, by its class and place alone. Each case's head is read through `heads`, with no credential, and - a case whose head cannot be read whole is not there either, `head_unreadable`. Heads are unpacked in `heads_place`, - apart from the cases (issue 623). A gate set's preflight is read first, the same way, into the scratch's own folder - of it, PREFLIGHT; its heads beside the set's, by their ids, which `preflight_of` holds apart (issue 613).""" + that raised raised, by its class and place alone. Each case's head is read from the home too, beside the case, by + `head_of`, and a case whose head cannot be read whole is not there either. No head is read with no credential + (issue 611). Heads are unpacked in `heads_place`, apart from the cases (issue 623). A gate set's preflight is read + first, the same way, into the scratch's own folder of it, PREFLIGHT; its heads beside the set's, by their ids, which + `preflight_of` holds apart (issue 613).""" kept, found, place = _planned(env, root, workspace) said = "" if kept["preflight"]: - ahead = _fetched(env, workspace, transport, heads, dict(found, cases=kept["preflight"]), place / PREFLIGHT) + ahead = _fetched(env, workspace, transport, dict(found, cases=kept["preflight"]), place / PREFLIGHT) said = ", %d of %d to preflight" % (ahead, len(kept["preflight"])) - return "fetched %d of %d cases" % (_fetched(env, workspace, transport, heads, found, place), found["count"]) + said + return "fetched %d of %d cases" % (_fetched(env, workspace, transport, found, place), found["count"]) + said -def _fetched(env: dict, workspace: pathlib.Path, transport, heads, found: dict, place: pathlib.Path) -> int: +def _fetched(env: dict, workspace: pathlib.Path, transport, found: dict, place: pathlib.Path) -> int: """How many of `found`'s cases `fetch` read into `place`, writing why each other was not read there.""" from adapters import heldout why, causes = {}, {} @@ -405,8 +422,8 @@ def _fetched(env: dict, workspace: pathlib.Path, transport, heads, found: dict, except Exception as err: data, code = None, "unanswered" causes[entry["id"]] = cause_of(err) - if data is not None and heads is not None: - code = head_of(data, heads_place(env, workspace), entry["id"], heads) + if data is not None: + code = head_of(env, data, heads_place(env, workspace), found["role"], entry["id"], transport) why[entry["id"]] = code if data is not None and code is None: path = within(place / "cases", "%s.json" % entry["id"]) diff --git a/harness/eval/canary.py b/harness/eval/canary.py index 8126a17d..378acecd 100644 --- a/harness/eval/canary.py +++ b/harness/eval/canary.py @@ -15,6 +15,7 @@ import json import os import pathlib +import shutil import subprocess import sys import tempfile @@ -49,10 +50,27 @@ def asked(args, env, timeout, **given): return asked +def head_of(data: bytes, folder: pathlib.Path, ident: str, transport): + """The example's head, its commit's tarball read through `transport`, with no credential, from this repo, which is + public, and unpacked into `folder` under its id; or `head_unreadable`. The eval job reads a case's head from the + held-out home instead, pinned by the case (issue 611): the example is no case of the home, and pins none it keeps.""" + from harness import head + case = json.loads(data) + repo, sha = case["facts"]["item"]["repo"], case["head"] + folder.mkdir(mode=0o700, exist_ok=True) + try: + head.unpack(head.fetch(transport, repo, sha), sha, job.within(folder, ident)) + except (head.Refused, OSError, job.Stop): + shutil.rmtree(str(folder / ident), ignore_errors=True) + return "head_unreadable" + return None + + def review(env: dict, call=live.cli_call, clock=time.time, heads=None) -> int: """One review by the eval job's path: 0 if the CLI answered in a status the Engine may write, its answer read and scored, and 1 otherwise, a `failed` Result among them. A scorer that raises fails the step. The example's head is - read through `heads`, by default with no credential, as the eval job's fetch reads a case's (decision 0138).""" + read through `heads`, by default with no credential, from this public repo (decision 0138); the eval job reads a case's + from the held-out home instead (issue 611).""" case = json.loads((job.ROOT / "examples" / "eval-case.json").read_text()) content = "sha256:" + hashlib.sha256(case["diff"].encode("utf-8")).hexdigest() case["facts"]["content_id"] = content @@ -72,7 +90,7 @@ def answered(given: dict) -> dict: if heads is None: from adapters import github heads = github.Live.anonymous() - fetched = job.head_of(data, pathlib.Path(unpacked), case["id"], heads) + fetched = head_of(data, pathlib.Path(unpacked), case["id"], heads) verdict, reasons = job.assess(found, {case["id"]: fetched}, place, answered)[case["id"]] status = results[0].get("status") if results else None print("status %s, verdict %s, reasons %s" % (status, verdict, ",".join(reasons) or "none")) diff --git a/harness/eval/fetch.py b/harness/eval/fetch.py index db558772..36411aba 100644 --- a/harness/eval/fetch.py +++ b/harness/eval/fetch.py @@ -1,7 +1,8 @@ """The eval job's fetch: each case from the private home, with the Maintainer's own token for it, read-only on the home alone (decision 0053), and nothing else. The token is used for reads alone: a request that is not a GET is refused -before it is sent, so no change to this step's code can write with it. Each case's head is read with no credential at -all, its tarball alone (decision 0138): the cases' repo is public, and the home's token is never sent anywhere else.""" +before it is sent, so no change to this step's code can write with it. Each case's head is read from the home as well, +its tarball and its commit's tree beside the case, each pinned by its sha256 there (issue 611): the home's token is +never sent anywhere else, and nothing is read with no credential.""" from harness.eval import Stop, exit_with, fetch @@ -27,8 +28,7 @@ def transport(env: dict): def step(env, root, workspace): - from adapters import github - return fetch(env, root, workspace, transport(env), github.Live.anonymous()) + return fetch(env, root, workspace, transport(env)) if __name__ == "__main__": diff --git a/harness/head.py b/harness/head.py index 6b92ed17..d00ad362 100644 --- a/harness/head.py +++ b/harness/head.py @@ -234,9 +234,15 @@ def tree(transport, repo: str, sha: str) -> dict: answer = transport.send(github.request("GET", path)) except github.Unanswered: raise Refused("the commit's tree could not be read") from None - body = answer.get("body") - if answer.get("status") != 200 or not isinstance(body, dict) or body.get("truncated") is not False \ - or not isinstance(body.get("tree"), list): + if answer.get("status") != 200: + raise Refused("the commit's tree was not answered whole") + return tree_of(answer.get("body")) + + +def tree_of(body) -> dict: + """A commit's tree, every level of it, as `tree` reads GitHub's answer, from the answer's body: so an eval case's + tree, kept beside it in the held-out home as GitHub answered it, is read by the same checks (issue 611).""" + if not isinstance(body, dict) or body.get("truncated") is not False or not isinstance(body.get("tree"), list): raise Refused("the commit's tree was not answered whole") files, links, seen = {}, set(), set() for entry in body["tree"]: diff --git a/schemas/eval-case.schema.json b/schemas/eval-case.schema.json index 2492886c..4eff02ed 100644 --- a/schemas/eval-case.schema.json +++ b/schemas/eval-case.schema.json @@ -57,6 +57,8 @@ "facts", "diff", "head", + "head_sha256", + "tree_sha256", "expected" ], "properties": { @@ -86,6 +88,16 @@ "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "head_sha256": { + "description": "The sha256 of the head's tarball as GitHub archived the commit, kept beside the case in the held-out home, `/.tar.gz`: the eval job reads the head there, with the home's token, and none with no credential (issue 611).", + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "tree_sha256": { + "description": "The sha256 of the commit's tree, every level of it, as GitHub answered it, kept beside the case in the held-out home, `/.tree.json`: the head is held to it file by file (issue 611; decision 0142).", + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, "issue": { "description": "The issue the change serves, as a live Run's Task holds it (decision 0110): optional, so a case without one is reviewed on its diff alone.", "type": "object", diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 9349abf4..7a2012ce 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -283,8 +283,8 @@ def open(self, req, timeout=None): self.assertNotIn("secret", held(refused.exception)) def test_a_transport_with_no_token_sends_a_commits_tarball_alone(self): - # Decision 0138: the eval job and the canary read the public cases' repo's heads with no credential at all, and - # a transport made for that sends nothing else, so no other read is ever made with no token by mistake. + # Decision 0138: the canary reads its example's head from this public repo with no credential at all (the eval + # job reads a case's head from the home, issue 611), and a transport made for that sends nothing else, so no other read is ever made with no token by mistake. anonymous = github.Live.anonymous() sent = [] @@ -3489,6 +3489,17 @@ def home_file(name: str, data: bytes, path: str = None, **change) -> dict: "response": {"status": 200, "headers": {"content-type": "application/json; charset=utf-8"}, "body": body}} +def home_beside(name: str, kind: str, data: bytes, status: int = 200, role: str = "pr-review", **change) -> dict: + """What a stand-in home, `o/home`, keeps beside a case, `/.`, its head's tarball or its tree, as the + contents API answers the raw media type: the bytes as they are, under its own type (trials/P-62.json).""" + from adapters import github, heldout + headers = dict({"content-type": heldout.RAW + "; charset=utf-8", "content-length": str(len(data))}, **change) + return {"request": {"via": "rest", "call": "GET /repos/o/home/contents/%s/%s.%s" % (role, name, kind), + "headers": {"accept": heldout.RAW}}, + "response": {"status": status, "headers": headers, + "body": data.decode("utf-8", errors="surrogateescape") if status == 200 else "404: Not Found"}} + + def home_set(*named) -> dict: return {"schema": "eval-set/1", "role": "pr-review", "intent": "review-change", "count": len(named), "cases": [{"id": n, "sha256": hashlib.sha256(d).hexdigest()} for n, d in named]} @@ -3553,6 +3564,62 @@ def send(self, req): self.assertEqual(asked, []) self.assertEqual(heldout.case(Anything(), "o", "home", "pr-review", entry), (self.ONE, None)) self.assertEqual(asked, ["/repos/o/home/contents/pr-review/c-1.json"]) + + def test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins(self): # issue 611 + # A head's tarball and its tree, beside the case, are asked with the raw media type, whose answer is the file's + # bytes whole however large (trials/P-62.json), and handed back only as the bytes the case pinned. + tar, tree = bytes(range(256)) * 9, b'{"tree": [], "truncated": false}\n' + pins = {"tar.gz": hashlib.sha256(tar).hexdigest(), "tree.json": hashlib.sha256(tree).hexdigest()} + # The answer's headers as GitHub gave them to the Maintainer's login for a file of 1.5 MiB (P-62's exchange 5). + p62 = json.loads((ROOT / "trials" / "P-62.json").read_text())["exchanges"][5] + self.assertEqual(p62["request"]["headers"]["accept"], heldout.RAW) + grounded = {k: v for k, v in p62["response"]["headers"].items() if k in ("content-type",)} + home = github.Replay([home_beside("c-1", "tar.gz", tar, **grounded), home_beside("c-1", "tree.json", tree)]) + self.assertEqual([heldout.beside(home, "o", "home", "pr-review", "c-1", k, pins[k]) for k in ("tar.gz", "tree.json")], + [(tar, None), (tree, None)]) + + def read(found, kind="tar.gz", pin=pins["tar.gz"], owner="o", name="home", role="pr-review", ident="c-1"): + return heldout.beside(github.Replay(found), owner, name, role, ident, kind, pin) + json_typed = home_beside("c-1", "tar.gz", tar, **{"content-type": "application/json; charset=utf-8"}) + raw_json = home_beside("c-1", "tar.gz", tar, **{"content-type": heldout.RAW + "+json; charset=utf-8"}) + untyped = home_beside("c-1", "tar.gz", tar) + del untyped["response"]["headers"]["content-type"] + failed = home_beside("c-1", "tar.gz", tar) + failed["response"]["status"] = 500 + large = home_beside("c-1", "tar.gz", tar) + bounded = mock.patch.dict(heldout.BESIDE, {"tar.gz": len(tar) - 1, "tree.json": len(tree) - 1}) + for found, code in (([home_beside("c-1", "tar.gz", tar[1:])], "hash_differs"), + ([home_beside("c-1", "tar.gz", tar, status=404)], "not_found"), + ([failed], "unanswered"), ([], "unanswered"), ([json_typed], "unanswered"), + ([raw_json], "unanswered"), ([untyped], "unanswered"), + ([home_beside("c-1", "tree.json", tree)], "unanswered")): + self.assertEqual(read(found), (None, code), code) + with bounded: + self.assertEqual(read([large]), (None, "too_large")) + self.assertEqual(read([home_beside("c-1", "tree.json", tree)], kind="tree.json", pin=pins["tree.json"]), + (None, "too_large")) + self.assertEqual(read([home_beside("c-1", "tree.json", tree)], kind="tree.json", pin=pins["tar.gz"]), + (None, "hash_differs")) + # A name, a kind or a pin that could name another file, or a home that could name another repo, is refused + # before anything is asked. + asked = [] + + class Anything: + def send(self, req): + asked.append((req["path"], req["accept"])) + return home_beside("c-1", "tar.gz", tar)["response"] + for kind, role, ident, pin in (("json", "pr-review", "c-1", pins["tar.gz"]), ("tar.gz", "pr-review/..", "c-1", pins["tar.gz"]), + ("tar.gz", "pr-review", "../c-1", pins["tar.gz"]), ("tar.gz", "pr-review", "c-1.json", pins["tar.gz"]), + ("tar.gz", "pr-review", "c-1", "x"), ("tar.gz", "pr-review", "c-1", None), + ("tar.gz", None, "c-1", pins["tar.gz"]), (None, "pr-review", "c-1", pins["tar.gz"])): + self.assertEqual(heldout.beside(Anything(), "o", "home", role, ident, kind, pin), (None, "not_this_set"), + (kind, role, ident, pin)) + for owner, name in (("o", ".."), ("o/x", "home"), (None, "home"), ("o", None)): + self.assertEqual(heldout.beside(Anything(), owner, name, "pr-review", "c-1", "tar.gz", pins["tar.gz"]), + (None, "unanswered"), (owner, name)) + self.assertEqual(asked, []) + self.assertEqual(heldout.beside(Anything(), "o", "home", "pr-review", "c-1", "tar.gz", pins["tar.gz"]), (tar, None)) + self.assertEqual(asked, [("/repos/o/home/contents/pr-review/c-1.tar.gz", heldout.RAW)]) def per_hundred(found: dict) -> dict: """A fixture's exchange asked as the port asks a list: a full page of a hundred, GitHub's answer kept.""" found = copy.deepcopy(found) diff --git a/tests/test_evals.py b/tests/test_evals.py index 0a0c69a4..d23f21e9 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -75,8 +75,9 @@ def opened(sealed: pathlib.Path, cert: pathlib.Path, key: pathlib.Path) -> bytes class Heads: - """A transport standing in for the anonymous one `fetch` reads each case's head with: each commit's tarball as - codeload serves one (P-60), or, for a commit in `refused`, a 404. What it was asked is kept by path.""" + """A transport standing in for the anonymous one the canary reads its example's head with: each commit's tarball as + codeload serves one (P-60), or, for a commit in `refused`, a 404. What it was asked is kept by path. The eval job + reads a case's head from the home instead (issue 611).""" def __init__(self, refused=()): self.refused, self.asked = set(refused), [] @@ -91,6 +92,46 @@ def send(self, req: dict) -> dict: return {"status": 200, "headers": {}, "body": support.tarball(sha).decode("utf-8", errors="surrogateescape")} +def stand_in_head(sha: str) -> tuple: + """`(tarball, tree)`: what the home keeps beside a case whose head is `sha`, its tarball as codeload serves one (P-60) + and its commit's tree as the API answers it (trials/P-60.tree.json), each byte for byte as the eval job reads it + there (issue 611). The head holds one file, as `tests/support.tarball` writes it.""" + from tests import support + body = b"# a repo\n" + blob = hashlib.sha1(b"blob %d\0" % len(body) + body).hexdigest() + tree = {"sha": "e" * 40, "truncated": False, + "tree": [{"path": "README.md", "mode": "100644", "type": "blob", "sha": blob, "size": len(body)}]} + return support.tarball(sha), (json.dumps(tree, indent=2) + "\n").encode() + + +def pinned(case: dict) -> dict: + """`case`, its head pinned by the sha256 of its stand-in tarball and tree, as the Maintainer's tool pins a case's.""" + tar, tree = stand_in_head(case["head"]) + return dict(case, head_sha256=hashlib.sha256(tar).hexdigest(), tree_sha256=hashlib.sha256(tree).hexdigest()) + + +def home_of(cases: dict, kept=None) -> list: + """The exchanges of a stand-in home for each of `cases`, in order: the case, and, for one that pins a head, what the + home keeps beside it, `kept(id, sha)` where given, as far as the job asks: the tree is not asked once the tarball is + not the one pinned.""" + from tests.test_adapters import home_beside, home_file + exchanges = [] + for ident, data in cases.items(): + exchanges.append(home_file(ident, data)) + try: + case = json.loads(data) + sha, pins = case["head"], (case["head_sha256"], case["tree_sha256"]) + except (ValueError, TypeError, KeyError, RecursionError): + continue + if not (isinstance(sha, str) and re.fullmatch(r"[0-9a-f]{40}", sha)): + continue + tar, tree = (kept or (lambda i, h: stand_in_head(h)))(ident, sha) + exchanges.append(home_beside(ident, "tar.gz", tar)) + if hashlib.sha256(tar).hexdigest() == pins[0]: + exchanges.append(home_beside(ident, "tree.json", tree)) + return exchanges + + def github_tarred() -> int: from adapters import github return github.TARRED @@ -924,7 +965,7 @@ def setUp(self): self.root, self.work, self.temp = base / "root", base / "work", base / "temp" for place in (self.root, self.work, self.temp): place.mkdir() - one = load("examples/eval-case.json") + one = pinned(load("examples/eval-case.json")) # A case's facts name the content its diff is, as the set's must. one["facts"]["content_id"] = "sha256:" + hashlib.sha256(one["diff"].encode()).hexdigest() two = dict(copy.deepcopy(one), id="poc-two") @@ -978,19 +1019,16 @@ def path(self) -> pathlib.Path: def step(self, step, *given, env=None): from harness import eval as job - if getattr(step, "__module__", None) == job.__name__ and getattr(step, "__name__", None) == "fetch" \ - and len(given) == 1: - given += (Heads(),) # each case's head, read as codeload would answer it (decision 0138) try: return step(dict(self.env, **(env or {})), self.root, self.work, *given) except job.Stop as stop: return stop.code def home(self, cases: dict, gate: bool = False): - """A home that answers for each of `cases`, in order; for a gate set's run, for its preflight first.""" + """A home that answers for each of `cases`, in order, with what it keeps beside each; for a gate set's run, for + its preflight first.""" from adapters import github - from tests.test_adapters import home_file - return github.Replay([home_file(i, d) for i, d in (dict(self.pre, **cases) if gate else cases).items()]) + return github.Replay(home_of(dict(self.pre, **cases) if gate else cases)) def test_a_set_of_no_case_ends_the_run_before_anything_is_read(self): import os @@ -1597,7 +1635,7 @@ def test_the_job_writes_each_kinds_word_and_the_judge_holds_it(self): # decisio for raw in self.cases.values(): if json.loads(raw)["expected"]["verdict"] == "met": met = json.loads(raw) - met = met or load("examples/eval-case.met.json") + met = met or pinned(load("examples/eval-case.met.json")) met["facts"] = dict(met["facts"], content_id="sha256:" + hashlib.sha256(met["diff"].encode()).hexdigest()) def run(miss): @@ -1649,7 +1687,7 @@ def reviewed(self): """A held-out case of the review Intent, as #183 writes them: its facts name the content its diff is, so its Task is a live Run's Task of that content. Made from tests/test_harness.py's change; nothing of the home.""" from tests import test_harness as live - case = dict(load("examples/eval-case.json"), id="poc-reviewed", facts=live.facts(), diff=live.DIFF.decode()) + case = pinned(dict(load("examples/eval-case.json"), id="poc-reviewed", facts=live.facts(), diff=live.DIFF.decode())) return case, live def test_the_facts_the_set_was_made_with_read_as_todays_once_filled(self): # decision 0105 @@ -1717,13 +1755,17 @@ def answering(the_task, *given, **named): built.append(json.dumps(the_task, sort_keys=True)) return answered(the_task, *given, **named) - def scored(heads=None, case=case): + def scored(kept=None, case=case): + from adapters import github seen["files"], seen["folders"] = {}, [] self.clean() # each run's scratch and heads are its own + if kept is not None: # the head the home keeps, and the case pins it + tar, tree = kept(case["id"], case["head"]) + case = dict(case, head_sha256=hashlib.sha256(tar).hexdigest(), tree_sha256=hashlib.sha256(tree).hexdigest()) cases = {case["id"]: (json.dumps(case, indent=2) + "\n").encode()} self.set(cases) self.step(job.plan) - self.step(job.fetch, self.home(cases), *([heads] if heads else [])) + self.step(job.fetch, github.Replay(home_of(cases, kept))) kept_plan = json.loads((self.temp / "heldout" / "plan.json").read_text()) (self.temp / "heldout" / "plan.json").write_text(json.dumps(dict(kept_plan, planned_at=1000))) cli = Reading(said=live.answer(live.judged(findings=[], effects={}))) @@ -1768,14 +1810,11 @@ def words(value): self.assertEqual(given[2], given[0]) # A head whose archive holds git's folder is refused whole: the case is unreadable by its code, and asks nothing. - class Historied(Heads): - def send(self, req): - from adapters import github - from tests import support - sha = github.TARBALL.fullmatch(req["path"]).group(3) - return {"status": 200, "headers": {}, "body": support.tarball(sha, [("README.md", b"0", b"# a repo\n"), - (".git/HEAD", b"0", b"ref: refs/heads/main\n")]).decode("utf-8", errors="surrogateescape")} - said, cli = scored(Historied()) + def historied(ident, sha): + from tests import support + return (support.tarball(sha, [("README.md", b"0", b"# a repo\n"), (".git/HEAD", b"0", b"ref: refs/heads/main\n")]), + stand_in_head(sha)[1]) + said, cli = scored(historied) self.assertEqual((said, cli.asked()), ("scored 1 of 1 cases, 0 passed", [])) self.assertEqual([c["reasons"] for c in json.loads((self.work / job.REPORT).read_text())["cases"]], [["head_unreadable"]]) @@ -2547,7 +2586,7 @@ def test_each_head_is_unpacked_apart_from_the_cases(self): # issue 623 # The plan makes the heads folder, as closed as the scratch (issue 626). self.assertTrue((self.temp / "heads").is_dir(), "the plan makes the heads folder") self.assertEqual(os.stat(str(self.temp / "heads")).st_mode & 0o777, 0o700) - self.assertEqual(self.step(fetch, self.home(cases), Heads()), "fetched 1 of 1 cases") + self.assertEqual(self.step(fetch, self.home(cases)), "fetched 1 of 1 cases") place = self.temp / "heldout" kept_plan = json.loads((place / "plan.json").read_text()) (place / "plan.json").write_text(json.dumps(dict(kept_plan, planned_at=1000))) @@ -2587,48 +2626,192 @@ def __call__(self, args, env, timeout, **given): (self.temp / "heldout").mkdir() self.assertEqual(self.step(plan), "scratch_unreadable") - def test_each_cases_head_is_read_with_no_credential_and_one_not_read_closes_the_gate(self): # decision 0138 - # The fetch reads each case's head, the commit its diff was taken at, as its tarball through a transport of its - # own, never the home's, and unpacks it under heads/; the run step reads it as the case's folder and removes it - # once the case is asked. A head that cannot be read whole is the case's `head_unreadable`, an Engine's or the - # Factory's fault, never the reviewer's: the gate closes. + def test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate(self): # issue 611 + # The fetch reads each case's head from the home, with the home's token, beside the case: its tarball, then its + # commit's tree, each by the raw media type and the sha256 the case pins; unpacks it under heads/ and holds it to + # the tree file by file (decision 0142). Nothing is read with no credential. The run step reads the head as the + # case's folder and removes it once the case is asked. A head not the one pinned is `hash_differs`, and one + # that cannot be read, unpacked or held whole `head_unreadable`, a fault of the run or the Engine, never the + # reviewer's: the gate closes. + from adapters import github, heldout from harness import eval as job from harness.eval import run as step + from tests import support + from tests.test_adapters import home_beside, home_file case, live = self.reviewed() - cases = {i: (json.dumps(dict(case, id=i, head=h), indent=2) + "\n").encode() - for i, h in (("poc-one", "1" * 40), ("poc-two", "2" * 40))} + tar, tree = stand_in_head(case["head"]) + other = {"sha": "e" * 40, "truncated": False, "tree": [dict(json.loads(tree)["tree"][0], path="OTHER.md")]} + truncated = dict(json.loads(tree), truncated=True) + kept = {"poc-one": (tar, tree), "poc-two": (None, None), "poc-three": (support.tarball("3" * 40), None), + "poc-four": (tar, (json.dumps(other) + "\n").encode()), "poc-five": (tar, b"not json\n"), + "poc-six": (tar, (json.dumps(truncated) + "\n").encode())} + cases, exchanges = {}, [] + for ident, (has, holds) in kept.items(): + # Each pins what the home keeps for poc-one, but poc-four, -five and -six pin the tree the home keeps. + pin = dict(case, id=ident) + if holds is not None: + pin["tree_sha256"] = hashlib.sha256(holds).hexdigest() + cases[ident] = (json.dumps(pin, indent=2) + "\n").encode() + exchanges.append(home_file(ident, cases[ident])) + exchanges.append(home_beside(ident, "tar.gz", b"", status=404) if has is None else home_beside(ident, "tar.gz", has)) + if has is not None and has == tar: + exchanges.append(home_beside(ident, "tree.json", holds)) + # A pin that is no sha256 is no case of the set, as a pin left out is: nothing is read beside it. + cases["poc-seven"] = (json.dumps(dict(case, id="poc-seven", head_sha256="x"), indent=2) + "\n").encode() + exchanges.append(home_file("poc-seven", cases["poc-seven"])) self.set(cases) self.step(job.plan) - heads = Heads(refused={"2" * 40}) - self.assertEqual(self.step(job.fetch, self.home(cases), heads), "fetched 1 of 2 cases") - repo = case["facts"]["item"]["repo"] - self.assertEqual(heads.asked, [("/repos/%s/tarball/%s" % (repo, "1" * 40), True, github_tarred()), - ("/repos/%s/tarball/%s" % (repo, "2" * 40), True, github_tarred())]) + asked = [] + + class Home(github.Replay): + def send(self, req): + asked.append((req["path"].rsplit("/", 1)[-1], req["accept"])) + return super().send(req) + self.assertEqual(self.step(getattr(job.fetch, "fetch", job.fetch), Home(exchanges)), "fetched 2 of 7 cases") + self.assertEqual(asked[:3], [("poc-one.json", github.JSON), ("poc-one.tar.gz", heldout.RAW), + ("poc-one.tree.json", heldout.RAW)]) + self.assertEqual(len(asked), len(exchanges), "each answer the home keeps is asked, once, in its order") place = self.temp / "heldout" - self.assertEqual(json.loads((place / "fetched.json").read_text()), {"poc-one": None, "poc-two": "head_unreadable"}) + self.assertEqual(json.loads((place / "fetched.json").read_text()), + {"poc-one": None, "poc-two": "head_unreadable", "poc-three": "hash_differs", + "poc-four": "head_unreadable", "poc-five": "head_unreadable", "poc-six": "head_unreadable", + "poc-seven": None}) self.assertEqual(sorted(p.name for p in (self.temp / "heads").iterdir()), ["poc-one"]) - self.assertTrue((self.temp / "heads" / "poc-one" / "README.md").is_file()) + self.assertEqual((self.temp / "heads" / "poc-one" / "README.md").read_bytes(), b"# a repo\n") cli = live.CLI(said=live.answer(live.judged(findings=[], effects={}))) kept_plan = json.loads((place / "plan.json").read_text()) (place / "plan.json").write_text(json.dumps(dict(kept_plan, planned_at=1000))) self.step(step.step, cli, lambda: 1000.0, env=live.JOB) - asked, = cli.asked() - self.assertEqual(pathlib.Path(asked["head"]).parts[-2:], ("heads", "poc-one")) + asked_of, = cli.asked() + self.assertEqual(pathlib.Path(asked_of["head"]).parts[-2:], ("heads", "poc-one")) self.assertFalse((self.temp / "heads" / "poc-one").exists(), "removed once its case is asked") report = json.loads((self.work / job.REPORT).read_text()) + self.assertEqual({c["id"]: c["reasons"] for c in report["cases"]}["poc-three"], ["hash_differs"]) self.assertEqual({c["id"]: c["reasons"] for c in report["cases"]}["poc-two"], ["head_unreadable"]) + self.assertEqual({c["id"]: c["reasons"] for c in report["cases"]}["poc-seven"], ["not_this_set"]) self.assertIn("head_unreadable", job.ENGINE_FAULTS) self.assertEqual(self.step(job.judge.step if hasattr(job.judge, "step") else job.judge), "engine_fault") - # The fetch step's own transport for the heads sends no token: the home's is never sent anywhere else. + # The fetch step's one transport is the home's, read-only, with its token: no transport with no credential. import importlib - from adapters import github fetching = importlib.import_module("harness.eval.fetch") made = [] - with mock.patch.object(fetching, "fetch", lambda env, root, work, home, heads: made.append((home, heads)) or "x"): + with mock.patch.object(fetching, "fetch", lambda env, root, work, *given: made.append(given) or "x"): fetching.step({"HELDOUT_TOKEN": "a-stand-in-held-out-token"}, self.root, self.work) - (home, anonymous), = made - self.assertIsNone(anonymous._token) - self.assertIsInstance(anonymous, github.Live) + (home,), = made + self.assertIsInstance(home, fetching.ReadOnly) + self.assertEqual(home._transport._token, "a-stand-in-held-out-token") + + def test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone(self): # issue 611 + # Run at his terminal in his checkout of the home, the tool reads each case's head, its tarball and its tree, + # holds the one to the other, writes both beside the case and pins them in it; a case it cannot pin is left as + # it was. It prints counts, then each pinned case's id and new sha256, never a byte of a case, and what it + # writes is what the eval job's fetch reads. + from adapters import github + from harness import eval as job + from harness import head + from tests import support + from tests.test_adapters import home_beside, home_file + from tools import heads_into_home as tool + case, live = self.reviewed() + plain = {k: v for k, v in case.items() if k not in ("head_sha256", "tree_sha256")} + tar, tree = stand_in_head(case["head"]) + home = self.temp / "home" + (home / "pr-review").mkdir(parents=True) + written = {"poc-one": (json.dumps(dict(plain, id="poc-one"), indent=2) + "\n").encode(), + "poc-two": (json.dumps(dict(plain, id="poc-two", head="2" * 40), indent=2) + "\n").encode(), + "poc-three": b"no case\n", "poc-four": (json.dumps(dict(plain, id="poc-other"), indent=2) + "\n").encode(), + "poc-five": (json.dumps(dict(plain, id="poc-five", head="5" * 40), indent=2) + "\n").encode()} + for ident, data in written.items(): + (home / "pr-review" / ("%s.json" % ident)).write_bytes(data) + (home / "pr-review" / "poc-old.tree.json").write_bytes(b"{}\n") # a tree an earlier run wrote: no case + # A case that is a link is read as none, and a link where a head is written is replaced, never written through. + outside = self.temp / "outside" + outside.write_bytes(b"kept\n") + (self.temp / "linked.json").write_bytes((json.dumps(dict(plain, id="poc-six"), indent=2) + "\n").encode()) + os.symlink(str(self.temp / "linked.json"), str(home / "pr-review" / "poc-six.json")) + os.symlink(str(outside), str(home / "pr-review" / "poc-one.tar.gz")) + asked = [] + + class GitHub: + """The repo the cases are of: each commit's tarball as codeload serves one, and its tree as the API answers; + but commit 2..2's tree lists a file its tarball does not hold, and 5..5's is answered 404.""" + def send(self, req): + asked.append(req["path"]) + tarred, listed = github.TARBALL.fullmatch(req["path"]), github.TREE.fullmatch(req["path"]) + if tarred: + return {"status": 200, "headers": {}, "body": support.tarball(tarred.group(3)).decode("utf-8", errors="surrogateescape")} + body = json.loads(stand_in_head(listed.group(3))[1]) + if listed.group(3) == "2" * 40: + body["tree"].append(dict(body["tree"][0], path="MISSING.md")) + return {"status": 404 if listed.group(3) == "5" * 40 else 200, "headers": {}, "body": body} + out = io.StringIO() + self.assertEqual(tool.main([str(home), "pr-review"], {}, GitHub(), out), 1) + new = (home / "pr-review" / "poc-one.json").read_bytes() + self.assertEqual(out.getvalue(), "pinned 1 of 6 cases; head_unreadable 2; not_a_case 3\n%s\n" + % json.dumps({"id": "poc-one", "sha256": hashlib.sha256(new).hexdigest()})) + for word in (case["diff"][:40], case["facts"]["item"]["repo"], case["head"]): + self.assertNotIn(word, out.getvalue()) + # The case pins what was written beside it, after its head, and is otherwise as it was. + self.assertEqual((home / "pr-review" / "poc-one.tar.gz").read_bytes(), tar) + self.assertFalse((home / "pr-review" / "poc-one.tar.gz").is_symlink()) + self.assertEqual(outside.read_bytes(), b"kept\n") + self.assertEqual([p.name for p in (home / "pr-review").iterdir() if p.name.startswith(".")], [], "nothing half written") + self.assertEqual(json.loads((home / "pr-review" / "poc-one.tree.json").read_bytes()), json.loads(tree)) + self.assertEqual(json.loads(new), dict(plain, id="poc-one", head_sha256=hashlib.sha256(tar).hexdigest(), + tree_sha256=hashlib.sha256((home / "pr-review" / "poc-one.tree.json").read_bytes()).hexdigest())) + keys = list(json.loads(new)) + self.assertEqual(keys[keys.index("head"):keys.index("head") + 3], ["head", "head_sha256", "tree_sha256"]) + for ident in ("poc-two", "poc-three", "poc-four", "poc-five"): + self.assertEqual((home / "pr-review" / ("%s.json" % ident)).read_bytes(), written[ident], ident) + self.assertFalse((home / "pr-review" / ("%s.tar.gz" % ident)).exists(), ident) + repo = case["facts"]["item"]["repo"] + self.assertEqual(asked, [head.path_of(repo, "5" * 40), head.tree_path(repo, "5" * 40), + head.path_of(repo, case["head"]), head.tree_path(repo, case["head"]), + head.path_of(repo, "2" * 40), head.tree_path(repo, "2" * 40)]) + # Run again, it pins the case anew, to the same bytes. + for ident in ("poc-two", "poc-three", "poc-four", "poc-five", "poc-six"): + (home / "pr-review" / ("%s.json" % ident)).unlink() + out = io.StringIO() + self.assertEqual(tool.main([str(home), "pr-review"], {}, GitHub(), out), 0) + self.assertEqual((home / "pr-review" / "poc-one.json").read_bytes(), new) + # What it wrote is what the eval job's fetch reads, with the home's token, beside the case. + cases = {"poc-one": new} + self.set(cases) + self.step(job.plan) + files = [home_file("poc-one", new), home_beside("poc-one", "tar.gz", (home / "pr-review" / "poc-one.tar.gz").read_bytes()), + home_beside("poc-one", "tree.json", (home / "pr-review" / "poc-one.tree.json").read_bytes())] + fetch = getattr(job.fetch, "fetch", job.fetch) # the step, whether or not its module was imported + self.assertEqual(self.step(fetch, github.Replay(files)), "fetched 1 of 1 cases") + # A head over what the job reads is not pinned; nor is a case whose write fails, by its code alone. + from adapters import heldout + with mock.patch.dict(heldout.BESIDE, {"tree.json": 10}): + out = io.StringIO() + self.assertEqual(tool.main([str(home), "pr-review"], {}, GitHub(), out), 1) + self.assertEqual(out.getvalue(), "pinned 0 of 1 cases; head_unreadable 1\n") + with mock.patch.object(tool, "put", mock.Mock(side_effect=OSError("a path"))): + out = io.StringIO() + self.assertEqual(tool.main([str(home), "pr-review"], {}, GitHub(), out), 1) + self.assertEqual(out.getvalue(), "pinned 0 of 1 cases; not_written 1\n") + # A name that is no file, here a folder where the tree goes, fails the case before its tarball is written. + (home / "pr-review" / "poc-one.tar.gz").unlink() + (home / "pr-review" / "poc-one.tree.json").unlink() + (home / "pr-review" / "poc-one.tree.json").mkdir() + (home / "pr-review" / "poc-one.tree.json" / "kept").write_bytes(b"") + out = io.StringIO() + self.assertEqual(tool.main([str(home), "pr-review"], {}, GitHub(), out), 1) + self.assertEqual(out.getvalue(), "pinned 0 of 1 cases; not_written 1\n") + self.assertFalse((home / "pr-review" / "poc-one.tar.gz").exists(), "no head half written") + shutil.rmtree(str(home / "pr-review" / "poc-one.tree.json")) + self.assertEqual(tool.main([str(home), "pr-review"], {}, GitHub(), io.StringIO()), 0) + with mock.patch.object(tool.os, "replace", mock.Mock(side_effect=OSError("a path"))): + self.assertEqual(tool.main([str(home), "pr-review"], {}, GitHub(), io.StringIO()), 1) + self.assertEqual([p.name for p in (home / "pr-review").iterdir() if p.name.startswith(".")], [], "nothing half written") + self.assertEqual((home / "pr-review" / "poc-one.json").read_bytes(), new) + # Asked wrongly, of a folder that is a link, or with no token, it reads nothing. + os.symlink(str(home / "pr-review"), str(home / "linked")) + for argv in ([str(home)], [str(home), "pr-review/.."], [str(home), "absent"], [str(home), "linked"]): + self.assertEqual(tool.main(argv, {}, GitHub(), io.StringIO()), 2, argv) + self.assertEqual(tool.main([str(home), "pr-review"], {}, None, io.StringIO()), 2) def test_the_step_gives_each_case_the_release_dispatched_and_a_fresh_run(self): # issue 309, ruling 5 # Through the step as the workflow runs it: each case's Task names the Release the dispatch named, which plan diff --git a/tools/heads_into_home.py b/tools/heads_into_home.py new file mode 100644 index 00000000..3ecfe6b0 --- /dev/null +++ b/tools/heads_into_home.py @@ -0,0 +1,149 @@ +"""Each case's head, put beside the case in the held-out home and pinned by it (issue 611). Run at the Maintainer's +terminal, in his checkout of the home, never in a session: a session never sees a case. + + GH_TOKEN="$(gh auth token)" python3 -m tools.heads_into_home + +For each case `/.json`, it reads the commit the case names as its head, `head`, of the repo its facts +name, through the one transport with the token given: the commit's tarball, through codeload as a live Run reads one +(decision 0138), and its tree, every level of it (decision 0142). It holds the one to the other, as the eval job will, +and only then writes `.tar.gz` and `.tree.json` beside the case, and adds to the case the sha256 of each, +`head_sha256` and `tree_sha256`, after its `head`, writing the case anew. Each file is written whole into the folder and +moved over its name, so a link there is replaced and never written through; a folder that is a link is refused. It +takes a role folder and touches nothing else: so the review Intent's set and a gate set made later are pinned alike. + +It prints counts and hashes alone, never a byte of a case: how many cases it pinned and, by a code, how many it did not, +then each pinned case's id and the sha256 of its new bytes, one JSON object to a line, which is what the set's manifest +under `evals/` takes. A case it did not pin is left as it was. It exits 0 only when every case was pinned.""" +from __future__ import annotations + +import contextlib +import hashlib +import json +import os +import pathlib +import re +import sys +import tempfile + +PART = re.compile(r"[a-z0-9][a-z0-9-]*") # a Role or a case id, as eval-set/1 holds them: no dot, no slash +COMMIT = re.compile(r"[0-9a-f]{40}") +REPO = re.compile(r"[A-Za-z0-9_.-]{1,100}/[A-Za-z0-9_.-]{1,100}") + + +def put(folder: pathlib.Path, files: dict) -> None: + """Each of `files`, `{name: bytes}`, written in `folder`, all or none: each name held to be nothing yet, a link or a + regular file, each written whole into a new file beside it, and only then each moved over its name, so that a link + there is replaced, never written through, and no case is left with its head half written.""" + for name in files: + target = str(folder / name) + if os.path.lexists(target) and not (os.path.islink(target) or os.path.isfile(target)): + raise OSError("not a file") + staged = {} + try: + for name, data in files.items(): + fd, staged[name] = tempfile.mkstemp(dir=str(folder), prefix=".heads-into-home-") + with os.fdopen(fd, "wb") as out: + out.write(data) + for name, temporary in staged.items(): + os.replace(temporary, str(folder / name)) + except BaseException: + for temporary in staged.values(): + with contextlib.suppress(OSError): + os.unlink(temporary) + raise + + +def read(path: pathlib.Path): + """The bytes of the case at `path`, or None for one that is a link or cannot be read as a file.""" + try: + fd = os.open(str(path), os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except OSError: + return None + try: + with os.fdopen(fd, "rb") as f: + return f.read() + except OSError: + return None + + +def pinned(data: bytes, transport, folder: pathlib.Path, ident: str): + """`({name: bytes}, None)`: the head's tarball and tree to write beside the case in `folder`, and the case anew, + pinning them, which `put` writes all or none; or `(None, code)`: + `not_a_case` for bytes that are no case with a head of a repo, and `head_unreadable` for a head not read, unpacked, + held to its tree, or within what the eval job reads, whole. Nothing is written here.""" + from adapters import github, heldout + from harness import head + try: + case = json.loads(data) + repo, sha = case["facts"]["item"]["repo"], case["head"] + except (ValueError, TypeError, KeyError, RecursionError): + return None, "not_a_case" + if not (isinstance(case, dict) and case.get("id") == ident and isinstance(sha, str) and COMMIT.fullmatch(sha) + and isinstance(repo, str) and REPO.fullmatch(repo)): + return None, "not_a_case" + if not github.TREE.fullmatch(head.tree_path(repo, sha)): + return None, "not_a_case" + try: + tar = head.fetch(transport, repo, sha) + answer = transport.send(github.request("GET", head.tree_path(repo, sha))) + if answer.get("status") != 200: + raise head.Refused("the commit's tree was not answered") + tree = head.tree_of(answer.get("body")) + with tempfile.TemporaryDirectory() as scratch: + into = os.path.join(os.path.realpath(scratch), ident) + head.unpack(tar, sha, into) + head.held(into, tree) + except (head.Refused, github.Unanswered, OSError): + return None, "head_unreadable" + # The tree as GitHub's answer held it, written compact, so that a tree GitHub gave whole is within what the job reads. + listed = (json.dumps(answer["body"], separators=(",", ":")) + "\n").encode("utf-8") + if len(tar) > heldout.BESIDE["tar.gz"] or len(listed) > heldout.BESIDE["tree.json"]: + return None, "head_unreadable" + out = {} + for key, value in case.items(): + if key not in ("head_sha256", "tree_sha256"): + out[key] = value + if key == "head": + out["head_sha256"], out["tree_sha256"] = hashlib.sha256(tar).hexdigest(), hashlib.sha256(listed).hexdigest() + new = (json.dumps(out, indent=2, ensure_ascii=False) + "\n").encode("utf-8") + return {"%s.tar.gz" % ident: tar, "%s.tree.json" % ident: listed, "%s.json" % ident: new}, None + + +def main(argv: list, env=None, transport=None, out=None) -> int: + from adapters import github + env, out = os.environ if env is None else env, sys.stdout if out is None else out + if len(argv) != 2 or not PART.fullmatch(argv[1]) or not (pathlib.Path(argv[0]) / argv[1]).is_dir() \ + or (pathlib.Path(argv[0]) / argv[1]).is_symlink(): + out.write("usage: GH_TOKEN=... python3 -m tools.heads_into_home \n") + return 2 + folder = pathlib.Path(argv[0]) / argv[1] + if transport is None: + if not env.get("GH_TOKEN"): + out.write("no token: give one as GH_TOKEN\n") + return 2 + from harness.eval.fetch import ReadOnly + transport = ReadOnly(github.Live(env["GH_TOKEN"])) + said, entries = {"pinned": 0}, [] + for path in sorted(p for p in folder.iterdir() if p.suffix == ".json" and not p.name.endswith(".tree.json")): + ident = path.name[:-len(".json")] + data = read(path) if PART.fullmatch(ident) else None + try: + files, code = pinned(data, transport, folder, ident) if data is not None else (None, "not_a_case") + new = None + if files is not None: + put(folder, files) + new = files[path.name] + except Exception: # by its code alone: no message, which could quote a case + new, code = None, "not_written" + said[code or "pinned"] = said.get(code or "pinned", 0) + 1 + if new is not None: + entries.append({"id": ident, "sha256": hashlib.sha256(new).hexdigest()}) + out.write("pinned %d of %d cases%s\n" % (said["pinned"], sum(said.values()), "".join( + "; %s %d" % (code, n) for code, n in sorted(said.items()) if code != "pinned"))) + for entry in entries: + out.write(json.dumps(entry) + "\n") + return 0 if said["pinned"] == sum(said.values()) else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/mutants.py b/tools/mutants.py index 2964fd16..5a194866 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -7202,20 +7202,17 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' head_.unpack(tarred, sha, os.path.join(folder, "head"))', ' pass', ('tests.test_steps.Reader.test_the_reader_writes_the_tasks_inputs',)), ("the eval fetch reads no case's head", 'harness/eval/__init__.py', - ' if data is not None and heads is not None:\n code = head_of(data, heads_place(env, workspace), entry["id"], heads)\n', - '', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_with_no_credential_and_one_not_read_closes_the_gate',)), + ' if data is not None:\n code = head_of(env, data, heads_place(env, workspace), found["role"], entry["id"], transport)\n', + '', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), ('a case is asked without its head', 'harness/eval/run.py', ' head = os.path.join(str(heads), case["id"]) if heads is not None else None', - ' head = None', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_with_no_credential_and_one_not_read_closes_the_gate',)), + ' head = None', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), ("a case's head is kept after it is asked", 'harness/eval/run.py', ' shutil.rmtree(head, ignore_errors=True)', - ' pass', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_with_no_credential_and_one_not_read_closes_the_gate',)), + ' pass', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), ("a head not read is the reviewer's fault", 'harness/eval/__init__.py', '"rubric_unreadable", "no_answer", "head_unreadable")', - '"rubric_unreadable", "no_answer")', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_with_no_credential_and_one_not_read_closes_the_gate',)), - ("the eval reads the heads with the home's token", 'harness/eval/fetch.py', - ' return fetch(env, root, workspace, transport(env), github.Live.anonymous())', - ' return fetch(env, root, workspace, transport(env), transport(env))', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_with_no_credential_and_one_not_read_closes_the_gate',)), + '"rubric_unreadable", "no_answer")', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), ("a tarball's owner is not quoted whole", 'harness/head.py', 'urllib.parse.quote(owner, safe=""), urllib.parse.quote(name, safe="")', 'owner, urllib.parse.quote(name, safe="")', ('tests.test_harness.Head.test_the_head_of_p60_is_unpacked_as_codeload_served_it',)), @@ -7997,8 +7994,9 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if hashlib.sha256(data).hexdigest() != entry["sha256"]:\n return None, "hash_differs"\n', '', ('tests.test_adapters.Heldout.test_a_held_out_case_not_read_whole_says_why',)), ('a held-out case GitHub did not find is unanswered', 'adapters/heldout.py', - ' if answer["status"] == 404:\n return None, "not_found"\n', - '', ('tests.test_adapters.Heldout.test_a_held_out_case_not_read_whole_says_why',)), + ' if answer["status"] == 404:\n return None, "not_found"\n body = answer["body"]\n if answer["status"] != 200 or not isinstance(body, dict):\n', + ' body = answer["body"]\n if answer["status"] != 200 or not isinstance(body, dict):\n', + ('tests.test_adapters.Heldout.test_a_held_out_case_not_read_whole_says_why',)), ('a held-out case of any size is read', 'adapters/heldout.py', ' if size > LARGEST:\n', ' if False:\n', ('tests.test_adapters.Heldout.test_a_held_out_case_not_read_whole_says_why',)), @@ -8027,8 +8025,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' and isinstance(role, str) and PART.fullmatch(role)):', ' and isinstance(role, str)):', ('tests.test_adapters.Heldout.test_a_held_out_case_not_read_whole_says_why',)), ('a home of any name is asked', 'adapters/heldout.py', - ' if not home(owner, name):\n', - ' if False:\n', ('tests.test_adapters.Heldout.test_a_held_out_case_not_read_whole_says_why',)), + ' return None, "not_this_set"\n if not home(owner, name):\n return None, "unanswered"\n path = ', + ' return None, "not_this_set"\n if False:\n return None, "unanswered"\n path = ', ('tests.test_adapters.Heldout.test_a_held_out_case_not_read_whole_says_why',)), ('a home that climbs out is asked', 'adapters/heldout.py', ' and name not in (".", "..")', '', ('tests.test_adapters.Heldout.test_a_held_out_case_not_read_whole_says_why',)), @@ -8599,7 +8597,7 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if os.path.commonpath([heads, other]) in (heads, other):\n', ' if heads == other or heads.startswith(other + os.sep) or other.startswith(heads + os.sep):\n', APART), ("the fetch unpacks each head in the scratch", 'harness/eval/__init__.py', - 'code = head_of(data, heads_place(env, workspace), entry["id"], heads)', 'code = head_of(data, place / "heads", entry["id"], heads)', APART), + 'code = head_of(env, data, heads_place(env, workspace), found["role"], entry["id"], transport)', 'code = head_of(env, data, place / "heads", found["role"], entry["id"], transport)', APART), ("the run step looks for each head in the scratch", 'harness/eval/run.py', 'reviewer(env, kept["release"], began, call, clock, heads_place(env, workspace))', 'reviewer(env, kept["release"], began, call, clock, place / "heads")', APART), # Issue 626: the scratch is read part by part, the plan makes the heads folder new, and the canary's head is apart. @@ -8632,15 +8630,15 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' return live.answer(the_task, env, config, watched, clock, head=head)\n', ' return live.answer(the_task, env, config, lambda a, **k: watched(a if a[1:] == ["--version"] else a + [case["expected"]["criterion"]], **k), clock, head=head)\n', NO_ANSWER), ("the head holds the case's answer", 'harness/eval/__init__.py', - ' head.unpack(head.fetch(transport, repo, sha), sha, within(folder, ident))\n', - ' head.unpack(head.fetch(transport, repo, sha), sha, within(folder, ident))\n (folder / ident / "answer.json").write_text(json.dumps(case["expected"]))\n', NO_ANSWER), + ' head.held(into, tree)\n', + ' head.held(into, tree)\n (folder / ident / "answer.json").write_text(json.dumps(case["expected"]))\n', NO_ANSWER), ("a folder of the head is named for the case's answer", 'harness/eval/__init__.py', - ' head.unpack(head.fetch(transport, repo, sha), sha, within(folder, ident))\n', - ' head.unpack(head.fetch(transport, repo, sha), sha, within(folder, ident))\n (folder / ident / case["expected"]["criterion"]).mkdir()\n', NO_ANSWER), + ' head.held(into, tree)\n', + ' head.held(into, tree)\n (folder / ident / case["expected"]["criterion"]).mkdir()\n', NO_ANSWER), # Issue 622: the head is compared byte for byte, severity varies, and the whole call is compared. ("a byte of the head is keyed on the case's verdict", 'harness/eval/__init__.py', - ' head.unpack(head.fetch(transport, repo, sha), sha, within(folder, ident))\n', - ' head.unpack(head.fetch(transport, repo, sha), sha, within(folder, ident))\n (folder / ident / "mark").write_bytes(b"\\x80" if case["expected"]["verdict"] == "finding" else b"\\x81")\n', NO_ANSWER), + ' head.held(into, tree)\n', + ' head.held(into, tree)\n (folder / ident / "mark").write_bytes(b"\\x80" if case["expected"]["verdict"] == "finding" else b"\\x81")\n', NO_ANSWER), ("the Task's texts say whether the expected Finding is blocking", 'harness/eval/run.py', ' if "issue" in case else []\n', ' if "issue" in case else [{"id": "issue:1", "by": {"kind": "human", "name": "a", "can_write": True}, "body": str((case["expected"]["finding"] or {}).get("severity") == "blocking")}]\n', NO_ANSWER), @@ -9257,6 +9255,85 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ('the fewest passes are read at the ends of the split alone', 'harness/eval/__init__.py', 'for cases in range(least, count - least + 1))', 'for cases in (least, count - least))', ('tests.test_evals.EvalJob.test_a_gate_set_gives_its_own_bars_and_least_and_its_record_states_them',)), + # Issue 611: a case's head is read from the held-out home, its tarball and its tree each by the sha256 the case pins. + ('a head beside a case of another hash is read', 'adapters/heldout.py', + ' if hashlib.sha256(data).hexdigest() != sha256:\n return None, "hash_differs"\n', '', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('the eval blob is asked for as raw+json', 'adapters/heldout.py', + 'RAW = "application/vnd.github.raw"', 'RAW = "application/vnd.github.raw+json"', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('what the home keeps beside a case is asked as JSON', 'adapters/heldout.py', + ' accept=RAW))', ' ))', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a head of any size is read', 'adapters/heldout.py', + ' if len(data) > BESIDE[kind]:\n', ' if False:\n', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a head answered under another type is read', 'adapters/heldout.py', + ' \\\n or not str(answer["headers"].get("content-type", "")).startswith(RAW + ";"):', ':', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ("a head answered as raw JSON is read", 'adapters/heldout.py', + '.startswith(RAW + ";")', '.startswith(RAW)', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a head GitHub did not find is unanswered', 'adapters/heldout.py', + ' if answer["status"] == 404:\n return None, "not_found"\n body = answer["body"]\n if answer["status"] != 200 or not isinstance(body, str)', + ' body = answer["body"]\n if answer["status"] != 200 or not isinstance(body, str)', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a head GitHub refused is read', 'adapters/heldout.py', + ' if answer["status"] != 200 or not isinstance(body, str) \\', ' if not isinstance(body, str) \\', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a file of any kind is asked beside a case', 'adapters/heldout.py', + ' and kind in BESIDE and isinstance(sha256, str)', ' and isinstance(sha256, str)', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a pin of any shape is asked by', 'adapters/heldout.py', + ' and isinstance(sha256, str) and SHA256.fullmatch(sha256)):', '):', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a file beside a case of any name is asked', 'adapters/heldout.py', + ' and isinstance(ident, str) and PART.fullmatch(ident)\n', '\n', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a home of any name is asked beside a case', 'adapters/heldout.py', + ' return None, "not_this_set"\n if not home(owner, name):\n return None, "unanswered"\n try:', + ' return None, "not_this_set"\n if False:\n return None, "unanswered"\n try:', ('tests.test_adapters.Heldout.test_what_the_home_keeps_beside_a_case_is_read_raw_by_the_hash_the_case_pins',)), + ('a head is unpacked and never held to its tree', 'harness/eval/__init__.py', + ' head.held(into, tree)\n except', ' except', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), + ('a head not the one pinned is read as unreadable', 'harness/eval/__init__.py', + 'return "hash_differs" if code == "hash_differs" else "head_unreadable"', 'return "head_unreadable"', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), + ("the tree's pin is never read", 'harness/eval/__init__.py', + '(case["head_sha256"], case["tree_sha256"])', '(case["head_sha256"], case["head_sha256"])', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), + ('a head is read beside another Role', 'harness/eval/__init__.py', + 'heads_place(env, workspace), found["role"], entry["id"], transport)', 'heads_place(env, workspace), "pr-review-gate", entry["id"], transport)', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), + ('a tree that cannot be read stops the fetch', 'harness/eval/__init__.py', + ' except (head.Refused, OSError, Stop, ValueError, RecursionError):', ' except (head.Refused, OSError, Stop):', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), + ('a case need pin no tree', 'schemas/eval-case.schema.json', ' "head_sha256",\n "tree_sha256",\n', ' "head_sha256",\n', ('tests.test_evals.TheSet.test_a_case_dropped_under_evals_in_any_file_is_caught',)), + ('a head pin may be cut short', 'schemas/eval-case.schema.json', + '"head_sha256": {\n "description": "The sha256 of the head\'s tarball as GitHub archived the commit, kept beside the case in the held-out home, `/.tar.gz`: the eval job reads the head there, with the home\'s token, and none with no credential (issue 611).",\n "type": "string",\n "pattern": "^[0-9a-f]{64}$"', + '"head_sha256": {\n "description": "The sha256 of the head\'s tarball as GitHub archived the commit, kept beside the case in the held-out home, `/.tar.gz`: the eval job reads the head there, with the home\'s token, and none with no credential (issue 611).",\n "type": "string",\n "pattern": "^[0-9a-f]{63,64}$"', ('tests.test_evals.TheSet.test_a_case_dropped_under_evals_in_any_file_is_caught',)), + ("the Maintainer's tool writes a head beside a case it cannot pin", 'tools/heads_into_home.py', + ' if os.path.lexists(target) and not (os.path.islink(target) or os.path.isfile(target)):\n', + ' if False:\n', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ('a case need pin no head', 'schemas/eval-case.schema.json', ' "head_sha256",\n "tree_sha256",\n', ' "tree_sha256",\n', ('tests.test_evals.TheSet.test_a_case_dropped_under_evals_in_any_file_is_caught',)), + ('a tree pin may be cut short', 'schemas/eval-case.schema.json', + '"tree_sha256": {\n "description": "The sha256 of the commit\'s tree, every level of it, as GitHub answered it, kept beside the case in the held-out home, `/.tree.json`: the head is held to it file by file (issue 611; decision 0142).",\n "type": "string",\n "pattern": "^[0-9a-f]{64}$"', + '"tree_sha256": {\n "description": "The sha256 of the commit\'s tree, every level of it, as GitHub answered it, kept beside the case in the held-out home, `/.tree.json`: the head is held to it file by file (issue 611; decision 0142).",\n "type": "string",\n "pattern": "^[0-9a-f]{63,64}$"', ('tests.test_evals.TheSet.test_a_case_dropped_under_evals_in_any_file_is_caught',)), + ("the Maintainer's tool writes a head not held to its tree", 'tools/heads_into_home.py', + ' head.held(into, tree)\n', '', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool pins a case of another id", 'tools/heads_into_home.py', + 'case.get("id") == ident and ', '', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool reads a tree answered otherwise", 'tools/heads_into_home.py', + ' if answer.get("status") != 200:\n', ' if False:\n', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool says every case was pinned", 'tools/heads_into_home.py', + ' return 0 if said["pinned"] == sum(said.values()) else 1\n', ' return 0\n', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool reads a tree it wrote as a case", 'tools/heads_into_home.py', + ' and not p.name.endswith(".tree.json")', '', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool pins a tree it did not write", 'tools/heads_into_home.py', + 'hashlib.sha256(listed).hexdigest()', 'hashlib.sha256(json.dumps(answer["body"]).encode()).hexdigest()', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool reads with no token", 'tools/heads_into_home.py', + ' if not env.get("GH_TOKEN"):\n', ' if False:\n', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ('a head not held is left in the heads folder', 'harness/eval/__init__.py', + ' shutil.rmtree(into, ignore_errors=True)\n', ' pass\n', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), + ('a case whose pin is no sha256 has its head read', 'harness/eval/__init__.py', + ' and all(isinstance(p, str) and SHA256.fullmatch(p) for p in pins)', '', ('tests.test_evals.EvalJob.test_each_cases_head_is_read_from_the_home_by_its_pins_and_one_not_read_closes_the_gate',)), + ("the Maintainer's tool writes through a link", 'tools/heads_into_home.py', + ' os.replace(temporary, str(folder / name))\n', + ' (folder / name).write_bytes(pathlib.Path(temporary).read_bytes())\n os.unlink(temporary)\n', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool reads a case that is a link", 'tools/heads_into_home.py', + ' | getattr(os, "O_NOFOLLOW", 0)', '', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool pins a head the job will refuse", 'tools/heads_into_home.py', + ' if len(tar) > heldout.BESIDE["tar.gz"] or len(listed) > heldout.BESIDE["tree.json"]:\n', ' if False:\n', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool reads a folder that is a link", 'tools/heads_into_home.py', + ' \\\n or (pathlib.Path(argv[0]) / argv[1]).is_symlink():', ':', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("a write that fails stops the Maintainer's tool", 'tools/heads_into_home.py', + ' except Exception: # by its code alone', ' except ZeroDivisionError: # by its code alone', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), + ("the Maintainer's tool leaves a file half written", 'tools/heads_into_home.py', + ' with contextlib.suppress(OSError):\n os.unlink(temporary)\n', ' pass\n', ('tests.test_evals.EvalJob.test_the_maintainers_tool_pins_each_head_beside_its_case_and_prints_counts_and_hashes_alone',)), # Issue 613, the preflight: each dispatch of a gate set asks a few development cases first, and a broken pipeline stops it. ('a gate set need name no preflight', 'harness/eval/__init__.py', 'OWN = ("bars", "kind_least", "record", "preflight")', @@ -15248,8 +15325,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: 'body.get("truncated") is not False', 'body.get("truncated") is True', ('tests.test_harness.Head.test_a_tree_not_answered_whole_or_holding_what_an_archive_cannot_is_refused',)), ('a tree answered otherwise than 200 is read', 'harness/head.py', - 'if answer.get("status") != 200 or not isinstance(body, dict) or body.get("truncated")', - 'if not isinstance(body, dict) or body.get("truncated")', ('tests.test_harness.Head.test_a_tree_not_answered_whole_or_holding_what_an_archive_cannot_is_refused',)), + ' if answer.get("status") != 200:\n raise Refused("the commit\'s tree was not answered whole")\n', + '', ('tests.test_harness.Head.test_a_tree_not_answered_whole_or_holding_what_an_archive_cannot_is_refused',)), ('a path the tree names twice is read', 'harness/head.py', 'if not isinstance(path, str) or path in seen:', 'if not isinstance(path, str):', ('tests.test_harness.Head.test_a_tree_not_answered_whole_or_holding_what_an_archive_cannot_is_refused',)), @@ -15302,8 +15379,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if os.path.islink(full) or not os.path.isfile(full):\n raise Refused("the head holds what is not a regular file")', ' if os.path.islink(full):\n raise Refused("the head holds what is not a regular file")', ('tests.test_harness.Head.test_the_head_of_p60_is_held_to_its_commits_tree',)), ('a tree read short is read', 'harness/head.py', - 'or not isinstance(body, dict) or body.get("truncated") is not False', - 'or not isinstance(body, dict)', ('tests.test_harness.Head.test_a_tree_not_answered_whole_or_holding_what_an_archive_cannot_is_refused',)), + 'if not isinstance(body, dict) or body.get("truncated") is not False or', + 'if not isinstance(body, dict) or', ('tests.test_harness.Head.test_a_tree_not_answered_whole_or_holding_what_an_archive_cannot_is_refused',)), ('a blob by a sha that only starts as one is read', 'harness/head.py', 'isinstance(blob, str) and BLOB.fullmatch(blob)', 'isinstance(blob, str) and BLOB.match(blob)', ('tests.test_harness.Head.test_a_tree_not_answered_whole_or_holding_what_an_archive_cannot_is_refused',)),