A dispatch's run is split into a preflight job, shards and a merge, and a gate set's shards are drawn by a salt (#611, record NNNN)

.github/workflows/eval.yml

@@ -1,14 +1,17 @@
 # The eval job, `factory / eval` (design §4.9 and §3.14, issue 309; decision 0053): the review Intent's held-out set, run
 # on the commit it judges, main's own head, for the Release the dispatch names. Dispatched alone, by the Maintainer
-# before a cut; the Release's gate reads its report (#197's ruling 4″). Its steps are harness.eval's modules and pinned
-# actions, in the order tools/check_workflows.py holds:
-# the set planned before any credential is used; the CLI installed at the Engine's pin by its checksum, with the package
-# the scrub needs, as root and before anything is here; each case fetched with the Maintainer's read-only token for the
-# home, held by that step alone, into a scratch directory outside the workspace; each case given to the review Intent with the model's credential alone, as a live Run is; for a
-# gate set, its ledger sealed to the Maintainer's certificates and the ciphertext uploaded; the report, of hashes,
-# verdicts and codes alone, or of counts alone for a gate set, uploaded whatever the verdicts; and the run judged by the
-# gate's own test. The dispatch names the set, the review Intent's where it names none (issue 612).
-# Nothing of a case, a Result or the home reaches the log: each step prints counts and codes.
+# before a cut; the Release's gate reads its report (#197's ruling 4″). The dispatch names the set, the review Intent's
+# where it names none (issue 612). The run is split into parts, each a job of its own (issue 611): the preflight's job,
+# which asks a gate set's preflight and fails where it stops the dispatch, so no shard runs; a matrix of shards behind
+# it, each with its own time limit, which asks its share of the set; and the merge job, which holds no credential and
+# writes the one report from what each part uploaded, whatever became of them. Each job's steps are harness.eval's
+# modules and pinned actions, in the order tools/check_workflows.py holds: the part planned before any credential is
+# used; the CLI installed at the Engine's pin by its checksum, with the package the scrub needs, as root and before
+# anything is here; each case, and its head, fetched with the Maintainer's read-only token for the home, held by that
+# step alone, into a scratch directory outside the workspace; each case given to the review Intent with the model's
+# credential alone, as a live Run is; for a gate set, the part's ledger sealed to the Maintainer's certificates and the
+# ciphertext uploaded; and what the part found uploaded, counts alone for a gate set. Nothing of a case, a Result or
+# the home reaches the log: each step prints counts and codes.
 name: factory
 on:
   workflow_dispatch:
@@ -22,14 +25,76 @@ on:
         default: pr-review
 permissions: {}
 jobs:
+  preflight:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    permissions:
+      contents: read
+    environment: factory-eval
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - run: python3 -m harness.eval.plan
+        timeout-minutes: 1
+        env:
+          RELEASE: ${{ inputs.release }}
+          EVAL_SET: ${{ inputs.set }}
+          EVAL_PART: preflight
+      - env:
+          CLI_VERSION: "2.1.306"
+          CLI_CHECKSUM: "520bc6c1f40af466bd03a278b5ae1af1a2938bc7de5b9e5cb15526da20d71cfe"
+        timeout-minutes: 3
+        run: |
+          url="https://downloads.claude.ai/claude-code-releases/${CLI_VERSION}/linux-x64/claude"
+          curl -sSfL --proto '=https' --proto-redir '=https' -o /tmp/claude "${url}"
+          echo "${CLI_CHECKSUM}  /tmp/claude" | sha256sum -c
+          chmod 0755 /tmp/claude
+          sudo mv /tmp/claude /usr/local/bin/claude
+          sudo apt-get update -q
+          sudo apt-get install -y --no-install-recommends bubblewrap
+      - run: python3 -m harness.eval.fetch
+        timeout-minutes: 3
+        env:
+          HELDOUT_TOKEN: ${{ secrets.HELDOUT_TOKEN }}
+          HELDOUT_OWNER: ${{ secrets.HELDOUT_OWNER }}
+          HELDOUT_REPO: ${{ secrets.HELDOUT_REPO }}
+      - run: python3 -m harness.eval.run
+        timeout-minutes: 109
+        env:
+          CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          CLAUDE_CLI: /usr/local/bin/claude
+      - run: python3 -m harness.eval.seal
+        timeout-minutes: 1
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: eval-ledger-preflight
+          path: eval-ledger.cms
+          if-no-files-found: ignore
+          retention-days: 90
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: eval-part-preflight
+          path: eval-part.json
+          if-no-files-found: error
+          retention-days: 90
+      - run: python3 -m harness.eval.judge
+        timeout-minutes: 1
   eval:
+    needs: preflight
+    strategy:
+      fail-fast: false
+      matrix:
+        shard:
+          - 0
+          - 1
+          - 2
+          - 3
     runs-on: ubuntu-24.04
     timeout-minutes: 120
     permissions:
       contents: read
     environment: factory-eval
-    outputs:
-      ledger_sha256: ${{ steps.seal.outputs.ledger_sha256 }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -39,6 +104,7 @@ jobs:
         env:
           RELEASE: ${{ inputs.release }}
           EVAL_SET: ${{ inputs.set }}
+          EVAL_PART: ${{ matrix.shard }}
       - env:
           CLI_VERSION: "2.1.306"
           CLI_CHECKSUM: "520bc6c1f40af466bd03a278b5ae1af1a2938bc7de5b9e5cb15526da20d71cfe"
@@ -63,14 +129,47 @@ jobs:
           CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
           CLAUDE_CLI: /usr/local/bin/claude
       - run: python3 -m harness.eval.seal
-        id: seal
         timeout-minutes: 1
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: eval-ledger
+          name: eval-ledger-${{ matrix.shard }}
           path: eval-ledger.cms
           if-no-files-found: ignore
           retention-days: 90
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: eval-part-${{ matrix.shard }}
+          path: eval-part.json
+          if-no-files-found: error
+          retention-days: 90
+  merge:
+    needs:
+      - preflight
+      - eval
+    if: always()
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    outputs:
+      ledger_sha256: ${{ steps.merge.outputs.ledger_sha256 }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - run: python3 -m harness.eval.plan
+        timeout-minutes: 1
+        env:
+          RELEASE: ${{ inputs.release }}
+          EVAL_SET: ${{ inputs.set }}
+          EVAL_PART: merge
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          pattern: eval-part-*
+          path: parts
+      - run: python3 -m harness.eval.merge
+        id: merge
+        timeout-minutes: 1
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: eval-report

adapters/heldout.py

@@ -79,11 +79,37 @@ def beside(transport, owner: str, name: str, role: str, ident: str, kind: str, s
     if not (isinstance(role, str) and PART.fullmatch(role) and isinstance(ident, str) and PART.fullmatch(ident)
             and kind in BESIDE and isinstance(sha256, str) and SHA256.fullmatch(sha256)):
         return None, "not_this_set"
+    data, code = _raw(transport, owner, name, "%s/%s.%s" % (role, ident, kind))
+    if code is not None:
+        return None, code
+    if len(data) > BESIDE[kind]:
+        return None, "too_large"
+    if hashlib.sha256(data).hexdigest() != sha256:
+        return None, "hash_differs"
+    return data, None
+
+
+
+def salt(transport, owner: str, name: str, role: str, set_name: str):
+    """The salt a gate set's shards are drawn by, `{role}/{set_name}.salt` in the home, read raw with the home's token,
+    16 to 256 bytes; or None where there is none to read whole (issue 611). It is no case and pins nothing: it keeps a
+    shard's cases from being told by their place in the public manifest, so a shard's counts say nothing of which
+    case is which kind."""
+    if not (isinstance(role, str) and PART.fullmatch(role) and isinstance(set_name, str) and PART.fullmatch(set_name)):
+        return None
+    data, _ = _raw(transport, owner, name, "%s/%s.salt" % (role, set_name))
+    return data if data is not None and 16 <= len(data) <= 256 else None
+
+
+def _raw(transport, owner: str, name: str, path: str) -> tuple:
+    """`(bytes, None)` of the file at `path` in the home, read whole with the raw media type and the home's token, or
+    `(None, code)`: `unanswered` for a home that could name another repo or an answer not of that type, and
+    `not_found`. The raw media type answers a file of more than 1 MiB whole, in one answer and no redirect
+    (trials/P-62.json)."""
     if not home(owner, name):
         return None, "unanswered"
     try:
-        answer = transport.send(request("GET", "/repos/%s/%s/contents/%s/%s.%s" % (owner, name, role, ident, kind),
-                                        accept=RAW))
+        answer = transport.send(request("GET", "/repos/%s/%s/contents/%s" % (owner, name, path), accept=RAW))
     except Unanswered:
         return None, "unanswered"
     if answer["status"] == 404:
@@ -92,10 +118,4 @@ def beside(transport, owner: str, name: str, role: str, ident: str, kind: str, s
     if answer["status"] != 200 or not isinstance(body, str) \
             or not str(answer["headers"].get("content-type", "")).startswith(RAW + ";"):
         return None, "unanswered"
-    data = raw(body)
-    if len(data) > BESIDE[kind]:
-        return None, "too_large"
-    if hashlib.sha256(data).hexdigest() != sha256:
-        return None, "hash_differs"
-    return data, None
-
+    return raw(body), None

config/harness.json

@@ -26,5 +26,6 @@
   },
   "advisory": [
     "coderabbit"
-  ]
+  ],
+  "eval_shards": 4
 }