evals: split the v2 sweep across three CI runners

.github/workflows/skill-evals.yml

@@ -146,15 +146,37 @@ jobs:
   # below are limited to the two this job installs a CLI for.
   # ----------------------------------------------------------------------------
   live-eval:
-    name: Judged sweep (real agent + judge, over the Tailnet)
+    name: Sweep ${{ matrix.suite }} (real agent + judge, over the Tailnet)
+    # Dispatch-only, deliberately. Nothing schedules this and nothing should: it
+    # drives a real agent against a real gateway for hours, and a run nobody is
+    # waiting on is a run nobody reads.
     if: github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
-    # 42 cases across the three suites, so `trials` N means 42N agent runs plus
-    # one judge call per correctness trial. At 60 minutes even trials=1 did not
-    # finish; trials=5 is 210 runs and needs the headroom. This is a backstop
-    # against a wedged agent, not a target -- the preflights below are what catch
-    # a misconfiguration, in seconds rather than by running out the clock.
-    timeout-minutes: 180
+    strategy:
+      # One suite's failure must not cancel the others. They are independent
+      # measurements and a partial sweep is still worth its artifact -- the
+      # scorecard job below is what decides whether the whole is scoreable.
+      fail-fast: false
+      matrix:
+        # 42 cases across the three suites, so `trials` N means 42N agent runs
+        # plus one judge call per correctness trial. Run 33914096408 measured
+        # trials=1 at 76 min sequentially, which puts trials=5 near 6.3h --
+        # past GitHub's HARD 6h per-job ceiling, which `timeout-minutes` cannot
+        # raise. The suites share nothing until the scorecard, so each gets its
+        # own runner and the long pole drops to consultation alone.
+        #
+        # Timeouts are backstops against a wedged agent, not targets; they are
+        # per-suite so a hung restraint run (5 min observed) is caught in
+        # minutes instead of hiding inside a ceiling sized for consultation.
+        # Ordered longest-first so the critical path starts first.
+        include:
+          - suite: consultation   # 22 cases; 52 min at trials=1
+            timeout: 330
+          - suite: correctness    # 12 cases, judged; 18 min at trials=1
+            timeout: 180
+          - suite: restraint      #  8 cases;  5 min at trials=1
+            timeout: 90
+    timeout-minutes: ${{ matrix.timeout }}
     permissions:
       contents: read
       # Required by tailscale/github-action to mint an OIDC token for OAuth.
@@ -407,10 +429,56 @@ jobs:
           root="${LLM_GATEWAY_URL%/}"; root="${root%/v1}"
           export ANTHROPIC_BASE_URL="$root"
           export AIKIT_LLM_URL="$root/v1"
-          ./evals/v2/run.sh "${{ inputs.agent }}" ./eval-runs/v2 "${{ inputs.trials }}"
+          # Naming the suite runs only that one AND suppresses the scorecard:
+          # every metric in metrics.toml is scoped to a case prefix, and a metric
+          # matching nothing is a hard error, so a subset cannot score itself.
+          # The scorecard job below folds the three back together.
+          ./evals/v2/run.sh "${{ inputs.agent }}" ./eval-runs/v2 \
+            "${{ inputs.trials }}" "${{ matrix.suite }}"
 
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: eval-run-${{ inputs.agent }}
+          name: eval-run-${{ inputs.agent }}-${{ matrix.suite }}
           path: eval-runs/
+
+  # ----------------------------------------------------------------------------
+  # Fold the per-suite runs into one scorecard and apply the gates. This is the
+  # only job with an opinion: its exit status is the sweep's verdict.
+  #
+  # It needs no gateway, no key and no Tailnet -- the judge already ran inside
+  # the correctness job and its verdicts are recorded in that suite's artifact.
+  # Scoring is pure local computation over the downloaded run directories.
+  # ----------------------------------------------------------------------------
+  scorecard:
+    name: Scorecard (fold the suites, apply the gates)
+    needs: live-eval
+    # Runs even when a suite job failed, so a partial sweep produces a named
+    # diagnosis instead of a silently skipped job. scorecard.sh refuses a root
+    # that is missing a suite and says which one.
+    if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install FastSkill CLI
+        run: |
+          curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash
+          fastskill --version
+
+      # Each suite job uploaded the contents of its own eval-runs/, so the three
+      # artifacts hold disjoint paths under v2/ and merge back into the exact
+      # layout run.sh writes for a single-machine sweep.
+      - name: Reassemble the run directories
+        uses: actions/download-artifact@v4
+        with:
+          pattern: eval-run-${{ inputs.agent }}-*
+          merge-multiple: true
+          path: eval-runs
+
+      - name: Show what came back
+        run: find eval-runs -maxdepth 2 -mindepth 1 | sort
+
+      - name: Score the sweep
+        run: ./evals/v2/scorecard.sh ./eval-runs/v2

evals/README.md

@@ -10,7 +10,8 @@ Two suites live here. They answer different questions and neither replaces the o
 | Agents | `claude` only | any (baselined on `pi`) |
 | Judged by an LLM | no | one advisory judge, on correctness |
 | Wired into CI | yes — `eval validate` + `eval score` on every PR | the suites are validated on every PR; the sweep runs on demand |
-| Cost per run | $0 (deterministic jobs) | ~$6 for the agent, ~1 h, plus one judge call per correctness trial |
+| Cost per run | $0 (deterministic jobs) | 42N agent runs + one judge call per correctness trial; free against a self-hosted gateway, metered against a paid API |
+| Wall clock | seconds | 76 min measured at `trials=1`; ~6.3 h at 5 sequentially, ~4.4 h split across three runners |
 
 v1 is the smoke test the release pipeline depends on. v2 is the measurement. Run v1 before
 every PR — CI does it for you. Run v2 when you change `SKILL.md` in a way that could move
@@ -24,6 +25,8 @@ evals/
 ├── agent-eval.Dockerfile     # reproducible container for a live v1 run
 └── v2/                       # spec 001 suite — see v2/README.md
     ├── stage.sh              # one suite → a throwaway skill tree; used by run.sh and CI
+    ├── run.sh                # runs the suites; naming one runs only that one
+    ├── scorecard.sh          # folds suite runs into metrics.toml and applies the gates
     └── correctness/judge-prompt.md   # the judge's rubric prompt, hand-written
 ```
 
@@ -62,12 +65,13 @@ python3 evals/v2/guard_vacuity.py         # must pass before you trust any numbe
 export AIKIT_LLM_URL=https://…/v1         # the judge's endpoint — there is no default
 export JUDGE_API_KEY=…                    # its key, named by api_key_env in checks.toml
 export JUDGE_MODEL=glm-5.3                # optional; overrides the model checks.toml declares
-./evals/v2/run.sh pi ./eval-runs/v2 5     # 42 cases x 5 trials, ~1 h, ~$6 + the judge's calls
+./evals/v2/run.sh pi ./eval-runs/v2 5     # 42 cases x 5 trials, ~6.3 h + one judge call per correctness trial
 ```
 
 `run.sh` runs the guard itself as a hard precondition, stages a throwaway skill tree and
 manifest per suite via `v2/stage.sh`, leaves the repo's own manifest untouched, and finishes
-by folding all three runs into the metrics in `v2/metrics.toml`. The suites are generated —
+by folding all three runs into the metrics in `v2/metrics.toml` — that last step is
+`v2/scorecard.sh`, and its exit status is the sweep's verdict. The suites are generated —
 edit `v2/patterns.json` and re-run `python3 evals/v2/build.py`, never the `checks.toml`
 files. The one exception is `v2/correctness/judge-prompt.md`, which is prose and is written
 by hand.
@@ -76,12 +80,29 @@ The correctness suite declares a judge, so it runs with `--judge` and needs the
 above. `run.sh` checks for them before the first suite starts: correctness runs last, and a
 key missing at its turn would surface only after the other two had been paid for in full.
 
-To re-fold a sweep already on disk without re-running it:
+### Running one suite at a time
+
+Name a suite and only that suite runs. CI does this to put each on its own runner: the
+sweep is ~6.3 h at `trials=5` on one machine, past GitHub's hard 6 h per-job ceiling.
 
 ```bash
-fastskill eval scorecard --root ./eval-runs/v2 --metrics evals/v2/metrics.toml
+./evals/v2/run.sh pi ./eval-runs/v2 5 consultation
+./evals/v2/run.sh pi ./eval-runs/v2 5 restraint correctness
 ```
 
+A subset run deliberately does **not** score itself. Every metric in `metrics.toml` is
+scoped to a case prefix, and a metric that matches nothing is a hard error, so scoring
+`restraint` alone would fail on the two `judge_score` metrics rather than tell you
+anything. Fold the parts once they are all present:
+
+```bash
+./evals/v2/scorecard.sh ./eval-runs/v2
+```
+
+That is also how to re-fold a sweep already on disk without re-running it. It refuses a
+root missing a suite and names which one, rather than letting the scorecard report an
+empty metric that reads like a `metrics.toml` bug.
+
 Raw artifacts are gitignored (`/eval-runs/`) — only the folded result is committed, as
 [`v2/baseline/pi-glm-5.3-2026-09-01.txt`](v2/baseline/pi-glm-5.3-2026-09-01.txt).
 Internals and per-suite detail: [`v2/README.md`](v2/README.md). Design rationale:
@@ -258,8 +279,13 @@ never matched against a trace.
   `eval validate` parses the judge and its prompt without calling anything. Advisory, not a
   required check — requiring one is a repository setting, and a required check that has never
   reported blocks every open pull request.
-- **live-eval** — opt-in via *Run workflow*, v1 only, needs `OPENAI_API_KEY` /
-  `ANTHROPIC_API_KEY`.
-
-No CI job runs a v2 *sweep*. At ~$6 and ~1 h it belongs on a schedule or a manual trigger,
-not on a pull request, and no job enforces the gates above yet.
+- **live-eval** — the v2 sweep, over the Tailnet, against a real agent and a real judge.
+  Dispatch-only: nothing schedules it and nothing should, because it runs for hours and a
+  run nobody is waiting on is a run nobody reads. One job per suite, in parallel, each with
+  its own `timeout-minutes` — the sweep is ~6.3 h sequentially at `trials=5`, past GitHub's
+  hard 6 h per-job ceiling, which no `timeout-minutes` can raise.
+- **scorecard** — `needs: live-eval`, downloads the three suites' artifacts, and runs
+  `v2/scorecard.sh` over the reassembled root. This is the job that applies the gates, so
+  it is the job whose red means the skill regressed rather than the harness broke.
+
+A sweep never runs on a pull request. It is a measurement, not a check.

evals/v2/README.md

@@ -32,13 +32,28 @@ export JUDGE_MODEL=glm-5.3         # optional; overrides the model checks.toml d
 the judged one, because correctness is last: a key missing at its turn would surface only
 after the other two suites had been paid for in full.
 
-`run.sh` ends by folding every run into the metrics in `metrics.toml`, so there is no
-separate aggregation step. To re-fold an existing sweep without re-running it:
+A full sweep ends by folding every run into the metrics in `metrics.toml`, so there is no
+separate aggregation step to remember. That fold is `scorecard.sh`, and its exit status is
+the sweep's verdict — the one place the measurement is allowed to have an opinion.
+
+Name a suite to run only that one:
+
+```bash
+./evals/v2/run.sh pi ./eval-runs/v2 5 correctness
+```
+
+A subset does **not** score itself: every metric here is scoped to a case prefix and a
+metric matching nothing is a hard error (`EVAL_SCORECARD_EMPTY_METRIC`), so scoring
+`restraint` alone would fail on the two `judge_score` metrics rather than measure anything.
+CI runs the three suites on parallel runners for the wall clock — 42 cases at `trials=5` is
+~6.3 h sequentially, past GitHub's hard 6 h per-job ceiling — and folds them at the end:
 
 ```bash
-fastskill eval scorecard --root ./eval-runs/v2 --metrics evals/v2/metrics.toml
+./evals/v2/scorecard.sh ./eval-runs/v2
 ```
 
+which is also how to re-fold a sweep already on disk without re-running it.
+
 ## Three suites, not fourteen
 
 Correctness used to be twelve single-case suites. A suite's checks applied to all of its
@@ -124,6 +139,7 @@ Three layers, because a broken measurement is worse than no measurement:
 | Script | What it proves | When it runs |
 |---|---|---|
 | `guard_vacuity.py` | no pattern occurs in the skill payload, so no check passes on a mere read | hard precondition of every `run.sh` |
+| `scorecard.sh` | a root missing a suite is named as such, not reported as an empty metric | folding a sweep, in CI and by hand |
 | `negctl.sh` | the consultation check *fails* when the skill read is deleted from a real trace | after a sweep, against real artifacts |
 | `stage.sh` + `eval validate` | every suite parses, and the judge's prompt uses the variables the engine requires | the `validate-v2` CI job, every pull request |
 

evals/v2/run.sh

@@ -12,15 +12,41 @@
 #   JUDGE_MODEL     optional; overrides the model every judge declares. The committed default
 #                   is a placeholder and has to exist on whatever AIKIT_LLM_URL serves.
 #
-# Usage: run.sh <agent> <out-dir> [trials]
+# Naming a suite runs only that one. CI does this to put each suite on its own runner --
+# 42 cases at trials=5 is ~6.3h sequentially, past GitHub's hard 6h per-job ceiling, and the
+# suites share nothing until the scorecard. A subset run deliberately does NOT score itself:
+# the metrics in metrics.toml span all three suites, and a metric matching no case is a hard
+# error (EVAL_SCORECARD_EMPTY_METRIC), so scoring `restraint` alone would fail on the two
+# judge_score metrics rather than tell you anything. Fold the parts with scorecard.sh instead.
+#
+# Usage: run.sh <agent> <out-dir> [trials] [suite...]
 set -euo pipefail
 
-AGENT="${1:?usage: run.sh <agent> <out-dir> [trials]}"
-OUT="${2:?usage: run.sh <agent> <out-dir> [trials]}"
+AGENT="${1:?usage: run.sh <agent> <out-dir> [trials] [suite...]}"
+OUT="${2:?usage: run.sh <agent> <out-dir> [trials] [suite...]}"
 TRIALS="${3:-5}"
+shift $(( $# < 3 ? $# : 3 ))
 
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+all_suites=(consultation restraint correctness)
+
+# No suite named => the full sweep, which scores itself. Any subset => the caller is
+# assembling a sweep out of parts and scorecard.sh is what has the opinion.
+if (($#)); then
+  suites=("$@")
+  for suite in "${suites[@]}"; do
+    if [[ ! -d "$HERE/$suite" ]]; then
+      echo "no such suite: '$suite' (have: ${all_suites[*]})" >&2
+      exit 2
+    fi
+  done
+  score_at_end=0
+else
+  suites=("${all_suites[@]}")
+  score_at_end=1
+fi
+
 # Anchor OUT before anything derives from it. Every suite below runs from inside a throwaway
 # skill tree staged elsewhere (stage.sh), so a relative out-dir resolves against *that* tree
 # rather than the caller's cwd: the run lands somewhere unrelated and the log redirect fails
@@ -37,8 +63,6 @@ echo "=== vacuity guard ==="
 python3 "$HERE/guard_vacuity.py"
 echo
 
-suites=(consultation restraint correctness)
-
 # Which suites are judged, decided before anything runs. correctness is last, so a missing
 # key found at its turn would surface only after the other two had been paid for in full.
 judged=()
@@ -65,7 +89,7 @@ for suite in "${suites[@]}"; do
 
   printf '  %-14s ' "$suite"
   # --no-fail: this is a measurement, not a gate (spec 001 P6). A failing case is data.
-  # The gates live in metrics.toml and are applied once, below, over all three runs.
+  # The gates live in metrics.toml and are applied once, by scorecard.sh, over all three.
   if (cd "$skill_dir" && fastskill eval run \
         --agent "$AGENT" \
         --output-dir "$OUT/$suite" \
@@ -77,11 +101,11 @@ for suite in "${suites[@]}"; do
   fi
 done
 
+if ((score_at_end)); then
+  echo
+  exec "$HERE/scorecard.sh" "$OUT"
+fi
+
 echo
-echo "=== scorecard ==="
-# Exit status is the scorecard's: a gate below its bar, or a metric that matched no case,
-# fails this script. That is the one place the sweep is allowed to have an opinion.
-#
-# The two `judge_score` metrics are the reason `--judge` above is not optional in practice: a
-# metric that matches nothing is a hard error, so an unjudged correctness run fails here.
-fastskill eval scorecard --root "$OUT" --metrics "$HERE/metrics.toml"
+echo "=== ran ${#suites[@]} of ${#all_suites[@]} suite(s); not scoring a subset ==="
+echo "fold this with the others: $HERE/scorecard.sh <dir containing all three>"

evals/v2/scorecard.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Fold one or more spec-001 suite runs into the metrics and apply the gates.
+#
+# This is the one place the sweep is allowed to have an opinion: the exit status is the
+# scorecard's, so a gate below its bar -- or a metric that matched no case -- fails here.
+#
+# It is a separate script from run.sh because CI runs the suites on three parallel runners
+# (42 cases at trials=5 is ~6.3h sequentially, past GitHub's hard 6h per-job ceiling) and
+# only reassembles them at the end. Scoring therefore has to be callable against a directory
+# nothing in this job produced.
+#
+# The layout it expects is exactly what run.sh writes -- one directory per suite:
+#
+#   <root>/consultation/<timestamp>/<agent>/<case>/...
+#   <root>/restraint/...
+#   <root>/correctness/...
+#
+# Usage: scorecard.sh <root>
+set -euo pipefail
+
+ROOT="${1:?usage: scorecard.sh <root>}"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+[[ -d "$ROOT" ]] || { echo "no such directory: $ROOT" >&2; exit 2; }
+ROOT="$(cd "$ROOT" && pwd)"
+
+# metrics.toml spans all three suites and a metric that matches nothing is a hard error
+# (EVAL_SCORECARD_EMPTY_METRIC). When a suite's runner failed outright its directory is
+# simply absent, and the scorecard's own error names a metric rather than the missing
+# suite -- which reads as a metrics.toml bug and is not one. Say which suite is missing.
+missing=()
+for suite in consultation restraint correctness; do
+  [[ -d "$ROOT/$suite" ]] || missing+=("$suite")
+done
+if ((${#missing[@]})); then
+  echo "::error::cannot score: no run directory for ${missing[*]} under $ROOT."
+  echo "Every metric in metrics.toml is scoped to a case prefix from one of the three suites," >&2
+  echo "so a missing suite surfaces here as EVAL_SCORECARD_EMPTY_METRIC naming the metric" >&2
+  echo "rather than the gap. Check whether that suite's job ran and uploaded its artifact." >&2
+  echo "present:" >&2
+  find "$ROOT" -mindepth 1 -maxdepth 1 -type d -printf '  %f\n' 2>/dev/null | sort >&2 || true
+  exit 1
+fi
+
+echo "=== scorecard ==="
+fastskill eval scorecard --root "$ROOT" --metrics "$HERE/metrics.toml"