From 14af6e842dc174fe3baf913cd6af5bca84d44092 Mon Sep 17 00:00:00 2001 From: Alex Oliveira <4482374+aroff@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:51:57 +0000 Subject: [PATCH] evals: split the v2 sweep across three CI runners At trials=5 the sweep is ~6.3 h on one machine, past GitHub's hard 6 h per-job ceiling -- a platform limit no timeout-minutes can raise, and `fastskill eval run` 0.9.212 has no concurrency flag, so trials are sequential. The three suites share nothing until the scorecard, so put each on its own runner: the longest (consultation) is ~4.4 h. run.sh takes optional suite names and validates them before any side effect. A subset run deliberately does NOT score itself: every metric in metrics.toml is scoped to a case prefix and a metric matching nothing is a hard error (EVAL_SCORECARD_EMPTY_METRIC), so scoring `restraint` alone would fail on the two judge_score metrics rather than measure anything. scorecard.sh is the fold, extracted so it can run against a directory it did not produce -- in CI, the reassembled artifacts of three jobs. It refuses a root missing a suite and names which one, instead of letting the scorecard blame a metric for a gap in the inputs. live-eval stays dispatch-only. Nothing schedules it and nothing should. Docs: correct the sweep's cost and wall clock. The old "~$6, ~1 h" was an Anthropic-rate token estimate, which is fiction against a self-hosted gateway; 76 min at trials=1 is measured (run 33914096408). Also refresh the CI section, which still said no CI job runs a v2 sweep. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DobwJ8YTnfowwqwhspU1kQ --- .github/workflows/skill-evals.yml | 86 +++++++++++++++++++++++++++---- evals/README.md | 46 +++++++++++++---- evals/v2/README.md | 22 ++++++-- evals/v2/run.sh | 50 +++++++++++++----- evals/v2/scorecard.sh | 46 +++++++++++++++++ 5 files changed, 215 insertions(+), 35 deletions(-) create mode 100755 evals/v2/scorecard.sh diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 1f876ae..6bd4ece 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -146,15 +146,37 @@ jobs: # below are limited to the two this job installs a CLI for. # ---------------------------------------------------------------------------- live-eval: - name: Judged sweep (real agent + judge, over the Tailnet) + name: Sweep ${{ matrix.suite }} (real agent + judge, over the Tailnet) + # Dispatch-only, deliberately. Nothing schedules this and nothing should: it + # drives a real agent against a real gateway for hours, and a run nobody is + # waiting on is a run nobody reads. if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest - # 42 cases across the three suites, so `trials` N means 42N agent runs plus - # one judge call per correctness trial. At 60 minutes even trials=1 did not - # finish; trials=5 is 210 runs and needs the headroom. This is a backstop - # against a wedged agent, not a target -- the preflights below are what catch - # a misconfiguration, in seconds rather than by running out the clock. - timeout-minutes: 180 + strategy: + # One suite's failure must not cancel the others. They are independent + # measurements and a partial sweep is still worth its artifact -- the + # scorecard job below is what decides whether the whole is scoreable. + fail-fast: false + matrix: + # 42 cases across the three suites, so `trials` N means 42N agent runs + # plus one judge call per correctness trial. Run 33914096408 measured + # trials=1 at 76 min sequentially, which puts trials=5 near 6.3h -- + # past GitHub's HARD 6h per-job ceiling, which `timeout-minutes` cannot + # raise. The suites share nothing until the scorecard, so each gets its + # own runner and the long pole drops to consultation alone. + # + # Timeouts are backstops against a wedged agent, not targets; they are + # per-suite so a hung restraint run (5 min observed) is caught in + # minutes instead of hiding inside a ceiling sized for consultation. + # Ordered longest-first so the critical path starts first. + include: + - suite: consultation # 22 cases; 52 min at trials=1 + timeout: 330 + - suite: correctness # 12 cases, judged; 18 min at trials=1 + timeout: 180 + - suite: restraint # 8 cases; 5 min at trials=1 + timeout: 90 + timeout-minutes: ${{ matrix.timeout }} permissions: contents: read # Required by tailscale/github-action to mint an OIDC token for OAuth. @@ -407,10 +429,56 @@ jobs: root="${LLM_GATEWAY_URL%/}"; root="${root%/v1}" export ANTHROPIC_BASE_URL="$root" export AIKIT_LLM_URL="$root/v1" - ./evals/v2/run.sh "${{ inputs.agent }}" ./eval-runs/v2 "${{ inputs.trials }}" + # Naming the suite runs only that one AND suppresses the scorecard: + # every metric in metrics.toml is scoped to a case prefix, and a metric + # matching nothing is a hard error, so a subset cannot score itself. + # The scorecard job below folds the three back together. + ./evals/v2/run.sh "${{ inputs.agent }}" ./eval-runs/v2 \ + "${{ inputs.trials }}" "${{ matrix.suite }}" - uses: actions/upload-artifact@v4 if: always() with: - name: eval-run-${{ inputs.agent }} + name: eval-run-${{ inputs.agent }}-${{ matrix.suite }} path: eval-runs/ + + # ---------------------------------------------------------------------------- + # Fold the per-suite runs into one scorecard and apply the gates. This is the + # only job with an opinion: its exit status is the sweep's verdict. + # + # It needs no gateway, no key and no Tailnet -- the judge already ran inside + # the correctness job and its verdicts are recorded in that suite's artifact. + # Scoring is pure local computation over the downloaded run directories. + # ---------------------------------------------------------------------------- + scorecard: + name: Scorecard (fold the suites, apply the gates) + needs: live-eval + # Runs even when a suite job failed, so a partial sweep produces a named + # diagnosis instead of a silently skipped job. scorecard.sh refuses a root + # that is missing a suite and says which one. + if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Install FastSkill CLI + run: | + curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash + fastskill --version + + # Each suite job uploaded the contents of its own eval-runs/, so the three + # artifacts hold disjoint paths under v2/ and merge back into the exact + # layout run.sh writes for a single-machine sweep. + - name: Reassemble the run directories + uses: actions/download-artifact@v4 + with: + pattern: eval-run-${{ inputs.agent }}-* + merge-multiple: true + path: eval-runs + + - name: Show what came back + run: find eval-runs -maxdepth 2 -mindepth 1 | sort + + - name: Score the sweep + run: ./evals/v2/scorecard.sh ./eval-runs/v2 diff --git a/evals/README.md b/evals/README.md index ca00b6f..8cc256e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -10,7 +10,8 @@ Two suites live here. They answer different questions and neither replaces the o | Agents | `claude` only | any (baselined on `pi`) | | Judged by an LLM | no | one advisory judge, on correctness | | Wired into CI | yes — `eval validate` + `eval score` on every PR | the suites are validated on every PR; the sweep runs on demand | -| Cost per run | $0 (deterministic jobs) | ~$6 for the agent, ~1 h, plus one judge call per correctness trial | +| Cost per run | $0 (deterministic jobs) | 42N agent runs + one judge call per correctness trial; free against a self-hosted gateway, metered against a paid API | +| Wall clock | seconds | 76 min measured at `trials=1`; ~6.3 h at 5 sequentially, ~4.4 h split across three runners | v1 is the smoke test the release pipeline depends on. v2 is the measurement. Run v1 before every PR — CI does it for you. Run v2 when you change `SKILL.md` in a way that could move @@ -24,6 +25,8 @@ evals/ ├── agent-eval.Dockerfile # reproducible container for a live v1 run └── v2/ # spec 001 suite — see v2/README.md ├── stage.sh # one suite → a throwaway skill tree; used by run.sh and CI + ├── run.sh # runs the suites; naming one runs only that one + ├── scorecard.sh # folds suite runs into metrics.toml and applies the gates └── correctness/judge-prompt.md # the judge's rubric prompt, hand-written ``` @@ -62,12 +65,13 @@ python3 evals/v2/guard_vacuity.py # must pass before you trust any numbe export AIKIT_LLM_URL=https://…/v1 # the judge's endpoint — there is no default export JUDGE_API_KEY=… # its key, named by api_key_env in checks.toml export JUDGE_MODEL=glm-5.3 # optional; overrides the model checks.toml declares -./evals/v2/run.sh pi ./eval-runs/v2 5 # 42 cases x 5 trials, ~1 h, ~$6 + the judge's calls +./evals/v2/run.sh pi ./eval-runs/v2 5 # 42 cases x 5 trials, ~6.3 h + one judge call per correctness trial ``` `run.sh` runs the guard itself as a hard precondition, stages a throwaway skill tree and manifest per suite via `v2/stage.sh`, leaves the repo's own manifest untouched, and finishes -by folding all three runs into the metrics in `v2/metrics.toml`. The suites are generated — +by folding all three runs into the metrics in `v2/metrics.toml` — that last step is +`v2/scorecard.sh`, and its exit status is the sweep's verdict. The suites are generated — edit `v2/patterns.json` and re-run `python3 evals/v2/build.py`, never the `checks.toml` files. The one exception is `v2/correctness/judge-prompt.md`, which is prose and is written by hand. @@ -76,12 +80,29 @@ The correctness suite declares a judge, so it runs with `--judge` and needs the above. `run.sh` checks for them before the first suite starts: correctness runs last, and a key missing at its turn would surface only after the other two had been paid for in full. -To re-fold a sweep already on disk without re-running it: +### Running one suite at a time + +Name a suite and only that suite runs. CI does this to put each on its own runner: the +sweep is ~6.3 h at `trials=5` on one machine, past GitHub's hard 6 h per-job ceiling. ```bash -fastskill eval scorecard --root ./eval-runs/v2 --metrics evals/v2/metrics.toml +./evals/v2/run.sh pi ./eval-runs/v2 5 consultation +./evals/v2/run.sh pi ./eval-runs/v2 5 restraint correctness ``` +A subset run deliberately does **not** score itself. Every metric in `metrics.toml` is +scoped to a case prefix, and a metric that matches nothing is a hard error, so scoring +`restraint` alone would fail on the two `judge_score` metrics rather than tell you +anything. Fold the parts once they are all present: + +```bash +./evals/v2/scorecard.sh ./eval-runs/v2 +``` + +That is also how to re-fold a sweep already on disk without re-running it. It refuses a +root missing a suite and names which one, rather than letting the scorecard report an +empty metric that reads like a `metrics.toml` bug. + Raw artifacts are gitignored (`/eval-runs/`) — only the folded result is committed, as [`v2/baseline/pi-glm-5.3-2026-09-01.txt`](v2/baseline/pi-glm-5.3-2026-09-01.txt). Internals and per-suite detail: [`v2/README.md`](v2/README.md). Design rationale: @@ -258,8 +279,13 @@ never matched against a trace. `eval validate` parses the judge and its prompt without calling anything. Advisory, not a required check — requiring one is a repository setting, and a required check that has never reported blocks every open pull request. -- **live-eval** — opt-in via *Run workflow*, v1 only, needs `OPENAI_API_KEY` / - `ANTHROPIC_API_KEY`. - -No CI job runs a v2 *sweep*. At ~$6 and ~1 h it belongs on a schedule or a manual trigger, -not on a pull request, and no job enforces the gates above yet. +- **live-eval** — the v2 sweep, over the Tailnet, against a real agent and a real judge. + Dispatch-only: nothing schedules it and nothing should, because it runs for hours and a + run nobody is waiting on is a run nobody reads. One job per suite, in parallel, each with + its own `timeout-minutes` — the sweep is ~6.3 h sequentially at `trials=5`, past GitHub's + hard 6 h per-job ceiling, which no `timeout-minutes` can raise. +- **scorecard** — `needs: live-eval`, downloads the three suites' artifacts, and runs + `v2/scorecard.sh` over the reassembled root. This is the job that applies the gates, so + it is the job whose red means the skill regressed rather than the harness broke. + +A sweep never runs on a pull request. It is a measurement, not a check. diff --git a/evals/v2/README.md b/evals/v2/README.md index d7af05c..fb698a7 100644 --- a/evals/v2/README.md +++ b/evals/v2/README.md @@ -32,13 +32,28 @@ export JUDGE_MODEL=glm-5.3 # optional; overrides the model checks.toml d the judged one, because correctness is last: a key missing at its turn would surface only after the other two suites had been paid for in full. -`run.sh` ends by folding every run into the metrics in `metrics.toml`, so there is no -separate aggregation step. To re-fold an existing sweep without re-running it: +A full sweep ends by folding every run into the metrics in `metrics.toml`, so there is no +separate aggregation step to remember. That fold is `scorecard.sh`, and its exit status is +the sweep's verdict — the one place the measurement is allowed to have an opinion. + +Name a suite to run only that one: + +```bash +./evals/v2/run.sh pi ./eval-runs/v2 5 correctness +``` + +A subset does **not** score itself: every metric here is scoped to a case prefix and a +metric matching nothing is a hard error (`EVAL_SCORECARD_EMPTY_METRIC`), so scoring +`restraint` alone would fail on the two `judge_score` metrics rather than measure anything. +CI runs the three suites on parallel runners for the wall clock — 42 cases at `trials=5` is +~6.3 h sequentially, past GitHub's hard 6 h per-job ceiling — and folds them at the end: ```bash -fastskill eval scorecard --root ./eval-runs/v2 --metrics evals/v2/metrics.toml +./evals/v2/scorecard.sh ./eval-runs/v2 ``` +which is also how to re-fold a sweep already on disk without re-running it. + ## Three suites, not fourteen Correctness used to be twelve single-case suites. A suite's checks applied to all of its @@ -124,6 +139,7 @@ Three layers, because a broken measurement is worse than no measurement: | Script | What it proves | When it runs | |---|---|---| | `guard_vacuity.py` | no pattern occurs in the skill payload, so no check passes on a mere read | hard precondition of every `run.sh` | +| `scorecard.sh` | a root missing a suite is named as such, not reported as an empty metric | folding a sweep, in CI and by hand | | `negctl.sh` | the consultation check *fails* when the skill read is deleted from a real trace | after a sweep, against real artifacts | | `stage.sh` + `eval validate` | every suite parses, and the judge's prompt uses the variables the engine requires | the `validate-v2` CI job, every pull request | diff --git a/evals/v2/run.sh b/evals/v2/run.sh index bf006ea..d43b4d2 100755 --- a/evals/v2/run.sh +++ b/evals/v2/run.sh @@ -12,15 +12,41 @@ # JUDGE_MODEL optional; overrides the model every judge declares. The committed default # is a placeholder and has to exist on whatever AIKIT_LLM_URL serves. # -# Usage: run.sh [trials] +# Naming a suite runs only that one. CI does this to put each suite on its own runner -- +# 42 cases at trials=5 is ~6.3h sequentially, past GitHub's hard 6h per-job ceiling, and the +# suites share nothing until the scorecard. A subset run deliberately does NOT score itself: +# the metrics in metrics.toml span all three suites, and a metric matching no case is a hard +# error (EVAL_SCORECARD_EMPTY_METRIC), so scoring `restraint` alone would fail on the two +# judge_score metrics rather than tell you anything. Fold the parts with scorecard.sh instead. +# +# Usage: run.sh [trials] [suite...] set -euo pipefail -AGENT="${1:?usage: run.sh [trials]}" -OUT="${2:?usage: run.sh [trials]}" +AGENT="${1:?usage: run.sh [trials] [suite...]}" +OUT="${2:?usage: run.sh [trials] [suite...]}" TRIALS="${3:-5}" +shift $(( $# < 3 ? $# : 3 )) HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +all_suites=(consultation restraint correctness) + +# No suite named => the full sweep, which scores itself. Any subset => the caller is +# assembling a sweep out of parts and scorecard.sh is what has the opinion. +if (($#)); then + suites=("$@") + for suite in "${suites[@]}"; do + if [[ ! -d "$HERE/$suite" ]]; then + echo "no such suite: '$suite' (have: ${all_suites[*]})" >&2 + exit 2 + fi + done + score_at_end=0 +else + suites=("${all_suites[@]}") + score_at_end=1 +fi + # Anchor OUT before anything derives from it. Every suite below runs from inside a throwaway # skill tree staged elsewhere (stage.sh), so a relative out-dir resolves against *that* tree # rather than the caller's cwd: the run lands somewhere unrelated and the log redirect fails @@ -37,8 +63,6 @@ echo "=== vacuity guard ===" python3 "$HERE/guard_vacuity.py" echo -suites=(consultation restraint correctness) - # Which suites are judged, decided before anything runs. correctness is last, so a missing # key found at its turn would surface only after the other two had been paid for in full. judged=() @@ -65,7 +89,7 @@ for suite in "${suites[@]}"; do printf ' %-14s ' "$suite" # --no-fail: this is a measurement, not a gate (spec 001 P6). A failing case is data. - # The gates live in metrics.toml and are applied once, below, over all three runs. + # The gates live in metrics.toml and are applied once, by scorecard.sh, over all three. if (cd "$skill_dir" && fastskill eval run \ --agent "$AGENT" \ --output-dir "$OUT/$suite" \ @@ -77,11 +101,11 @@ for suite in "${suites[@]}"; do fi done +if ((score_at_end)); then + echo + exec "$HERE/scorecard.sh" "$OUT" +fi + echo -echo "=== scorecard ===" -# Exit status is the scorecard's: a gate below its bar, or a metric that matched no case, -# fails this script. That is the one place the sweep is allowed to have an opinion. -# -# The two `judge_score` metrics are the reason `--judge` above is not optional in practice: a -# metric that matches nothing is a hard error, so an unjudged correctness run fails here. -fastskill eval scorecard --root "$OUT" --metrics "$HERE/metrics.toml" +echo "=== ran ${#suites[@]} of ${#all_suites[@]} suite(s); not scoring a subset ===" +echo "fold this with the others: $HERE/scorecard.sh " diff --git a/evals/v2/scorecard.sh b/evals/v2/scorecard.sh new file mode 100755 index 0000000..a2613a0 --- /dev/null +++ b/evals/v2/scorecard.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Fold one or more spec-001 suite runs into the metrics and apply the gates. +# +# This is the one place the sweep is allowed to have an opinion: the exit status is the +# scorecard's, so a gate below its bar -- or a metric that matched no case -- fails here. +# +# It is a separate script from run.sh because CI runs the suites on three parallel runners +# (42 cases at trials=5 is ~6.3h sequentially, past GitHub's hard 6h per-job ceiling) and +# only reassembles them at the end. Scoring therefore has to be callable against a directory +# nothing in this job produced. +# +# The layout it expects is exactly what run.sh writes -- one directory per suite: +# +# /consultation////... +# /restraint/... +# /correctness/... +# +# Usage: scorecard.sh +set -euo pipefail + +ROOT="${1:?usage: scorecard.sh }" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +[[ -d "$ROOT" ]] || { echo "no such directory: $ROOT" >&2; exit 2; } +ROOT="$(cd "$ROOT" && pwd)" + +# metrics.toml spans all three suites and a metric that matches nothing is a hard error +# (EVAL_SCORECARD_EMPTY_METRIC). When a suite's runner failed outright its directory is +# simply absent, and the scorecard's own error names a metric rather than the missing +# suite -- which reads as a metrics.toml bug and is not one. Say which suite is missing. +missing=() +for suite in consultation restraint correctness; do + [[ -d "$ROOT/$suite" ]] || missing+=("$suite") +done +if ((${#missing[@]})); then + echo "::error::cannot score: no run directory for ${missing[*]} under $ROOT." + echo "Every metric in metrics.toml is scoped to a case prefix from one of the three suites," >&2 + echo "so a missing suite surfaces here as EVAL_SCORECARD_EMPTY_METRIC naming the metric" >&2 + echo "rather than the gap. Check whether that suite's job ran and uploaded its artifact." >&2 + echo "present:" >&2 + find "$ROOT" -mindepth 1 -maxdepth 1 -type d -printf ' %f\n' 2>/dev/null | sort >&2 || true + exit 1 +fi + +echo "=== scorecard ===" +fastskill eval scorecard --root "$ROOT" --metrics "$HERE/metrics.toml"