feat(evals): add lightweight smoke tests and grading safeguards

.github/workflows/skill-evals.yml

@@ -24,9 +24,9 @@ on:
       agent:
         description: 'Agent CLI to run live evals with'
         type: choice
-        # Both installed runtimes can supply consultation evidence. Check the
-        # installed FastSkill version's runtime observability before a sweep.
-        options: [claude, codex]
+        # This hosted workflow currently configures only the Claude gateway route.
+        # Codex needs its own provider/auth/model setup before being offered in CI.
+        options: [claude]
         default: claude
       trials:
         description: 'Trials per case (5 is the spec-001 default)'
@@ -59,7 +59,7 @@ jobs:
           cd fastskill
           fastskill eval validate
       - name: Validate authoring assets and packaged example
-        run: python3 -m unittest scripts/test_eval_authoring.py scripts/test_eval_scripts.py
+        run: python3 -m unittest discover -s scripts -p 'test_*.py'
 
   # ----------------------------------------------------------------------------
   # Deterministic, always-on: the spec-001 v2 suites. `validate` above reads the
@@ -92,7 +92,7 @@ jobs:
           fastskill --version
       - name: Validate each suite
         run: |
-          for suite in consultation restraint correctness; do
+          for suite in consultation restraint correctness smoke; do
             echo "--- $suite"
             dir="$(evals/v2/stage.sh "$suite" "$RUNNER_TEMP/stage-$suite" 1 | tail -1)"
             (cd "$dir" && fastskill eval validate)
@@ -110,15 +110,8 @@ jobs:
       - name: Install FastSkill CLI
         run: |
           curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash
-      - name: PASS fixture must score green
-        run: fastskill eval score --run-dir evals/fixtures/pass
-      - name: FAIL fixture must score red
-        run: |
-          if fastskill eval score --run-dir evals/fixtures/fail; then
-            echo "::error::FAIL fixture unexpectedly passed — checks engine regression"
-            exit 1
-          fi
-          echo "FAIL fixture correctly reported a failing suite."
+      - name: Fixtures must produce their intended check verdicts
+        run: python3 -m unittest scripts/test_eval_scoring.py
 
   # ----------------------------------------------------------------------------
   # Opt-in (workflow_dispatch): run the spec-001 v2 sweep against a REAL agent
@@ -249,12 +242,10 @@ jobs:
           curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash
           fastskill --version
 
-      - name: Install agent CLI (${{ inputs.agent }})
+      - name: Install and record the Claude agent CLI version
         run: |
-          case "${{ inputs.agent }}" in
-            codex)  npm install -g @openai/codex ;;
-            claude) npm install -g @anthropic-ai/claude-code ;;
-          esac
+          npm install -g @anthropic-ai/claude-code
+          claude --version
 
       # run.sh already refuses to start a sweep it cannot judge, but it only
       # checks that the variables are SET. Whether the Tailnet grant exists,
@@ -409,6 +400,7 @@ jobs:
           # ANTHROPIC_BASE_URL is derived from the root below, not passed here.
           ANTHROPIC_AUTH_TOKEN: ${{ secrets.LLM_GATEWAY_KEY }}
           ANTHROPIC_MODEL: ${{ secrets.LLM_GATEWAY_MODEL }}
+          TARGET_MODEL: ${{ secrets.LLM_GATEWAY_MODEL }}
           # The judge's half: same gateway, same key, OpenAI-compatible route.
           # JUDGE_MODEL overrides the placeholder each checks.toml declares.
           LLM_GATEWAY_URL: ${{ secrets.LLM_GATEWAY_URL }}

CONTRIBUTING.md

@@ -41,9 +41,13 @@ repeat the eval validation and fixture scoring below with the candidate FastSkil
 The skill ships an eval suite so its trigger behavior and guidance stay correct. Config lives
 in `[tool.fastskill.eval]` in `fastskill/skill-project.toml` and points at `../evals/`.
 
+For the smallest live check run `bash evals/smoke.sh claude ./new-smoke-results` from the
+repository root (six cases, one trial, no judge). This measures consultation and fragment
+evidence, not semantic accuracy. Authoring outcome exercises are documented in the eval guide.
+
 The commands below cover the default (v1) suite, which is what CI gates. There is a second,
 larger suite under `evals/v2/` that measures recall, restraint and answer correctness across
-five trials per case; it runs on demand rather than in CI. **[`evals/README.md`](evals/README.md)
+five trials per case; live runs are opt-in, including manual CI dispatch. **[`evals/README.md`](evals/README.md)
 covers both suites and the methodology** — read it before adding or changing an eval case.
 
 ### Validate (deterministic, no agent)

README.md

@@ -24,6 +24,19 @@ included in the released skill ZIP. See the [assessment and ownership record](sp
 
 See [CONTRIBUTING.md](CONTRIBUTING.md) for development and release details.
 
+## Quick self-test
+
+From a clone, with FastSkill and an authenticated agent installed:
+
+```bash
+bash evals/smoke.sh claude ./smoke-results
+```
+
+Six cases, one trial each, no native judge. The output directory must be new. An optional
+third argument selects the target model. This checks consultation, restraint and trace
+fragments, **not semantic answer accuracy**, and costs agent tokens. See the
+[eval guide](evals/README.md) for the optional benchmark and authoring workflow exercises.
+
 ## License
 
 Apache-2.0

evals/README.md

@@ -1,13 +1,49 @@
 # Evals
 
+## Start small
+
+```bash
+bash evals/smoke.sh claude ./smoke-results                 # from the repository root
+bash evals/smoke.sh claude ./another-run <target-model>    # explicit model, fresh output
+```
+
+This stages six cases (two consultation, two restraint, two command-fragment cases),
+validates the runtime, and runs one trial each without a judge. Agent authentication
+and token usage still apply. A failing check or runtime exits nonzero. Keep the output
+directory: it includes the staged project and saved results. No existing directory is overwritten.
+
+**A green smoke is not semantic correctness.** Text checks search the trace, not just
+the final answer. The benchmark metric is now named “Required-fragment match rate
+(not semantic accuracy)”. The no-publish case is judge-only for correctness: a correct
+refusal may mention a nonexistent command. The optional judge remains advisory and needs
+calibration; `v2/calibration.json` supplies labelled contrasts, not a measured pass rate.
+
+## Exercise the authoring workflow
+
+Use your chosen agent to perform a small real task; the helper does not launch models:
+
+```bash
+python3 evals/authoring/workflow.py prepare create ./authoring-create
+# Ask your agent to read authoring-create/task.md and perform that task.
+python3 evals/authoring/workflow.py verify ./authoring-create
+```
+
+Repeat with `repair` or `extend` and a fresh destination. The verifier runs the actual
+FastSkill validator, checks positive/negative cases where relevant, and verifies preserved
+files/rows and review notes. It does not certify the semantic quality of generated cases:
+review those yourself. No target pilot or judge call is authorized by these task prompts.
+Deterministic harness tests use controlled edits and must not be reported as live-agent success.
+
+## Existing benchmark suites
+
 Two suites live here. They answer different questions and neither replaces the other.
 
 | | `prompts.csv` + `checks.toml` (v1) | `v2/` (spec 001) |
 |---|---|---|
 | Question | does the skill still trigger at all? | how well does it actually perform? |
 | Cases | 14, all positive | 42 — 22 consultation, 8 restraint, 12 correctness |
 | Trials per case | 1 | 5 |
-| Agents | `claude` only | any (baselined on `pi`) |
+| Agents | backend with observable consultation | supported local backend (historically baselined on `pi`) |
 | Judged by an LLM | no | one advisory judge, on correctness |
 | Wired into CI | yes — `eval validate` + `eval score` on every PR | the suites are validated on every PR; the sweep runs on demand |
 | Cost per run | $0 (deterministic jobs) | 42N agent runs + one judge call per correctness trial; free against a self-hosted gateway, metered against a paid API |
@@ -137,9 +173,9 @@ inherent and two have since been fixed:
    a case whose explicit checks contradict the column is rejected rather than silently
    resolved.
 
-The consequence for writing a positive check: pair a real flag with a value invented for
-that prompt. `--tag` is vacuous; `--tag v2.1.0` can only come from the agent synthesizing
-an answer.
+Pairing a flag with a scenario-specific value avoids matches in shipped documentation.
+It does not prove correctness: an incorrect operation or unrelated tool output can still
+contain that fragment. The guard is a useful contamination check, not a correctness oracle.
 
 ### A substring check cannot see the command around it
 
@@ -209,7 +245,7 @@ rather than by the directory a suite happens to live in.
 |---|---|---|---|
 | Skill-open rate | `op-*` `skill_invoked` | ≥ 0.85 | 100.0% (106/106) |
 | Restraint rate | `off-*` `skill_invoked` | ≥ 0.90 | 100.0% (40/40) |
-| Answer accuracy | `c-*` `command_contains` + `trigger_expectation` | ≥ 0.80 | 96.7% (58/60) |
+| Historical fragment score (formerly “Answer accuracy”) | old `c-*` text checks | old ≥ 0.80 | 96.7% (58/60); not comparable to revised 11-case fragment metric |
 | Tool-budget compliance | `op-*` `max_tool_calls` | ≥ 0.90 | 92.5% (98/106) |
 | Efficiency | p95 tool calls per `op-*` trial | ≤ 25 | 30 (median 8, max 50) |
 | Cost | USD per sweep | not gated | $5.81 / 210 trials |
@@ -253,10 +289,11 @@ python3 evals/v2/guard_vacuity.py                     # no check can pass on a m
 evals/v2/negctl.sh ./eval-runs/v2/consultation/<timestamp>/pi   # needs one real run
 ```
 
-`negctl.sh` is the red-green proof: it copies a real completed case, deletes the trace
-line carrying the skill read from one copy, and re-scores both. The consultation check
-must go from pass to fail while `max_tool_calls` stays put. It reads the path to delete
-out of each trial's `result.json`, so it removes the evidence the check actually reads.
+`negctl.sh` copies a real completed case, preserves its metadata, removes consultation
+tool calls and paired results from one copy, and re-scores both. Each trial's invocation
+check must flip while its tool-budget verdict stays unchanged. It uses each trial's own
+skill path (or an identifying Skill call), not one shared path for all trials. Tool-call
+counts can decrease; the invariant is the budget verdict, not an identical count.
 
 ## Adding a case
 
@@ -284,7 +321,10 @@ never matched against a trace.
   `eval validate` parses the judge and its prompt without calling anything. Advisory, not a
   required check — requiring one is a repository setting, and a required check that has never
   reported blocks every open pull request.
-- **live-eval** — the v2 sweep, over the Tailnet, against a real agent and a real judge.
+- **live-eval** — the v2 sweep, over the Tailnet, against Claude and a real judge.
+  CI offers Claude only until another agent has its own tested authentication/provider
+  configuration. Local scripts still accept supported installed runtimes. `TARGET_MODEL`
+  explicitly selects the target model for `run.sh`; it is distinct from `JUDGE_MODEL`.
   Dispatch-only: nothing schedules it and nothing should, because it runs for hours and a
   run nobody is waiting on is a run nobody reads. One job per suite, in parallel, each with
   its own `timeout-minutes` — the sweep is ~6.3 h sequentially at `trials=5`, past GitHub's

evals/authoring/workflow.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Prepare and verify small authoring tasks for the user's chosen agent.
+
+No model invocation or fabricated live-run verdicts. A prepared task stays pending
+until an agent edits target/ and the real validator and preservation checks pass.
+"""
+import argparse
+import csv
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[2]
+SCENARIOS = ("create", "repair", "extend")
+
+
+def digest(path):
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def rows(path):
+    with path.open(newline="") as handle:
+        return list(csv.DictReader(handle))
+
+
+def prepare(scenario, destination):
+    destination.mkdir()  # Refuse an existing destination, including a symlink.
+    source = ROOT / "evals/authoring/fixtures" / (
+        "invoice-no-evals" if scenario == "create" else "existing-suite")
+    if scenario == "repair":
+        source = ROOT / "fastskill/examples/invoice-extraction"
+    target = destination / "target"
+    shutil.copytree(source, target)
+    if scenario == "repair":
+        (target / "evals/checks.toml").write_text(
+            '[[check]]\nname = "skill_invoked"\nexpected = false\n'
+            'cases = ["invoice-basic"]\nrequired = true\n')
+    protected = {}
+    for path in target.rglob("*"):
+        if path.is_file() and path.relative_to(target).parts[0] != "evals":
+            protected[str(path.relative_to(target))] = digest(path)
+    if scenario == "extend":
+        for name in ("checks.toml", "user-note.md"):
+            path = target / "evals" / name
+            protected[str(path.relative_to(target))] = digest(path)
+    state = {"scenario": scenario, "protected": protected,
+             "original_rows": rows(target / "evals/prompts.csv") if scenario != "create" else []}
+    (destination / "baseline.json").write_text(json.dumps(state, indent=2))
+    shutil.copytree(ROOT / "fastskill", destination / "authoring-skill")
+    instructions = {
+        "create": "Create a small valid invoice-extraction suite with at least one positive and one unrelated negative case. Expected outcomes must derive from the fixtures, not be included as answers in the task prompts.",
+        "repair": "Diagnose and repair the invalid eval configuration without changing any prompt rows, target instructions, manifest or fixtures.",
+        "extend": "Add an unrelated negative-trigger case while preserving every existing row, check, fixture and user note.",
+    }
+    (destination / "task.md").write_text(
+        "Read authoring-skill/SKILL.md and its eval-authoring reference. Work only in target/. "
+        + instructions[scenario] + " Write target/evals/review-notes.md explaining the change and remaining limitations. "
+        "Run fastskill eval validate from target/. Do not run target agents or judges; "
+        "authoring only is authorized. Do not edit baseline.json or authoring-skill/.\n")
+    return destination / "task.md"
+
+
+def verify(destination, cli="fastskill"):
+    state = json.loads((destination / "baseline.json").read_text())
+    target = destination / "target"
+    for relative, expected in state["protected"].items():
+        path = target / relative
+        if not path.is_file() or digest(path) != expected:
+            raise ValueError(f"protected file changed: {relative}")
+    result = subprocess.run([cli, "eval", "validate", "--json"], cwd=target,
+                            capture_output=True, text=True)
+    if result.returncode:
+        raise ValueError("validation failed: " + result.stdout + result.stderr)
+    validation = json.loads(result.stdout)
+    if not validation.get("valid"):
+        raise ValueError("validator did not confirm validity")
+    current = rows(target / "evals/prompts.csv")
+    original = state["original_rows"]
+    scenario = state["scenario"]
+    if scenario == "repair" and current != original:
+        raise ValueError("repair changed prompt rows")
+    if scenario == "extend" and (len(current) <= len(original) or current[:len(original)] != original):
+        raise ValueError("extension must preserve original rows and add a case")
+    relevant = current[len(original):] if scenario == "extend" else current
+    if scenario != "repair" and not any(r.get("should_trigger", "").lower() in ("false", "0") for r in relevant):
+        raise ValueError("missing explicit negative-trigger case")
+    if scenario == "create" and not any(r.get("should_trigger", "").lower() in ("true", "1") for r in current):
+        raise ValueError("missing positive-trigger case")
+    notes = target / "evals/review-notes.md"
+    if not notes.is_file() or not notes.read_text().strip():
+        raise ValueError("missing review notes")
+    return {"scenario": scenario, "validated_cases": len(current), "preservation": "passed",
+            "semantic_quality": "requires human review", "target_pilot": "not run"}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    commands = parser.add_subparsers(dest="action", required=True)
+    setup = commands.add_parser("prepare")
+    setup.add_argument("scenario", choices=SCENARIOS)
+    setup.add_argument("destination", type=Path)
+    check = commands.add_parser("verify")
+    check.add_argument("destination", type=Path)
+    check.add_argument("--cli", default="fastskill")
+    args = parser.parse_args()
+    try:
+        if args.action == "prepare":
+            # Do not follow a dangling destination symlink before mkdir refuses it.
+            print(prepare(args.scenario, args.destination.absolute()))
+        else:
+            print(json.dumps(verify(args.destination.resolve(), args.cli), indent=2))
+    except (ValueError, OSError, KeyError) as error:
+        parser.exit(1, str(error) + "\n")
+
+
+if __name__ == "__main__":
+    main()

evals/checks.toml

@@ -12,9 +12,9 @@
 # *mentions* the skill; `skill_invoked` only passes when a `Skill` tool call
 # with an identifying field equal to "fastskill" is present.
 #
-# NOTE: this requires an agent whose backend emits structured Skill tool-use
-# events (claude). codex has no Skill tool concept, so its traces can never
-# satisfy this check — run live evals with --agent claude.
+# A supported backend can also provide consultation evidence through tool inputs
+# referencing the recorded skill document path. Validate the selected runtime;
+# local runtime support does not establish CI authentication for that runtime.
 [[check]]
 name = "skill_invoked"
 skill = "fastskill"

evals/smoke.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Six cases, one trial, no native judge. Agent authentication is still required.
+# Usage: bash evals/smoke.sh <agent> <new-output-dir> [model]
+set -euo pipefail
+AGENT="${1:?usage: smoke.sh <agent> <new-output-dir> [model]}"
+OUT="${2:?usage: smoke.sh <agent> <new-output-dir> [model]}"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Atomic creation refuses existing directories, files and symlinks.
+mkdir -- "$OUT"
+OUT="$(cd "$OUT" && pwd)"
+python3 "$HERE/v2/guard_vacuity.py"
+skill_dir="$(bash "$HERE/v2/stage.sh" smoke "$OUT/project" 1)"
+model=()
+if [[ -n "${3:-}" ]]; then model=(--model "$3"); fi
+cd "$skill_dir"
+fastskill eval validate --agent "$AGENT"
+echo "Smoke only: consultation, restraint, tool budget and trace fragments; NOT semantic accuracy."
+echo "Artifacts and staged project are retained in $OUT"
+fastskill eval run --agent "$AGENT" --trials 1 "${model[@]}" --output-dir "$OUT/results"