From be9e9edaf23f9ff14a702e384b6de0f69e78c5a1 Mon Sep 17 00:00:00 2001
From: Alex Oliveira <4482374+aroff@users.noreply.github.com>
Date: Tue, 29 Sep 2026 19:20:04 +0000
Subject: [PATCH] feat(evals): add lightweight smoke and trustworthy grading
 signals

---
 .github/workflows/skill-evals.yml     |  30 +++----
 CONTRIBUTING.md                       |   6 +-
 README.md                             |  13 +++
 evals/README.md                       |  60 ++++++++++---
 evals/authoring/workflow.py           | 119 +++++++++++++++++++++++++
 evals/checks.toml                     |   6 +-
 evals/smoke.sh                        |  19 ++++
 evals/v2/README.md                    |  26 +++---
 evals/v2/build.py                     |  18 +++-
 evals/v2/calibration.json             |  11 +++
 evals/v2/correctness/checks.toml      |   9 +-
 evals/v2/metrics.toml                 |  11 ++-
 evals/v2/negctl.py                    | 101 ++++++++++++++++++++++
 evals/v2/negctl.sh                    | 120 +-------------------------
 evals/v2/patterns.json                |   6 +-
 evals/v2/run.sh                       |   3 +
 evals/v2/smoke/checks.toml            |  21 +++++
 evals/v2/smoke/prompts.csv            |   7 ++
 evals/v2/stage.sh                     |   2 +-
 scripts/test_authoring_workflows.py   |  93 ++++++++++++++++++++
 scripts/test_eval_generation.py       |  39 +++++++++
 scripts/test_eval_negctl.py           |  84 ++++++++++++++++++
 scripts/test_eval_scoring.py          |  95 ++++++++++++++++++++
 scripts/test_eval_scripts.py          |   4 +-
 scripts/test_eval_smoke.py            |  51 +++++++++++
 specs/003-simple-eval-enhancements.md |  72 ++++++++++++++++
 26 files changed, 844 insertions(+), 182 deletions(-)
 create mode 100644 evals/authoring/workflow.py
 create mode 100644 evals/smoke.sh
 create mode 100644 evals/v2/calibration.json
 create mode 100644 evals/v2/negctl.py
 create mode 100644 evals/v2/smoke/checks.toml
 create mode 100644 evals/v2/smoke/prompts.csv
 create mode 100644 scripts/test_authoring_workflows.py
 create mode 100644 scripts/test_eval_generation.py
 create mode 100644 scripts/test_eval_negctl.py
 create mode 100644 scripts/test_eval_scoring.py
 create mode 100644 scripts/test_eval_smoke.py
 create mode 100644 specs/003-simple-eval-enhancements.md

diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml
index 5ab8767..3db6625 100644
--- a/.github/workflows/skill-evals.yml
+++ b/.github/workflows/skill-evals.yml
@@ -24,9 +24,9 @@ on:
       agent:
         description: 'Agent CLI to run live evals with'
         type: choice
-        # Both installed runtimes can supply consultation evidence. Check the
-        # installed FastSkill version's runtime observability before a sweep.
-        options: [claude, codex]
+        # This hosted workflow currently configures only the Claude gateway route.
+        # Codex needs its own provider/auth/model setup before being offered in CI.
+        options: [claude]
         default: claude
       trials:
         description: 'Trials per case (5 is the spec-001 default)'
@@ -59,7 +59,7 @@ jobs:
           cd fastskill
           fastskill eval validate
       - name: Validate authoring assets and packaged example
-        run: python3 -m unittest scripts/test_eval_authoring.py scripts/test_eval_scripts.py
+        run: python3 -m unittest discover -s scripts -p 'test_*.py'
 
   # ----------------------------------------------------------------------------
   # Deterministic, always-on: the spec-001 v2 suites. `validate` above reads the
@@ -92,7 +92,7 @@ jobs:
           fastskill --version
       - name: Validate each suite
         run: |
-          for suite in consultation restraint correctness; do
+          for suite in consultation restraint correctness smoke; do
             echo "--- $suite"
             dir="$(evals/v2/stage.sh "$suite" "$RUNNER_TEMP/stage-$suite" 1 | tail -1)"
             (cd "$dir" && fastskill eval validate)
@@ -110,15 +110,8 @@ jobs:
       - name: Install FastSkill CLI
         run: |
           curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash
-      - name: PASS fixture must score green
-        run: fastskill eval score --run-dir evals/fixtures/pass
-      - name: FAIL fixture must score red
-        run: |
-          if fastskill eval score --run-dir evals/fixtures/fail; then
-            echo "::error::FAIL fixture unexpectedly passed — checks engine regression"
-            exit 1
-          fi
-          echo "FAIL fixture correctly reported a failing suite."
+      - name: Fixtures must produce their intended check verdicts
+        run: python3 -m unittest scripts/test_eval_scoring.py
 
   # ----------------------------------------------------------------------------
   # Opt-in (workflow_dispatch): run the spec-001 v2 sweep against a REAL agent
@@ -249,12 +242,10 @@ jobs:
           curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash
           fastskill --version
 
-      - name: Install agent CLI (${{ inputs.agent }})
+      - name: Install and record the Claude agent CLI version
         run: |
-          case "${{ inputs.agent }}" in
-            codex)  npm install -g @openai/codex ;;
-            claude) npm install -g @anthropic-ai/claude-code ;;
-          esac
+          npm install -g @anthropic-ai/claude-code
+          claude --version
 
       # run.sh already refuses to start a sweep it cannot judge, but it only
       # checks that the variables are SET. Whether the Tailnet grant exists,
@@ -409,6 +400,7 @@ jobs:
           # ANTHROPIC_BASE_URL is derived from the root below, not passed here.
           ANTHROPIC_AUTH_TOKEN: ${{ secrets.LLM_GATEWAY_KEY }}
           ANTHROPIC_MODEL: ${{ secrets.LLM_GATEWAY_MODEL }}
+          TARGET_MODEL: ${{ secrets.LLM_GATEWAY_MODEL }}
           # The judge's half: same gateway, same key, OpenAI-compatible route.
           # JUDGE_MODEL overrides the placeholder each checks.toml declares.
           LLM_GATEWAY_URL: ${{ secrets.LLM_GATEWAY_URL }}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 57a6bc4..e28bc8c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -41,9 +41,13 @@ repeat the eval validation and fixture scoring below with the candidate FastSkil
 The skill ships an eval suite so its trigger behavior and guidance stay correct. Config lives
 in `[tool.fastskill.eval]` in `fastskill/skill-project.toml` and points at `../evals/`.
 
+For the smallest live check run `bash evals/smoke.sh claude ./new-smoke-results` from the
+repository root (six cases, one trial, no judge). This measures consultation and fragment
+evidence, not semantic accuracy. Authoring outcome exercises are documented in the eval guide.
+
 The commands below cover the default (v1) suite, which is what CI gates. There is a second,
 larger suite under `evals/v2/` that measures recall, restraint and answer correctness across
-five trials per case; it runs on demand rather than in CI. **[`evals/README.md`](evals/README.md)
+five trials per case; live runs are opt-in, including manual CI dispatch. **[`evals/README.md`](evals/README.md)
 covers both suites and the methodology** — read it before adding or changing an eval case.
 
 ### Validate (deterministic, no agent)
diff --git a/README.md b/README.md
index 7a6bdeb..98db4db 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,19 @@ included in the released skill ZIP. See the [assessment and ownership record](sp
 
 See [CONTRIBUTING.md](CONTRIBUTING.md) for development and release details.
 
+## Quick self-test
+
+From a clone, with FastSkill and an authenticated agent installed:
+
+```bash
+bash evals/smoke.sh claude ./smoke-results
+```
+
+Six cases, one trial each, no native judge. The output directory must be new. An optional
+third argument selects the target model. This checks consultation, restraint and trace
+fragments, **not semantic answer accuracy**, and costs agent tokens. See the
+[eval guide](evals/README.md) for the optional benchmark and authoring workflow exercises.
+
 ## License
 
 Apache-2.0
diff --git a/evals/README.md b/evals/README.md
index 472b562..51f9994 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -1,5 +1,41 @@
 # Evals
 
+## Start small
+
+```bash
+bash evals/smoke.sh claude ./smoke-results                 # from the repository root
+bash evals/smoke.sh claude ./another-run <target-model>    # explicit model, fresh output
+```
+
+This stages six cases (two consultation, two restraint, two command-fragment cases),
+validates the runtime, and runs one trial each without a judge. Agent authentication
+and token usage still apply. A failing check or runtime exits nonzero. Keep the output
+directory: it includes the staged project and saved results. No existing directory is overwritten.
+
+**A green smoke is not semantic correctness.** Text checks search the trace, not just
+the final answer. The benchmark metric is now named “Required-fragment match rate
+(not semantic accuracy)”. The no-publish case is judge-only for correctness: a correct
+refusal may mention a nonexistent command. The optional judge remains advisory and needs
+calibration; `v2/calibration.json` supplies labelled contrasts, not a measured pass rate.
+
+## Exercise the authoring workflow
+
+Use your chosen agent to perform a small real task; the helper does not launch models:
+
+```bash
+python3 evals/authoring/workflow.py prepare create ./authoring-create
+# Ask your agent to read authoring-create/task.md and perform that task.
+python3 evals/authoring/workflow.py verify ./authoring-create
+```
+
+Repeat with `repair` or `extend` and a fresh destination. The verifier runs the actual
+FastSkill validator, checks positive/negative cases where relevant, and verifies preserved
+files/rows and review notes. It does not certify the semantic quality of generated cases:
+review those yourself. No target pilot or judge call is authorized by these task prompts.
+Deterministic harness tests use controlled edits and must not be reported as live-agent success.
+
+## Existing benchmark suites
+
 Two suites live here. They answer different questions and neither replaces the other.
 
 | | `prompts.csv` + `checks.toml` (v1) | `v2/` (spec 001) |
@@ -7,7 +43,7 @@ Two suites live here. They answer different questions and neither replaces the o
 | Question | does the skill still trigger at all? | how well does it actually perform? |
 | Cases | 14, all positive | 42 — 22 consultation, 8 restraint, 12 correctness |
 | Trials per case | 1 | 5 |
-| Agents | `claude` only | any (baselined on `pi`) |
+| Agents | backend with observable consultation | supported local backend (historically baselined on `pi`) |
 | Judged by an LLM | no | one advisory judge, on correctness |
 | Wired into CI | yes — `eval validate` + `eval score` on every PR | the suites are validated on every PR; the sweep runs on demand |
 | Cost per run | $0 (deterministic jobs) | 42N agent runs + one judge call per correctness trial; free against a self-hosted gateway, metered against a paid API |
@@ -137,9 +173,9 @@ inherent and two have since been fixed:
    a case whose explicit checks contradict the column is rejected rather than silently
    resolved.
 
-The consequence for writing a positive check: pair a real flag with a value invented for
-that prompt. `--tag` is vacuous; `--tag v2.1.0` can only come from the agent synthesizing
-an answer.
+Pairing a flag with a scenario-specific value avoids matches in shipped documentation.
+It does not prove correctness: an incorrect operation or unrelated tool output can still
+contain that fragment. The guard is a useful contamination check, not a correctness oracle.
 
 ### A substring check cannot see the command around it
 
@@ -209,7 +245,7 @@ rather than by the directory a suite happens to live in.
 |---|---|---|---|
 | Skill-open rate | `op-*` `skill_invoked` | ≥ 0.85 | 100.0% (106/106) |
 | Restraint rate | `off-*` `skill_invoked` | ≥ 0.90 | 100.0% (40/40) |
-| Answer accuracy | `c-*` `command_contains` + `trigger_expectation` | ≥ 0.80 | 96.7% (58/60) |
+| Historical fragment score (formerly “Answer accuracy”) | old `c-*` text checks | old ≥ 0.80 | 96.7% (58/60); not comparable to revised 11-case fragment metric |
 | Tool-budget compliance | `op-*` `max_tool_calls` | ≥ 0.90 | 92.5% (98/106) |
 | Efficiency | p95 tool calls per `op-*` trial | ≤ 25 | 30 (median 8, max 50) |
 | Cost | USD per sweep | not gated | $5.81 / 210 trials |
@@ -253,10 +289,11 @@ python3 evals/v2/guard_vacuity.py                     # no check can pass on a m
 evals/v2/negctl.sh ./eval-runs/v2/consultation/<timestamp>/pi   # needs one real run
 ```
 
-`negctl.sh` is the red-green proof: it copies a real completed case, deletes the trace
-line carrying the skill read from one copy, and re-scores both. The consultation check
-must go from pass to fail while `max_tool_calls` stays put. It reads the path to delete
-out of each trial's `result.json`, so it removes the evidence the check actually reads.
+`negctl.sh` copies a real completed case, preserves its metadata, removes consultation
+tool calls and paired results from one copy, and re-scores both. Each trial's invocation
+check must flip while its tool-budget verdict stays unchanged. It uses each trial's own
+skill path (or an identifying Skill call), not one shared path for all trials. Tool-call
+counts can decrease; the invariant is the budget verdict, not an identical count.
 
 ## Adding a case
 
@@ -284,7 +321,10 @@ never matched against a trace.
   `eval validate` parses the judge and its prompt without calling anything. Advisory, not a
   required check — requiring one is a repository setting, and a required check that has never
   reported blocks every open pull request.
-- **live-eval** — the v2 sweep, over the Tailnet, against a real agent and a real judge.
+- **live-eval** — the v2 sweep, over the Tailnet, against Claude and a real judge.
+  CI offers Claude only until another agent has its own tested authentication/provider
+  configuration. Local scripts still accept supported installed runtimes. `TARGET_MODEL`
+  explicitly selects the target model for `run.sh`; it is distinct from `JUDGE_MODEL`.
   Dispatch-only: nothing schedules it and nothing should, because it runs for hours and a
   run nobody is waiting on is a run nobody reads. One job per suite, in parallel, each with
   its own `timeout-minutes` — the sweep is ~6.3 h sequentially at `trials=5`, past GitHub's
diff --git a/evals/authoring/workflow.py b/evals/authoring/workflow.py
new file mode 100644
index 0000000..6f06b36
--- /dev/null
+++ b/evals/authoring/workflow.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Prepare and verify small authoring tasks for the user's chosen agent.
+
+No model invocation or fabricated live-run verdicts. A prepared task stays pending
+until an agent edits target/ and the real validator and preservation checks pass.
+"""
+import argparse
+import csv
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[2]
+SCENARIOS = ("create", "repair", "extend")
+
+
+def digest(path):
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def rows(path):
+    with path.open(newline="") as handle:
+        return list(csv.DictReader(handle))
+
+
+def prepare(scenario, destination):
+    destination.mkdir()  # Refuse an existing destination, including a symlink.
+    source = ROOT / "evals/authoring/fixtures" / (
+        "invoice-no-evals" if scenario == "create" else "existing-suite")
+    if scenario == "repair":
+        source = ROOT / "fastskill/examples/invoice-extraction"
+    target = destination / "target"
+    shutil.copytree(source, target)
+    if scenario == "repair":
+        (target / "evals/checks.toml").write_text(
+            '[[check]]\nname = "skill_invoked"\nexpected = false\n'
+            'cases = ["invoice-basic"]\nrequired = true\n')
+    protected = {}
+    for path in target.rglob("*"):
+        if path.is_file() and path.relative_to(target).parts[0] != "evals":
+            protected[str(path.relative_to(target))] = digest(path)
+    if scenario == "extend":
+        for name in ("checks.toml", "user-note.md"):
+            path = target / "evals" / name
+            protected[str(path.relative_to(target))] = digest(path)
+    state = {"scenario": scenario, "protected": protected,
+             "original_rows": rows(target / "evals/prompts.csv") if scenario != "create" else []}
+    (destination / "baseline.json").write_text(json.dumps(state, indent=2))
+    shutil.copytree(ROOT / "fastskill", destination / "authoring-skill")
+    instructions = {
+        "create": "Create a small valid invoice-extraction suite with at least one positive and one unrelated negative case. Expected outcomes must derive from the fixtures, not be included as answers in the task prompts.",
+        "repair": "Diagnose and repair the invalid eval configuration without changing any prompt rows, target instructions, manifest or fixtures.",
+        "extend": "Add an unrelated negative-trigger case while preserving every existing row, check, fixture and user note.",
+    }
+    (destination / "task.md").write_text(
+        "Read authoring-skill/SKILL.md and its eval-authoring reference. Work only in target/. "
+        + instructions[scenario] + " Write target/evals/review-notes.md explaining the change and remaining limitations. "
+        "Run fastskill eval validate from target/. Do not run target agents or judges; "
+        "authoring only is authorized. Do not edit baseline.json or authoring-skill/.\n")
+    return destination / "task.md"
+
+
+def verify(destination, cli="fastskill"):
+    state = json.loads((destination / "baseline.json").read_text())
+    target = destination / "target"
+    for relative, expected in state["protected"].items():
+        path = target / relative
+        if not path.is_file() or digest(path) != expected:
+            raise ValueError(f"protected file changed: {relative}")
+    result = subprocess.run([cli, "eval", "validate", "--json"], cwd=target,
+                            capture_output=True, text=True)
+    if result.returncode:
+        raise ValueError("validation failed: " + result.stdout + result.stderr)
+    validation = json.loads(result.stdout)
+    if not validation.get("valid"):
+        raise ValueError("validator did not confirm validity")
+    current = rows(target / "evals/prompts.csv")
+    original = state["original_rows"]
+    scenario = state["scenario"]
+    if scenario == "repair" and current != original:
+        raise ValueError("repair changed prompt rows")
+    if scenario == "extend" and (len(current) <= len(original) or current[:len(original)] != original):
+        raise ValueError("extension must preserve original rows and add a case")
+    relevant = current[len(original):] if scenario == "extend" else current
+    if scenario != "repair" and not any(r.get("should_trigger", "").lower() in ("false", "0") for r in relevant):
+        raise ValueError("missing explicit negative-trigger case")
+    if scenario == "create" and not any(r.get("should_trigger", "").lower() in ("true", "1") for r in current):
+        raise ValueError("missing positive-trigger case")
+    notes = target / "evals/review-notes.md"
+    if not notes.is_file() or not notes.read_text().strip():
+        raise ValueError("missing review notes")
+    return {"scenario": scenario, "validated_cases": len(current), "preservation": "passed",
+            "semantic_quality": "requires human review", "target_pilot": "not run"}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    commands = parser.add_subparsers(dest="action", required=True)
+    setup = commands.add_parser("prepare")
+    setup.add_argument("scenario", choices=SCENARIOS)
+    setup.add_argument("destination", type=Path)
+    check = commands.add_parser("verify")
+    check.add_argument("destination", type=Path)
+    check.add_argument("--cli", default="fastskill")
+    args = parser.parse_args()
+    try:
+        if args.action == "prepare":
+            # Do not follow a dangling destination symlink before mkdir refuses it.
+            print(prepare(args.scenario, args.destination.absolute()))
+        else:
+            print(json.dumps(verify(args.destination.resolve(), args.cli), indent=2))
+    except (ValueError, OSError, KeyError) as error:
+        parser.exit(1, str(error) + "\n")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/evals/checks.toml b/evals/checks.toml
index 74c40cf..a269724 100644
--- a/evals/checks.toml
+++ b/evals/checks.toml
@@ -12,9 +12,9 @@
 # *mentions* the skill; `skill_invoked` only passes when a `Skill` tool call
 # with an identifying field equal to "fastskill" is present.
 #
-# NOTE: this requires an agent whose backend emits structured Skill tool-use
-# events (claude). codex has no Skill tool concept, so its traces can never
-# satisfy this check — run live evals with --agent claude.
+# A supported backend can also provide consultation evidence through tool inputs
+# referencing the recorded skill document path. Validate the selected runtime;
+# local runtime support does not establish CI authentication for that runtime.
 [[check]]
 name = "skill_invoked"
 skill = "fastskill"
diff --git a/evals/smoke.sh b/evals/smoke.sh
new file mode 100644
index 0000000..e5672f1
--- /dev/null
+++ b/evals/smoke.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Six cases, one trial, no native judge. Agent authentication is still required.
+# Usage: bash evals/smoke.sh <agent> <new-output-dir> [model]
+set -euo pipefail
+AGENT="${1:?usage: smoke.sh <agent> <new-output-dir> [model]}"
+OUT="${2:?usage: smoke.sh <agent> <new-output-dir> [model]}"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Atomic creation refuses existing directories, files and symlinks.
+mkdir -- "$OUT"
+OUT="$(cd "$OUT" && pwd)"
+python3 "$HERE/v2/guard_vacuity.py"
+skill_dir="$(bash "$HERE/v2/stage.sh" smoke "$OUT/project" 1)"
+model=()
+if [[ -n "${3:-}" ]]; then model=(--model "$3"); fi
+cd "$skill_dir"
+fastskill eval validate --agent "$AGENT"
+echo "Smoke only: consultation, restraint, tool budget and trace fragments; NOT semantic accuracy."
+echo "Artifacts and staged project are retained in $OUT"
+fastskill eval run --agent "$AGENT" --trials 1 "${model[@]}" --output-dir "$OUT/results"
diff --git a/evals/v2/README.md b/evals/v2/README.md
index 6b4a7eb..c398d13 100644
--- a/evals/v2/README.md
+++ b/evals/v2/README.md
@@ -10,7 +10,13 @@ engine constraints it works around.
 |---|---|---|
 | `consultation/` | Is the skill opened when it should be? | `should_trigger = true` on every case, which generates a `skill_invoked` check |
 | `restraint/` | Is it left alone when it should be? | `should_trigger = false`, generating the same check with inverted polarity |
-| `correctness/` | Is the answer right? | one `command_contains` per case, scoped to it, on a scenario-unique flag and value — plus a judge, scored against a reference answer |
+| `correctness/` | Do expected fragments occur; what does a semantic reviewer think? | eleven scoped fragment checks plus an advisory final-answer judge on all twelve cases; no-publish is judge-only for correctness |
+
+For a quick start use `bash evals/smoke.sh claude ./new-smoke-results` from the repository
+root: six cases, one trial, no judge. The full benchmark below remains optional. Fragment
+matches are not semantic accuracy and may come from tool output rather than the final answer.
+Use `calibration.json` to challenge a selected judge before trusting its scores; those
+examples are labelled test inputs, not recorded judge results.
 
 ## Layout
 
@@ -74,8 +80,8 @@ checks file is now a header and nothing else, and it is still fully scored.
 
 Correctness cases are on-topic, so they keep `should_trigger = true` and gain the generated
 check too. That is a second, independent assertion rather than a dilution of the first: the
-scorecard keeps a rate per check type, so consulting the skill and answering correctly are
-reported as the separate things they are.
+scorecard keeps consultation and trace-fragment matches separate. Neither proves the answer
+is correct. Semantic correctness is reported separately by the optional advisory judge.
 
 ## The judge
 
@@ -153,19 +159,19 @@ Three layers, because a broken measurement is worse than no measurement:
 evals/v2/negctl.sh ./eval-runs/v2/consultation/<timestamp>/pi
 ```
 
-`negctl.sh` copies one real completed case twice, strips every trace line naming the staged
-skill path from one copy, and re-scores both with `fastskill eval score`. Measured on the
-first pi sweep, when the check was still a text expectation:
+`negctl.sh` now preserves the recorded summary, removes only consultation calls and paired
+results from a copy, and checks the specific verdict on every trial. Each trial may have
+a different skill path. The following output is historical, from the first pi sweep when
+the check was still a text expectation:
 
 ```
   present  passed 1.00 [('trigger_expectation', True),  ('max_tool_calls', True)]
   absent   failed 0.00 [('trigger_expectation', False), ('max_tool_calls', True)]
 ```
 
-Exactly one trace line per trial carries the read, and `max_tool_calls` is unmoved, so the
-flip is attributable to the consultation check alone. The check is now `skill_invoked` and
-the control reads the path it matches out of each trial's `result.json`, so it deletes the
-evidence the check reads rather than a string that resembles it.
+The current control requires `skill_invoked` to flip and the `max_tool_calls` verdict to
+stay unchanged. Removing tool calls changes the count, not necessarily the budget verdict.
+It refuses missing consultation evidence or unobservable/error trials.
 
 ## Agent support
 
diff --git a/evals/v2/build.py b/evals/v2/build.py
index a86ab28..dc9f87e 100755
--- a/evals/v2/build.py
+++ b/evals/v2/build.py
@@ -173,8 +173,24 @@ def main():
     body += judge(pats["judge"], d)
     (d / "checks.toml").write_text(body)
 
+    # Small, unjudged profile. This measures consultation/restraint and fragments,
+    # deliberately not semantic correctness. Keep it independent of judge secrets.
+    smoke_ids = {"op-init", "op-eval", "off-python-list", "off-npm",
+                 "c-tag-pin", "c-serve-port"}
+    rows = []
+    for suite, triggered in (("consultation", "true"), ("restraint", "false")):
+        rows.extend([cid, prompt, triggered, "smoke", ""]
+                    for cid, prompt, _ in pats[suite] if cid in smoke_ids)
+    selected = [case for case in pats["correctness"] if case["id"] in smoke_ids]
+    rows.extend([case["id"], case["prompt"], "true", "smoke", ""] for case in selected)
+    d = HERE / "smoke"
+    write_csv(d / "prompts.csv", rows)
+    (d / "checks.toml").write_text(HEADER + tool_cap(cap) + "".join(
+        command_contains(pattern, case["id"])
+        for case in selected for pattern in case["require"]))
+
     print(
-        f"generated 3 suites: consultation ({len(pats['consultation'])} cases), "
+        f"generated smoke ({len(rows)} cases) and 3 benchmark suites: consultation ({len(pats['consultation'])} cases), "
         f"restraint ({len(pats['restraint'])} cases), "
         f"correctness ({len(pats['correctness'])} cases, "
         f"1 judge with {len(pats['judge']['criteria'])} criteria)"
diff --git a/evals/v2/calibration.json b/evals/v2/calibration.json
new file mode 100644
index 0000000..511dbcb
--- /dev/null
+++ b/evals/v2/calibration.json
@@ -0,0 +1,11 @@
+{
+  "purpose": "Human-labelled contrasts for calibrating the optional final-answer judge. Not a measured judge pass rate; do not feed these answers to the evaluated agent.",
+  "examples": [
+    {"case": "c-tag-pin", "answer": "fastskill skill remove widgets --tag v2.1.0", "acceptable": false, "reason": "Wrong operation despite matching flag."},
+    {"case": "c-tag-pin", "answer": "fastskill skill add https://github.com/acme/widgets.git --tag v2.1.0", "acceptable": true, "reason": "Correct operation, repository and tag."},
+    {"case": "c-serve-port", "answer": "fastskill server serve --port 9123", "acceptable": false, "reason": "Requested write endpoints are not enabled."},
+    {"case": "c-serve-port", "answer": "fastskill server serve --port 9123 --enable-write", "acceptable": true, "reason": "Includes both requested settings."},
+    {"case": "c-no-publish", "answer": "There is no fastskill publish command. Use a Git or ZIP URL instead.", "acceptable": true, "reason": "Explicit refusal plus supported alternative; mentioning is not recommending."},
+    {"case": "c-no-publish", "answer": "I do not know.", "acceptable": false, "reason": "Does not explain the limitation or a supported alternative."}
+  ]
+}
diff --git a/evals/v2/correctness/checks.toml b/evals/v2/correctness/checks.toml
index 0fbf35a..3525688 100644
--- a/evals/v2/correctness/checks.toml
+++ b/evals/v2/correctness/checks.toml
@@ -69,13 +69,6 @@ pattern = "--priority 7"
 required = true
 cases = ["c-repos-priority"]
 
-[[check]]
-name = "trigger_expectation"
-pattern = "fastskill publish"
-expected = false
-required = true
-cases = ["c-no-publish"]
-
 [[check]]
 name = "max_tool_calls"
 limit = 25
@@ -95,7 +88,7 @@ description = "The answer gives a command that does what the prompt asked: the r
 [[judge.criterion]]
 name = "would_run"
 kind = "bool"
-description = "Every subcommand, flag and argument in the answer exists in fastskill and is spelled correctly, so the command as written would execute rather than be rejected as an unknown argument. A flag the tool does not have makes this no, even when the rest of the command is right."
+description = "Every recommended subcommand, flag and argument exists in fastskill and is spelled correctly. A nonexistent flag makes this no. For an unsupported operation, an explicit refusal with a supported distribution alternative is valid; mentioning a nonexistent command to explain that it does not exist is not recommending it. This is a reference-based judgment, not execution proof."
 
 [[judge.criterion]]
 name = "no_unrequested_side_effects"
diff --git a/evals/v2/metrics.toml b/evals/v2/metrics.toml
index 3adc16c..e5b9212 100644
--- a/evals/v2/metrics.toml
+++ b/evals/v2/metrics.toml
@@ -36,14 +36,13 @@ cases = ["off-*"]
 checks = ["skill_invoked"]
 min_rate = 0.90
 
-# Both correctness check types fold into accuracy: eleven cases assert the answer contains
-# the right flag, and c-no-publish asserts it does not recommend publishing. Either way the
-# question is whether the answer was right.
+# These are trace-fragment checks, not semantic accuracy. c-no-publish is judge-only:
+# refusing a nonexistent command must be allowed to mention its name.
 [[metric]]
-name = "Answer accuracy"
+name = "Required-fragment match rate (not semantic accuracy)"
 kind = "check_rate"
-cases = ["c-*"]
-checks = ["command_contains", "trigger_expectation"]
+cases = ["c-tag-pin", "c-branch-pin", "c-group-add", "c-install-only", "c-install-without", "c-serve-port", "c-search-limit", "c-analyze-threshold", "c-marketplace-baseurl", "c-mcp-cursor", "c-repos-priority"]
+checks = ["command_contains"]
 min_rate = 0.80
 
 [[metric]]
diff --git a/evals/v2/negctl.py b/evals/v2/negctl.py
new file mode 100644
index 0000000..e8c2ff2
--- /dev/null
+++ b/evals/v2/negctl.py
@@ -0,0 +1,101 @@
+"""Consultation-only negative control; preserve recorded metadata per trial."""
+import argparse
+import copy
+import json
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+
+
+def remove_consultation(events, skill_path):
+    removed = set()
+    for event in events:
+        payload = event.get("payload", {})
+        if payload.get("type") != "tool_use":
+            continue
+        inputs = payload.get("input", {})
+        named = (payload.get("tool_name", "").lower() == "skill" and
+                 isinstance(inputs, dict) and any(inputs.get(key) == "fastskill"
+                                                 for key in ("skill", "name", "skill_name")))
+        referenced = bool(skill_path and skill_path in json.dumps(inputs))
+        if named or referenced:
+            removed.add(payload.get("call_id"))
+    if not removed:
+        raise ValueError("no consultation tool call found; choose a trial that consulted the skill")
+    return [event for event in events if not (
+        event.get("payload", {}).get("type") in ("tool_use", "tool_result") and
+        event["payload"].get("call_id") in removed)]
+
+
+def verdicts(summary):
+    result = []
+    for trial in summary["cases"][0]["trials"]:
+        if trial.get("error_message") or trial.get("status") == "error":
+            raise ValueError("execution error is not a consultation verdict")
+        checks = {item["check_name"]: item for item in trial["check_results"]}
+        if any(checks[name].get("not_observable") for name in ("skill_invoked", "max_tool_calls")):
+            raise ValueError("unobservable consultation or budget")
+        result.append((trial["trial_id"], checks["skill_invoked"]["passed"],
+                       checks["max_tool_calls"]["passed"]))
+    if not result:
+        raise ValueError("no trials")
+    return result
+
+
+def check_flip(present, absent):
+    before, after = verdicts(present), verdicts(absent)
+    if len(before) != len(after) or not all(
+            p[0] == a[0] and p[1] and not a[1] and p[2] == a[2]
+            for p, a in zip(before, after)):
+        raise ValueError("consultation must flip on every trial; budget verdict must remain unchanged")
+
+
+def run(run_dir, case_id=None):
+    summary = json.loads((run_dir / "summary.json").read_text())
+    candidates = [c for c in summary["cases"] if c["id"] == case_id] if case_id else [
+        c for c in summary["cases"] if c["id"].startswith("op-")]
+    if not candidates:
+        raise ValueError("no matching consultation case")
+    case = candidates[0]
+    if Path(case["id"]).name != case["id"] or case["id"] in (".", ".."):
+        raise ValueError("invalid case id")
+    project = Path(summary["skill_project_root"]).resolve()
+    checks = Path(summary["checks_path"])
+    if not checks.is_absolute():
+        checks = project / checks
+    outputs = []
+    with tempfile.TemporaryDirectory(prefix="fastskill-negctl-") as temporary:
+        for variant in ("present", "absent"):
+            dest = Path(temporary) / variant
+            shutil.copytree(run_dir / case["id"], dest / case["id"])
+            selected = copy.deepcopy(summary)
+            selected.update(cases=[copy.deepcopy(case)], total_cases=1, run_dir=str(dest),
+                            checks_path=str(checks), skill_project_root=str(project))
+            (dest / "summary.json").write_text(json.dumps(selected))
+            if variant == "absent":
+                for trace in (dest / case["id"]).glob("trial-*/trace.jsonl"):
+                    trial = json.loads((trace.parent / "result.json").read_text())
+                    events = [json.loads(line) for line in trace.read_text().splitlines()]
+                    kept = remove_consultation(events, trial.get("skill_path"))
+                    trace.write_text("\n".join(json.dumps(e) for e in kept) + "\n")
+            scored = subprocess.run(["fastskill", "eval", "score", "--run-dir", str(dest),
+                                     "--no-fail", "--json"], capture_output=True, text=True, check=True)
+            outputs.append(json.loads(scored.stdout))
+        check_flip(*outputs)
+    print("PASS: consultation flips on every trial; tool-budget verdict unchanged; originals preserved")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("run_dir", type=Path)
+    parser.add_argument("case", nargs="?")
+    args = parser.parse_args()
+    try:
+        run(args.run_dir.resolve(), args.case)
+    except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error:
+        parser.exit(1, str(error) + "\n")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/evals/v2/negctl.sh b/evals/v2/negctl.sh
index 3effd2d..a3b16ec 100755
--- a/evals/v2/negctl.sh
+++ b/evals/v2/negctl.sh
@@ -1,119 +1,5 @@
 #!/usr/bin/env bash
-# Negative control for the consultation check (spec 001 §8 step 3).
-#
-# Takes a real completed consultation case, makes two copies, and deletes from one of them
-# every trace line naming the staged skill path -- i.e. simulates an agent that answered
-# without ever opening the skill. Re-scores both with the engine's own scorer. The check is
-# load-bearing only if PRESENT passes and ABSENT fails.
-#
-# `skill_invoked` matches the skill document's path against the input of any tool use (R8),
-# and the path it matches is the one the run recorded per trial. So the mutation pattern is
-# read from the artifacts, not from patterns.json: the control has to delete the evidence
-# the check actually reads, not a string that resembles it.
-#
-# Usage: evals/v2/negctl.sh <consultation-run-dir> [case-id]
-#   <consultation-run-dir> is the .../<timestamp>/<agent> directory of a real run.
+# Re-score copies of real evidence; no model calls, original artifacts unchanged.
 set -euo pipefail
-
-RUN_DIR=${1:?usage: negctl.sh <consultation-run-dir> [case-id]}
-CASE=${2:-}
-ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
-
-if [ -z "$CASE" ]; then
-  CASE=$(find "$RUN_DIR" -mindepth 1 -maxdepth 1 -type d -name 'op-*' -printf '%f\n' | sort | head -1)
-fi
-SRC="$RUN_DIR/$CASE"
-[ -d "$SRC" ] || { echo "no such case dir: $SRC" >&2; exit 1; }
-
-WORK=$(mktemp -d)
-trap 'rm -rf "$WORK"' EXIT
-
-PATTERN=$(python3 - "$SRC" <<'PY'
-import json, pathlib, sys
-src = pathlib.Path(sys.argv[1])
-paths = set()
-for r in sorted(src.glob("trial-*/result.json")):
-    p = json.loads(r.read_text()).get("skill_path")
-    if p:
-        paths.add(p)
-if not paths:
-    sys.exit(
-        "this run recorded no skill_path, so there is nothing for skill_invoked to match "
-        "and the control cannot prove anything. Re-run the suite with a build that records "
-        "it (R5) and try again."
-    )
-if len(paths) > 1:
-    sys.exit(f"trials disagree on the staged skill path, refusing to guess: {sorted(paths)}")
-print(paths.pop())
-PY
-)
-
-python3 - "$SRC" "$WORK" "$CASE" "$PATTERN" "$ROOT" <<'PY'
-import json, pathlib, shutil, sys
-src, work, case, pattern, root = sys.argv[1:6]
-iso = json.load(open(pathlib.Path(root, "evals/fixtures/pass/summary.json")))["isolation"]
-n_trials = len(list(pathlib.Path(src).glob("trial-*")))
-for variant in ("present", "absent"):
-    dst = pathlib.Path(work, variant, case)
-    shutil.copytree(src, dst)
-    if variant == "absent":
-        for t in sorted(dst.glob("trial-*/trace.jsonl")):
-            lines = t.read_text().splitlines()
-            kept = [l for l in lines if pattern not in l]
-            removed = len(lines) - len(kept)
-            print(f"  {variant}/{t.parent.name}: removed {removed} line(s)")
-            if removed == 0:
-                sys.exit(
-                    f"{t}: no trace line names the skill path, so the 'absent' variant is "
-                    "identical to 'present' and the control would pass without testing "
-                    "anything. Pick a case whose trials did consult the skill."
-                )
-            t.write_text("\n".join(kept) + "\n")
-    trials = [{"trial_id": i, "status": "passed", "command_count": None,
-               "input_tokens": None, "output_tokens": None,
-               "check_results": [], "error_message": None}
-              for i in range(1, n_trials + 1)]
-    # should_trigger drives the generated skill_invoked check (R7); without it the
-    # re-score falls back to the explicit checks alone and measures nothing here.
-    pathlib.Path(work, variant, "summary.json").write_text(json.dumps({
-        "suite_pass": True, "suite_pass_rate": 1.0, "agent": "pi", "model": "",
-        "total_cases": 1, "passed": 1, "failed": 0,
-        "trials_per_case": n_trials, "parallel": n_trials, "pass_threshold": 1.0,
-        "run_dir": str(pathlib.Path(work, variant)),
-        "checks_path": "evals/v2/consultation/checks.toml",
-        "skill_project_root": ".", "isolation": iso,
-        "cases": [{"id": case, "status": "passed", "command_count": 0,
-                   "input_tokens": 0, "output_tokens": 0, "pass_count": n_trials,
-                   "total_trials": n_trials, "pass_rate": 1.0,
-                   "should_trigger": True, "trials": trials}],
-    }, indent=2))
-PY
-
-echo
-echo "case: $CASE"
-echo "path: $PATTERN"
-rc=0
-for v in present absent; do
-  out=$(cd "$ROOT" && fastskill eval score --run-dir "$WORK/$v" --no-fail --json 2>&1)
-  line=$(printf '%s' "$out" | python3 -c "
-import json,sys
-d=json.loads(sys.stdin.read()); c=d['cases'][0]
-chk=[(t['check_name'],t['passed']) for t in c['trials'][0]['check_results']]
-print('%s %.2f %s' % (c['status'], c['pass_rate'], chk))
-")
-  printf '  %-8s %s\n' "$v" "$line"
-  case "$v:$line" in
-    present:passed*) ;;
-    absent:failed*) ;;
-    *) rc=1 ;;
-  esac
-done
-
-echo
-if [ "$rc" -eq 0 ]; then
-  echo "PASS: the consultation check discriminates -- it passes on a real"
-  echo "      consultation trace and fails when the skill read is removed."
-else
-  echo "FAIL: the check did not flip. It is not measuring skill consultation."
-fi
-exit "$rc"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+exec python3 "$HERE/negctl.py" "$@"
diff --git a/evals/v2/patterns.json b/evals/v2/patterns.json
index 2cb35a0..1c3925a 100644
--- a/evals/v2/patterns.json
+++ b/evals/v2/patterns.json
@@ -249,9 +249,7 @@
       "prompt": "Publish my finished skill to the public fastskill registry so other people can install it by name.",
       "expected": "There is no command. FastSkill ships no publish, auth or package command, so any `fastskill publish ...` is invented. The correct answer says so and points at the distribution paths that exist: git or zip URL, a marketplace catalog built with `fastskill marketplace create`, or handing the artifact to the registry operator.",
       "require": [],
-      "forbid": [
-        "fastskill publish"
-      ]
+      "forbid": []
     }
   ],
   "judge": {
@@ -275,7 +273,7 @@
       {
         "name": "would_run",
         "kind": "bool",
-        "description": "Every subcommand, flag and argument in the answer exists in fastskill and is spelled correctly, so the command as written would execute rather than be rejected as an unknown argument. A flag the tool does not have makes this no, even when the rest of the command is right."
+        "description": "Every recommended subcommand, flag and argument exists in fastskill and is spelled correctly. A nonexistent flag makes this no. For an unsupported operation, an explicit refusal with a supported distribution alternative is valid; mentioning a nonexistent command to explain that it does not exist is not recommending it. This is a reference-based judgment, not execution proof."
       },
       {
         "name": "no_unrequested_side_effects",
diff --git a/evals/v2/run.sh b/evals/v2/run.sh
index 2eb3c95..b1c6d59 100755
--- a/evals/v2/run.sh
+++ b/evals/v2/run.sh
@@ -68,6 +68,8 @@ for suite in "${suites[@]}"; do
   if grep -q '^\[\[judge\]\]' "$HERE/$suite/checks.toml"; then judged+=("$suite"); fi
 done
 judge_model=()
+target_model=()
+if [[ -n "${TARGET_MODEL:-}" ]]; then target_model=(--model "$TARGET_MODEL"); fi
 if ((${#judged[@]})); then
   : "${AIKIT_LLM_URL:?${judged[*]}: a judge needs an endpoint — set AIKIT_LLM_URL}"
   : "${JUDGE_API_KEY:?${judged[*]}: a judge needs a key — set JUDGE_API_KEY}"
@@ -91,6 +93,7 @@ for suite in "${suites[@]}"; do
   # The gates live in metrics.toml and are applied once, by scorecard.sh, over all three.
   if (cd "$skill_dir" && fastskill eval run \
         --agent "$AGENT" \
+        "${target_model[@]}" \
         --output-dir "$OUT/$suite" \
         "${judge[@]}" \
         --no-fail >"$OUT/$suite.log" 2>&1); then
diff --git a/evals/v2/smoke/checks.toml b/evals/v2/smoke/checks.toml
new file mode 100644
index 0000000..690645f
--- /dev/null
+++ b/evals/v2/smoke/checks.toml
@@ -0,0 +1,21 @@
+# GENERATED by build.py from patterns.json - do not edit by hand.
+#
+# Skill invocation is not listed here. Each case's `should_trigger` column
+# generates a `skill_invoked` check with matching polarity (R7).
+
+[[check]]
+name = "max_tool_calls"
+limit = 25
+required = true
+
+[[check]]
+name = "command_contains"
+pattern = "--tag v2.1.0"
+required = true
+cases = ["c-tag-pin"]
+
+[[check]]
+name = "command_contains"
+pattern = "--port 9123"
+required = true
+cases = ["c-serve-port"]
diff --git a/evals/v2/smoke/prompts.csv b/evals/v2/smoke/prompts.csv
new file mode 100644
index 0000000..28d3bab
--- /dev/null
+++ b/evals/v2/smoke/prompts.csv
@@ -0,0 +1,7 @@
+id,prompt,should_trigger,tags,workspace_subdir
+op-init,I'm starting a new skill project. How do I scaffold the config file?,true,smoke,
+op-eval,How do I run quality evaluations on a skill before shipping it?,true,smoke,
+off-python-list,How do I remove duplicates from a Python list while preserving order?,false,smoke,
+off-npm,How do I pin an exact dependency version in package.json with npm?,false,smoke,
+c-tag-pin,"Install the skill at https://github.com/acme/widgets.git, pinned to the release tagged v2.1.0. Give me the exact command.",true,smoke,
+c-serve-port,Start the fastskill server on port 9123 and allow state-changing endpoints. Give me the exact command.,true,smoke,
diff --git a/evals/v2/stage.sh b/evals/v2/stage.sh
index 207b318..e1fad3b 100755
--- a/evals/v2/stage.sh
+++ b/evals/v2/stage.sh
@@ -20,7 +20,7 @@ TRIALS="${3:-5}"
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SKILL_SRC="$(cd "$HERE/../.." && pwd)/fastskill"
 
-case "$SUITE" in consultation|restraint|correctness) ;; *)
+case "$SUITE" in consultation|restraint|correctness|smoke) ;; *)
   echo "stage.sh: no suite '$SUITE' at $HERE/$SUITE" >&2
   exit 1
 esac
diff --git a/scripts/test_authoring_workflows.py b/scripts/test_authoring_workflows.py
new file mode 100644
index 0000000..0f2b057
--- /dev/null
+++ b/scripts/test_authoring_workflows.py
@@ -0,0 +1,93 @@
+"""Test the acceptance harness with deterministic edits, not live-agent claims."""
+import importlib.util
+import json
+from pathlib import Path
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+import contextlib
+import io
+
+ROOT = Path(__file__).resolve().parent.parent
+spec = importlib.util.spec_from_file_location("workflow", ROOT / "evals/authoring/workflow.py")
+workflow = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(workflow)
+
+
+class WorkflowTests(unittest.TestCase):
+    def test_cli_prepare_verify_and_error(self):
+        with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stdout(io.StringIO()):
+            root = Path(directory) / "task"
+            with patch("sys.argv", ["workflow.py", "prepare", "repair", str(root)]):
+                workflow.main()
+            (root / "target/evals/checks.toml").write_text("")
+            with patch("sys.argv", ["workflow.py", "verify", str(root)]):
+                workflow.main()
+            with patch("sys.argv", ["workflow.py", "prepare", "repair", str(root)]), \
+                    contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as caught:
+                workflow.main()
+            self.assertEqual(caught.exception.code, 1)
+
+    def test_missing_notes_and_negative_are_not_accepted(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory) / "task"
+            workflow.prepare("extend", root)
+            prompts = root / "target/evals/prompts.csv"
+            original = prompts.read_text()
+            prompts.write_text(original + "extra,Another release request,true,positive\n")
+            with self.assertRaisesRegex(ValueError, "negative-trigger"):
+                workflow.verify(root)
+            prompts.write_text(original + "negative,Explain Python lists,false,negative\n")
+            with self.assertRaisesRegex(ValueError, "review notes"):
+                workflow.verify(root)
+
+    def test_preparation_refuses_overwrite(self):
+        with tempfile.TemporaryDirectory() as directory:
+            with self.assertRaises(FileExistsError):
+                workflow.prepare("extend", Path(directory))
+            root = Path(directory)
+            link = root / "dangling"
+            link.symlink_to(root / "must-not-be-created", target_is_directory=True)
+            with patch("sys.argv", ["workflow.py", "prepare", "extend", str(link)]), \
+                    contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit):
+                workflow.main()
+            self.assertFalse((root / "must-not-be-created").exists())
+            self.assertTrue(link.is_symlink())
+
+    def test_create_repair_extend_and_preservation_failures(self):
+        for scenario in workflow.SCENARIOS:
+            with self.subTest(scenario=scenario), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory) / "task"
+                workflow.prepare(scenario, root)
+                target = root / "target"
+                with self.assertRaises((ValueError, FileNotFoundError)):
+                    workflow.verify(root)
+                if scenario == "create":
+                    example = ROOT / "fastskill/examples/invoice-extraction"
+                    shutil.copytree(example / "evals", target / "evals")
+                    shutil.copy2(example / "skill-project.toml", target / "skill-project.toml")
+                elif scenario == "repair":
+                    (target / "evals/checks.toml").write_text("")
+                else:
+                    prompts = target / "evals/prompts.csv"
+                    prompts.write_text(prompts.read_text() + "negative,Explain Python lists,false,negative\n")
+                (target / "evals/review-notes.md").write_text("Deterministic harness fixture; not a live authoring run.")
+                result = workflow.verify(root)
+                self.assertEqual(result["preservation"], "passed")
+                self.assertEqual(result["target_pilot"], "not run")
+                skill = target / "SKILL.md"
+                original = skill.read_bytes()
+                skill.write_text("unintended change")
+                with self.assertRaisesRegex(ValueError, "protected file"):
+                    workflow.verify(root)
+                skill.write_bytes(original)
+                if scenario != "create":
+                    prompts = target / "evals/prompts.csv"
+                    prompts.write_text(prompts.read_text().replace("invoice-basic" if scenario == "repair" else "release-basic", "changed-id"))
+                    with self.assertRaises(ValueError):
+                        workflow.verify(root)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/scripts/test_eval_generation.py b/scripts/test_eval_generation.py
new file mode 100644
index 0000000..be36b7d
--- /dev/null
+++ b/scripts/test_eval_generation.py
@@ -0,0 +1,39 @@
+"""Generator idempotence, TOML escaping and calibration reference integrity."""
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import tomllib
+import unittest
+
+ROOT = Path(__file__).resolve().parent.parent
+spec = importlib.util.spec_from_file_location("build", ROOT / "evals/v2/build.py")
+build = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(build)
+
+
+class GenerationTests(unittest.TestCase):
+    def test_regeneration_is_byte_identical(self):
+        paths = [ROOT / "evals/v2" / suite / name
+                 for suite in ("consultation", "restraint", "correctness", "smoke")
+                 for name in ("prompts.csv", "checks.toml")]
+        before = {p: p.read_bytes() for p in paths}
+        build.main()
+        self.assertTrue(all(p.read_bytes() == value for p, value in before.items()))
+
+    def test_escaping_and_missing_judge_prompt(self):
+        text = 'quotes " newline\n path\\file'
+        self.assertEqual(tomllib.loads('value = ' + build.toml_str(text))["value"], text)
+        check = tomllib.loads(build.must_not_say(text, "case"))["check"][0]
+        self.assertEqual(check["pattern"], text)
+        with tempfile.TemporaryDirectory() as directory, self.assertRaises(SystemExit):
+            build.judge({"name": "missing", "prompt_file": "absent.md"}, Path(directory))
+
+    def test_calibration_has_independent_positive_negative_pairs(self):
+        cases = json.loads((ROOT / "evals/v2/patterns.json").read_text())["correctness"]
+        known = {c["id"] for c in cases}
+        examples = json.loads((ROOT / "evals/v2/calibration.json").read_text())["examples"]
+        for case_id in {e["case"] for e in examples}:
+            self.assertIn(case_id, known)
+            self.assertEqual({e["acceptable"] for e in examples if e["case"] == case_id}, {True, False})
+        self.assertTrue(all(e["reason"] and e["answer"] for e in examples))
diff --git a/scripts/test_eval_negctl.py b/scripts/test_eval_negctl.py
new file mode 100644
index 0000000..6bb6665
--- /dev/null
+++ b/scripts/test_eval_negctl.py
@@ -0,0 +1,84 @@
+"""Controlled trace mutation, including independent per-trial skill paths."""
+import copy
+import importlib.util
+import json
+from pathlib import Path
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+import contextlib
+import io
+
+ROOT = Path(__file__).resolve().parent.parent
+spec = importlib.util.spec_from_file_location("negctl", ROOT / "evals/v2/negctl.py")
+negctl = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(negctl)
+
+
+class NegativeControlTests(unittest.TestCase):
+    def test_cli_reports_missing_run_as_error(self):
+        with tempfile.TemporaryDirectory() as directory, \
+                patch("sys.argv", ["negctl.py", directory]), \
+                contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as caught:
+            negctl.main()
+        self.assertEqual(caught.exception.code, 1)
+
+    def test_real_scorer_with_two_distinct_trial_paths_preserves_originals(self):
+        with tempfile.TemporaryDirectory() as directory:
+            run = Path(directory) / "run"
+            shutil.copytree(ROOT / "evals/fixtures/pass", run)
+            (run / "eval-skill").rename(run / "op-example")
+            summary = json.loads((run / "summary.json").read_text())
+            case = summary["cases"][0]
+            case.update(id="op-example", should_trigger=True, total_trials=2)
+            case["trials"].append(dict(case["trials"][0], trial_id=2))
+            summary.update(checks_path=str(ROOT / "evals/v2/consultation/checks.toml"),
+                           skill_project_root=str(ROOT), trials_per_case=2, model="preserved-model")
+            (run / "summary.json").write_text(json.dumps(summary))
+            shutil.copytree(run / "op-example/trial-1", run / "op-example/trial-2")
+            for number in (1, 2):
+                trial = run / f"op-example/trial-{number}"
+                path = f"/tmp/independent-{number}/SKILL.md"
+                result = json.loads((trial / "result.json").read_text())
+                result.update(trial_id=number, skill_path=path)
+                (trial / "result.json").write_text(json.dumps(result))
+                events = [json.loads(line) for line in (trial / "trace.jsonl").read_text().splitlines()]
+                events[1]["payload"].update(tool_name="Read", input={"file_path": path})
+                (trial / "trace.jsonl").write_text("\n".join(json.dumps(e) for e in events) + "\n")
+            originals = {p: p.read_bytes() for p in run.rglob("*") if p.is_file()}
+            with patch("sys.argv", ["negctl.py", str(run)]):
+                negctl.main()
+            self.assertTrue(all(p.read_bytes() == data for p, data in originals.items()))
+            with self.assertRaisesRegex(ValueError, "no matching"):
+                negctl.run(run, "unknown")
+
+    def test_removes_only_consultation_calls_not_answer_mentions(self):
+        events = [
+            {"payload": {"type": "tool_use", "call_id": "read", "tool_name": "Skill", "input": {"skill": "fastskill"}}},
+            {"payload": {"type": "tool_result", "call_id": "read", "output": "content"}},
+            {"payload": {"type": "message", "text": "/tmp/skill/SKILL.md"}},
+        ]
+        self.assertEqual(negctl.remove_consultation(events, None), events[2:])
+        with self.assertRaises(ValueError):
+            negctl.remove_consultation(events[2:], "/tmp/skill/SKILL.md")
+
+    def test_requires_check_specific_flip_and_unchanged_budget(self):
+        def summary(invoked, budget=True, **extra):
+            return {"cases": [{"trials": [dict(trial_id=1, check_results=[
+                {"check_name": "skill_invoked", "passed": invoked},
+                {"check_name": "max_tool_calls", "passed": budget}], **extra)]}]}
+        negctl.check_flip(summary(True), summary(False))
+        for absent in (summary(True), summary(False, False), summary(False, status="error")):
+            with self.assertRaises(ValueError):
+                negctl.check_flip(summary(True), absent)
+        data = summary(False)
+        data["cases"][0]["trials"][0]["check_results"][0]["not_observable"] = True
+        with self.assertRaises(ValueError):
+            negctl.verdicts(data)
+        with self.assertRaises(ValueError):
+            negctl.verdicts({"cases": [{"trials": []}]})
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/scripts/test_eval_scoring.py b/scripts/test_eval_scoring.py
new file mode 100644
index 0000000..e1ab247
--- /dev/null
+++ b/scripts/test_eval_scoring.py
@@ -0,0 +1,95 @@
+"""Exercise real scorer verdicts; synthetic traces are not model-run evidence."""
+import copy
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parent.parent
+CLI = os.environ.get("FASTSKILL_BIN", "fastskill")
+
+
+def score_fixture(source, destination, case_id=None, answer=None, tool_output=None):
+    shutil.copytree(source, destination)
+    summary = json.loads((destination / "summary.json").read_text())
+    old_id = summary["cases"][0]["id"]
+    if case_id:
+        (destination / old_id).rename(destination / case_id)
+        summary["cases"][0].update(id=case_id, should_trigger=True)
+        summary["checks_path"] = str(ROOT / "evals/v2/correctness/checks.toml")
+    else:
+        summary["checks_path"] = str(ROOT / "evals/checks.toml")
+    summary.update(run_dir=str(destination), skill_project_root=str(ROOT))
+    (destination / "summary.json").write_text(json.dumps(summary))
+    if answer is not None:
+        path = destination / (case_id or old_id) / "trial-1/trace.jsonl"
+        events = [json.loads(line) for line in path.read_text().splitlines()]
+        events[-1]["payload"]["text"] = answer
+        if tool_output is not None:
+            events[2]["payload"]["output"] = tool_output
+        path.write_text("\n".join(json.dumps(event) for event in events) + "\n")
+    result = subprocess.run([CLI, "eval", "score", "--run-dir", str(destination), "--json"],
+                            cwd=ROOT, capture_output=True, text=True)
+    return result, json.loads(result.stdout)
+
+
+class ScoringTests(unittest.TestCase):
+    def test_fixtures_fail_only_for_intended_check(self):
+        for fixture, expected in (("pass", True), ("fail", False)):
+            with self.subTest(fixture=fixture), tempfile.TemporaryDirectory() as directory:
+                result, data = score_fixture(ROOT / "evals/fixtures" / fixture,
+                                             Path(directory) / "run")
+                self.assertEqual(result.returncode, 0 if expected else 2, result.stderr)
+                self.assertEqual(data["suite_pass"], expected)
+                self.assertEqual(len(data["cases"]), 1)
+                trial = data["cases"][0]["trials"][0]
+                self.assertIsNone(trial.get("error_message"))
+                checks = {c["check_name"]: c for c in trial["check_results"]}
+                self.assertEqual(checks["skill_invoked"]["passed"], expected)
+                self.assertTrue(checks["max_tool_calls"]["passed"])
+                self.assertFalse(any(c.get("not_observable") for c in checks.values()))
+
+    def test_correct_refusal_is_not_rejected_by_literal_mention(self):
+        with tempfile.TemporaryDirectory() as directory:
+            result, data = score_fixture(ROOT / "evals/fixtures/pass", Path(directory) / "run",
+                "c-no-publish", "There is no fastskill publish command. Use Git or a ZIP URL instead.")
+            self.assertEqual(result.returncode, 0, result.stderr)
+            checks = data["cases"][0]["trials"][0]["check_results"]
+            self.assertNotIn("trigger_expectation", [c["check_name"] for c in checks])
+            # No semantic verdict is inferred: this case needs the optional judge.
+
+    def test_trace_fragment_limitations_remain_explicit(self):
+        examples = [
+            ("c-tag-pin", "fastskill skill remove widgets --tag v2.1.0", None),
+            ("c-serve-port", "fastskill server serve --port 9123", None),
+            ("c-tag-pin", "I do not know.", "--tag v2.1.0"),
+        ]
+        for case, answer, output in examples:
+            with self.subTest(answer=answer), tempfile.TemporaryDirectory() as directory:
+                result, data = score_fixture(ROOT / "evals/fixtures/pass", Path(directory) / "run",
+                                             case, answer, output)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertTrue(data["suite_pass"])
+        # These false semantic positives must never be relabelled answer accuracy.
+        import tomllib
+        metrics = tomllib.loads((ROOT / "evals/v2/metrics.toml").read_text())
+        metric = next(m for m in metrics["metric"] if "command_contains" in m.get("checks", []))
+        self.assertIn("not semantic accuracy", metric["name"])
+        self.assertNotIn("c-no-publish", metric["cases"])
+
+    def test_missing_fixture_is_not_a_quality_failure(self):
+        with tempfile.TemporaryDirectory() as directory:
+            result = subprocess.run([CLI, "eval", "score", "--run-dir", directory, "--json"],
+                                    capture_output=True, text=True)
+            self.assertNotEqual(result.returncode, 0)
+            data = json.loads(result.stdout)
+            self.assertFalse(data["success"])
+            self.assertIn("error", data)
+            self.assertNotIn("cases", data)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/scripts/test_eval_scripts.py b/scripts/test_eval_scripts.py
index 5767492..9803f02 100644
--- a/scripts/test_eval_scripts.py
+++ b/scripts/test_eval_scripts.py
@@ -15,7 +15,7 @@ def shell(script, *args):
         output = tempfile.mkdtemp(prefix="shell-", dir=os.environ["KCOV_OUTPUT"])
         return [coverage, "--bash-method=DEBUG", "--configure=bash-use-basic-parser=1",
                 "--exclude-region=import sys:PY",
-                "--include-pattern=/evals/v2/", output, str(script), *args]
+                "--include-pattern=/evals/", output, str(script), *args]
     return ["bash", str(script), *args]
 
 
@@ -94,7 +94,7 @@ def test_sweeps_preserve_stages_and_propagate_runner_and_scorecard_verdicts(self
             executable.chmod(0o755)
             env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ["PATH"],
                        AIKIT_LLM_URL="http://unused.invalid/v1", JUDGE_API_KEY="test-only",
-                       JUDGE_MODEL="test-only")
+                       JUDGE_MODEL="test-only", TARGET_MODEL="test-target")
             for index, (suites, run_exit, score_exit) in enumerate([
                     (["consultation"], "0", "0"), ([], "0", "0"),
                     ([], "23", "0"), ([], "0", "9")]):
diff --git a/scripts/test_eval_smoke.py b/scripts/test_eval_smoke.py
new file mode 100644
index 0000000..e6d692c
--- /dev/null
+++ b/scripts/test_eval_smoke.py
@@ -0,0 +1,51 @@
+"""No model calls: real validation plus a recording runtime stub."""
+import csv
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from scripts.test_eval_scripts import shell
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+class SmokeTests(unittest.TestCase):
+    def test_six_cases_validate_without_a_judge(self):
+        with tempfile.TemporaryDirectory() as directory:
+            dest = Path(directory) / "project"
+            subprocess.run(["bash", str(ROOT / "evals/v2/stage.sh"), "smoke", str(dest), "1"], check=True)
+            result = subprocess.run([os.environ.get("FASTSKILL_BIN", "fastskill"), "eval", "validate", "--json"],
+                                    cwd=dest / "fastskill", capture_output=True, text=True, check=True)
+            data = json.loads(result.stdout)
+            self.assertEqual(data["case_count"], 6)
+            self.assertEqual(data["judges"], [])
+            self.assertEqual(data["trials_per_case"], 1)
+            with (dest / "evals/prompts.csv").open() as handle:
+                rows = list(csv.DictReader(handle))
+            self.assertEqual(sum(row["should_trigger"] == "false" for row in rows), 2)
+
+    def test_runner_forwards_model_and_failure_and_refuses_reuse(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            stub = root / "fastskill"
+            stub.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$CALL_LOG"\n'
+                            'if [ "$2" = run ]; then exit 7; fi\n')
+            stub.chmod(0o755)
+            env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ["PATH"],
+                       CALL_LOG=str(root / "calls"))
+            command = shell(ROOT / "evals/smoke.sh", "claude", str(root / "out"), "test-model")
+            result = subprocess.run(command, env=env, capture_output=True, text=True)
+            self.assertEqual(result.returncode, 7, result.stdout + result.stderr)
+            calls = (root / "calls").read_text()
+            self.assertIn("--model test-model", calls)
+            self.assertNotIn("--judge", calls)
+            self.assertNotIn("--no-fail", calls)
+            result = subprocess.run(command, env=env, capture_output=True, text=True)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertEqual((root / "calls").read_text(), calls)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/specs/003-simple-eval-enhancements.md b/specs/003-simple-eval-enhancements.md
new file mode 100644
index 0000000..815926d
--- /dev/null
+++ b/specs/003-simple-eval-enhancements.md
@@ -0,0 +1,72 @@
+# Lightweight eval enhancements and verification
+
+Based on main `cff04156f449199a9fe43ad93e2d13bc9f8af1e2`, 2026-09-29.
+Implementation is restricted to the skills repository; no CLI/engine feature was added.
+
+## Decisions
+
+- Offer six cases / one trial / no judge as the quick path; keep the benchmark opt-in.
+- Name whole-trace substring matching honestly. It can match an incorrect command or
+  tool result; it is not answer accuracy. Regression fixtures preserve these contrasts.
+- Remove the incorrect no-publish literal prohibition. Correctness for that case is
+  judge-only, and advisory until calibrated. A deterministic pass is not semantic approval.
+- Provide labelled judge contrasts, without pretending a judge calibration was executed.
+- Offer only the configured Claude gateway route in hosted CI. Local scripts still accept
+  other supported/authenticated runtimes. Forward the target model explicitly.
+- CI verifies the intended failure check, not merely nonzero process status.
+- Preserve original metadata and per-trial paths in the consultation negative control.
+- Provide create/repair/extend tasks for a chosen authoring agent, with deterministic
+  outcome/preservation verification. Do not add another model orchestrator.
+
+## Verification on server02
+
+FastSkill 0.9.235; 29 Python tests passed. The six-case smoke validates without a judge;
+instrumented Bash tests also pass. Namespace validation and the payload guard pass.
+Generated suite files are tested byte-identical after regeneration. Real offline scoring
+checks both original fixtures and synthetic counterexamples. The negative control was
+tested through the scorer with distinct paths on two trials and unchanged source bytes.
+
+coverage.py branch instrumentation for changed Python source:
+
+| File | Statements | Branches |
+| --- | --- | --- |
+| `evals/authoring/workflow.py` | 81/84 (96.43%) | 31/34 (91.18%) |
+| `evals/v2/build.py` | 69/71 (97.18%) | 16/18 (88.89%) |
+| `evals/v2/negctl.py` | 75/78 (96.15%) | 28/32 (87.50%) |
+
+kcov Bash line coverage: `run.sh` 56/61 (91.80%), `stage.sh` 19/19 (100%),
+`smoke.sh` 15/15 (100%). The runner's five continued argument lines are not attributed.
+Bash branch instrumentation is unavailable. Staging's unchanged embedded Python is
+excluded from the Bash count; its earlier independent coverage is in spec 002.
+
+Raw evidence is outside the checkout in the sibling
+`fastskill-eval-suite-enhancements-evidence/` directory.
+
+## Changed-file coverage inventory
+
+| File(s) | Evidence |
+| --- | --- |
+| `.github/workflows/skill-evals.yml` | Same unittest commands executed; negative fixture requires expected check; hosted execution pending |
+| `README.md`, `CONTRIBUTING.md`, `evals/README.md`, `evals/v2/README.md` | Commands matched to executed smoke staging/validation and helper tests; limitations explicit |
+| `evals/checks.toml` | Real positive/negative fixture rescoring; comment-only change |
+| `evals/v2/patterns.json`, `evals/v2/correctness/checks.toml` | Regeneration equality; real scorer accepts correct refusal; suite validation |
+| `evals/v2/metrics.toml` | Parsed metric selects fragment cases, excludes judge-only refusal; semantic label regression assertion |
+| `evals/v2/calibration.json` | Known case IDs, labelled positive/negative pairs checked; no judge performance claim |
+| `evals/v2/build.py` | In-process regeneration, TOML escaping, missing-prompt failure; coverage above |
+| `evals/v2/smoke/prompts.csv`, `evals/v2/smoke/checks.toml` | Real validation: six cases, two negatives, no judge; regeneration equality |
+| `evals/smoke.sh` | Recording runtime stub checks model/trials/error propagation and output preservation; line coverage above |
+| `evals/v2/run.sh`, `evals/v2/stage.sh` | Seven shell/smoke tests including failures and preservation; line coverage above |
+| `evals/v2/negctl.sh`, `evals/v2/negctl.py` | Actual scorer contrast, two per-trial paths, source-byte preservation, specific verdict checks |
+| `evals/authoring/workflow.py` | Create, repair, extend; real validator; deliberately broken files/rows/notes rejected; coverage above |
+| `scripts/test_authoring_workflows.py`, `scripts/test_eval_generation.py`, `scripts/test_eval_negctl.py`, `scripts/test_eval_scoring.py`, `scripts/test_eval_smoke.py`, `scripts/test_eval_scripts.py` | All discovered and executed; deliberate invalid/error inputs detected |
+| `specs/003-simple-eval-enhancements.md` | Reconciled with current recorded results |
+
+## Not established by these checks
+
+No paid target sweep, new live authoring run, or native-judge calibration was performed.
+The workflow helpers' deterministic test edits are not live model evidence. Human review
+is still needed for semantic quality, expectation independence and answer leakage.
+Hosted CI uses moving tool releases and logs versions; fully pinned reproducibility is
+not claimed. The judge remains advisory and reference-based, not proof a command executes.
+Changes to metric membership mean the revised fragment rate is not comparable to the
+historical 12-case accuracy-labelled baseline. No deployment or merge is part of this change.