aroff · Sep 8, 2026 · Sep 8, 2026
diff --git a/fastskill/SKILL.md b/fastskill/SKILL.md
@@ -1,7 +1,7 @@
 ---
 name: fastskill
-version: 1.2.0
-description: Package manager and operational toolkit for Claude Code-compatible skills. Use this skill when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; running skill evaluations (`fastskill eval`) or the optimization loop (`fastskill optimize`); serving skills over HTTP/MCP; or building marketplace catalogs. See references/eval.md for eval setup in full.
+version: 1.3.0
+description: Package manager and operational toolkit for Claude Code-compatible skills. Use this skill when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; running skill evaluations (`fastskill eval validate/run/judge/report/score/scorecard`) or the optimization loop (`fastskill optimize`); serving skills over HTTP/MCP; or building marketplace catalogs. See references/eval.md for eval setup in full.
 license: Apache-2.0
 ---
 
@@ -54,16 +54,26 @@ Semantic search is the only feature that needs the embedding section. Everything
 
 ```toml
 # skill-project.toml  (minimum for reindex + search)
+schema_version = "1"
+
+[dependencies]
+
+[tool.fastskill]
+skills_directory = ".claude/skills"
+
 [tool.fastskill.embedding]
 openai_base_url = "https://api.openai.com/v1"
 embedding_model = "text-embedding-3-small"
 ```
 
-Required environment variable for embeddings: **`OPENAI_API_KEY`**. `skills_directory` is not required for `reindex`/`search` if skills are already installed under the default `.claude/skills/` path.
+Required environment variable for embeddings: **`OPENAI_API_KEY`**. Local search falls back to
+keyword matching when embeddings are not configured.
 
 ### Configuration file structure
 
 ```toml
+schema_version = "1"
+
 [metadata]
 id = "my-skill"
 version = "1.0.0"
@@ -89,6 +99,7 @@ priority = 0
 
 ```
 skill-project.toml
+├── schema_version = "1"    — current on-disk manifest schema; write this in new manifests
 ├── [metadata]              — skill identity (required when authoring/publishing a skill)
 │     id                   — string, unique skill ID (no slashes; scope is a separate concept)
 │     version              — semver string
@@ -131,16 +142,20 @@ Configuration is stored **only** in `skill-project.toml`. FastSkill does not rea
 
 ```bash
 export OPENAI_API_KEY="your-key-here"        # embeddings / semantic search
-export FASTSKILL_API_URL="https://registry.example.com"   # default registry target
 ```
 
-Additional: `REGISTRY_INDEX_PATH` (override index path), `FASTSKILL_NO_PROGRESS` (disable progress bars), `FASTSKILL_AUTH_TOKEN` / `FASTSKILL_TOKEN` (registry auth token, checked by `doctor`). Repository PAT auth reads a per-repo `env_var` (default `PAT_TOKEN`).
+Additional: `REGISTRY_INDEX_PATH` overrides the local index path and `FASTSKILL_NO_PROGRESS`
+disables reindex progress bars. `doctor` recognizes `FASTSKILL_AUTH_TOKEN` or `FASTSKILL_TOKEN`;
+repository authentication reads the `env_var` configured on that repository (default
+`PAT_TOKEN`).
 
 ## Skill evaluations (evals)
 
 The CLI runs **skill evaluations**: prompts from a CSV, optional deterministic **checks** (TOML), agent execution via aikit-sdk, and timestamped artifact directories. Configuration lives in `[tool.fastskill.eval]` inside `skill-project.toml`. Cases run **isolated by default** — a per-case scratch workspace containing only the skill under test (needs `SKILL.md` + `[metadata].id`; opt out with `--no-isolation`) — so trigger rates measure the skill, not the machine.
 
-**You MUST** follow the dedicated guide for CSV and checks schema, CLI commands (`eval validate`, `eval run`, `eval report`, `eval score`), pass/fail rules, and packaging notes:
+**You MUST** follow the dedicated guide for CSV, checks, judges, scorecards, CLI commands
+(`eval validate`, `eval run`, `eval judge`, `eval report`, `eval score`, `eval scorecard`), and
+pass/fail rules:
 
 - **[Skill evals guide](references/eval.md)** — setup, prompts CSV, `checks.toml`, agents, artifacts, CI.
 
@@ -210,6 +225,8 @@ A bundle is a versioned ZIP containing selected skills, their dependency closure
 content digests. Declare every member in both `[bundle.members]` and `[dependencies]`:
 
 ```toml
+schema_version = "1"
+
 [bundle]
 format = "fastskill-bundle-v1"
 id = "platform-team"
@@ -271,11 +288,17 @@ fastskill reindex --force         # force a full re-index
 
 fastskill search "powerpoint presentation"        # remote catalogs (default)
 fastskill search "data processing" --local        # installed skills
-fastskill search "charts" --local --limit 5 --format json
+fastskill search "charts" --local --limit 5 --json
+fastskill search "deploy service" --local --paths --json --content preview
+fastskill read deployment-skill                    # retrieve the full selected skill
 ```
 
 Search needs an embedding provider (`[tool.fastskill.embedding]` + `OPENAI_API_KEY`) for semantic ranking; without it, local search falls back to keyword matching. Add/install/update/remove auto-reindex when `auto_reindex = true` (skipped silently if no embedding provider is configured); use `--no-reindex` to opt out.
 
+For agent workflows, prefer `--local --paths --json --content preview` to discover candidate skill
+files, then `fastskill read <id>` for the selected skill. Use `--content full` only when the caller
+needs all matching documents in one response.
+
 ## Repository management
 
 FastSkill manages skill sources through the `repos` command group. Repositories are stored in `[[tool.fastskill.repositories]]` in `skill-project.toml`.
@@ -412,8 +435,9 @@ All application routes are versioned under **`/api/v1/…`** (requests to `/api/
 ### MCP server (expose FastSkill to your agent)
 
 ```bash
-# Run an MCP server that exposes every CLI command as a tool `fastskill.<path>`
+# Read tools are exposed by default; mutating tools require --enable-write
 fastskill mcp serve --transport stdio
+fastskill mcp serve --transport stdio --enable-write
 fastskill mcp serve --transport http --port 8080 --path /mcp
 
 # Write MCP server config into an agent's config
@@ -423,6 +447,25 @@ fastskill mcp list
 
 `mcp install` supports agents `claude`, `cursor`, `gemini`, `copilot`, `opencode`, and `codex`, at `--scope project` or `global`.
 
+Without `--enable-write`, mutating tools are hidden from `tools/list` and rejected if called.
+
+## Cache and automation utilities
+
+```bash
+fastskill cache info                  # cache path, entries, and disk usage
+fastskill cache info --json
+fastskill cache clean                 # clear every cached source
+fastskill cache clean --source git    # git | registry | local | zip
+fastskill cache clean --json
+
+fastskill completion bash             # bash | zsh | fish | powershell | pwsh
+fastskill spec --format json          # authoritative machine-readable CLI surface
+fastskill spec --format markdown --output fastskill-cli.md
+```
+
+Use `fastskill spec` when an integration needs current command paths, arguments, defaults, or
+examples. Do not maintain a hand-written command schema in an agent integration.
+
 ## Diagnostics
 
 ```bash

diff --git a/fastskill/references/eval.md b/fastskill/references/eval.md
@@ -180,6 +180,7 @@ fastskill eval run --agent claude --output-dir /tmp/evals --case smoke-1
 fastskill eval run --agent codex --output-dir ./evals --tag smoke --model gpt-4o
 fastskill eval run --agent codex --output-dir ./evals --trials 5 --threshold 0.8
 fastskill eval run --all --output-dir ./evals --ci
+fastskill eval run --agent claude --output-dir ./evals --judge
 fastskill eval run --agent codex --output-dir ./evals --json
 ```
 
@@ -189,6 +190,8 @@ fastskill eval run --agent codex --output-dir ./evals --json
 - **`--ci`**: pass/fail the suite on its overall pass rate vs the threshold instead of requiring zero failed cases.
 - **`--no-fail`**: still prints failures but exits `0` (for CI that only collects artifacts).
 - **`--no-isolation`**: run in the project root against the ambient agent environment (see below).
+- **`--judge`**: after deterministic scoring, run every `[[judge]]` declared in the checks file.
+- **`--judge-model <model>`**: override every judge model and record the override.
 
 Non-zero exit on suite failure unless `--no-fail`.
 
@@ -209,10 +212,44 @@ Under each run directory (one per agent):
 - **`<case-id>/trial-<n>/`**: `stdout.txt`, `stderr.txt`, `trace.jsonl`, `result.json` (`TrialResult`, including `check_results`).
 - **`<case-id>/aggregated.json`**: per-case trial aggregation (pass count, pass rate, status).
 - **`workspaces/`**: retained scratch workspaces of failed cases (isolation only).
+- **`<case-id>/trial-<n>/judgments.json`**: full judge exchanges and normalized criterion scores,
+  when the run has been judged.
 
 `eval score` needs `summary.json` to include `checks_path` (written on `eval run` when checks were configured).
 
-## 6. Report and re-score
+## 6. Judge, report, re-score, and build scorecards
+
+Declare judges in the checks file. The prompt or prompt file must render `{{output_contract}}`;
+`fastskill eval validate` checks the declaration before a paid request is made.
+
+```toml
+[judge_defaults]
+model = "gpt-4.1"
+base_url = "https://api.openai.com/v1"
+api_key_env = "JUDGE_API_KEY"
+temperature = 0.0
+
+[[judge]]
+name = "quality"
+prompt_file = "judge-prompt.md"
+
+[[judge.criterion]]
+name = "clarity"
+kind = "scale"
+scale = 5
+description = "The response is direct, precise, and easy to apply."
+```
+
+Judge a completed per-agent run, or pass `--judge` to `eval run`:
+
+```bash
+fastskill eval judge --run-dir ./eval-runs/2026-04-07T12-00-00Z/codex
+fastskill eval judge --run-dir ./eval-runs/latest/codex --judge-model gpt-4.1
+fastskill eval judge --run-dir ./eval-runs/latest/codex --rejudge
+```
+
+Judging caches an identical request unless `--rejudge` is set. A judge error still exits non-zero
+when `--no-fail` is present because the measurement is missing.
 
 ```bash
 fastskill eval report --run-dir ./eval-runs/2026-04-07T12-00-00Z/codex
@@ -229,6 +266,35 @@ fastskill eval score --run-dir ./eval-runs/2026-04-07T12-00-00Z/codex
 
 **Note:** `eval score` applies `file_exists` relative to **`skill_project_root`** from `summary.json` — not per-case `workspace_subdir`, and not the (deleted) scratch workspace the case originally ran in. If you rely on workspace-specific files, prefer re-running `eval run` or align check paths with that root. A run whose `summary.json` contains zero cases refuses to re-score (`EVAL_EMPTY_SUITE`).
 
+Fold multiple run directories into named metrics and threshold gates:
+
+```bash
+fastskill eval scorecard --root ./eval-runs --metrics ./evals/metrics.toml
+fastskill eval scorecard --root ./eval-runs --metrics ./evals/metrics.toml --format html -o report.html
+fastskill eval scorecard --format html --from june.json --from july.json -o trend.html
+```
+
+```toml
+[[metric]]
+name = "Skill-open rate"
+kind = "check_rate"
+checks = ["skill_invoked"]
+min_rate = 0.85
+
+[[metric]]
+name = "Answer quality"
+kind = "judge_score"
+judges = ["quality"]
+min_score = 0.7
+
+suites = ["./suites/consultation", "./suites/restraint"]
+```
+
+The other metric kind is `tool_calls_p95` with `max`. Scorecards reject mixed targets, mixed
+judge identities, and duplicate case IDs unless the matching `--allow-*` option is explicit. Use
+`--format json` for an archival `fastskill.scorecard/1` document or `--format html` for a
+self-contained report.
+
 ## 7. Packaging and CI
 
 - The CLI has no `package`/`publish` command. If you ship the skill by other means (Git, ZIP, a registry pipeline), keep prompts/checks under `evals/` or another path you leave out of the shipped artifact if you treat evals as repo-only.
@@ -239,7 +305,9 @@ fastskill eval score --run-dir ./eval-runs/2026-04-07T12-00-00Z/codex
 1. `[tool.fastskill.eval]` with existing `prompts` CSV path.
 2. `SKILL.md` beside `skill-project.toml` and `[metadata].id` set (required by default-on isolation).
 3. CSV header: `id`, `prompt`, `should_trigger` (+ optional `tags`, `workspace_subdir`).
-4. Optional `checks` TOML with `[[check]]` and correct `name` / fields — use `skill_invoked` (not a bare-name `trigger_expectation`) to assert your skill fired.
+4. Optional checks TOML with `[[check]]`; use `skill_invoked` to assert your skill fired. Add
+   `[[judge]]` and criteria when deterministic evidence does not measure answer quality.
 5. `fastskill eval validate` (and optionally `--agent <key>`).
 6. `fastskill eval run --agent <key> --output-dir <dir>`.
-7. Inspect `summary.json` (including its `isolation` report) and per-case `trial-<n>/result.json` / `trace.jsonl`.
+7. Inspect `summary.json` (including its `isolation` report) and per-case `trial-<n>/result.json` /
+   `trace.jsonl`; use `eval scorecard` for durable release gates and trend reports.
diff --git a/fastskill/skill-project.toml b/fastskill/skill-project.toml
@@ -1,6 +1,8 @@
+schema_version = "1"
+
 [metadata]
 id = "fastskill"
-version = "1.2.0"
+version = "1.3.0"
 name = "fastskill"
 description = "Package manager and operational toolkit for AI Agent skills. This skill should be used when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; performing semantic search; running skill evaluations or the optimization loop; serving skills over HTTP/MCP; or building marketplace catalogs."
 author = "Alexandre Oliveira"