From b85e70d2bc7088a0b41cce4af0ad15982929dd15 Mon Sep 17 00:00:00 2001 From: Alex Oliveira <4482374+aroff@users.noreply.github.com> Date: Tue, 8 Sep 2026 06:18:23 +0000 Subject: [PATCH] docs(fastskill): align skill with current CLI --- fastskill/SKILL.md | 59 ++++++++++++++++++++++++---- fastskill/references/eval.md | 74 ++++++++++++++++++++++++++++++++++-- fastskill/skill-project.toml | 4 +- 3 files changed, 125 insertions(+), 12 deletions(-) diff --git a/fastskill/SKILL.md b/fastskill/SKILL.md index b3cbe59..ae7285c 100644 --- a/fastskill/SKILL.md +++ b/fastskill/SKILL.md @@ -1,7 +1,7 @@ --- name: fastskill -version: 1.2.0 -description: Package manager and operational toolkit for Claude Code-compatible skills. Use this skill when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; running skill evaluations (`fastskill eval`) or the optimization loop (`fastskill optimize`); serving skills over HTTP/MCP; or building marketplace catalogs. See references/eval.md for eval setup in full. +version: 1.3.0 +description: Package manager and operational toolkit for Claude Code-compatible skills. Use this skill when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; running skill evaluations (`fastskill eval validate/run/judge/report/score/scorecard`) or the optimization loop (`fastskill optimize`); serving skills over HTTP/MCP; or building marketplace catalogs. See references/eval.md for eval setup in full. license: Apache-2.0 --- @@ -54,16 +54,26 @@ Semantic search is the only feature that needs the embedding section. Everything ```toml # skill-project.toml (minimum for reindex + search) +schema_version = "1" + +[dependencies] + +[tool.fastskill] +skills_directory = ".claude/skills" + [tool.fastskill.embedding] openai_base_url = "https://api.openai.com/v1" embedding_model = "text-embedding-3-small" ``` -Required environment variable for embeddings: **`OPENAI_API_KEY`**. `skills_directory` is not required for `reindex`/`search` if skills are already installed under the default `.claude/skills/` path. +Required environment variable for embeddings: **`OPENAI_API_KEY`**. Local search falls back to +keyword matching when embeddings are not configured. ### Configuration file structure ```toml +schema_version = "1" + [metadata] id = "my-skill" version = "1.0.0" @@ -89,6 +99,7 @@ priority = 0 ``` skill-project.toml +├── schema_version = "1" — current on-disk manifest schema; write this in new manifests ├── [metadata] — skill identity (required when authoring/publishing a skill) │ id — string, unique skill ID (no slashes; scope is a separate concept) │ version — semver string @@ -131,16 +142,20 @@ Configuration is stored **only** in `skill-project.toml`. FastSkill does not rea ```bash export OPENAI_API_KEY="your-key-here" # embeddings / semantic search -export FASTSKILL_API_URL="https://registry.example.com" # default registry target ``` -Additional: `REGISTRY_INDEX_PATH` (override index path), `FASTSKILL_NO_PROGRESS` (disable progress bars), `FASTSKILL_AUTH_TOKEN` / `FASTSKILL_TOKEN` (registry auth token, checked by `doctor`). Repository PAT auth reads a per-repo `env_var` (default `PAT_TOKEN`). +Additional: `REGISTRY_INDEX_PATH` overrides the local index path and `FASTSKILL_NO_PROGRESS` +disables reindex progress bars. `doctor` recognizes `FASTSKILL_AUTH_TOKEN` or `FASTSKILL_TOKEN`; +repository authentication reads the `env_var` configured on that repository (default +`PAT_TOKEN`). ## Skill evaluations (evals) The CLI runs **skill evaluations**: prompts from a CSV, optional deterministic **checks** (TOML), agent execution via aikit-sdk, and timestamped artifact directories. Configuration lives in `[tool.fastskill.eval]` inside `skill-project.toml`. Cases run **isolated by default** — a per-case scratch workspace containing only the skill under test (needs `SKILL.md` + `[metadata].id`; opt out with `--no-isolation`) — so trigger rates measure the skill, not the machine. -**You MUST** follow the dedicated guide for CSV and checks schema, CLI commands (`eval validate`, `eval run`, `eval report`, `eval score`), pass/fail rules, and packaging notes: +**You MUST** follow the dedicated guide for CSV, checks, judges, scorecards, CLI commands +(`eval validate`, `eval run`, `eval judge`, `eval report`, `eval score`, `eval scorecard`), and +pass/fail rules: - **[Skill evals guide](references/eval.md)** — setup, prompts CSV, `checks.toml`, agents, artifacts, CI. @@ -210,6 +225,8 @@ A bundle is a versioned ZIP containing selected skills, their dependency closure content digests. Declare every member in both `[bundle.members]` and `[dependencies]`: ```toml +schema_version = "1" + [bundle] format = "fastskill-bundle-v1" id = "platform-team" @@ -271,11 +288,17 @@ fastskill reindex --force # force a full re-index fastskill search "powerpoint presentation" # remote catalogs (default) fastskill search "data processing" --local # installed skills -fastskill search "charts" --local --limit 5 --format json +fastskill search "charts" --local --limit 5 --json +fastskill search "deploy service" --local --paths --json --content preview +fastskill read deployment-skill # retrieve the full selected skill ``` Search needs an embedding provider (`[tool.fastskill.embedding]` + `OPENAI_API_KEY`) for semantic ranking; without it, local search falls back to keyword matching. Add/install/update/remove auto-reindex when `auto_reindex = true` (skipped silently if no embedding provider is configured); use `--no-reindex` to opt out. +For agent workflows, prefer `--local --paths --json --content preview` to discover candidate skill +files, then `fastskill read ` for the selected skill. Use `--content full` only when the caller +needs all matching documents in one response. + ## Repository management FastSkill manages skill sources through the `repos` command group. Repositories are stored in `[[tool.fastskill.repositories]]` in `skill-project.toml`. @@ -412,8 +435,9 @@ All application routes are versioned under **`/api/v1/…`** (requests to `/api/ ### MCP server (expose FastSkill to your agent) ```bash -# Run an MCP server that exposes every CLI command as a tool `fastskill.` +# Read tools are exposed by default; mutating tools require --enable-write fastskill mcp serve --transport stdio +fastskill mcp serve --transport stdio --enable-write fastskill mcp serve --transport http --port 8080 --path /mcp # Write MCP server config into an agent's config @@ -423,6 +447,25 @@ fastskill mcp list `mcp install` supports agents `claude`, `cursor`, `gemini`, `copilot`, `opencode`, and `codex`, at `--scope project` or `global`. +Without `--enable-write`, mutating tools are hidden from `tools/list` and rejected if called. + +## Cache and automation utilities + +```bash +fastskill cache info # cache path, entries, and disk usage +fastskill cache info --json +fastskill cache clean # clear every cached source +fastskill cache clean --source git # git | registry | local | zip +fastskill cache clean --json + +fastskill completion bash # bash | zsh | fish | powershell | pwsh +fastskill spec --format json # authoritative machine-readable CLI surface +fastskill spec --format markdown --output fastskill-cli.md +``` + +Use `fastskill spec` when an integration needs current command paths, arguments, defaults, or +examples. Do not maintain a hand-written command schema in an agent integration. + ## Diagnostics ```bash diff --git a/fastskill/references/eval.md b/fastskill/references/eval.md index 61aad8b..cbbfe9d 100644 --- a/fastskill/references/eval.md +++ b/fastskill/references/eval.md @@ -180,6 +180,7 @@ fastskill eval run --agent claude --output-dir /tmp/evals --case smoke-1 fastskill eval run --agent codex --output-dir ./evals --tag smoke --model gpt-4o fastskill eval run --agent codex --output-dir ./evals --trials 5 --threshold 0.8 fastskill eval run --all --output-dir ./evals --ci +fastskill eval run --agent claude --output-dir ./evals --judge fastskill eval run --agent codex --output-dir ./evals --json ``` @@ -189,6 +190,8 @@ fastskill eval run --agent codex --output-dir ./evals --json - **`--ci`**: pass/fail the suite on its overall pass rate vs the threshold instead of requiring zero failed cases. - **`--no-fail`**: still prints failures but exits `0` (for CI that only collects artifacts). - **`--no-isolation`**: run in the project root against the ambient agent environment (see below). +- **`--judge`**: after deterministic scoring, run every `[[judge]]` declared in the checks file. +- **`--judge-model `**: override every judge model and record the override. Non-zero exit on suite failure unless `--no-fail`. @@ -209,10 +212,44 @@ Under each run directory (one per agent): - **`/trial-/`**: `stdout.txt`, `stderr.txt`, `trace.jsonl`, `result.json` (`TrialResult`, including `check_results`). - **`/aggregated.json`**: per-case trial aggregation (pass count, pass rate, status). - **`workspaces/`**: retained scratch workspaces of failed cases (isolation only). +- **`/trial-/judgments.json`**: full judge exchanges and normalized criterion scores, + when the run has been judged. `eval score` needs `summary.json` to include `checks_path` (written on `eval run` when checks were configured). -## 6. Report and re-score +## 6. Judge, report, re-score, and build scorecards + +Declare judges in the checks file. The prompt or prompt file must render `{{output_contract}}`; +`fastskill eval validate` checks the declaration before a paid request is made. + +```toml +[judge_defaults] +model = "gpt-4.1" +base_url = "https://api.openai.com/v1" +api_key_env = "JUDGE_API_KEY" +temperature = 0.0 + +[[judge]] +name = "quality" +prompt_file = "judge-prompt.md" + +[[judge.criterion]] +name = "clarity" +kind = "scale" +scale = 5 +description = "The response is direct, precise, and easy to apply." +``` + +Judge a completed per-agent run, or pass `--judge` to `eval run`: + +```bash +fastskill eval judge --run-dir ./eval-runs/2026-04-07T12-00-00Z/codex +fastskill eval judge --run-dir ./eval-runs/latest/codex --judge-model gpt-4.1 +fastskill eval judge --run-dir ./eval-runs/latest/codex --rejudge +``` + +Judging caches an identical request unless `--rejudge` is set. A judge error still exits non-zero +when `--no-fail` is present because the measurement is missing. ```bash fastskill eval report --run-dir ./eval-runs/2026-04-07T12-00-00Z/codex @@ -229,6 +266,35 @@ fastskill eval score --run-dir ./eval-runs/2026-04-07T12-00-00Z/codex **Note:** `eval score` applies `file_exists` relative to **`skill_project_root`** from `summary.json` — not per-case `workspace_subdir`, and not the (deleted) scratch workspace the case originally ran in. If you rely on workspace-specific files, prefer re-running `eval run` or align check paths with that root. A run whose `summary.json` contains zero cases refuses to re-score (`EVAL_EMPTY_SUITE`). +Fold multiple run directories into named metrics and threshold gates: + +```bash +fastskill eval scorecard --root ./eval-runs --metrics ./evals/metrics.toml +fastskill eval scorecard --root ./eval-runs --metrics ./evals/metrics.toml --format html -o report.html +fastskill eval scorecard --format html --from june.json --from july.json -o trend.html +``` + +```toml +[[metric]] +name = "Skill-open rate" +kind = "check_rate" +checks = ["skill_invoked"] +min_rate = 0.85 + +[[metric]] +name = "Answer quality" +kind = "judge_score" +judges = ["quality"] +min_score = 0.7 + +suites = ["./suites/consultation", "./suites/restraint"] +``` + +The other metric kind is `tool_calls_p95` with `max`. Scorecards reject mixed targets, mixed +judge identities, and duplicate case IDs unless the matching `--allow-*` option is explicit. Use +`--format json` for an archival `fastskill.scorecard/1` document or `--format html` for a +self-contained report. + ## 7. Packaging and CI - The CLI has no `package`/`publish` command. If you ship the skill by other means (Git, ZIP, a registry pipeline), keep prompts/checks under `evals/` or another path you leave out of the shipped artifact if you treat evals as repo-only. @@ -239,7 +305,9 @@ fastskill eval score --run-dir ./eval-runs/2026-04-07T12-00-00Z/codex 1. `[tool.fastskill.eval]` with existing `prompts` CSV path. 2. `SKILL.md` beside `skill-project.toml` and `[metadata].id` set (required by default-on isolation). 3. CSV header: `id`, `prompt`, `should_trigger` (+ optional `tags`, `workspace_subdir`). -4. Optional `checks` TOML with `[[check]]` and correct `name` / fields — use `skill_invoked` (not a bare-name `trigger_expectation`) to assert your skill fired. +4. Optional checks TOML with `[[check]]`; use `skill_invoked` to assert your skill fired. Add + `[[judge]]` and criteria when deterministic evidence does not measure answer quality. 5. `fastskill eval validate` (and optionally `--agent `). 6. `fastskill eval run --agent --output-dir `. -7. Inspect `summary.json` (including its `isolation` report) and per-case `trial-/result.json` / `trace.jsonl`. +7. Inspect `summary.json` (including its `isolation` report) and per-case `trial-/result.json` / + `trace.jsonl`; use `eval scorecard` for durable release gates and trend reports. diff --git a/fastskill/skill-project.toml b/fastskill/skill-project.toml index 71a7570..bc4c647 100644 --- a/fastskill/skill-project.toml +++ b/fastskill/skill-project.toml @@ -1,6 +1,8 @@ +schema_version = "1" + [metadata] id = "fastskill" -version = "1.2.0" +version = "1.3.0" name = "fastskill" description = "Package manager and operational toolkit for AI Agent skills. This skill should be used when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; performing semantic search; running skill evaluations or the optimization loop; serving skills over HTTP/MCP; or building marketplace catalogs." author = "Alexandre Oliveira"