From b21bcaa087a7ff7041f495a644428dd0c932218c Mon Sep 17 00:00:00 2001 From: Alex Oliveira <4482374+aroff@users.noreply.github.com> Date: Thu, 10 Sep 2026 00:42:41 +0000 Subject: [PATCH] feat(skill): teach command namespaces --- .github/workflows/publish-skill.yml | 2 +- .github/workflows/skill-evals.yml | 10 + CLAUDE.md | 2 +- CONTRIBUTING.md | 13 +- README.md | 11 +- evals/README.md | 5 + .../fixtures/pass/eval-skill/aggregated.json | 6 +- .../pass/eval-skill/trial-1/result.json | 6 +- .../pass/eval-skill/trial-1/stdout.txt | 22 +- .../pass/eval-skill/trial-1/trace.jsonl | 14 +- evals/fixtures/pass/summary.json | 6 +- evals/v2/README.md | 1 + evals/v2/correctness/prompts.csv | 18 +- evals/v2/patterns.json | 18 +- fastskill/SKILL.md | 221 ++++++++++-------- fastskill/references/eval.md | 3 +- fastskill/skill-project.toml | 2 +- scripts/check-command-namespaces.py | 144 ++++++++++++ scripts/test_command_namespaces.py | 116 +++++++++ specs/001-enhanced-eval.md | 36 +-- 20 files changed, 469 insertions(+), 187 deletions(-) create mode 100644 scripts/check-command-namespaces.py create mode 100644 scripts/test_command_namespaces.py diff --git a/.github/workflows/publish-skill.yml b/.github/workflows/publish-skill.yml index 6795a13..71a7f94 100644 --- a/.github/workflows/publish-skill.yml +++ b/.github/workflows/publish-skill.yml @@ -100,7 +100,7 @@ jobs: ```bash # Install this skill - fastskill add https://github.com/${{ github.repository }}.git + fastskill skill add https://github.com/${{ github.repository }}.git ``` ## Contents diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 6bd4ece..772cf12 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -4,9 +4,15 @@ on: push: branches: [main] paths: + - 'README.md' + - 'CONTRIBUTING.md' + - 'CLAUDE.md' + - 'fastskill/SKILL.md' - 'fastskill/skill-project.toml' - 'fastskill/references/eval.md' - 'evals/**' + - 'scripts/**' + - 'specs/**' - '.github/workflows/skill-evals.yml' # No `paths:` filter here on purpose. `validate` and `score-fixtures` are # required status checks, and a required check that is skipped by a path @@ -45,6 +51,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Validate canonical command namespaces + run: | + python3 -m unittest scripts/test_command_namespaces.py + python3 scripts/check-command-namespaces.py - name: Install FastSkill CLI run: | curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash diff --git a/CLAUDE.md b/CLAUDE.md index 48102b0..63f750f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # CLAUDE.md -This repository contains the FastSkill skill for AI Agent Skills, installed and used via the FastSkill CLI (`fastskill add https://github.com/gofastskill/skill.git`). +This repository contains the FastSkill skill for AI Agent Skills, installed and used via the FastSkill CLI (`fastskill skill add https://github.com/gofastskill/skill.git`). ## Pull request automation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 340d9aa..e102818 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,9 +24,15 @@ from the `fastskill/` directory so path resolution finds that manifest and `../e ## Development 1. Edit files in the `fastskill/` subdirectory (and `evals/` when extending the suite) -2. Open a pull request from a branch off the latest `origin/main` (see `CLAUDE.md`) — +2. Run `python3 -m unittest scripts/test_command_namespaces.py` and + `python3 scripts/check-command-namespaces.py` from the repository root +3. Open a pull request from a branch off the latest `origin/main` (see `CLAUDE.md`) — changes land via PR with required status checks, not by pushing to `main` directly -3. On merge, the CI/CD workflow automatically packages and publishes a new release +4. On merge, the CI/CD workflow automatically packages and publishes a new release + +The syntax check is deterministic and does not need a FastSkill binary. Before releasing, +repeat the eval validation and fixture scoring below with the candidate FastSkill binary on +`PATH`; the installed public release may still expose the previous breaking command tree. ## Evaluations @@ -91,7 +97,8 @@ a check that contradicts the column is rejected rather than silently resolved. S `.github/workflows/skill-evals.yml` gates changes to the skill/eval files: -- **validate** and **score-fixtures** run on every push/PR — deterministic, no agent, no tokens. +- **namespace syntax**, **validate**, and **score-fixtures** run on every push/PR — deterministic, + no agent, no tokens. - **live-eval** is opt-in via *Run workflow* (`workflow_dispatch`), runs the v1 suite against a real agent, and needs `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` secrets. diff --git a/README.md b/README.md index 3bfaf1c..04c98fb 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,22 @@ # FastSkill Skill Repository -This repository contains the FastSkill skill for AI Agent Skills. Use the FastSkill CLI to install and use it. +This repository contains the FastSkill skill for AI Agent Skills. Install it with the +FastSkill CLI to give agents current guidance for skill projects, packages, bundles, +repositories, evaluations, and operations. ## Installation From your project root: ```bash -fastskill add https://github.com/gofastskill/skill.git +fastskill project init +fastskill skill add https://github.com/gofastskill/skill.git ``` +Read the installed skill with `fastskill skill read fastskill`. The skill documents the +canonical command namespaces introduced by the breaking CLI migration; use a matching +FastSkill release. + See [CONTRIBUTING.md](CONTRIBUTING.md) for development and release details. ## License diff --git a/evals/README.md b/evals/README.md index 8cc256e..472b562 100644 --- a/evals/README.md +++ b/evals/README.md @@ -36,6 +36,8 @@ evals/ from `fastskill/` and the fixture check runs from the repo root. ```bash +python3 scripts/check-command-namespaces.py # canonical CLI syntax, no binary + cd fastskill fastskill eval validate # schema + config, no agent cd .. @@ -47,6 +49,9 @@ fastskill eval run --agent claude --output-dir ../eval-runs fastskill eval report --run-dir ../eval-runs//claude ``` +Run the FastSkill commands with the candidate binary on `PATH` when validating a breaking +CLI release. The namespace syntax check remains useful before that binary exists. + `eval score` is read-only. Every number it reports is a function of the run's artifacts, and scoring the committed fixtures leaves the tree clean. The backfill of `command_count` and the token counts now happens in `eval run`, at write time, where the writer owns the diff --git a/evals/fixtures/pass/eval-skill/aggregated.json b/evals/fixtures/pass/eval-skill/aggregated.json index 327e1db..5255209 100644 --- a/evals/fixtures/pass/eval-skill/aggregated.json +++ b/evals/fixtures/pass/eval-skill/aggregated.json @@ -4,9 +4,9 @@ { "trial_id": 1, "status": "passed", - "command_count": 2, + "command_count": 1, "input_tokens": 6, - "output_tokens": 1778, + "output_tokens": 35, "check_results": [ { "check_name": "skill_invoked", @@ -28,4 +28,4 @@ "pass_count": 1, "total_trials": 1, "pass_rate": 1.0 -} \ No newline at end of file +} diff --git a/evals/fixtures/pass/eval-skill/trial-1/result.json b/evals/fixtures/pass/eval-skill/trial-1/result.json index 1a6c4af..245413e 100644 --- a/evals/fixtures/pass/eval-skill/trial-1/result.json +++ b/evals/fixtures/pass/eval-skill/trial-1/result.json @@ -1,9 +1,9 @@ { "trial_id": 1, "status": "passed", - "command_count": 2, + "command_count": 1, "input_tokens": 6, - "output_tokens": 1778, + "output_tokens": 35, "check_results": [ { "check_name": "skill_invoked", @@ -19,4 +19,4 @@ } ], "error_message": null -} \ No newline at end of file +} diff --git a/evals/fixtures/pass/eval-skill/trial-1/stdout.txt b/evals/fixtures/pass/eval-skill/trial-1/stdout.txt index 5ca18f9..46a563f 100644 --- a/evals/fixtures/pass/eval-skill/trial-1/stdout.txt +++ b/evals/fixtures/pass/eval-skill/trial-1/stdout.txt @@ -1,20 +1,2 @@ -{"type":"system","subtype":"init","cwd":"/tmp/.tmpaKU6vT","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","tools":["Task","Bash","CronCreate","CronDelete","CronList","DesignSync","Edit","EnterWorktree","ExitWorktree","Monitor","NotebookEdit","PushNotification","Read","RemoteTrigger","ReportFindings","ScheduleWakeup","SendMessage","Skill","TaskCreate","TaskGet","TaskList","TaskOutput","TaskStop","TaskUpdate","ToolSearch","WebFetch","WebSearch","Workflow","Write"],"mcp_servers":[{"name":"claude.ai Google Calendar","status":"needs-auth"},{"name":"claude.ai Google Drive","status":"needs-auth"},{"name":"claude.ai Gmail","status":"needs-auth"}],"model":"claude-opus-4-8[1m]","permissionMode":"bypassPermissions","slash_commands":["fastskill","deep-research","design-sync","dataviz","update-config","verify","debug","code-review","simplify","batch","fewer-permission-prompts","doctor","loop","schedule","claude-api","run","run-skill-generator","agents","clear","color","compact","config","context","effort","fast","heapdump","init","mcp","import","model","__remote-workflow","reload-skills","rename","review","security-review","usage-credits","extra-usage","usage","insights","recap","goal","design","design-consent","design-revoke","team-onboarding"],"apiKeySource":"none","claude_code_version":"2.1.215","output_style":"default","agents":["claude","Explore","general-purpose","Plan","statusline-setup"],"skills":["fastskill","deep-research","design-sync","dataviz","update-config","verify","debug","code-review","simplify","batch","fewer-permission-prompts","doctor","loop","schedule","claude-api","run","run-skill-generator"],"plugins":[],"capabilities":["interrupt_receipt_v1","msg_lifecycle_v1"],"analytics_disabled":false,"product_feedback_disabled":false,"uuid":"e4167944-1194-4d3a-8bbf-86fb8dad735a","memory_paths":{"auto":"/home/sysuser/.claude/projects/-tmp/memory/"},"fast_mode_state":"off"} -{"type":"rate_limit_event","rate_limit_info":{"status":"allowed","resetsAt":1787845800,"rateLimitType":"five_hour","overageStatus":"rejected","overageDisabledReason":"org_level_disabled","isUsingOverage":false},"uuid":"e41bb638-1954-47c4-ab43-5bec8ccdd915","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee"} -{"type":"system","subtype":"thinking_tokens","estimated_tokens":50,"estimated_tokens_delta":50,"uuid":"81d1e1d7-9580-466c-845f-699c1d10516d","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee"} -{"type":"system","subtype":"thinking_tokens","estimated_tokens":119,"estimated_tokens_delta":69,"uuid":"e6999b81-a110-428b-ae85-d205e77fcb3b","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee"} -{"type":"assistant","message":{"model":"claude-opus-4-8","id":"msg_011CeTQHnGS4Y7W6zEgmNPeS","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"EtMDCqgBCBEYAipATP0F2IBmgTwoIq+o7sscCAMBq+v8IMEDooqNirkcN5/GbMUgwBluSSwkzGJgemb+AfBpnhg7zUi6sljf4YjbfDIPY2xhdWRlLW9wdXMtNC04OABCCHRoaW5raW5nWiRjNmNlZmU5NC00YjA1LTQzMjgtYmRiZS02MTY0MGFhZDVmMDhyEFkopPIZT30Lu4m6CEUJoOyIAQGoAf+PwdQGsAECEgzz9hA65fhzjTBYmYkaDOJB1WHeSYSJCi0UnyIw//uSU7z8z1hBmQ2iom0Y2lCFKwhdf9iUGxKenhINo9RfDpGhtBQW7XQ1fEDJLJMBKtcBpNiU7XHxMWPjnA5NHAC1/ozCNw5gR7QWqG9d1xv7BML7LhH6rZF0TmxYXb93IRx/1O1uxxuP3hV0zN/ZjlPwZp8Eo7Z6FJ7ZdW3IxCcZdqlyFJ3J+vlCRJUI1reoXp/J6f1WNtIo3/myU9EDsqnJx0JLV5A6CggdKd9N8R/35BuRtwBh/g7rTqe5d2yEtRxoyvvKpxOgeWVGGUpgIDGjYfKWpRe8RoTlvwMKy/I/qObMG024HEDmroTk1nFTXFbqohXQ2EkMz7JUS++Q0PuP0xqdLzn4daUYAQ=="}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":5326,"cache_read_input_tokens":15581,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5326},"output_tokens":2,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"82698b00-713a-4cef-8212-cdec9d060b2e","timestamp":"2026-08-27T14:21:51.489Z","request_id":"req_011CeTQHmK8b72pu7Wqd7P6x"} -{"type":"assistant","message":{"model":"claude-opus-4-8","id":"msg_011CeTQHnGS4Y7W6zEgmNPeS","type":"message","role":"assistant","content":[{"type":"text","text":"This maps directly to the `fastskill` toolkit, which has an eval command and a dedicated reference for setup. Let me pull it up."}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":5326,"cache_read_input_tokens":15581,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5326},"output_tokens":2,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"bce8b011-83bb-499f-a0a7-2fa978b5d97d","timestamp":"2026-08-27T14:21:52.275Z","request_id":"req_011CeTQHmK8b72pu7Wqd7P6x"} -{"type":"assistant","message":{"model":"claude-opus-4-8","id":"msg_011CeTQHnGS4Y7W6zEgmNPeS","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_018oBohn6g2ijtAafjuWcEb8","name":"Skill","input":{"skill":"fastskill","args":"eval"},"caller":{"type":"direct"}}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":5326,"cache_read_input_tokens":15581,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5326},"output_tokens":2,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"983b921b-5773-49a8-bc6d-2a26f3f20ef1","timestamp":"2026-08-27T14:21:52.276Z","request_id":"req_011CeTQHmK8b72pu7Wqd7P6x"} -{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_018oBohn6g2ijtAafjuWcEb8","content":"Launching skill: fastskill"}]},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"6e83cc1b-db1b-4377-b900-4a75cf055dfd","timestamp":"2026-08-27T14:21:52.293Z","tool_use_result":{"success":true,"commandName":"fastskill"}} -{"type":"user","message":{"role":"user","content":[{"type":"text","text":"Base directory for this skill: /tmp/.tmpaKU6vT/.claude/skills/fastskill\n\n# FastSkill\n\nFastSkill is a package manager and operational toolkit for Claude Code-compatible skills. It provides discovery, installation, versioning, semantic search, quality evals, and a local HTTP/MCP server for skills at scale.\n\n## Overview\n\nFastSkill follows Anthropic's standardized `SKILL.md` skill layout and adds a manifest (`skill-project.toml`), a lockfile (`skills.lock`), semantic search, evaluation suites, an optimization loop, and repository/marketplace tooling. Modern agents (Claude Code, Cursor, …) read installed skills directly from the skills directory — **there is no metadata-file sync step**.\n\n## Installation\n\n```bash\n# Quick install (recommended)\ncurl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash\n\n# Or via Homebrew (macOS & Linux)\nbrew install gofastskill/cli/fastskill\n\n# Or via Scoop (Windows)\nscoop bucket add gofastskill https://github.com/gofastskill/scoop-bucket\nscoop install fastskill\n\n# Or via Cargo\ncargo install fastskill\n```\n\nVerify:\n\n```bash\nfastskill --version\n```\n\n## Configuration\n\nFastSkill uses **`skill-project.toml`** as the single configuration file for both project-level and skill-level contexts. FastSkill walks up the directory tree from the current directory to find it.\n\n### Quick setup\n\n```bash\nfastskill init\n```\n\nThis creates a `skill-project.toml` with the standard sections.\n\n### Minimal configuration (enables reindex / search)\n\nSemantic search is the only feature that needs the embedding section. Everything else works without it.\n\n```toml\n# skill-project.toml (minimum for reindex + search)\n[tool.fastskill.embedding]\nopenai_base_url = \"https://api.openai.com/v1\"\nembedding_model = \"text-embedding-3-small\"\n```\n\nRequired environment variable for embeddings: **`OPENAI_API_KEY`**. `skills_directory` is not required for `reindex`/`search` if skills are already installed under the default `.claude/skills/` path.\n\n### Configuration file structure\n\n```toml\n[metadata]\nid = \"my-skill\"\nversion = \"1.0.0\"\n\n[dependencies]\n# Add your skill dependencies here\n\n[tool.fastskill]\nskills_directory = \".claude/skills\"\n\n[tool.fastskill.embedding]\nopenai_base_url = \"https://api.openai.com/v1\"\nembedding_model = \"text-embedding-3-small\"\n\n[[tool.fastskill.repositories]]\nname = \"anthropic\"\ntype = \"git-marketplace\"\nurl = \"https://github.com/anthropics/skills\"\npriority = 0\n```\n\n### Full schema reference\n\n```\nskill-project.toml\n├── [metadata] — skill identity (required when authoring/publishing a skill)\n│ id — string, unique skill ID (no slashes; scope is a separate concept)\n│ version — semver string\n│ name — human-readable name\n│ description — one-line description\n│ author — optional\n│ compatibility — optional, Claude version constraint\n│ download_url — optional\n│ tags / capabilities — optional string arrays\n├── [dependencies] — skill dependencies (optional, may be empty table)\n├── [tool.fastskill] — fastskill runtime config (optional section)\n│ skills_directory — path where installed skills are stored (default: .claude/skills/)\n│ install_depth — max transitive dependency depth (default: 5)\n│ skip_transitive — skip transitive deps (default: false)\n│ auto_reindex — reindex automatically after add/install/update/remove (default: true)\n├── [tool.fastskill.embedding] — required only for reindex / search\n│ openai_base_url — REQUIRED string, OpenAI-compatible API base URL\n│ embedding_model — REQUIRED string, model name\n│ index_path — optional path, default: .claude/.fastskill/index.db\n├── [tool.fastskill.eval] — skill evaluation config (see references/eval.md)\n│ prompts / checks / timeout_seconds / fail_on_missing_agent\n├── [tool.fastskill.server] — serve CORS config: allowed_origins, allowed_headers\n└── [[tool.fastskill.repositories]] — zero or more repository entries\n name — REQUIRED string, unique repo label\n type — REQUIRED enum: git-marketplace | http-registry | zip-url | local\n REQUIRED source-type-specific location — exactly one, matching `type`:\n git-marketplace → url http-registry → index_url\n zip-url → zip_url local → path\n branch — optional, git branch (git-marketplace only)\n priority — REQUIRED integer, lower = higher priority\n auth — optional table: { type = \"pat\", env_var = \"VAR\" }\n **http-registry only** — other source types\n authenticate differently and ignore or reject it.\n See \"Authentication\" below.\n```\n\nConfiguration is stored **only** in `skill-project.toml`. FastSkill does not read a separate `.fastskill/config.yaml` file.\n\n### Environment variables\n\n```bash\nexport OPENAI_API_KEY=\"your-key-here\" # embeddings / semantic search\nexport FASTSKILL_API_URL=\"https://registry.example.com\" # default registry target\n```\n\nAdditional: `REGISTRY_INDEX_PATH` (override index path), `FASTSKILL_NO_PROGRESS` (disable progress bars), `FASTSKILL_AUTH_TOKEN` / `FASTSKILL_TOKEN` (registry auth token, checked by `doctor`). Repository PAT auth reads a per-repo `env_var` (default `PAT_TOKEN`).\n\n## Skill evaluations (evals)\n\nThe CLI runs **skill evaluations**: prompts from a CSV, optional deterministic **checks** (TOML), agent execution via aikit-sdk, and timestamped artifact directories. Configuration lives in `[tool.fastskill.eval]` inside `skill-project.toml`. Cases run **isolated by default** — a per-case scratch workspace containing only the skill under test (needs `SKILL.md` + `[metadata].id`; opt out with `--no-isolation`) — so trigger rates measure the skill, not the machine.\n\n**You MUST** follow the dedicated guide for CSV and checks schema, CLI commands (`eval validate`, `eval run`, `eval report`, `eval score`), pass/fail rules, and packaging notes:\n\n- **[Skill evals guide](references/eval.md)** — setup, prompts CSV, `checks.toml`, agents, artifacts, CI.\n\n## Basic usage\n\n### Adding skills\n\n```bash\n# From git repository\nfastskill add https://github.com/org/skill.git\n\n# From a git repository subdirectory (GitHub tree URL: tree//)\nfastskill add \"https://github.com/org/repo/tree/main/path/to/skill\"\n\n# From a local folder\nfastskill add ./local-skill\n\n# Editable mode (symlink a local folder for development)\nfastskill add ./local-skill -e\n\n# Every skill under a local folder (recursive)\nfastskill add ./skills -r\n\n# From a registry with version\nfastskill add scope/pptx@1.0.0\n\n# From git with branch/tag\nfastskill add https://github.com/org/skill.git --branch main\nfastskill add https://github.com/org/skill.git --tag v1.0.0\n\n# Add to a group\nfastskill add https://github.com/org/skill.git --group dev\n```\n\n### Installing from the manifest\n\nDeclare dependencies in `skill-project.toml`:\n\n```toml\n[dependencies]\nweb-scraper = { origin = { type = \"git\", url = \"https://github.com/org/web-scraper.git\" } }\ndata-processor = { origin = { type = \"git\", url = \"https://github.com/org/data-processor.git\" }, groups = [\"prod\"] }\n```\n\nPin a branch, tag, or commit with `ref`:\n\n```toml\nweb-scraper = { origin = { type = \"git\", url = \"https://github.com/org/web-scraper.git\", ref = { branch = \"main\" } } }\n```\n\n> The older `{ source = \"git\", url = ... }` shape (no `origin` wrapper) still reads and is\n> silently upgraded in memory, but it's slated for removal — write new manifests with\n> `origin`.\n\nInstall:\n\n```bash\nfastskill install # apply the manifest, update skills.lock\nfastskill install --lock # install exact versions from skills.lock (reproducible)\nfastskill install --only prod # only the prod group\nfastskill install --without dev # everything except the dev group\n```\n\n### Listing, reading, and removing skills\n\n```bash\nfastskill list # list installed skills with reconciliation status\nfastskill list --json # machine-readable\nfastskill read my-skill-id # print the skill's SKILL.md\nfastskill read my-skill-id --meta # metadata only\nfastskill read my-skill-id --tree # dependency tree\nfastskill remove my-skill-id # uninstall and update manifest + lock\n```\n\n`fastskill ` is shorthand for `fastskill read `.\n\n### Updating skills\n\n```bash\nfastskill update # update all skills from their recorded source\nfastskill update my-skill-id # update one skill\nfastskill update --check # report what would change without writing\nfastskill update --dry-run # preview\n```\n\n### Semantic search\n\n```bash\nfastskill reindex # build/refresh the local vector index\nfastskill reindex --force # force a full re-index\n\nfastskill search \"powerpoint presentation\" # remote catalogs (default)\nfastskill search \"data processing\" --local # installed skills\nfastskill search \"charts\" --local --limit 5 --format json\n```\n\nSearch needs an embedding provider (`[tool.fastskill.embedding]` + `OPENAI_API_KEY`) for semantic ranking; without it, local search falls back to keyword matching. Add/install/update/remove auto-reindex when `auto_reindex = true` (skipped silently if no embedding provider is configured); use `--no-reindex` to opt out.\n\n## Repository management\n\nFastSkill manages skill sources through the `repos` command group. Repositories are stored in `[[tool.fastskill.repositories]]` in `skill-project.toml`.\n\n```bash\n# Add repositories (type: git-marketplace | http-registry | zip-url | local)\nfastskill repos add team-skills --repo-type git-marketplace https://github.com/org/team-skills.git\nfastskill repos add prod-registry --repo-type http-registry https://api.fastskill.io/index\nfastskill repos add official --repo-type zip-url https://example.com/skills/\nfastskill repos add local-dev --repo-type local ./local-skills\n\n# Inspect and maintain\nfastskill repos list # list configured repositories\nfastskill repos info team-skills # repository details\nfastskill repos test team-skills # connectivity check\nfastskill repos update team-skills --branch main --priority 1\nfastskill repos refresh # refresh cached catalog metadata\nfastskill repos remove team-skills\n\n# Browse catalogs\nfastskill repos skills --repository prod-registry [--scope engineering] [--all-versions]\nfastskill repos show engineering/data-analyzer --repository prod-registry\nfastskill repos versions engineering/data-analyzer --repository prod-registry\n```\n\nSearch across configured registries with `fastskill search \"\" --repository `.\n\n**git-marketplace requirements**: the repository must contain a `marketplace.json` at `.claude-plugin/marketplace.json` (Claude Code standard) or `marketplace.json` (root, legacy).\n\n**Authentication** uses env-var indirection — never store plaintext tokens:\n\n```toml\n[[tool.fastskill.repositories]]\nname = \"production-registry\"\ntype = \"http-registry\"\nindex_url = \"https://api.fastskill.io/index\"\nauth = { type = \"pat\", env_var = \"FASTSKILL_TOKEN\" }\n```\n\nThe `auth` block applies to **`http-registry` sources only**. Every other source type\nauthenticates by a different mechanism, and `auth` is not consulted for them:\n\n| Source type | How to authenticate |\n|---|---|\n| `http-registry` | `auth = { type = \"pat\", env_var = \"...\" }` (above). |\n| `git-marketplace` | The system git credential helper (`gh auth login`, `git config credential.helper`) or an SSH remote plus a key in your SSH agent. FastSkill shells out to `git` and never injects PATs — setting `auth` here is rejected with an error. |\n| `zip-url` | A **pre-signed URL** (S3/GCS), which carries the credential in the URL itself. A zip-url fetch is a plain HTTP GET that sends no auth headers. |\n| `local` | Filesystem permissions. |\n\n> Do not rely on `auth` to protect a private `zip-url` artifact — it is not applied to\n> the request. Use a pre-signed URL instead.\n\n## Building marketplace catalogs\n\nGenerate a `marketplace.json` from a folder of skills so others can consume them as a `git-marketplace` repository.\n\n```bash\nfastskill marketplace create --path ./skills --name \"My Marketplace\"\nfastskill marketplace create -p . -o .claude-plugin/marketplace.json --name \"My Marketplace\" --base-url https://example.com/skills/\n```\n\n| Option | Description |\n|--------|-------------|\n| `-p, --path ` | Root to scan for skills (default `.`) |\n| `-o, --output ` | Output path (default `.claude-plugin/marketplace.json`) |\n| `--name ` | **Required.** Marketplace name |\n| `--base-url ` | Base URL for download links |\n| `--owner-name` / `--owner-email` / `--description` / `--version` | Optional metadata |\n\n> **Distribution note:** the `fastskill` CLI does not include `publish`/`auth`/`package` commands. Skill distribution to a hosted registry is handled by the platform operator (a managed deploy workflow), not a self-hosted CLI publish path. Author locally, share via git/zip/marketplace catalogs, or hand artifacts to your registry operator.\n\n## Analyzing a skill collection\n\n`analyze` uses the semantic index to find overlap across skills (requires an embedding provider and an index — run `reindex` first).\n\n```bash\nfastskill analyze matrix --threshold 0.8 # pairwise similarity\nfastskill analyze cluster -k 8 --min-size 2 # semantic clusters\nfastskill analyze duplicates --threshold 0.92 --severity high # near-duplicates\n```\n\n## Optimizing skills\n\n`optimize` runs an automated text-gradient loop that improves a skill document against eval cases: propose a patch → run the target agent → grade → accept if the score improves → repeat, writing the best version to disk.\n\n```bash\nfastskill optimize run --config optimize.toml --out-dir ./optimize-runs\nfastskill optimize status ./optimize-runs/ [--watch]\nfastskill optimize resume ./optimize-runs/\nfastskill optimize inspect ./optimize-runs/ --step 3 --show all\nfastskill optimize export ./optimize-runs/ --out ./best_skill.md\n```\n\nScoring passes (rollouts, gate, baseline, final) are **isolated by default**: each case scores in a scratch workspace containing only the candidate skill. Set `isolate = false` in the config or pass `optimize run --no-isolation` to score against the machine's ambient environment instead; the decision is persisted into the run's provenance config so `resume` replays it. Baselines recorded before isolation existed will shift on the first isolated run.\n\n## Serving skills (HTTP API and MCP)\n\nStart a local server exposing an HTTP API and web UI:\n\n```bash\nfastskill serve # read-only by default\nfastskill serve --host 0.0.0.0 --port 8080\nfastskill serve --enable-write # enable state-changing endpoints\n```\n\nThe server is **read-only by default** (ADR-0003): read endpoints (list/get skills, project, search, resolve, status, registry browse, dashboard) are always available; every write endpoint (install/update/remove/reindex/manifest edits) returns **HTTP 403** unless you start the server with `--enable-write`. `serve` enforces **no authentication of its own** — run it local-first, or place an authenticating reverse proxy in front if you expose it.\n\nAll application routes are versioned under **`/api/v1/…`** (requests to `/api/…` redirect 308 to `/api/v1/…`).\n\n| Endpoint | Method | Access | Description |\n|----------|--------|--------|-------------|\n| `/api/v1/skills` | GET | read | List installed skills |\n| `/api/v1/skills/{id}` | GET | read | Get a skill |\n| `/api/v1/skills/{id}/content` | GET | read | Get `SKILL.md` content |\n| `/api/v1/project` | GET | read | Project manifest view |\n| `/api/v1/search` | POST | read | Search skills (`{\"query\":\"…\",\"limit\":N}`) |\n| `/api/v1/resolve` | POST | read | Resolve the most relevant skills for a prompt |\n| `/api/v1/status` | GET | read | Service status + capability flags |\n| `/api/v1/registry/*` | GET | read | Browse registry sources / index / versions |\n| `/api/v1/skills/install` | POST | **write** | Install a skill from an origin |\n| `/api/v1/skills/update` | POST | **write** | Update one or all skills |\n| `/api/v1/skills/{id}` | DELETE | **write** | Remove a skill |\n| `/api/v1/reindex` | POST | **write** | Reindex |\n| `/api/v1/registry/refresh` | POST | **write** | Refresh registry sources |\n| `/api/v1/manifest/skills` | POST/PUT/DELETE | **write** | Manifest skill management |\n| `/healthz`, `/readyz` | GET | read | Liveness / readiness probes |\n\n### MCP server (expose FastSkill to your agent)\n\n```bash\n# Run an MCP server that exposes every CLI command as a tool `fastskill.`\nfastskill mcp serve --transport stdio\nfastskill mcp serve --transport http --port 8080 --path /mcp\n\n# Write MCP server config into an agent's config\nfastskill mcp install --agent claude --scope project --stdio\nfastskill mcp list\n```\n\n`mcp install` supports agents `claude`, `cursor`, `gemini`, `copilot`, `opencode`, and `codex`, at `--scope project` or `global`.\n\n## Diagnostics\n\n```bash\nfastskill doctor # check skills dir, skill-project.toml, embedding config, OPENAI_API_KEY, auth token\nfastskill doctor --json\n```\n\nAdd `-v` / `--verbose` to any command for more detail.\n\n## CI/CD integration\n\n### GitHub Actions\n\n```yaml\nname: Skills CI\n\non:\n push:\n branches: [main]\n paths: ['skills/**', 'skill-project.toml', 'skills.lock']\n\njobs:\n verify:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n\n - name: Install FastSkill\n run: curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash\n\n - name: Reproducible install\n run: fastskill install --lock\n\n - name: Validate evals\n run: fastskill eval validate\n```\n\nSet `OPENAI_API_KEY` and any repository PAT env vars as CI secrets when your workflow uses search or private repositories.\n\n## Troubleshooting\n\n### \"Embedding configuration required but not found\"\n\n1. Run `fastskill init` (or add `[tool.fastskill.embedding]` to `skill-project.toml`).\n2. Set `OPENAI_API_KEY`.\n3. Re-run `fastskill reindex`.\n\n### Repository source not appearing\n\n- Verify `marketplace.json` exists at the expected location for `git-marketplace`.\n- Confirm the repo `type` and URL are correct; run `fastskill repos test `.\n\n### Search returns nothing\n\n- Run `fastskill reindex` to (re)build the index.\n- Confirm `OPENAI_API_KEY` and `[tool.fastskill.embedding]` are set (`fastskill doctor`).\n\n## Version management\n\nFastSkill uses semantic versioning. A skill's version is read from the `[metadata]` section of its `skill-project.toml`.\n\n## Additional resources\n\n- **[Skill evals (`fastskill eval`)](references/eval.md)** — eval config, prompts CSV, checks TOML, running/scoring, artifacts.\n- Full documentation: \n\n\nARGUMENTS: eval"}]},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"ba04b0bf-280e-44c9-abcd-162138de8b1d","timestamp":"2026-08-27T14:21:52.291Z","isSynthetic":true} -{"type":"system","subtype":"thinking_tokens","estimated_tokens":50,"estimated_tokens_delta":50,"uuid":"29e531f7-3ff4-4f61-808f-bcea5bfa0fcb","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee"} -{"type":"system","subtype":"thinking_tokens","estimated_tokens":103,"estimated_tokens_delta":53,"uuid":"0c340f1a-ce0b-4525-adb2-047e2d3190a5","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee"} -{"type":"assistant","message":{"model":"claude-opus-4-8","id":"msg_011CeTQJ5L6qSe7pRjF2cobX","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"EpYDCqgBCBEYAipAKIigCuA7Zs/gIerURntgU/Y3x0oC1jyxp+JUvxpcwL8Iwp4duZR7rlkuq3mY1jToUNsn/IXJg6pQwOi0fcuWkzIPY2xhdWRlLW9wdXMtNC04OABCCHRoaW5raW5nWiRjNmNlZmU5NC00YjA1LTQzMjgtYmRiZS02MTY0MGFhZDVmMDhyEFkopPIZT30Lu4m6CEUJoOyIAQGoAYSQwdQGsAECEgw2Fd8UCUZodrJAGRkaDO9brSsSdk27dee1sCIwaZE+hCIFDMHAV/1o7k5GA2eU+tDl2Lwk34T3SGFsudPocoWV8fAU5EsLwaf1CSaRKpoBnjfKJ1Qjciy4hSzg2UAI8HxAmcoHWR/dfrj12ynLXT7HS1k5ynGAx2fWkZrlcMnKyVn65a6+BGTnswC5S+xBIyxCIJVMw9FYV1QZM0fy5lvjNgML9/ptZPjrEoilN6UPplUAEu9696OMafzLio86eZWpdArNrQnfWnF3PyaAZrLQD3ZLS3WlgAF+up4Lih8WGcCvnnFFJhdixxgB"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":7394,"cache_read_input_tokens":20907,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":7394},"output_tokens":6,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"4b1b3f46-82dd-4832-bbf7-1cc1e7504105","timestamp":"2026-08-27T14:21:56.714Z","request_id":"req_011CeTQJ3cQtxsBEyqkrknKT"} -{"type":"assistant","message":{"model":"claude-opus-4-8","id":"msg_011CeTQJ5L6qSe7pRjF2cobX","type":"message","role":"assistant","content":[{"type":"text","text":"Let me read the dedicated eval guide, which the skill instructs me to follow for the full schema and commands."}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":7394,"cache_read_input_tokens":20907,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":7394},"output_tokens":6,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"7f094057-46ef-4adf-a4a9-d160c35bbaf4","timestamp":"2026-08-27T14:21:57.478Z","request_id":"req_011CeTQJ3cQtxsBEyqkrknKT"} -{"type":"assistant","message":{"model":"claude-opus-4-8","id":"msg_011CeTQJ5L6qSe7pRjF2cobX","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_0182ZtNA5ZUoB6f3GstTUjho","name":"Read","input":{"file_path":"/tmp/.tmpaKU6vT/.claude/skills/fastskill/references/eval.md"},"caller":{"type":"direct"}}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":7394,"cache_read_input_tokens":20907,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":7394},"output_tokens":6,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"73ebdd4c-c8c2-4867-95f7-0361964eb0ee","timestamp":"2026-08-27T14:21:58.038Z","request_id":"req_011CeTQJ3cQtxsBEyqkrknKT"} -{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_0182ZtNA5ZUoB6f3GstTUjho","type":"tool_result","content":"1\t# Skill evaluations (`fastskill eval`)\n2\t\n3\tThis guide matches the FastSkill CLI implementation: configuration is read from `skill-project.toml`, cases from a prompts CSV, optional scoring rules from a checks TOML, and execution uses **aikit-sdk** (`run_agent_events`) with a supported agent CLI.\n4\t\n5\t## What evals are for\n6\t\n7\tEvals run repeatable prompts against a real agent. By default each case runs **isolated**: the skill under test is deployed alone into a per-case scratch workspace, and the agent's user-level skill discovery is suppressed where the backend supports it — so trigger rates measure your skill, not whatever else is installed on the machine. Each run records stdout, stderr, a JSONL trace, and per-case results under a timestamped run directory. Optional **checks** assert properties of that output (skill invocation, substring expectations, file presence, tool-call count).\n8\t\n9\t## Prerequisites\n10\t\n11\t1. **FastSkill CLI** installed (`fastskill --version`).\n12\t2. A **`skill-project.toml`** reachable from the current directory (FastSkill walks up the tree to find it).\n13\t3. **`[tool.fastskill.eval]`** present with a valid `prompts` path (see below).\n14\t4. For **`fastskill eval run`**: a **runnable agent** installed and on `PATH`. Agent keys are a closed set defined by aikit-sdk: `claude`, `codex`, `cursor`, `gemini`, `opencode`, `pi`, `aikit` — the last runs in-process and needs no CLI. If `fail_on_missing_agent = true` (default), the CLI refuses to run when the chosen agent is missing.\n15\t\n16\t `--agent` is validated against the agents **detected on the current machine**, not against the full key list. An agent is \"detected\" when its CLI is on `PATH` and answers `--version`. So a supported-but-not-installed key fails exactly like a typo:\n17\t\n18\t ```\n19\t RUNTIME_UNKNOWN_ID: Unknown runtime ID(s): opencode. Available: aikit, claude, codex, cursor, gemini, pi\n20\t ```\n21\t\n22\t If you get this for a key spelled correctly, install that agent's CLI — the `Available:` list reports what was found locally, so it differs from machine to machine and from the list above.\n23\t\n24\tEvals do **not** require embedding or `OPENAI_API_KEY` unless your workflow also uses `reindex` / `search`.\n25\t\n26\t## 1. Configure `skill-project.toml`\n27\t\n28\tAdd a `[tool.fastskill.eval]` table. Paths are resolved relative to the **skill project root** (directory containing `skill-project.toml`) unless absolute.\n29\t\n30\t| Field | Required | Default | Meaning |\n31\t|--------|----------|---------|---------|\n32\t| `prompts` | yes | — | Path to the prompts CSV file. |\n33\t| `checks` | no | none | Path to checks TOML. If omitted, checks are not loaded. |\n34\t| `timeout_seconds` | no | `900` | Per-case timeout passed to the agent run. |\n35\t| `trials_per_case` | no | `1` | How many times each case runs; results aggregate into a per-case pass rate. Overridable with `eval run --trials ` (1–1000). |\n36\t| `parallel` | no | CPU cores | Maximum concurrent trials within one case. |\n37\t| `pass_threshold` | no | `1.0` | Fraction of a case's trials that must pass for the case to pass (0.0–1.0). Overridable with `--threshold`. |\n38\t| `fail_on_missing_agent` | no | `true` | If true, `eval run` and `eval validate --agent ` error when the agent is not available. |\n39\t\n40\tExample:\n41\t\n42\t```toml\n43\t[tool.fastskill.eval]\n44\tprompts = \"evals/prompts.csv\"\n45\tchecks = \"evals/checks.toml\"\n46\ttimeout_seconds = 600\n47\tfail_on_missing_agent = true\n48\t```\n49\t\n50\t### Validate configuration\n51\t\n52\tFrom the skill project (or any subdirectory under it):\n53\t\n54\t```bash\n55\tfastskill eval validate\n56\tfastskill eval validate --agent codex\n57\tfastskill eval validate --json\n58\t```\n59\t\n60\tErrors use stable prefixes: `EVAL_CONFIG_MISSING`, `EVAL_PROMPTS_NOT_FOUND`, `EVAL_AGENT_UNAVAILABLE`.\n61\t\n62\t## 2. Define cases: prompts CSV\n63\t\n64\tThe suite loader expects a header row and **required** columns: `id`, `prompt`, `should_trigger`.\n65\t\n66\tOptional columns: `tags`, `workspace_subdir`.\n67\t\n68\t- **`id`**: non-empty identifier per row (used for `--case`, artifact folder names, and reports).\n69\t- **`prompt`**: text sent to the agent; use CSV quoting for commas or newlines (`\"...\"`, escaped `\"\"` inside quotes).\n70\t- **`should_trigger`**: `true` / `false` / `1` / other (treated false). Documented intent only; **pass/fail is not driven by this column**. When no checks file is configured, a case **passes** if the agent exits with code `0`, **fails** otherwise. When checks are loaded, a case **passes** only if **every** check passes (see below).\n71\t- **`tags`**: comma-separated tags inside the cell (e.g. `smoke,basic`). Used with `eval run --tag `.\n72\t- **`workspace_subdir`**: relative path that becomes the agent's working directory for the case. Under the default isolation it is created **inside the case's scratch workspace**, with fixture files copied in from the same path under the project root; under `--no-isolation` it resolves directly under the project root. If empty or column omitted, cwd is the workspace root (or the project root under `--no-isolation`).\n73\t\n74\tExample:\n75\t\n76\t```csv\n77\tid,prompt,should_trigger,tags,workspace_subdir\n78\tsmoke-1,\"List fastskill subcommands briefly\",true,smoke,\n79\tdeep-1,\"Explain eval validate flags\",true,docs,\n80\t```\n81\t\n82\tEmpty lines are skipped.\n83\t\n84\t## 3. Define checks: `checks.toml` (optional)\n85\t\n86\tPoint `checks` in TOML at a file containing one or more `[[check]]` tables. Each check must include **`name`**, selecting the check type. Optional **`required`** (boolean, default `true`) is honoured on all types: a case passes when every **required** check passes; a `required = false` check still runs and is reported in `check_results`, but cannot fail the case.\n87\t\n88\t### `skill_invoked`\n89\t\n90\tThe check you almost certainly want for \"did my skill fire?\". Matches a **structured `Skill` tool invocation** in the trace, not a substring of prose. When `skill` is given, it must **exactly equal** the invocation's skill-identifying input field (`skill`, `name`, or `skillName`) — `\"foo\"` does not match a `\"foo-bar\"` invocation, and a skill name merely mentioned in argument text does not count. Omit `skill` to match any skill invocation. With `expected = false` this is the only reliable way to assert non-triggering.\n91\t\n92\t```toml\n93\t[[check]]\n94\tname = \"skill_invoked\"\n95\tskill = \"my-skill\"\n96\texpected = true\n97\t```\n98\t\n99\t### `trigger_expectation`\n100\t\n101\tSubstring search in the **trace JSONL only** (agent stdout is not searched — claude's startup event lists every installed skill by name there, which made bare-skill-name patterns pass vacuously). `expected = true` means the pattern must appear; `expected = false` means it must not. Prefer `skill_invoked` for triggering assertions; use this for matching arbitrary text the agent produced.\n102\t\n103\t```toml\n104\t[[check]]\n105\tname = \"trigger_expectation\"\n106\tpattern = \"fastskill eval\"\n107\texpected = true\n108\trequired = true\n109\t```\n110\t\n111\t### `command_contains`\n112\t\n113\tTrue if the pattern appears anywhere in the same trace-only haystack (literal substring).\n114\t\n115\t```toml\n116\t[[check]]\n117\tname = \"command_contains\"\n118\tpattern = \"validate\"\n119\trequired = true\n120\t```\n121\t\n122\t### `file_exists`\n123\t\n124\tTrue if `path` exists under the case **working directory** (project root or `project_root / workspace_subdir` for that case).\n125\t\n126\t```toml\n127\t[[check]]\n128\tname = \"file_exists\"\n129\tpath = \"report.txt\"\n130\trequired = true\n131\t```\n132\t\n133\t### `max_tool_calls`\n134\t\n135\tCounts **tool calls** in `trace.jsonl` — trace events whose payload is typed `tool_use` (a structured tool invocation the agent made) or `raw_json` (a line the agent's backend emitted that FastSkill's aikit version does not yet model). Passes if count ≤ `limit`.\n136\t\n137\tAgent prose is **not** a tool call: `message` and `reasoning` events are never counted.\n138\t\n139\t```toml\n140\t[[check]]\n141\tname = \"max_tool_calls\"\n142\tlimit = 20\n143\trequired = true\n144\t```\n145\t\n146\t> **Renamed.** This check was `max_command_count`. The old name still parses, so existing `checks.toml` files keep working, but results report the check as `max_tool_calls`. Use the new name in new files.\n147\t>\n148\t> **Re-baseline your limits if you set them before FastSkill 0.9.192.** Earlier versions counted only `raw_json`, which for some agents (notably `codex`) meant *every* trace line — including plain text — while structured tool calls counted as zero. The same agent behaviour can therefore produce a very different number now, usually a much smaller one. A limit tuned against the old count is not measuring what you think it is.\n149\t\n150\tParse/load errors: `EVAL_CHECKS_INVALID`.\n151\t\n152\t### Scoring rules when checks are present\n153\t\n154\t- Timeout, or failure to launch the agent at all: case status **`error`** with a message (checks are not used to override that).\n155\t- If **`checks` is empty** (no file or file loads zero checks): pass iff exit code is `0`.\n156\t- If checks are loaded: case **`passed`** only if every **required** check passed; `required = false` results are reported but advisory.\n157\t- A suite that parses to **zero cases** refuses to run (`EVAL_EMPTY_SUITE`) instead of reporting a vacuous `0/0 PASSED`.\n158\t\n159\t## 4. Run evals\n160\t\n161\t```bash\n162\tfastskill eval run --agent codex --output-dir ./eval-runs\n163\tfastskill eval run --agent claude --output-dir /tmp/evals --case smoke-1\n164\tfastskill eval run --agent codex --output-dir ./evals --tag smoke --model gpt-4o\n165\tfastskill eval run --agent codex --output-dir ./evals --trials 5 --threshold 0.8\n166\tfastskill eval run --all --output-dir ./evals --ci\n167\tfastskill eval run --agent codex --output-dir ./evals --json\n168\t```\n169\t\n170\t- **`--output-dir`**: base directory; a subdirectory is created per run (`YYYY-MM-DDTHH-MM-SSZ`, with numeric suffix if needed), containing one directory per agent.\n171\t- **`--all`**: run against every detected agent (mutually exclusive with `--agent`).\n172\t- **`--trials `** / **`--threshold `**: override `trials_per_case` / `pass_threshold` for this run.\n173\t- **`--ci`**: pass/fail the suite on its overall pass rate vs the threshold instead of requiring zero failed cases.\n174\t- **`--no-fail`**: still prints failures but exits `0` (for CI that only collects artifacts).\n175\t- **`--no-isolation`**: run in the project root against the ambient agent environment (see below).\n176\t\n177\tNon-zero exit on suite failure unless `--no-fail`.\n178\t\n179\t### Environment isolation (default on)\n180\t\n181\tEvery case runs in a per-case scratch workspace containing only the skill under test, with user-scope skill discovery suppressed per backend (claude `--setting-sources project`; codex via a scratch `CODEX_HOME`; pi `--no-skills --skill`). Isolation needs a **`SKILL.md`** beside `skill-project.toml` and a **`[metadata].id`** naming the skill — missing either is a loud `EVAL_ISOLATION_NO_SKILL` error, never a silent fallback.\n182\t\n183\t- The output's `isolation:` line and the `isolation` object in `summary.json` report what each run actually got (per-scope fidelity, mechanism, degrade reason, ambient skills when observable). Backends without a suppression mechanism (e.g. gemini, cursor) run with user scope reported `unsupported`; opencode degrades to inherit with a recorded reason.\n184\t- A **failed** case's scratch workspace is kept under `/workspaces/` for debugging; successful ones are deleted.\n185\t- `--no-isolation` restores the legacy behaviour: cwd is the project root and the agent sees everything installed on the machine.\n186\t- Trigger rates measured before isolation existed were taken against the ambient environment — expect scores to re-baseline on the first isolated run.\n187\t\n188\t## 5. Artifacts\n189\t\n190\tUnder each run directory (one per agent):\n191\t\n192\t- **`summary.json`**: suite-level metadata (`suite_pass`, agent, model, paths, `isolation`, per-case summaries with trials).\n193\t- **`/trial-/`**: `stdout.txt`, `stderr.txt`, `trace.jsonl`, `result.json` (`CaseResult`, including `check_results`).\n194\t- **`/aggregated.json`**: per-case trial aggregation (pass count, pass rate, status).\n195\t- **`workspaces/`**: retained scratch workspaces of failed cases (isolation only).\n196\t\n197\t`eval score` needs `summary.json` to include `checks_path` (written on `eval run` when checks were configured).\n198\t\n199\t## 6. Report and re-score\n200\t\n201\t```bash\n202\tfastskill eval report --run-dir ./eval-runs/2026-04-07T12-00-00Z\n203\tfastskill eval report --run-dir ./eval-runs/2026-04-07T12-00-00Z --json\n204\t```\n205\t\n206\tRe-run deterministic checks on saved stdout/trace without invoking the agent:\n207\t\n208\t```bash\n209\tfastskill eval score --run-dir ./eval-runs/2026-04-07T12-00-00Z\n210\t```\n211\t\n212\t**Note:** `eval score` applies `file_exists` relative to **`skill_project_root`** from `summary.json` — not per-case `workspace_subdir`, and not the (deleted) scratch workspace the case originally ran in. If you rely on workspace-specific files, prefer re-running `eval run` or align check paths with that root. A run whose `summary.json` contains zero cases refuses to re-score (`EVAL_EMPTY_SUITE`).\n213\t\n214\t## 7. Packaging and CI\n215\t\n216\t- **`fastskill package`** excludes an `evals/` tree from published ZIPs (local eval suites are not shipped). Keep prompts/checks under `evals/` or paths you do not publish if you treat evals as repo-only.\n217\t- In CI, install the CLI and the agent you pass to `--agent`, add `[tool.fastskill.eval]`, then run `eval validate` and `eval run` with a writable `--output-dir`.\n218\t\n219\t## Quick checklist\n220\t\n221\t1. `[tool.fastskill.eval]` with existing `prompts` CSV path.\n222\t2. `SKILL.md` beside `skill-project.toml` and `[metadata].id` set (required by default-on isolation).\n223\t3. CSV header: `id`, `prompt`, `should_trigger` (+ optional `tags`, `workspace_subdir`).\n224\t4. Optional `checks` TOML with `[[check]]` and correct `name` / fields — use `skill_invoked` (not a bare-name `trigger_expectation`) to assert your skill fired.\n225\t5. `fastskill eval validate` (and optionally `--agent `).\n226\t6. `fastskill eval run --agent --output-dir `.\n227\t7. Inspect `summary.json` (including its `isolation` report) and per-case `trial-/result.json` / `trace.jsonl`.\n228\t"}]},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"d11c099a-b68e-4306-b494-ccd6a3708c7b","timestamp":"2026-08-27T14:21:58.045Z","tool_use_result":{"type":"text","file":{"filePath":"/tmp/.tmpaKU6vT/.claude/skills/fastskill/references/eval.md","content":"# Skill evaluations (`fastskill eval`)\n\nThis guide matches the FastSkill CLI implementation: configuration is read from `skill-project.toml`, cases from a prompts CSV, optional scoring rules from a checks TOML, and execution uses **aikit-sdk** (`run_agent_events`) with a supported agent CLI.\n\n## What evals are for\n\nEvals run repeatable prompts against a real agent. By default each case runs **isolated**: the skill under test is deployed alone into a per-case scratch workspace, and the agent's user-level skill discovery is suppressed where the backend supports it — so trigger rates measure your skill, not whatever else is installed on the machine. Each run records stdout, stderr, a JSONL trace, and per-case results under a timestamped run directory. Optional **checks** assert properties of that output (skill invocation, substring expectations, file presence, tool-call count).\n\n## Prerequisites\n\n1. **FastSkill CLI** installed (`fastskill --version`).\n2. A **`skill-project.toml`** reachable from the current directory (FastSkill walks up the tree to find it).\n3. **`[tool.fastskill.eval]`** present with a valid `prompts` path (see below).\n4. For **`fastskill eval run`**: a **runnable agent** installed and on `PATH`. Agent keys are a closed set defined by aikit-sdk: `claude`, `codex`, `cursor`, `gemini`, `opencode`, `pi`, `aikit` — the last runs in-process and needs no CLI. If `fail_on_missing_agent = true` (default), the CLI refuses to run when the chosen agent is missing.\n\n `--agent` is validated against the agents **detected on the current machine**, not against the full key list. An agent is \"detected\" when its CLI is on `PATH` and answers `--version`. So a supported-but-not-installed key fails exactly like a typo:\n\n ```\n RUNTIME_UNKNOWN_ID: Unknown runtime ID(s): opencode. Available: aikit, claude, codex, cursor, gemini, pi\n ```\n\n If you get this for a key spelled correctly, install that agent's CLI — the `Available:` list reports what was found locally, so it differs from machine to machine and from the list above.\n\nEvals do **not** require embedding or `OPENAI_API_KEY` unless your workflow also uses `reindex` / `search`.\n\n## 1. Configure `skill-project.toml`\n\nAdd a `[tool.fastskill.eval]` table. Paths are resolved relative to the **skill project root** (directory containing `skill-project.toml`) unless absolute.\n\n| Field | Required | Default | Meaning |\n|--------|----------|---------|---------|\n| `prompts` | yes | — | Path to the prompts CSV file. |\n| `checks` | no | none | Path to checks TOML. If omitted, checks are not loaded. |\n| `timeout_seconds` | no | `900` | Per-case timeout passed to the agent run. |\n| `trials_per_case` | no | `1` | How many times each case runs; results aggregate into a per-case pass rate. Overridable with `eval run --trials ` (1–1000). |\n| `parallel` | no | CPU cores | Maximum concurrent trials within one case. |\n| `pass_threshold` | no | `1.0` | Fraction of a case's trials that must pass for the case to pass (0.0–1.0). Overridable with `--threshold`. |\n| `fail_on_missing_agent` | no | `true` | If true, `eval run` and `eval validate --agent ` error when the agent is not available. |\n\nExample:\n\n```toml\n[tool.fastskill.eval]\nprompts = \"evals/prompts.csv\"\nchecks = \"evals/checks.toml\"\ntimeout_seconds = 600\nfail_on_missing_agent = true\n```\n\n### Validate configuration\n\nFrom the skill project (or any subdirectory under it):\n\n```bash\nfastskill eval validate\nfastskill eval validate --agent codex\nfastskill eval validate --json\n```\n\nErrors use stable prefixes: `EVAL_CONFIG_MISSING`, `EVAL_PROMPTS_NOT_FOUND`, `EVAL_AGENT_UNAVAILABLE`.\n\n## 2. Define cases: prompts CSV\n\nThe suite loader expects a header row and **required** columns: `id`, `prompt`, `should_trigger`.\n\nOptional columns: `tags`, `workspace_subdir`.\n\n- **`id`**: non-empty identifier per row (used for `--case`, artifact folder names, and reports).\n- **`prompt`**: text sent to the agent; use CSV quoting for commas or newlines (`\"...\"`, escaped `\"\"` inside quotes).\n- **`should_trigger`**: `true` / `false` / `1` / other (treated false). Documented intent only; **pass/fail is not driven by this column**. When no checks file is configured, a case **passes** if the agent exits with code `0`, **fails** otherwise. When checks are loaded, a case **passes** only if **every** check passes (see below).\n- **`tags`**: comma-separated tags inside the cell (e.g. `smoke,basic`). Used with `eval run --tag `.\n- **`workspace_subdir`**: relative path that becomes the agent's working directory for the case. Under the default isolation it is created **inside the case's scratch workspace**, with fixture files copied in from the same path under the project root; under `--no-isolation` it resolves directly under the project root. If empty or column omitted, cwd is the workspace root (or the project root under `--no-isolation`).\n\nExample:\n\n```csv\nid,prompt,should_trigger,tags,workspace_subdir\nsmoke-1,\"List fastskill subcommands briefly\",true,smoke,\ndeep-1,\"Explain eval validate flags\",true,docs,\n```\n\nEmpty lines are skipped.\n\n## 3. Define checks: `checks.toml` (optional)\n\nPoint `checks` in TOML at a file containing one or more `[[check]]` tables. Each check must include **`name`**, selecting the check type. Optional **`required`** (boolean, default `true`) is honoured on all types: a case passes when every **required** check passes; a `required = false` check still runs and is reported in `check_results`, but cannot fail the case.\n\n### `skill_invoked`\n\nThe check you almost certainly want for \"did my skill fire?\". Matches a **structured `Skill` tool invocation** in the trace, not a substring of prose. When `skill` is given, it must **exactly equal** the invocation's skill-identifying input field (`skill`, `name`, or `skillName`) — `\"foo\"` does not match a `\"foo-bar\"` invocation, and a skill name merely mentioned in argument text does not count. Omit `skill` to match any skill invocation. With `expected = false` this is the only reliable way to assert non-triggering.\n\n```toml\n[[check]]\nname = \"skill_invoked\"\nskill = \"my-skill\"\nexpected = true\n```\n\n### `trigger_expectation`\n\nSubstring search in the **trace JSONL only** (agent stdout is not searched — claude's startup event lists every installed skill by name there, which made bare-skill-name patterns pass vacuously). `expected = true` means the pattern must appear; `expected = false` means it must not. Prefer `skill_invoked` for triggering assertions; use this for matching arbitrary text the agent produced.\n\n```toml\n[[check]]\nname = \"trigger_expectation\"\npattern = \"fastskill eval\"\nexpected = true\nrequired = true\n```\n\n### `command_contains`\n\nTrue if the pattern appears anywhere in the same trace-only haystack (literal substring).\n\n```toml\n[[check]]\nname = \"command_contains\"\npattern = \"validate\"\nrequired = true\n```\n\n### `file_exists`\n\nTrue if `path` exists under the case **working directory** (project root or `project_root / workspace_subdir` for that case).\n\n```toml\n[[check]]\nname = \"file_exists\"\npath = \"report.txt\"\nrequired = true\n```\n\n### `max_tool_calls`\n\nCounts **tool calls** in `trace.jsonl` — trace events whose payload is typed `tool_use` (a structured tool invocation the agent made) or `raw_json` (a line the agent's backend emitted that FastSkill's aikit version does not yet model). Passes if count ≤ `limit`.\n\nAgent prose is **not** a tool call: `message` and `reasoning` events are never counted.\n\n```toml\n[[check]]\nname = \"max_tool_calls\"\nlimit = 20\nrequired = true\n```\n\n> **Renamed.** This check was `max_command_count`. The old name still parses, so existing `checks.toml` files keep working, but results report the check as `max_tool_calls`. Use the new name in new files.\n>\n> **Re-baseline your limits if you set them before FastSkill 0.9.192.** Earlier versions counted only `raw_json`, which for some agents (notably `codex`) meant *every* trace line — including plain text — while structured tool calls counted as zero. The same agent behaviour can therefore produce a very different number now, usually a much smaller one. A limit tuned against the old count is not measuring what you think it is.\n\nParse/load errors: `EVAL_CHECKS_INVALID`.\n\n### Scoring rules when checks are present\n\n- Timeout, or failure to launch the agent at all: case status **`error`** with a message (checks are not used to override that).\n- If **`checks` is empty** (no file or file loads zero checks): pass iff exit code is `0`.\n- If checks are loaded: case **`passed`** only if every **required** check passed; `required = false` results are reported but advisory.\n- A suite that parses to **zero cases** refuses to run (`EVAL_EMPTY_SUITE`) instead of reporting a vacuous `0/0 PASSED`.\n\n## 4. Run evals\n\n```bash\nfastskill eval run --agent codex --output-dir ./eval-runs\nfastskill eval run --agent claude --output-dir /tmp/evals --case smoke-1\nfastskill eval run --agent codex --output-dir ./evals --tag smoke --model gpt-4o\nfastskill eval run --agent codex --output-dir ./evals --trials 5 --threshold 0.8\nfastskill eval run --all --output-dir ./evals --ci\nfastskill eval run --agent codex --output-dir ./evals --json\n```\n\n- **`--output-dir`**: base directory; a subdirectory is created per run (`YYYY-MM-DDTHH-MM-SSZ`, with numeric suffix if needed), containing one directory per agent.\n- **`--all`**: run against every detected agent (mutually exclusive with `--agent`).\n- **`--trials `** / **`--threshold `**: override `trials_per_case` / `pass_threshold` for this run.\n- **`--ci`**: pass/fail the suite on its overall pass rate vs the threshold instead of requiring zero failed cases.\n- **`--no-fail`**: still prints failures but exits `0` (for CI that only collects artifacts).\n- **`--no-isolation`**: run in the project root against the ambient agent environment (see below).\n\nNon-zero exit on suite failure unless `--no-fail`.\n\n### Environment isolation (default on)\n\nEvery case runs in a per-case scratch workspace containing only the skill under test, with user-scope skill discovery suppressed per backend (claude `--setting-sources project`; codex via a scratch `CODEX_HOME`; pi `--no-skills --skill`). Isolation needs a **`SKILL.md`** beside `skill-project.toml` and a **`[metadata].id`** naming the skill — missing either is a loud `EVAL_ISOLATION_NO_SKILL` error, never a silent fallback.\n\n- The output's `isolation:` line and the `isolation` object in `summary.json` report what each run actually got (per-scope fidelity, mechanism, degrade reason, ambient skills when observable). Backends without a suppression mechanism (e.g. gemini, cursor) run with user scope reported `unsupported`; opencode degrades to inherit with a recorded reason.\n- A **failed** case's scratch workspace is kept under `/workspaces/` for debugging; successful ones are deleted.\n- `--no-isolation` restores the legacy behaviour: cwd is the project root and the agent sees everything installed on the machine.\n- Trigger rates measured before isolation existed were taken against the ambient environment — expect scores to re-baseline on the first isolated run.\n\n## 5. Artifacts\n\nUnder each run directory (one per agent):\n\n- **`summary.json`**: suite-level metadata (`suite_pass`, agent, model, paths, `isolation`, per-case summaries with trials).\n- **`/trial-/`**: `stdout.txt`, `stderr.txt`, `trace.jsonl`, `result.json` (`CaseResult`, including `check_results`).\n- **`/aggregated.json`**: per-case trial aggregation (pass count, pass rate, status).\n- **`workspaces/`**: retained scratch workspaces of failed cases (isolation only).\n\n`eval score` needs `summary.json` to include `checks_path` (written on `eval run` when checks were configured).\n\n## 6. Report and re-score\n\n```bash\nfastskill eval report --run-dir ./eval-runs/2026-04-07T12-00-00Z\nfastskill eval report --run-dir ./eval-runs/2026-04-07T12-00-00Z --json\n```\n\nRe-run deterministic checks on saved stdout/trace without invoking the agent:\n\n```bash\nfastskill eval score --run-dir ./eval-runs/2026-04-07T12-00-00Z\n```\n\n**Note:** `eval score` applies `file_exists` relative to **`skill_project_root`** from `summary.json` — not per-case `workspace_subdir`, and not the (deleted) scratch workspace the case originally ran in. If you rely on workspace-specific files, prefer re-running `eval run` or align check paths with that root. A run whose `summary.json` contains zero cases refuses to re-score (`EVAL_EMPTY_SUITE`).\n\n## 7. Packaging and CI\n\n- **`fastskill package`** excludes an `evals/` tree from published ZIPs (local eval suites are not shipped). Keep prompts/checks under `evals/` or paths you do not publish if you treat evals as repo-only.\n- In CI, install the CLI and the agent you pass to `--agent`, add `[tool.fastskill.eval]`, then run `eval validate` and `eval run` with a writable `--output-dir`.\n\n## Quick checklist\n\n1. `[tool.fastskill.eval]` with existing `prompts` CSV path.\n2. `SKILL.md` beside `skill-project.toml` and `[metadata].id` set (required by default-on isolation).\n3. CSV header: `id`, `prompt`, `should_trigger` (+ optional `tags`, `workspace_subdir`).\n4. Optional `checks` TOML with `[[check]]` and correct `name` / fields — use `skill_invoked` (not a bare-name `trigger_expectation`) to assert your skill fired.\n5. `fastskill eval validate` (and optionally `--agent `).\n6. `fastskill eval run --agent --output-dir `.\n7. Inspect `summary.json` (including its `isolation` report) and per-case `trial-/result.json` / `trace.jsonl`.\n","numLines":228,"startLine":1,"totalLines":228}}} -{"type":"system","subtype":"thinking_tokens","estimated_tokens":50,"estimated_tokens_delta":50,"uuid":"02b2db88-bdf6-4c44-8290-5ac7ef914ec8","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee"} -{"type":"system","subtype":"thinking_tokens","estimated_tokens":100,"estimated_tokens_delta":50,"uuid":"dd417749-5dcb-49c9-accc-1e1005e49250","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee"} -{"type":"assistant","message":{"model":"claude-opus-4-8","id":"msg_011CeTQJWyTSFMPBLR84WRaJ","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"EogDCqgBCBEYAipAPRTMsnQFBnV/sfFE+76fLIauCdPq7EY/b8qN7RgaXgHFuW9Qz7jhfj95xhDU5UBSqmqlOa2gZtu31/ntwrwLfDIPY2xhdWRlLW9wdXMtNC04OABCCHRoaW5raW5nWiRjNmNlZmU5NC00YjA1LTQzMjgtYmRiZS02MTY0MGFhZDVmMDhyEFkopPIZT30Lu4m6CEUJoOyIAQGoAYmQwdQGsAECEgxhXU4zlNB8sNWX4CMaDIVEwXS15WetQpnIoSIwA1jqoJqmFWlWYhg16Sxj9ucLg0ac+/wRJ2TsU2QKcJXx+k+YVqCNRTU+luNmiqs7KowBBwxFRConbaNRuhuEo73kzcJ9FCMIYSJFUCbgFQv1+boFXboZOSj+Q5018vG8/TJCBi7s8wFmn0KUnuu1FCuxI+TohRSumJRRqRLY+XY4xoMEB/T2EZgMkvPVfByaGpaVzI6e9pPtzkrmy2hpriyi5fOCJuUxBmwXchJJODT8V2CXMpjNIYLIoKZO8sgYAQ=="}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":5879,"cache_read_input_tokens":28301,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5879},"output_tokens":4,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"71534c31-edc7-4da6-867e-9e0b33026f40","timestamp":"2026-08-27T14:22:02.270Z","request_id":"req_011CeTQJUA3G8BLbxGL3Spyr"} -{"type":"assistant","message":{"model":"claude-opus-4-8","id":"msg_011CeTQJWyTSFMPBLR84WRaJ","type":"message","role":"assistant","content":[{"type":"text","text":"`fastskill eval` is the tool for this. It runs repeatable prompts against a real agent, checks whether your skill actually fires, and asserts properties of the output — all isolated by default so trigger rates measure *your* skill, not whatever else is on the machine. Here's the workflow end to end.\n\n## 1. Set up the eval config\n\nAdd a `[tool.fastskill.eval]` table to your `skill-project.toml` (paths are relative to the file's directory):\n\n```toml\n[tool.fastskill.eval]\nprompts = \"evals/prompts.csv\"\nchecks = \"evals/checks.toml\" # optional\ntimeout_seconds = 600\ntrials_per_case = 3 # run each case N times → per-case pass rate\npass_threshold = 0.8 # fraction of trials that must pass\nfail_on_missing_agent = true\n```\n\nIsolation (on by default) requires a `SKILL.md` beside `skill-project.toml` and a `[metadata].id` — otherwise you get a loud `EVAL_ISOLATION_NO_SKILL` error.\n\n## 2. Write the cases — `evals/prompts.csv`\n\nHeader row with required columns `id`, `prompt`, `should_trigger` (plus optional `tags`, `workspace_subdir`):\n\n```csv\nid,prompt,should_trigger,tags,workspace_subdir\nsmoke-1,\"List fastskill subcommands briefly\",true,smoke,\nneg-1,\"What's the capital of France?\",false,negative,\n```\n\nInclude **negative cases** (`should_trigger=false`) — a good skill fires when it should *and stays quiet when it shouldn't*. Note `should_trigger` is documentation of intent only; it does **not** drive pass/fail. Checks do that.\n\n## 3. Add checks — `evals/checks.toml` (optional but recommended)\n\nWithout a checks file, a case passes if the agent merely exits 0. With checks, a case passes only if **every** `required` check passes. The one you want for \"did my skill fire?\":\n\n```toml\n[[check]]\nname = \"skill_invoked\" # matches a structured Skill tool call, not prose\nskill = \"my-skill\"\nexpected = true\n\n# assert NON-triggering on a negative case:\n[[check]]\nname = \"skill_invoked\"\nskill = \"my-skill\"\nexpected = false\n```\n\nOther check types: `trigger_expectation` (substring in the trace), `command_contains`, `file_exists`, `max_tool_calls`. Use `skill_invoked` rather than a bare-name `trigger_expectation` for triggering — the latter can pass vacuously.\n\n## 4. Validate, then run\n\n```bash\nfastskill eval validate --agent claude # config/prompts/agent sanity check\nfastskill eval run --agent claude --output-dir ./eval-runs --trials 5 --threshold 0.8\n```\n\nThe agent (`claude`, `codex`, etc.) must be installed and on `PATH`. Each run writes a timestamped dir with `summary.json`, per-case `trial-/result.json` + `trace.jsonl`, and — for **failed** cases only — the retained scratch workspace under `workspaces/` for debugging.\n\n## 5. Inspect / re-score\n\n```bash\nfastskill eval report --run-dir ./eval-runs/\nfastskill eval score --run-dir ./eval-runs/ # re-run checks on saved traces, no agent call\n```\n\n## Ship gate\nWire it into CI so a regression blocks the release:\n\n```bash\nfastskill eval run --all --output-dir ./evals --ci\n```\n\n`--ci` fails the suite on overall pass rate vs threshold; `--no-fail` collects artifacts without failing the build.\n\n---\n\n**One step beyond evals:** if a skill scores poorly, `fastskill optimize run --config optimize.toml` runs an automated loop that proposes edits to the skill doc, re-runs these same eval cases, and keeps changes only when the score improves — so your eval suite doubles as the objective for improving the skill.\n\nWant me to scaffold the `evals/prompts.csv`, `checks.toml`, and the config block for a specific skill in this repo?"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":5879,"cache_read_input_tokens":28301,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5879},"output_tokens":4,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","uuid":"bf7350bc-dd66-4083-b1c5-ea44e980a145","timestamp":"2026-08-27T14:22:18.948Z","request_id":"req_011CeTQJUA3G8BLbxGL3Spyr"} -{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":30584,"duration_api_ms":31864,"ttft_ms":3121,"ttft_stream_ms":1593,"time_to_request_ms":50,"num_turns":4,"result":"`fastskill eval` is the tool for this. It runs repeatable prompts against a real agent, checks whether your skill actually fires, and asserts properties of the output — all isolated by default so trigger rates measure *your* skill, not whatever else is on the machine. Here's the workflow end to end.\n\n## 1. Set up the eval config\n\nAdd a `[tool.fastskill.eval]` table to your `skill-project.toml` (paths are relative to the file's directory):\n\n```toml\n[tool.fastskill.eval]\nprompts = \"evals/prompts.csv\"\nchecks = \"evals/checks.toml\" # optional\ntimeout_seconds = 600\ntrials_per_case = 3 # run each case N times → per-case pass rate\npass_threshold = 0.8 # fraction of trials that must pass\nfail_on_missing_agent = true\n```\n\nIsolation (on by default) requires a `SKILL.md` beside `skill-project.toml` and a `[metadata].id` — otherwise you get a loud `EVAL_ISOLATION_NO_SKILL` error.\n\n## 2. Write the cases — `evals/prompts.csv`\n\nHeader row with required columns `id`, `prompt`, `should_trigger` (plus optional `tags`, `workspace_subdir`):\n\n```csv\nid,prompt,should_trigger,tags,workspace_subdir\nsmoke-1,\"List fastskill subcommands briefly\",true,smoke,\nneg-1,\"What's the capital of France?\",false,negative,\n```\n\nInclude **negative cases** (`should_trigger=false`) — a good skill fires when it should *and stays quiet when it shouldn't*. Note `should_trigger` is documentation of intent only; it does **not** drive pass/fail. Checks do that.\n\n## 3. Add checks — `evals/checks.toml` (optional but recommended)\n\nWithout a checks file, a case passes if the agent merely exits 0. With checks, a case passes only if **every** `required` check passes. The one you want for \"did my skill fire?\":\n\n```toml\n[[check]]\nname = \"skill_invoked\" # matches a structured Skill tool call, not prose\nskill = \"my-skill\"\nexpected = true\n\n# assert NON-triggering on a negative case:\n[[check]]\nname = \"skill_invoked\"\nskill = \"my-skill\"\nexpected = false\n```\n\nOther check types: `trigger_expectation` (substring in the trace), `command_contains`, `file_exists`, `max_tool_calls`. Use `skill_invoked` rather than a bare-name `trigger_expectation` for triggering — the latter can pass vacuously.\n\n## 4. Validate, then run\n\n```bash\nfastskill eval validate --agent claude # config/prompts/agent sanity check\nfastskill eval run --agent claude --output-dir ./eval-runs --trials 5 --threshold 0.8\n```\n\nThe agent (`claude`, `codex`, etc.) must be installed and on `PATH`. Each run writes a timestamped dir with `summary.json`, per-case `trial-/result.json` + `trace.jsonl`, and — for **failed** cases only — the retained scratch workspace under `workspaces/` for debugging.\n\n## 5. Inspect / re-score\n\n```bash\nfastskill eval report --run-dir ./eval-runs/\nfastskill eval score --run-dir ./eval-runs/ # re-run checks on saved traces, no agent call\n```\n\n## Ship gate\nWire it into CI so a regression blocks the release:\n\n```bash\nfastskill eval run --all --output-dir ./evals --ci\n```\n\n`--ci` fails the suite on overall pass rate vs threshold; `--no-fail` collects artifacts without failing the build.\n\n---\n\n**One step beyond evals:** if a skill scores poorly, `fastskill optimize run --config optimize.toml` runs an automated loop that proposes edits to the skill doc, re-runs these same eval cases, and keeps changes only when the score improves — so your eval suite doubles as the objective for improving the skill.\n\nWant me to scaffold the `evals/prompts.csv`, `checks.toml`, and the config block for a specific skill in this repo?","stop_reason":"end_turn","session_id":"71b193b3-6966-40ec-ad3e-2f16db6488ee","total_cost_usd":0.2634765,"usage":{"input_tokens":6,"cache_creation_input_tokens":18599,"cache_read_input_tokens":64789,"output_tokens":1778,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":18599,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1429,"cache_read_input_tokens":28301,"cache_creation_input_tokens":5879,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5879},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-haiku-4-5-20251001":{"inputTokens":532,"outputTokens":16,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.000612,"contextWindow":200000,"maxOutputTokens":32000},"claude-opus-4-8[1m]":{"inputTokens":6,"outputTokens":1778,"cacheReadInputTokens":64789,"cacheCreationInputTokens":18599,"webSearchRequests":0,"costUSD":0.2628645,"contextWindow":1000000,"maxOutputTokens":64000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"e5c16fd5-cac7-43f6-b196-03f4717fa66b"} +{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Validate with `fastskill eval validate`, run with `fastskill eval run`, and inspect results with `fastskill eval report`. Use `fastskill optimization run` when you want an iterative improvement loop."}]}} +{"type":"result","subtype":"success","is_error":false,"result":"Validate with `fastskill eval validate`, run with `fastskill eval run`, and inspect results with `fastskill eval report`. Use `fastskill optimization run` when you want an iterative improvement loop."} diff --git a/evals/fixtures/pass/eval-skill/trial-1/trace.jsonl b/evals/fixtures/pass/eval-skill/trial-1/trace.jsonl index f7fef9a..5a86d93 100644 --- a/evals/fixtures/pass/eval-skill/trial-1/trace.jsonl +++ b/evals/fixtures/pass/eval-skill/trial-1/trace.jsonl @@ -1,10 +1,4 @@ -{"seq":0,"payload":{"type":"message","text":"This maps directly to the `fastskill` toolkit, which has an eval command and a dedicated reference for setup. Let me pull it up.","role":"assistant"}} -{"seq":1,"payload":{"type":"tool_use","call_id":"toolu_018oBohn6g2ijtAafjuWcEb8","tool_name":"Skill","input":{"args":"eval","skill":"fastskill"}}} -{"seq":2,"payload":{"type":"tool_result","call_id":"toolu_018oBohn6g2ijtAafjuWcEb8","output":"Launching skill: fastskill","is_error":false}} -{"seq":3,"payload":{"type":"message","text":"Let me read the dedicated eval guide, which the skill instructs me to follow for the full schema and commands.","role":"assistant"}} -{"seq":4,"payload":{"type":"tool_use","call_id":"toolu_0182ZtNA5ZUoB6f3GstTUjho","tool_name":"Read","input":{"file_path":"/tmp/.tmpaKU6vT/.claude/skills/fastskill/references/eval.md"}}} -{"seq":5,"payload":{"type":"tool_result","call_id":"toolu_0182ZtNA5ZUoB6f3GstTUjho","output":"1\t# Skill evaluations (`fastskill eval`)\n2\t\n3\tThis guide matches the FastSkill CLI implementation: configuration is read from `skill-project.toml`, cases from a prompts CSV, optional scoring rules from a checks TOML, and execution uses **aikit-sdk** (`run_agent_events`) with a supported agent CLI.\n4\t\n5\t## What evals are for\n6\t\n7\tEvals run repeatable prompts against a real agent. By default each case runs **isolated**: the skill under test is deployed alone into a per-case scratch workspace, and the agent's user-level skill discovery is suppressed where the backend supports it — so trigger rates measure your skill, not whatever else is installed on the machine. Each run records stdout, stderr, a JSONL trace, and per-case results under a timestamped run directory. Optional **checks** assert properties of that output (skill invocation, substring expectations, file presence, tool-call count).\n8\t\n9\t## Prerequisites\n10\t\n11\t1. **FastSkill CLI** installed (`fastskill --version`).\n12\t2. A **`skill-project.toml`** reachable from the current directory (FastSkill walks up the tree to find it).\n13\t3. **`[tool.fastskill.eval]`** present with a valid `prompts` path (see below).\n14\t4. For **`fastskill eval run`**: a **runnable agent** installed and on `PATH`. Agent keys are a closed set defined by aikit-sdk: `claude`, `codex`, `cursor`, `gemini`, `opencode`, `pi`, `aikit` — the last runs in-process and needs no CLI. If `fail_on_missing_agent = true` (default), the CLI refuses to run when the chosen agent is missing.\n15\t\n16\t `--agent` is validated against the agents **detected on the current machine**, not against the full key list. An agent is \"detected\" when its CLI is on `PATH` and answers `--version`. So a supported-but-not-installed key fails exactly like a typo:\n17\t\n18\t ```\n19\t RUNTIME_UNKNOWN_ID: Unknown runtime ID(s): opencode. Available: aikit, claude, codex, cursor, gemini, pi\n20\t ```\n21\t\n22\t If you get this for a key spelled correctly, install that agent's CLI — the `Available:` list reports what was found locally, so it differs from machine to machine and from the list above.\n23\t\n24\tEvals do **not** require embedding or `OPENAI_API_KEY` unless your workflow also uses `reindex` / `search`.\n25\t\n26\t## 1. Configure `skill-project.toml`\n27\t\n28\tAdd a `[tool.fastskill.eval]` table. Paths are resolved relative to the **skill project root** (directory containing `skill-project.toml`) unless absolute.\n29\t\n30\t| Field | Required | Default | Meaning |\n31\t|--------|----------|---------|---------|\n32\t| `prompts` | yes | — | Path to the prompts CSV file. |\n33\t| `checks` | no | none | Path to checks TOML. If omitted, checks are not loaded. |\n34\t| `timeout_seconds` | no | `900` | Per-case timeout passed to the agent run. |\n35\t| `trials_per_case` | no | `1` | How many times each case runs; results aggregate into a per-case pass rate. Overridable with `eval run --trials ` (1–1000). |\n36\t| `parallel` | no | CPU cores | Maximum concurrent trials within one case. |\n37\t| `pass_threshold` | no | `1.0` | Fraction of a case's trials that must pass for the case to pass (0.0–1.0). Overridable with `--threshold`. |\n38\t| `fail_on_missing_agent` | no | `true` | If true, `eval run` and `eval validate --agent ` error when the agent is not available. |\n39\t\n40\tExample:\n41\t\n42\t```toml\n43\t[tool.fastskill.eval]\n44\tprompts = \"evals/prompts.csv\"\n45\tchecks = \"evals/checks.toml\"\n46\ttimeout_seconds = 600\n47\tfail_on_missing_agent = true\n48\t```\n49\t\n50\t### Validate configuration\n51\t\n52\tFrom the skill project (or any subdirectory under it):\n53\t\n54\t```bash\n55\tfastskill eval validate\n56\tfastskill eval validate --agent codex\n57\tfastskill eval validate --json\n58\t```\n59\t\n60\tErrors use stable prefixes: `EVAL_CONFIG_MISSING`, `EVAL_PROMPTS_NOT_FOUND`, `EVAL_AGENT_UNAVAILABLE`.\n61\t\n62\t## 2. Define cases: prompts CSV\n63\t\n64\tThe suite loader expects a header row and **required** columns: `id`, `prompt`, `should_trigger`.\n65\t\n66\tOptional columns: `tags`, `workspace_subdir`.\n67\t\n68\t- **`id`**: non-empty identifier per row (used for `--case`, artifact folder names, and reports).\n69\t- **`prompt`**: text sent to the agent; use CSV quoting for commas or newlines (`\"...\"`, escaped `\"\"` inside quotes).\n70\t- **`should_trigger`**: `true` / `false` / `1` / other (treated false). Documented intent only; **pass/fail is not driven by this column**. When no checks file is configured, a case **passes** if the agent exits with code `0`, **fails** otherwise. When checks are loaded, a case **passes** only if **every** check passes (see below).\n71\t- **`tags`**: comma-separated tags inside the cell (e.g. `smoke,basic`). Used with `eval run --tag `.\n72\t- **`workspace_subdir`**: relative path that becomes the agent's working directory for the case. Under the default isolation it is created **inside the case's scratch workspace**, with fixture files copied in from the same path under the project root; under `--no-isolation` it resolves directly under the project root. If empty or column omitted, cwd is the workspace root (or the project root under `--no-isolation`).\n73\t\n74\tExample:\n75\t\n76\t```csv\n77\tid,prompt,should_trigger,tags,workspace_subdir\n78\tsmoke-1,\"List fastskill subcommands briefly\",true,smoke,\n79\tdeep-1,\"Explain eval validate flags\",true,docs,\n80\t```\n81\t\n82\tEmpty lines are skipped.\n83\t\n84\t## 3. Define checks: `checks.toml` (optional)\n85\t\n86\tPoint `checks` in TOML at a file containing one or more `[[check]]` tables. Each check must include **`name`**, selecting the check type. Optional **`required`** (boolean, default `true`) is honoured on all types: a case passes when every **required** check passes; a `required = false` check still runs and is reported in `check_results`, but cannot fail the case.\n87\t\n88\t### `skill_invoked`\n89\t\n90\tThe check you almost certainly want for \"did my skill fire?\". Matches a **structured `Skill` tool invocation** in the trace, not a substring of prose. When `skill` is given, it must **exactly equal** the invocation's skill-identifying input field (`skill`, `name`, or `skillName`) — `\"foo\"` does not match a `\"foo-bar\"` invocation, and a skill name merely mentioned in argument text does not count. Omit `skill` to match any skill invocation. With `expected = false` this is the only reliable way to assert non-triggering.\n91\t\n92\t```toml\n93\t[[check]]\n94\tname = \"skill_invoked\"\n95\tskill = \"my-skill\"\n96\texpected = true\n97\t```\n98\t\n99\t### `trigger_expectation`\n100\t\n101\tSubstring search in the **trace JSONL only** (agent stdout is not searched — claude's startup event lists every installed skill by name there, which made bare-skill-name patterns pass vacuously). `expected = true` means the pattern must appear; `expected = false` means it must not. Prefer `skill_invoked` for triggering assertions; use this for matching arbitrary text the agent produced.\n102\t\n103\t```toml\n104\t[[check]]\n105\tname = \"trigger_expectation\"\n106\tpattern = \"fastskill eval\"\n107\texpected = true\n108\trequired = true\n109\t```\n110\t\n111\t### `command_contains`\n112\t\n113\tTrue if the pattern appears anywhere in the same trace-only haystack (literal substring).\n114\t\n115\t```toml\n116\t[[check]]\n117\tname = \"command_contains\"\n118\tpattern = \"validate\"\n119\trequired = true\n120\t```\n121\t\n122\t### `file_exists`\n123\t\n124\tTrue if `path` exists under the case **working directory** (project root or `project_root / workspace_subdir` for that case).\n125\t\n126\t```toml\n127\t[[check]]\n128\tname = \"file_exists\"\n129\tpath = \"report.txt\"\n130\trequired = true\n131\t```\n132\t\n133\t### `max_tool_calls`\n134\t\n135\tCounts **tool calls** in `trace.jsonl` — trace events whose payload is typed `tool_use` (a structured tool invocation the agent made) or `raw_json` (a line the agent's backend emitted that FastSkill's aikit version does not yet model). Passes if count ≤ `limit`.\n136\t\n137\tAgent prose is **not** a tool call: `message` and `reasoning` events are never counted.\n138\t\n139\t```toml\n140\t[[check]]\n141\tname = \"max_tool_calls\"\n142\tlimit = 20\n143\trequired = true\n144\t```\n145\t\n146\t> **Renamed.** This check was `max_command_count`. The old name still parses, so existing `checks.toml` files keep working, but results report the check as `max_tool_calls`. Use the new name in new files.\n147\t>\n148\t> **Re-baseline your limits if you set them before FastSkill 0.9.192.** Earlier versions counted only `raw_json`, which for some agents (notably `codex`) meant *every* trace line — including plain text — while structured tool calls counted as zero. The same agent behaviour can therefore produce a very different number now, usually a much smaller one. A limit tuned against the old count is not measuring what you think it is.\n149\t\n150\tParse/load errors: `EVAL_CHECKS_INVALID`.\n151\t\n152\t### Scoring rules when checks are present\n153\t\n154\t- Timeout, or failure to launch the agent at all: case status **`error`** with a message (checks are not used to override that).\n155\t- If **`checks` is empty** (no file or file loads zero checks): pass iff exit code is `0`.\n156\t- If checks are loaded: case **`passed`** only if every **required** check passed; `required = false` results are reported but advisory.\n157\t- A suite that parses to **zero cases** refuses to run (`EVAL_EMPTY_SUITE`) instead of reporting a vacuous `0/0 PASSED`.\n158\t\n159\t## 4. Run evals\n160\t\n161\t```bash\n162\tfastskill eval run --agent codex --output-dir ./eval-runs\n163\tfastskill eval run --agent claude --output-dir /tmp/evals --case smoke-1\n164\tfastskill eval run --agent codex --output-dir ./evals --tag smoke --model gpt-4o\n165\tfastskill eval run --agent codex --output-dir ./evals --trials 5 --threshold 0.8\n166\tfastskill eval run --all --output-dir ./evals --ci\n167\tfastskill eval run --agent codex --output-dir ./evals --json\n168\t```\n169\t\n170\t- **`--output-dir`**: base directory; a subdirectory is created per run (`YYYY-MM-DDTHH-MM-SSZ`, with numeric suffix if needed), containing one directory per agent.\n171\t- **`--all`**: run against every detected agent (mutually exclusive with `--agent`).\n172\t- **`--trials `** / **`--threshold `**: override `trials_per_case` / `pass_threshold` for this run.\n173\t- **`--ci`**: pass/fail the suite on its overall pass rate vs the threshold instead of requiring zero failed cases.\n174\t- **`--no-fail`**: still prints failures but exits `0` (for CI that only collects artifacts).\n175\t- **`--no-isolation`**: run in the project root against the ambient agent environment (see below).\n176\t\n177\tNon-zero exit on suite failure unless `--no-fail`.\n178\t\n179\t### Environment isolation (default on)\n180\t\n181\tEvery case runs in a per-case scratch workspace containing only the skill under test, with user-scope skill discovery suppressed per backend (claude `--setting-sources project`; codex via a scratch `CODEX_HOME`; pi `--no-skills --skill`). Isolation needs a **`SKILL.md`** beside `skill-project.toml` and a **`[metadata].id`** naming the skill — missing either is a loud `EVAL_ISOLATION_NO_SKILL` error, never a silent fallback.\n182\t\n183\t- The output's `isolation:` line and the `isolation` object in `summary.json` report what each run actually got (per-scope fidelity, mechanism, degrade reason, ambient skills when observable). Backends without a suppression mechanism (e.g. gemini, cursor) run with user scope reported `unsupported`; opencode degrades to inherit with a recorded reason.\n184\t- A **failed** case's scratch workspace is kept under `/workspaces/` for debugging; successful ones are deleted.\n185\t- `--no-isolation` restores the legacy behaviour: cwd is the project root and the agent sees everything installed on the machine.\n186\t- Trigger rates measured before isolation existed were taken against the ambient environment — expect scores to re-baseline on the first isolated run.\n187\t\n188\t## 5. Artifacts\n189\t\n190\tUnder each run directory (one per agent):\n191\t\n192\t- **`summary.json`**: suite-level metadata (`suite_pass`, agent, model, paths, `isolation`, per-case summaries with trials).\n193\t- **`/trial-/`**: `stdout.txt`, `stderr.txt`, `trace.jsonl`, `result.json` (`CaseResult`, including `check_results`).\n194\t- **`/aggregated.json`**: per-case trial aggregation (pass count, pass rate, status).\n195\t- **`workspaces/`**: retained scratch workspaces of failed cases (isolation only).\n196\t\n197\t`eval score` needs `summary.json` to include `checks_path` (written on `eval run` when checks were configured).\n198\t\n199\t## 6. Report and re-score\n200\t\n201\t```bash\n202\tfastskill eval report --run-dir ./eval-runs/2026-04-07T12-00-00Z\n203\tfastskill eval report --run-dir ./eval-runs/2026-04-07T12-00-00Z --json\n204\t```\n205\t\n206\tRe-run deterministic checks on saved stdout/trace without invoking the agent:\n207\t\n208\t```bash\n209\tfastskill eval score --run-dir ./eval-runs/2026-04-07T12-00-00Z\n210\t```\n211\t\n212\t**Note:** `eval score` applies `file_exists` relative to **`skill_project_root`** from `summary.json` — not per-case `workspace_subdir`, and not the (deleted) scratch workspace the case originally ran in. If you rely on workspace-specific files, prefer re-running `eval run` or align check paths with that root. A run whose `summary.json` contains zero cases refuses to re-score (`EVAL_EMPTY_SUITE`).\n213\t\n214\t## 7. Packaging and CI\n215\t\n216\t- **`fastskill package`** excludes an `evals/` tree from published ZIPs (local eval suites are not shipped). Keep prompts/checks under `evals/` or paths you do not publish if you treat evals as repo-only.\n217\t- In CI, install the CLI and the agent you pass to `--agent`, add `[tool.fastskill.eval]`, then run `eval validate` and `eval run` with a writable `--output-dir`.\n218\t\n219\t## Quick checklist\n220\t\n221\t1. `[tool.fastskill.eval]` with existing `prompts` CSV path.\n222\t2. `SKILL.md` beside `skill-project.toml` and `[metadata].id` set (required by default-on isolation).\n223\t3. CSV header: `id`, `prompt`, `should_trigger` (+ optional `tags`, `workspace_subdir`).\n224\t4. Optional `checks` TOML with `[[check]]` and correct `name` / fields — use `skill_invoked` (not a bare-name `trigger_expectation`) to assert your skill fired.\n225\t5. `fastskill eval validate` (and optionally `--agent `).\n226\t6. `fastskill eval run --agent --output-dir `.\n227\t7. Inspect `summary.json` (including its `isolation` report) and per-case `trial-/result.json` / `trace.jsonl`.\n228\t","is_error":false}} -{"seq":6,"payload":{"type":"message","text":"`fastskill eval` is the tool for this. It runs repeatable prompts against a real agent, checks whether your skill actually fires, and asserts properties of the output — all isolated by default so trigger rates measure *your* skill, not whatever else is on the machine. Here's the workflow end to end.\n\n## 1. Set up the eval config\n\nAdd a `[tool.fastskill.eval]` table to your `skill-project.toml` (paths are relative to the file's directory):\n\n```toml\n[tool.fastskill.eval]\nprompts = \"evals/prompts.csv\"\nchecks = \"evals/checks.toml\" # optional\ntimeout_seconds = 600\ntrials_per_case = 3 # run each case N times → per-case pass rate\npass_threshold = 0.8 # fraction of trials that must pass\nfail_on_missing_agent = true\n```\n\nIsolation (on by default) requires a `SKILL.md` beside `skill-project.toml` and a `[metadata].id` — otherwise you get a loud `EVAL_ISOLATION_NO_SKILL` error.\n\n## 2. Write the cases — `evals/prompts.csv`\n\nHeader row with required columns `id`, `prompt`, `should_trigger` (plus optional `tags`, `workspace_subdir`):\n\n```csv\nid,prompt,should_trigger,tags,workspace_subdir\nsmoke-1,\"List fastskill subcommands briefly\",true,smoke,\nneg-1,\"What's the capital of France?\",false,negative,\n```\n\nInclude **negative cases** (`should_trigger=false`) — a good skill fires when it should *and stays quiet when it shouldn't*. Note `should_trigger` is documentation of intent only; it does **not** drive pass/fail. Checks do that.\n\n## 3. Add checks — `evals/checks.toml` (optional but recommended)\n\nWithout a checks file, a case passes if the agent merely exits 0. With checks, a case passes only if **every** `required` check passes. The one you want for \"did my skill fire?\":\n\n```toml\n[[check]]\nname = \"skill_invoked\" # matches a structured Skill tool call, not prose\nskill = \"my-skill\"\nexpected = true\n\n# assert NON-triggering on a negative case:\n[[check]]\nname = \"skill_invoked\"\nskill = \"my-skill\"\nexpected = false\n```\n\nOther check types: `trigger_expectation` (substring in the trace), `command_contains`, `file_exists`, `max_tool_calls`. Use `skill_invoked` rather than a bare-name `trigger_expectation` for triggering — the latter can pass vacuously.\n\n## 4. Validate, then run\n\n```bash\nfastskill eval validate --agent claude # config/prompts/agent sanity check\nfastskill eval run --agent claude --output-dir ./eval-runs --trials 5 --threshold 0.8\n```\n\nThe agent (`claude`, `codex`, etc.) must be installed and on `PATH`. Each run writes a timestamped dir with `summary.json`, per-case `trial-/result.json` + `trace.jsonl`, and — for **failed** cases only — the retained scratch workspace under `workspaces/` for debugging.\n\n## 5. Inspect / re-score\n\n```bash\nfastskill eval report --run-dir ./eval-runs/\nfastskill eval score --run-dir ./eval-runs/ # re-run checks on saved traces, no agent call\n```\n\n## Ship gate\nWire it into CI so a regression blocks the release:\n\n```bash\nfastskill eval run --all --output-dir ./evals --ci\n```\n\n`--ci` fails the suite on overall pass rate vs threshold; `--no-fail` collects artifacts without failing the build.\n\n---\n\n**One step beyond evals:** if a skill scores poorly, `fastskill optimize run --config optimize.toml` runs an automated loop that proposes edits to the skill doc, re-runs these same eval cases, and keeps changes only when the score improves — so your eval suite doubles as the objective for improving the skill.\n\nWant me to scaffold the `evals/prompts.csv`, `checks.toml`, and the config block for a specific skill in this repo?","role":"assistant"}} -{"seq":7,"payload":{"type":"message","text":"`fastskill eval` is the tool for this. It runs repeatable prompts against a real agent, checks whether your skill actually fires, and asserts properties of the output — all isolated by default so trigger rates measure *your* skill, not whatever else is on the machine. Here's the workflow end to end.\n\n## 1. Set up the eval config\n\nAdd a `[tool.fastskill.eval]` table to your `skill-project.toml` (paths are relative to the file's directory):\n\n```toml\n[tool.fastskill.eval]\nprompts = \"evals/prompts.csv\"\nchecks = \"evals/checks.toml\" # optional\ntimeout_seconds = 600\ntrials_per_case = 3 # run each case N times → per-case pass rate\npass_threshold = 0.8 # fraction of trials that must pass\nfail_on_missing_agent = true\n```\n\nIsolation (on by default) requires a `SKILL.md` beside `skill-project.toml` and a `[metadata].id` — otherwise you get a loud `EVAL_ISOLATION_NO_SKILL` error.\n\n## 2. Write the cases — `evals/prompts.csv`\n\nHeader row with required columns `id`, `prompt`, `should_trigger` (plus optional `tags`, `workspace_subdir`):\n\n```csv\nid,prompt,should_trigger,tags,workspace_subdir\nsmoke-1,\"List fastskill subcommands briefly\",true,smoke,\nneg-1,\"What's the capital of France?\",false,negative,\n```\n\nInclude **negative cases** (`should_trigger=false`) — a good skill fires when it should *and stays quiet when it shouldn't*. Note `should_trigger` is documentation of intent only; it does **not** drive pass/fail. Checks do that.\n\n## 3. Add checks — `evals/checks.toml` (optional but recommended)\n\nWithout a checks file, a case passes if the agent merely exits 0. With checks, a case passes only if **every** `required` check passes. The one you want for \"did my skill fire?\":\n\n```toml\n[[check]]\nname = \"skill_invoked\" # matches a structured Skill tool call, not prose\nskill = \"my-skill\"\nexpected = true\n\n# assert NON-triggering on a negative case:\n[[check]]\nname = \"skill_invoked\"\nskill = \"my-skill\"\nexpected = false\n```\n\nOther check types: `trigger_expectation` (substring in the trace), `command_contains`, `file_exists`, `max_tool_calls`. Use `skill_invoked` rather than a bare-name `trigger_expectation` for triggering — the latter can pass vacuously.\n\n## 4. Validate, then run\n\n```bash\nfastskill eval validate --agent claude # config/prompts/agent sanity check\nfastskill eval run --agent claude --output-dir ./eval-runs --trials 5 --threshold 0.8\n```\n\nThe agent (`claude`, `codex`, etc.) must be installed and on `PATH`. Each run writes a timestamped dir with `summary.json`, per-case `trial-/result.json` + `trace.jsonl`, and — for **failed** cases only — the retained scratch workspace under `workspaces/` for debugging.\n\n## 5. Inspect / re-score\n\n```bash\nfastskill eval report --run-dir ./eval-runs/\nfastskill eval score --run-dir ./eval-runs/ # re-run checks on saved traces, no agent call\n```\n\n## Ship gate\nWire it into CI so a regression blocks the release:\n\n```bash\nfastskill eval run --all --output-dir ./evals --ci\n```\n\n`--ci` fails the suite on overall pass rate vs threshold; `--no-fail` collects artifacts without failing the build.\n\n---\n\n**One step beyond evals:** if a skill scores poorly, `fastskill optimize run --config optimize.toml` runs an automated loop that proposes edits to the skill doc, re-runs these same eval cases, and keeps changes only when the score improves — so your eval suite doubles as the objective for improving the skill.\n\nWant me to scaffold the `evals/prompts.csv`, `checks.toml`, and the config block for a specific skill in this repo?","role":"assistant"}} -{"seq":8,"payload":{"type":"token_usage_line","usage":{"cache_creation_tokens":18599,"cache_read_tokens":64789,"input_tokens":6,"output_tokens":1778,"reasoning_tokens":null,"total_tokens":null},"source":"claude","raw_agent_line_seq":7}} -{"seq":9,"payload":{"type":"unknown","payload_type":"result","raw":"Result { text: \"`fastskill eval` is the tool for this. It runs repeatable prompts against a real agent, checks whether your skill actually fires, and asserts properties of the output — all isolated by default so trigger rates measure *your* skill, not whatever else is on the machine. Here's the workflow end to end.\\n\\n## 1. Set up the eval config\\n\\nAdd a `[tool.fastskill.eval]` table to your `skill-project.toml` (paths are relative to the file's directory):\\n\\n```toml\\n[tool.fastskill.eval]\\nprompts = \\\"evals/prompts.csv\\\"\\nchecks = \\\"evals/checks.toml\\\" # optional\\ntimeout_seconds = 600\\ntrials_per_case = 3 # run each case N times → per-case pass rate\\npass_threshold = 0.8 # fraction of trials that must pass\\nfail_on_missing_agent = true\\n```\\n\\nIsolation (on by default) requires a `SKILL.md` beside `skill-project.toml` and a `[metadata].id` — otherwise you get a loud `EVAL_ISOLATION_NO_SKILL` error.\\n\\n## 2. Write the cases — `evals/prompts.csv`\\n\\nHeader row with required columns `id`, `prompt`, `should_trigger` (plus optional `tags`, `workspace_subdir`):\\n\\n```csv\\nid,prompt,should_trigger,tags,workspace_subdir\\nsmoke-1,\\\"List fastskill subcommands briefly\\\",true,smoke,\\nneg-1,\\\"What's the capital of France?\\\",false,negative,\\n```\\n\\nInclude **negative cases** (`should_trigger=false`) — a good skill fires when it should *and stays quiet when it shouldn't*. Note `should_trigger` is documentation of intent only; it does **not** drive pass/fail. Checks do that.\\n\\n## 3. Add checks — `evals/checks.toml` (optional but recommended)\\n\\nWithout a checks file, a case passes if the agent merely exits 0. With checks, a case passes only if **every** `required` check passes. The one you want for \\\"did my skill fire?\\\":\\n\\n```toml\\n[[check]]\\nname = \\\"skill_invoked\\\" # matches a structured Skill tool call, not prose\\nskill = \\\"my-skill\\\"\\nexpected = true\\n\\n# assert NON-triggering on a negative case:\\n[[check]]\\nname = \\\"skill_invoked\\\"\\nskill = \\\"my-skill\\\"\\nexpected = false\\n```\\n\\nOther check types: `trigger_expectation` (substring in the trace), `command_contains`, `file_exists`, `max_tool_calls`. Use `skill_invoked` rather than a bare-name `trigger_expectation` for triggering — the latter can pass vacuously.\\n\\n## 4. Validate, then run\\n\\n```bash\\nfastskill eval validate --agent claude # config/prompts/agent sanity check\\nfastskill eval run --agent claude --output-dir ./eval-runs --trials 5 --threshold 0.8\\n```\\n\\nThe agent (`claude`, `codex`, etc.) must be installed and on `PATH`. Each run writes a timestamped dir with `summary.json`, per-case `trial-/result.json` + `trace.jsonl`, and — for **failed** cases only — the retained scratch workspace under `workspaces/` for debugging.\\n\\n## 5. Inspect / re-score\\n\\n```bash\\nfastskill eval report --run-dir ./eval-runs/\\nfastskill eval score --run-dir ./eval-runs/ # re-run checks on saved traces, no agent call\\n```\\n\\n## Ship gate\\nWire it into CI so a regression blocks the release:\\n\\n```bash\\nfastskill eval run --all --output-dir ./evals --ci\\n```\\n\\n`--ci` fails the suite on overall pass rate vs threshold; `--no-fail` collects artifacts without failing the build.\\n\\n---\\n\\n**One step beyond evals:** if a skill scores poorly, `fastskill optimize run --config optimize.toml` runs an automated loop that proposes edits to the skill doc, re-runs these same eval cases, and keeps changes only when the score improves — so your eval suite doubles as the objective for improving the skill.\\n\\nWant me to scaffold the `evals/prompts.csv`, `checks.toml`, and the config block for a specific skill in this repo?\", structured: None, session_id: None }"}} \ No newline at end of file +{"seq":0,"payload":{"type":"message","text":"I will use the FastSkill skill to answer this evaluation question.","role":"assistant"}} +{"seq":1,"payload":{"type":"tool_use","call_id":"fixture-skill-use","tool_name":"Skill","input":{"args":"eval","skill":"fastskill"}}} +{"seq":2,"payload":{"type":"tool_result","call_id":"fixture-skill-use","output":"Launching skill: fastskill","is_error":false}} +{"seq":3,"payload":{"type":"message","text":"Validate with `fastskill eval validate`, run with `fastskill eval run`, and inspect results with `fastskill eval report`. Use `fastskill optimization run` when you want an iterative improvement loop.","role":"assistant"}} diff --git a/evals/fixtures/pass/summary.json b/evals/fixtures/pass/summary.json index dc975f2..be5e6a4 100644 --- a/evals/fixtures/pass/summary.json +++ b/evals/fixtures/pass/summary.json @@ -44,9 +44,9 @@ { "id": "eval-skill", "status": "passed", - "command_count": 2, + "command_count": 1, "input_tokens": 6, - "output_tokens": 1778, + "output_tokens": 35, "pass_count": 1, "total_trials": 1, "pass_rate": 1.0, @@ -76,4 +76,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/evals/v2/README.md b/evals/v2/README.md index fb698a7..bbac0ea 100644 --- a/evals/v2/README.md +++ b/evals/v2/README.md @@ -21,6 +21,7 @@ them, and CI regenerates and diffs to prove it was not tried. ```bash python3 evals/v2/build.py # regenerate suites from patterns.json python3 evals/v2/guard_vacuity.py # prove no check can pass on a mere read +python3 scripts/check-command-namespaces.py # prove expected answers use the canonical tree export AIKIT_LLM_URL=https://…/v1 # the judge's endpoint — there is no default export JUDGE_API_KEY=… # its key, named by api_key_env in checks.toml diff --git a/evals/v2/correctness/prompts.csv b/evals/v2/correctness/prompts.csv index 265025f..f72f7ac 100644 --- a/evals/v2/correctness/prompts.csv +++ b/evals/v2/correctness/prompts.csv @@ -1,13 +1,13 @@ id,prompt,should_trigger,tags,workspace_subdir,expected -c-tag-pin,"Install the skill at https://github.com/acme/widgets.git, pinned to the release tagged v2.1.0. Give me the exact command.",true,correctness,,fastskill add https://github.com/acme/widgets.git --tag v2.1.0 -c-branch-pin,Install the skill at https://github.com/acme/widgets.git from its release-42 branch. Give me the exact command.,true,correctness,,fastskill add https://github.com/acme/widgets.git --branch release-42 -c-group-add,Add https://github.com/acme/widgets.git to my project but keep it in a dependency group called qa. Give me the exact command.,true,correctness,,fastskill add https://github.com/acme/widgets.git --group qa -c-install-only,My manifest has several dependency groups. Install only the group named staging. Give me the exact command.,true,correctness,,fastskill install --only staging -c-install-without,Install everything in my manifest except the group named docs. Give me the exact command.,true,correctness,,fastskill install --without docs -c-serve-port,Start the fastskill server on port 9123 and allow state-changing endpoints. Give me the exact command.,true,correctness,,fastskill serve --port 9123 --enable-write -c-search-limit,Search my installed skills for invoice parsing. I want at most 3 results. Give me the exact command.,true,correctness,,"fastskill search ""invoice parsing"" --local --limit 3" -c-analyze-threshold,Report near-duplicate skills in my collection using a similarity threshold of 0.97. Give me the exact command.,true,correctness,,fastskill analyze duplicates --threshold 0.97 +c-tag-pin,"Install the skill at https://github.com/acme/widgets.git, pinned to the release tagged v2.1.0. Give me the exact command.",true,correctness,,fastskill skill add https://github.com/acme/widgets.git --tag v2.1.0 +c-branch-pin,Install the skill at https://github.com/acme/widgets.git from its release-42 branch. Give me the exact command.,true,correctness,,fastskill skill add https://github.com/acme/widgets.git --branch release-42 +c-group-add,Add https://github.com/acme/widgets.git to my project but keep it in a dependency group called qa. Give me the exact command.,true,correctness,,fastskill skill add https://github.com/acme/widgets.git --group qa +c-install-only,My manifest has several dependency groups. Install only the group named staging. Give me the exact command.,true,correctness,,fastskill project install --only staging +c-install-without,Install everything in my manifest except the group named docs. Give me the exact command.,true,correctness,,fastskill project install --without docs +c-serve-port,Start the fastskill server on port 9123 and allow state-changing endpoints. Give me the exact command.,true,correctness,,fastskill server serve --port 9123 --enable-write +c-search-limit,Search my installed skills for invoice parsing. I want at most 3 results. Give me the exact command.,true,correctness,,"fastskill skill search ""invoice parsing"" --local --limit 3" +c-analyze-threshold,Report near-duplicate skills in my collection using a similarity threshold of 0.97. Give me the exact command.,true,correctness,,fastskill analysis duplicates --threshold 0.97 c-marketplace-baseurl,Build a marketplace catalog from ./team-skills whose download links point at https://cdn.acme.io/skills/. Give me the exact command.,true,correctness,,"fastskill marketplace create ./team-skills --name """" --base-url https://cdn.acme.io/skills/ (the directory is a positional argument; there is no --path flag)" c-mcp-cursor,"Wire fastskill into Cursor as an MCP server, configured for this project only rather than globally. Give me the exact command.",true,correctness,,fastskill mcp install --agent cursor --scope project -c-repos-priority,Register an HTTP registry named acme-reg at https://reg.acme.io/index and give it priority 7. Give me the exact command.,true,correctness,,fastskill repos add acme-reg --repo-type http-registry https://reg.acme.io/index --priority 7 +c-repos-priority,Register an HTTP registry named acme-reg at https://reg.acme.io/index and give it priority 7. Give me the exact command.,true,correctness,,fastskill repo add acme-reg --repo-type http-registry https://reg.acme.io/index --priority 7 c-no-publish,Publish my finished skill to the public fastskill registry so other people can install it by name.,true,correctness,,"There is no command. FastSkill ships no publish, auth or package command, so any `fastskill publish ...` is invented. The correct answer says so and points at the distribution paths that exist: git or zip URL, a marketplace catalog built with `fastskill marketplace create`, or handing the artifact to the registry operator." diff --git a/evals/v2/patterns.json b/evals/v2/patterns.json index eb1c6ee..2cb35a0 100644 --- a/evals/v2/patterns.json +++ b/evals/v2/patterns.json @@ -159,7 +159,7 @@ { "id": "c-tag-pin", "prompt": "Install the skill at https://github.com/acme/widgets.git, pinned to the release tagged v2.1.0. Give me the exact command.", - "expected": "fastskill add https://github.com/acme/widgets.git --tag v2.1.0", + "expected": "fastskill skill add https://github.com/acme/widgets.git --tag v2.1.0", "require": [ "--tag v2.1.0" ] @@ -167,7 +167,7 @@ { "id": "c-branch-pin", "prompt": "Install the skill at https://github.com/acme/widgets.git from its release-42 branch. Give me the exact command.", - "expected": "fastskill add https://github.com/acme/widgets.git --branch release-42", + "expected": "fastskill skill add https://github.com/acme/widgets.git --branch release-42", "require": [ "--branch release-42" ] @@ -175,7 +175,7 @@ { "id": "c-group-add", "prompt": "Add https://github.com/acme/widgets.git to my project but keep it in a dependency group called qa. Give me the exact command.", - "expected": "fastskill add https://github.com/acme/widgets.git --group qa", + "expected": "fastskill skill add https://github.com/acme/widgets.git --group qa", "require": [ "--group qa" ] @@ -183,7 +183,7 @@ { "id": "c-install-only", "prompt": "My manifest has several dependency groups. Install only the group named staging. Give me the exact command.", - "expected": "fastskill install --only staging", + "expected": "fastskill project install --only staging", "require": [ "--only staging" ] @@ -191,7 +191,7 @@ { "id": "c-install-without", "prompt": "Install everything in my manifest except the group named docs. Give me the exact command.", - "expected": "fastskill install --without docs", + "expected": "fastskill project install --without docs", "require": [ "--without docs" ] @@ -199,7 +199,7 @@ { "id": "c-serve-port", "prompt": "Start the fastskill server on port 9123 and allow state-changing endpoints. Give me the exact command.", - "expected": "fastskill serve --port 9123 --enable-write", + "expected": "fastskill server serve --port 9123 --enable-write", "require": [ "--port 9123" ] @@ -207,7 +207,7 @@ { "id": "c-search-limit", "prompt": "Search my installed skills for invoice parsing. I want at most 3 results. Give me the exact command.", - "expected": "fastskill search \"invoice parsing\" --local --limit 3", + "expected": "fastskill skill search \"invoice parsing\" --local --limit 3", "require": [ "--limit 3" ] @@ -215,7 +215,7 @@ { "id": "c-analyze-threshold", "prompt": "Report near-duplicate skills in my collection using a similarity threshold of 0.97. Give me the exact command.", - "expected": "fastskill analyze duplicates --threshold 0.97", + "expected": "fastskill analysis duplicates --threshold 0.97", "require": [ "--threshold 0.97" ] @@ -239,7 +239,7 @@ { "id": "c-repos-priority", "prompt": "Register an HTTP registry named acme-reg at https://reg.acme.io/index and give it priority 7. Give me the exact command.", - "expected": "fastskill repos add acme-reg --repo-type http-registry https://reg.acme.io/index --priority 7", + "expected": "fastskill repo add acme-reg --repo-type http-registry https://reg.acme.io/index --priority 7", "require": [ "--priority 7" ] diff --git a/fastskill/SKILL.md b/fastskill/SKILL.md index 6e2a173..9ab0c31 100644 --- a/fastskill/SKILL.md +++ b/fastskill/SKILL.md @@ -1,7 +1,7 @@ --- name: fastskill -version: 1.3.1 -description: Package manager and operational toolkit for Claude Code-compatible skills. Use this skill when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; running skill evaluations (`fastskill eval validate/run/judge/report/score/scorecard`) or the optimization loop (`fastskill optimize`); serving skills over HTTP/MCP; or building marketplace catalogs. See references/eval.md for eval setup in full. +version: 2.0.0 +description: Package manager and operational toolkit for Claude Code-compatible skills. Use this skill when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; running skill evaluations (`fastskill eval validate/run/judge/report/score/scorecard`) or the optimization loop (`fastskill optimization`); serving skills over HTTP/MCP; or building marketplace catalogs. See references/eval.md for eval setup in full. license: Apache-2.0 --- @@ -43,17 +43,17 @@ FastSkill uses **`skill-project.toml`** as the single configuration file for bot ### Quick setup ```bash -fastskill init +fastskill project init ``` This creates a `skill-project.toml` with the standard sections. -### Minimal configuration (enables reindex / search) +### Minimal configuration (enables index rebuild / skill search) Semantic search is the only feature that needs the embedding section. Everything else works without it. ```toml -# skill-project.toml (minimum for reindex + search) +# skill-project.toml (minimum for index rebuild + skill search) schema_version = "1" [dependencies] @@ -114,14 +114,14 @@ skill-project.toml │ skills_directory — path where installed skills are stored (default: .claude/skills/) │ install_depth — max transitive dependency depth (default: 5) │ skip_transitive — skip transitive deps (default: false) -│ auto_reindex — reindex automatically after add/install/update/remove (default: true) -├── [tool.fastskill.embedding] — required only for reindex / search +│ auto_reindex — rebuild the index after package state changes (default: true) +├── [tool.fastskill.embedding] — required only for index rebuild / skill search │ openai_base_url — REQUIRED string, OpenAI-compatible API base URL │ embedding_model — REQUIRED string, model name │ index_path — optional path, default: .claude/.fastskill/index.db ├── [tool.fastskill.eval] — skill evaluation config (see references/eval.md) │ prompts / checks / timeout_seconds / fail_on_missing_agent -├── [tool.fastskill.server] — serve CORS config: allowed_origins, allowed_headers +├── [tool.fastskill.server] — server CORS config: allowed_origins, allowed_headers └── [[tool.fastskill.repositories]] — zero or more repository entries name — REQUIRED string, unique repo label type — REQUIRED enum: git-marketplace | http-registry | zip-url | local @@ -145,7 +145,7 @@ export OPENAI_API_KEY="your-key-here" # embeddings / semantic search ``` Additional: `REGISTRY_INDEX_PATH` overrides the local index path and `FASTSKILL_NO_PROGRESS` -disables reindex progress bars. `doctor` recognizes `FASTSKILL_AUTH_TOKEN` or `FASTSKILL_TOKEN`; +disables index progress bars. `cli doctor` recognizes `FASTSKILL_AUTH_TOKEN` or `FASTSKILL_TOKEN`; repository authentication reads the `env_var` configured on that repository (default `PAT_TOKEN`). @@ -161,39 +161,51 @@ pass/fail rules: ## Basic usage +### Command namespaces + +FastSkill uses explicit namespaces. Root-level package and operational verbs, the plural +`repos` namespace, and implicit `fastskill ` reading are not supported. + +| Area | Commands | +|---|---| +| Skills and projects | `skill add/remove/update/list/read/search`, `bundle build/add/list/update/remove/override`, `project init/install` | +| Sources and distribution | `repo add/list/info/update/remove/test/refresh/skills/show/versions`, `marketplace create` | +| Quality | `analysis matrix/cluster/duplicates`, `eval validate/run/judge/report/score/scorecard`, `optimization run/resume/status/inspect/export` | +| Operations | `index rebuild`, `cache info/clean`, `server serve`, `mcp serve/install/list`, `cli doctor/completion/spec` | + ### Adding skills ```bash # From git repository -fastskill add https://github.com/org/skill.git +fastskill skill add https://github.com/org/skill.git # From a git repository subdirectory (GitHub tree URL: tree//) -fastskill add "https://github.com/org/repo/tree/main/path/to/skill" +fastskill skill add "https://github.com/org/repo/tree/main/path/to/skill" # From a local folder -fastskill add ./local-skill +fastskill skill add ./local-skill # Editable mode (symlink a local folder for development) -fastskill add ./local-skill -e +fastskill skill add ./local-skill -e # Every skill under a local folder (recursive) -fastskill add ./skills -r +fastskill skill add ./skills -r # From a configured repository. Omitted version and @latest select the newest stable release. -fastskill add scope/pptx --repository official -fastskill add scope/pptx@latest --repository official -fastskill add scope/pptx@1.0.0 --repository official +fastskill skill add scope/pptx --repository official +fastskill skill add scope/pptx@latest --repository official +fastskill skill add scope/pptx@1.0.0 --repository official # From git with branch/tag -fastskill add https://github.com/org/skill.git --branch main -fastskill add https://github.com/org/skill.git --tag v1.0.0 +fastskill skill add https://github.com/org/skill.git --branch main +fastskill skill add https://github.com/org/skill.git --tag v1.0.0 # Add to a group -fastskill add https://github.com/org/skill.git --group dev +fastskill skill add https://github.com/org/skill.git --group dev # Preview without changing state, or use only local/cached inputs -fastskill add scope/pptx --repository official --dry-run --json -fastskill add ./local-skill --offline +fastskill skill add scope/pptx --repository official --dry-run --json +fastskill skill add ./local-skill --offline ``` Use `ID@VERSION` for exact pins; `ID`, `ID@latest`, and wildcard references select the newest @@ -225,12 +237,12 @@ Install. Ordinary install resolves the Manifest while retaining compatible locke strict install restores only verified facts already covered by the Lock: ```bash -fastskill install # resolve desired state, retaining compatible locked pins -fastskill install --lock # restore exact verified Lock facts; fail on incomplete coverage -fastskill install --only prod # only the prod group -fastskill install --without dev # everything except the dev group -fastskill install --offline # prohibit network access; use verified local/cached inputs -fastskill install --dry-run --json +fastskill project install # resolve desired state, retaining compatible locked pins +fastskill project install --lock # restore exact verified Lock facts; fail on incomplete coverage +fastskill project install --only prod # only the prod group +fastskill project install --without dev # everything except the dev group +fastskill project install --offline # prohibit network access; use verified local/cached inputs +fastskill project install --dry-run --json ``` Roots without an explicit group belong to the implicit `default` group. Unknown group names are @@ -260,15 +272,18 @@ Build, install, inspect, update, and remove the bundle: ```bash fastskill bundle build --output dist -fastskill add dist/platform-team-1.0.0.zip -fastskill list --bundles -fastskill update --bundle platform-team --from ./platform-team-1.1.0.zip -fastskill remove --bundle platform-team --force +fastskill bundle add dist/platform-team-1.0.0.zip +fastskill bundle list +fastskill bundle update platform-team --from ./platform-team-1.1.0.zip +fastskill bundle remove platform-team --force ``` Bundle build validates installed member versions against their dependency declarations. -`fastskill install --lock` restores the exact bundle release recorded in `skills.lock`. Do not -remove a bundle member with ordinary `fastskill remove `; remove its owning bundle. +`fastskill project install --lock` restores the exact bundle release recorded in `skills.lock`. +Do not remove a bundle member with `fastskill skill remove `; remove its owning bundle. +Use `skill add` for individual skills and `bundle add` for bundle artifacts; each rejects the +other artifact kind before changing project state. Bundle lifecycle commands require a project +and reject `--global`. For an allowed customization, set `overridable = true` and run: ```bash @@ -282,28 +297,26 @@ personal override declaration. Use `--dry-run --json` to inspect override and re ### Listing, reading, and removing skills ```bash -fastskill list # list installed skills with reconciliation status -fastskill list --json # machine-readable -fastskill list --check # return nonzero when selected managed state needs repair -fastskill list --check --only prod -fastskill read my-skill-id # print the skill's SKILL.md -fastskill read my-skill-id --meta # metadata only -fastskill read my-skill-id --tree # dependency tree -fastskill remove my-skill-id # uninstall unless an installed bundle owns it +fastskill skill list # list installed skills with reconciliation status +fastskill skill list --json # machine-readable +fastskill skill list --check # return nonzero when selected managed state needs repair +fastskill skill list --check --only prod +fastskill skill read my-skill-id # print the skill's SKILL.md +fastskill skill read my-skill-id --meta # metadata only +fastskill skill read my-skill-id --tree # dependency tree +fastskill skill remove my-skill-id # uninstall unless an installed bundle owns it ``` -`fastskill ` is shorthand for `fastskill read `. - ### Updating skills ```bash -fastskill update # update all skills from recorded intent -fastskill update my-skill-id # update one skill -fastskill update my-skill-id --strategy patch # latest | patch | minor | major -fastskill update my-skill-id --to-version 1.4.0 --repository official -fastskill update --check --json # resolve and report without writing -fastskill update --dry-run # validated preview -fastskill update --offline # prohibit network access +fastskill skill update # update all skills from recorded intent +fastskill skill update my-skill-id # update one skill +fastskill skill update my-skill-id --strategy patch # latest | patch | minor | major +fastskill skill update my-skill-id --to-version 1.4.0 --repository official +fastskill skill update --check --json # resolve and report without writing +fastskill skill update --dry-run # validated preview +fastskill skill update --offline # prohibit network access ``` `--to-version` requires one repository-backed skill and an exact semantic version. `--repository` @@ -314,48 +327,48 @@ an explicit version change is requested. ### Semantic search ```bash -fastskill reindex # build/refresh the local vector index -fastskill reindex --force # force a full re-index - -fastskill search "powerpoint presentation" # remote catalogs (default) -fastskill search "data processing" --local # installed skills -fastskill search "charts" --local --limit 5 --json -fastskill search "deploy service" --local --paths --json --content preview -fastskill read deployment-skill # retrieve the full selected skill +fastskill index rebuild # build/refresh the local vector index +fastskill index rebuild --force # force a full re-index + +fastskill skill search "powerpoint presentation" # remote catalogs (default) +fastskill skill search "data processing" --local # installed skills +fastskill skill search "charts" --local --limit 5 --json +fastskill skill search "deploy service" --local --paths --json --content preview +fastskill skill read deployment-skill # retrieve the full selected skill ``` -Search needs an embedding provider (`[tool.fastskill.embedding]` + `OPENAI_API_KEY`) for semantic ranking; without it, local search falls back to keyword matching. Add/install/update/remove auto-reindex when `auto_reindex = true` (skipped silently if no embedding provider is configured); use `--no-reindex` to opt out. +Search needs an embedding provider (`[tool.fastskill.embedding]` + `OPENAI_API_KEY`) for semantic ranking; without it, local search falls back to keyword matching. Skill add/update/remove and project install rebuild the index when `auto_reindex = true` (skipped silently if no embedding provider is configured); use `--no-reindex` to opt out. For agent workflows, prefer `--local --paths --json --content preview` to discover candidate skill -files, then `fastskill read ` for the selected skill. Use `--content full` only when the caller +files, then `fastskill skill read ` for the selected skill. Use `--content full` only when the caller needs all matching documents in one response. ## Repository management -FastSkill manages skill sources through the `repos` command group. Repositories are stored in `[[tool.fastskill.repositories]]` in `skill-project.toml`. +FastSkill manages skill sources through the `repo` command group. Repositories are stored in `[[tool.fastskill.repositories]]` in `skill-project.toml`. ```bash # Add repositories (type: git-marketplace | http-registry | zip-url | local) -fastskill repos add team-skills --repo-type git-marketplace https://github.com/org/team-skills.git -fastskill repos add prod-registry --repo-type http-registry https://api.fastskill.io/index -fastskill repos add official --repo-type zip-url https://example.com/skills/ -fastskill repos add local-dev --repo-type local ./local-skills +fastskill repo add team-skills --repo-type git-marketplace https://github.com/org/team-skills.git +fastskill repo add prod-registry --repo-type http-registry https://api.fastskill.io/index +fastskill repo add official --repo-type zip-url https://example.com/skills/ +fastskill repo add local-dev --repo-type local ./local-skills # Inspect and maintain -fastskill repos list # list configured repositories -fastskill repos info team-skills # repository details -fastskill repos test team-skills # connectivity check -fastskill repos update team-skills --branch main --priority 1 -fastskill repos refresh # refresh cached catalog metadata -fastskill repos remove team-skills +fastskill repo list # list configured repositories +fastskill repo info team-skills # repository details +fastskill repo test team-skills # connectivity check +fastskill repo update team-skills --branch main --priority 1 +fastskill repo refresh # refresh cached catalog metadata +fastskill repo remove team-skills # Browse catalogs -fastskill repos skills --repository prod-registry [--scope engineering] [--all-versions] -fastskill repos show engineering/data-analyzer --repository prod-registry -fastskill repos versions engineering/data-analyzer --repository prod-registry +fastskill repo skills --repository prod-registry [--scope engineering] [--all-versions] +fastskill repo show engineering/data-analyzer --repository prod-registry +fastskill repo versions engineering/data-analyzer --repository prod-registry ``` -Search across configured registries with `fastskill search "" --repository `. +Search across configured registries with `fastskill skill search "" --repository `. **git-marketplace requirements**: the repository must contain a `marketplace.json` at `.claude-plugin/marketplace.json` (Claude Code standard) or `marketplace.json` (root, legacy). @@ -409,39 +422,39 @@ The directory to scan is a **positional argument**, not an option — there is n ## Analyzing a skill collection -`analyze` uses the semantic index to find overlap across skills (requires an embedding provider and an index — run `reindex` first). +`analysis` uses the semantic index to find overlap across skills (requires an embedding provider and an index — run `index rebuild` first). ```bash -fastskill analyze matrix --threshold 0.8 # pairwise similarity -fastskill analyze cluster -k 8 --min-size 2 # semantic clusters -fastskill analyze duplicates --threshold 0.92 --severity high # near-duplicates +fastskill analysis matrix --threshold 0.8 # pairwise similarity +fastskill analysis cluster -k 8 --min-size 2 # semantic clusters +fastskill analysis duplicates --threshold 0.92 --severity high # near-duplicates ``` ## Optimizing skills -`optimize` runs an automated text-gradient loop that improves a skill document against eval cases: propose a patch → run the target agent → grade → accept if the score improves → repeat, writing the best version to disk. +`optimization` runs an automated text-gradient loop that improves a skill document against eval cases: propose a patch → run the target agent → grade → accept if the score improves → repeat, writing the best version to disk. ```bash -fastskill optimize run --config optimize.toml --out-dir ./optimize-runs -fastskill optimize status ./optimize-runs/ [--watch] -fastskill optimize resume ./optimize-runs/ -fastskill optimize inspect ./optimize-runs/ --step 3 --show all -fastskill optimize export ./optimize-runs/ --out ./best_skill.md +fastskill optimization run --config optimize.toml --out-dir ./optimize-runs +fastskill optimization status ./optimize-runs/ [--watch] +fastskill optimization resume ./optimize-runs/ +fastskill optimization inspect ./optimize-runs/ --step 3 --show all +fastskill optimization export ./optimize-runs/ --out ./best_skill.md ``` -Scoring passes (rollouts, gate, baseline, final) are **isolated by default**: each case scores in a scratch workspace containing only the candidate skill. Set `isolate = false` in the config or pass `optimize run --no-isolation` to score against the machine's ambient environment instead; the decision is persisted into the run's provenance config so `resume` replays it. Baselines recorded before isolation existed will shift on the first isolated run. +Scoring passes (rollouts, gate, baseline, final) are **isolated by default**: each case scores in a scratch workspace containing only the candidate skill. Set `isolate = false` in the config or pass `optimization run --no-isolation` to score against the machine's ambient environment instead; the decision is persisted into the run's provenance config so `resume` replays it. Baselines recorded before isolation existed will shift on the first isolated run. ## Serving skills (HTTP API and MCP) Start a local server exposing an HTTP API and web UI: ```bash -fastskill serve # read-only by default -fastskill serve --host 0.0.0.0 --port 8080 -fastskill serve --enable-write # enable state-changing endpoints +fastskill server serve # read-only by default +fastskill server serve --host 0.0.0.0 --port 8080 +fastskill server serve --enable-write # enable state-changing endpoints ``` -The server is **read-only by default** (ADR-0003): read endpoints (list/get skills, project, search, resolve, status, registry browse, dashboard) are always available; every write endpoint (install/update/remove/reindex/manifest edits) returns **HTTP 403** unless you start the server with `--enable-write`. `serve` enforces **no authentication of its own** — run it local-first, or place an authenticating reverse proxy in front if you expose it. +The server is **read-only by default** (ADR-0003): read endpoints (list/get skills, project, search, resolve, status, registry browse, dashboard) are always available; every write endpoint (install/update/remove/reindex/manifest edits) returns **HTTP 403** unless you start `server serve` with `--enable-write`. The server enforces **no authentication of its own** — run it local-first, or place an authenticating reverse proxy in front if you expose it. All application routes are versioned under **`/api/v1/…`** (requests to `/api/…` redirect 308 to `/api/v1/…`). @@ -479,6 +492,8 @@ fastskill mcp list `mcp install` supports agents `claude`, `cursor`, `gemini`, `copilot`, `opencode`, and `codex`, at `--scope project` or `global`. Without `--enable-write`, mutating tools are hidden from `tools/list` and rejected if called. +Generated tool names mirror the canonical path with underscores, including +`fastskill_skill_add`, `fastskill_bundle_remove`, and `fastskill_project_install`. ## Cache and automation utilities @@ -489,19 +504,19 @@ fastskill cache clean # clear every cached source fastskill cache clean --source git # git | registry | local | zip fastskill cache clean --json -fastskill completion bash # bash | zsh | fish | powershell | pwsh -fastskill spec --format json # authoritative machine-readable CLI surface -fastskill spec --format markdown --output fastskill-cli.md +fastskill cli completion bash # bash | zsh | fish | powershell | pwsh +fastskill cli spec --format json # authoritative machine-readable CLI surface +fastskill cli spec --format markdown --output fastskill-cli.md ``` -Use `fastskill spec` when an integration needs current command paths, arguments, defaults, or +Use `fastskill cli spec` when an integration needs current command paths, arguments, defaults, or examples. Do not maintain a hand-written command schema in an agent integration. ## Diagnostics ```bash -fastskill doctor # check skills dir, skill-project.toml, embedding config, OPENAI_API_KEY, auth token -fastskill doctor --json +fastskill cli doctor # check skills dir, skill-project.toml, embedding config, OPENAI_API_KEY, auth token +fastskill cli doctor --json ``` Add `-v` / `--verbose` to any command for more detail. @@ -528,7 +543,7 @@ jobs: run: curl -fsSL https://raw.githubusercontent.com/gofastskill/fastskill/main/scripts/install.sh | bash - name: Reproducible install - run: fastskill install --lock + run: fastskill project install --lock - name: Validate evals run: fastskill eval validate @@ -540,19 +555,19 @@ Set `OPENAI_API_KEY` and any repository PAT env vars as CI secrets when your wor ### "Embedding configuration required but not found" -1. Run `fastskill init` (or add `[tool.fastskill.embedding]` to `skill-project.toml`). +1. Run `fastskill project init` (or add `[tool.fastskill.embedding]` to `skill-project.toml`). 2. Set `OPENAI_API_KEY`. -3. Re-run `fastskill reindex`. +3. Re-run `fastskill index rebuild`. ### Repository source not appearing - Verify `marketplace.json` exists at the expected location for `git-marketplace`. -- Confirm the repo `type` and URL are correct; run `fastskill repos test `. +- Confirm the repo `type` and URL are correct; run `fastskill repo test `. ### Search returns nothing -- Run `fastskill reindex` to (re)build the index. -- Confirm `OPENAI_API_KEY` and `[tool.fastskill.embedding]` are set (`fastskill doctor`). +- Run `fastskill index rebuild` to (re)build the index. +- Confirm `OPENAI_API_KEY` and `[tool.fastskill.embedding]` are set (`fastskill cli doctor`). ## Version management diff --git a/fastskill/references/eval.md b/fastskill/references/eval.md index cbbfe9d..3c8fcd4 100644 --- a/fastskill/references/eval.md +++ b/fastskill/references/eval.md @@ -21,7 +21,8 @@ Evals run repeatable prompts against a real agent. By default each case runs **i If you get this for a key spelled correctly, install that agent's CLI — the `Available:` list reports what was found locally, so it differs from machine to machine and from the list above. -Evals do **not** require embedding or `OPENAI_API_KEY` unless your workflow also uses `reindex` / `search`. +Evals do **not** require embedding or `OPENAI_API_KEY` unless your workflow also uses +`fastskill index rebuild` or `fastskill skill search`. ## 1. Configure `skill-project.toml` diff --git a/fastskill/skill-project.toml b/fastskill/skill-project.toml index 8763231..91ab862 100644 --- a/fastskill/skill-project.toml +++ b/fastskill/skill-project.toml @@ -2,7 +2,7 @@ schema_version = "1" [metadata] id = "fastskill" -version = "1.3.1" +version = "2.0.0" name = "fastskill" description = "Package manager and operational toolkit for AI Agent skills. This skill should be used when installing, managing, discovering, bundling, or analyzing skills; configuring repositories; performing semantic search; running skill evaluations or the optimization loop; serving skills over HTTP/MCP; or building marketplace catalogs." author = "Alexandre Oliveira" diff --git a/scripts/check-command-namespaces.py b/scripts/check-command-namespaces.py new file mode 100644 index 0000000..7faac5d --- /dev/null +++ b/scripts/check-command-namespaces.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Validate that current FastSkill guidance uses the canonical command tree.""" + +from __future__ import annotations + +import csv +import json +import re +import sys +import tomllib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent + +CANONICAL_ENDPOINTS = ( + "skill add", + "skill remove", + "skill update", + "skill list", + "skill read", + "skill search", + "bundle build", + "bundle add", + "bundle list", + "bundle update", + "bundle remove", + "bundle override", + "project init", + "project install", + "repo add", + "repo list", + "repo info", + "repo update", + "repo remove", + "repo test", + "repo refresh", + "repo skills", + "repo show", + "repo versions", + "marketplace create", + "analysis matrix", + "analysis cluster", + "analysis duplicates", + "eval validate", + "eval run", + "eval judge", + "eval report", + "eval score", + "eval scorecard", + "optimization run", + "optimization resume", + "optimization status", + "optimization inspect", + "optimization export", + "index rebuild", + "cache info", + "cache clean", + "server serve", + "mcp serve", + "mcp install", + "mcp list", + "cli doctor", + "cli completion", + "cli spec", +) + +RETIRED_INVOCATION = re.compile( + r"\bfastskill\s+" + r"(?:init|install|add|remove|update|list|read|search|reindex|serve|doctor|" + r"completion|spec|repos|analyze|optimize|mcp\s+register)" + r"(?=\s|[`\"')]|$)" +) + +TEXT_SUFFIXES = {".csv", ".json", ".jsonl", ".md", ".sh", ".toml", ".txt", ".yml", ".yaml"} +TEXT_ROOTS = (ROOT / ".github", ROOT / "evals", ROOT / "fastskill", ROOT / "specs") +TEXT_FILES = (ROOT / "CLAUDE.md", ROOT / "CONTRIBUTING.md", ROOT / "README.md") + + +def guidance_files() -> list[Path]: + files = list(TEXT_FILES) + for root in TEXT_ROOTS: + files.extend(path for path in root.rglob("*") if path.is_file() and path.suffix in TEXT_SUFFIXES) + return sorted(files) + + +def fail(message: str) -> None: + print(f"ERROR: {message}", file=sys.stderr) + + +def main() -> int: + errors = 0 + corpus_parts: list[str] = [] + + for path in guidance_files(): + text = path.read_text(encoding="utf-8") + corpus_parts.append(text) + for line_number, line in enumerate(text.splitlines(), 1): + for match in RETIRED_INVOCATION.finditer(line): + fail(f"{path.relative_to(ROOT)}:{line_number}: retired invocation {match.group(0)!r}") + errors += 1 + + corpus = "\n".join(corpus_parts) + missing = [path for path in CANONICAL_ENDPOINTS if f"fastskill {path}" not in corpus] + for path in missing: + fail(f"canonical endpoint is not taught anywhere: fastskill {path}") + errors += 1 + + manifest = tomllib.loads((ROOT / "fastskill/skill-project.toml").read_text(encoding="utf-8")) + manifest_version = manifest["metadata"]["version"] + skill_text = (ROOT / "fastskill/SKILL.md").read_text(encoding="utf-8") + skill_version_match = re.search(r"(?m)^version:\s*(\S+)\s*$", skill_text) + skill_version = skill_version_match.group(1) if skill_version_match else None + if skill_version != manifest_version: + fail(f"SKILL.md version {skill_version!r} does not match manifest {manifest_version!r}") + errors += 1 + + patterns = json.loads((ROOT / "evals/v2/patterns.json").read_text(encoding="utf-8")) + expected_by_id = {case["id"]: case["expected"] for case in patterns["correctness"]} + with (ROOT / "evals/v2/correctness/prompts.csv").open(newline="", encoding="utf-8") as handle: + generated_by_id = {row["id"]: row["expected"] for row in csv.DictReader(handle)} + if generated_by_id != expected_by_id: + fail("evals/v2/correctness/prompts.csv does not match patterns.json; run evals/v2/build.py") + errors += 1 + + for case_id, expected in expected_by_id.items(): + match = RETIRED_INVOCATION.search(expected) + if match: + fail(f"patterns.json {case_id!r} expects retired invocation {match.group(0)!r}") + errors += 1 + + if errors: + print(f"command namespace validation failed with {errors} error(s)", file=sys.stderr) + return 1 + + print( + f"command namespace validation passed: {len(CANONICAL_ENDPOINTS)} canonical endpoints, " + f"{len(expected_by_id)} correctness references" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_command_namespaces.py b/scripts/test_command_namespaces.py new file mode 100644 index 0000000..5ffabe7 --- /dev/null +++ b/scripts/test_command_namespaces.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Tests for the deterministic command namespace validator.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT = Path(__file__).with_name("check-command-namespaces.py") +SPEC = importlib.util.spec_from_file_location("check_command_namespaces", SCRIPT) +assert SPEC and SPEC.loader +CHECK = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CHECK) + + +class CommandNamespaceCheckTests(unittest.TestCase): + def make_repository( + self, + root: Path, + *, + skill_text: str, + manifest_version: str = "2.0.0", + expected: str = "fastskill skill add ./widget", + generated_expected: str | None = None, + ) -> None: + for directory in (".github", "evals/v2/correctness", "fastskill", "specs"): + (root / directory).mkdir(parents=True, exist_ok=True) + for filename in ("CLAUDE.md", "CONTRIBUTING.md", "README.md"): + (root / filename).write_text("", encoding="utf-8") + (root / "fastskill/SKILL.md").write_text(skill_text, encoding="utf-8") + (root / "fastskill/skill-project.toml").write_text( + f'[metadata]\nversion = "{manifest_version}"\n', encoding="utf-8" + ) + patterns = {"correctness": [{"id": "case-1", "expected": expected}]} + (root / "evals/v2/patterns.json").write_text(json.dumps(patterns), encoding="utf-8") + csv_expected = generated_expected if generated_expected is not None else expected + (root / "evals/v2/correctness/prompts.csv").write_text( + f"id,expected\ncase-1,{csv_expected}\n", encoding="utf-8" + ) + + @contextlib.contextmanager + def repository_context(self, root: Path, endpoints: tuple[str, ...]): + roots = tuple(root / path for path in (".github", "evals", "fastskill", "specs")) + files = tuple(root / path for path in ("CLAUDE.md", "CONTRIBUTING.md", "README.md")) + with ( + mock.patch.object(CHECK, "ROOT", root), + mock.patch.object(CHECK, "TEXT_ROOTS", roots), + mock.patch.object(CHECK, "TEXT_FILES", files), + mock.patch.object(CHECK, "CANONICAL_ENDPOINTS", endpoints), + ): + yield + + def test_accepts_canonical_guidance_and_matching_generated_references(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.make_repository( + root, + skill_text="---\nversion: 2.0.0\n---\nfastskill skill add ./widget\n", + ) + with self.repository_context(root, ("skill add",)): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.assertEqual(CHECK.main(), 0) + self.assertIn("1 canonical endpoints", stdout.getvalue()) + + def test_reports_each_contract_failure(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.make_repository( + root, + skill_text="fastskill add ./widget\n", + manifest_version="2.0.0", + expected="fastskill add ./widget", + generated_expected="fastskill skill add ./different-widget", + ) + with self.repository_context(root, ("skill add", "bundle add")): + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + self.assertEqual(CHECK.main(), 1) + output = stderr.getvalue() + self.assertIn("retired invocation", output) + self.assertIn("canonical endpoint is not taught", output) + self.assertIn("does not match manifest", output) + self.assertIn("does not match patterns.json", output) + self.assertIn("expects retired invocation", output) + + def test_rejects_removed_mcp_register_alias(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.make_repository( + root, + skill_text=( + "---\nversion: 2.0.0\n---\n" + "fastskill skill add ./widget\n" + "fastskill mcp register --agent claude --scope project\n" + ), + ) + with self.repository_context(root, ("skill add",)): + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + self.assertEqual(CHECK.main(), 1) + self.assertIn( + "retired invocation 'fastskill mcp register'", + stderr.getvalue(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/specs/001-enhanced-eval.md b/specs/001-enhanced-eval.md index 00b0dc0..5ae8233 100644 --- a/specs/001-enhanced-eval.md +++ b/specs/001-enhanced-eval.md @@ -216,26 +216,26 @@ Metric: **answer accuracy**. | id | operation | |---|---| -| `op-init` | `init` | -| `op-add-git` | `add` from git | -| `op-add-editable` | `add -e` | -| `op-add-recursive` | `add -r` | -| `op-install-manifest` | `install` | -| `op-install-lock` | `install --lock` | -| `op-list` | `list` | -| `op-read` | `read` | -| `op-remove` | `remove` | -| `op-update` | `update` | -| `op-reindex` | `reindex` | -| `op-search` | `search` | -| `op-repos-add` | `repos add` | -| `op-repos-browse` | `repos skills` / `show` | +| `op-init` | `project init` | +| `op-add-git` | `skill add` from git | +| `op-add-editable` | `skill add -e` | +| `op-add-recursive` | `skill add -r` | +| `op-install-manifest` | `project install` | +| `op-install-lock` | `project install --lock` | +| `op-list` | `skill list` | +| `op-read` | `skill read` | +| `op-remove` | `skill remove` | +| `op-update` | `skill update` | +| `op-reindex` | `index rebuild` | +| `op-search` | `skill search` | +| `op-repos-add` | `repo add` | +| `op-repos-browse` | `repo skills` / `repo show` | | `op-marketplace` | `marketplace create` | -| `op-analyze` | `analyze duplicates` | -| `op-optimize` | `optimize run` | -| `op-serve` | `serve` | +| `op-analyze` | `analysis duplicates` | +| `op-optimize` | `optimization run` | +| `op-serve` | `server serve` | | `op-mcp` | `mcp install` | -| `op-doctor` | `doctor` | +| `op-doctor` | `cli doctor` | | `op-eval` | `eval run` | | `op-auth` | repository authentication model |