diff --git a/AGENTS.md b/AGENTS.md index e8b55cda..368bd03c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,40 +4,43 @@ Instructions for AI coding agents (OpenAI Codex, Cursor, Gemini CLI, and any too ## Prerequisites -1. `apimetrics project show` — confirm a project is active. Every project-scoped command fails without one. If none is set, run `apimetrics project select`. -2. `apimetrics login` handles authentication (OAuth). **There is no `--api-key` flag.** +1. `apimetrics project show` — confirm a project is active. Every project-scoped command fails without one unless you pass `--project-id `. If none is set, run `apimetrics project select`. +2. Commands log in automatically when needed; `apimetrics login` does it up front (OAuth). For headless use, pass `--service-account `. ## Core facts about this CLI -- **Commands are flat, generated from the platform's live OpenAPI description**: `list-calls`, `create-call`, `run-call` — not `calls create`. The set can change with the server spec, so treat `apimetrics --help` and `apimetrics --help` as authoritative and inspect a command's help before assuming an option or body field. -- **Create/update commands read JSON from stdin** via heredoc. There is no `--body`, `--data`, or `-d`: +- **Commands are flat, generated from the platform's live OpenAPI description**: `list-calls`, `create-call`, `run-monitor`. The set can change with the server spec, so treat `apimetrics --help` and `apimetrics --help` as authoritative and inspect a command's help before assuming an option or body field. `run-monitor ` is the single command that starts a run for any monitor type — API call, browser, or MCP; it requires a body, so send at least `{}` on stdin, and returns the `result_id` to poll. +- **Deprecated commands still run but warn.** Where the server's spec deprecates a command, the CLI hides it from `--help` and prints `Command "X" is deprecated, do not use` on stderr (stdout JSON is unaffected). On servers that expose the `/api/3/monitors/` commands, `list-calls`/`create-call`/`get-call`/`update-call`/`delete-call` are deprecated in favour of `list-api-monitors` (or `list-monitors` for every type), `create-api-monitor`, `get-api-monitor`, `patch-api-monitor`/`update-api-monitor`, and `delete-api-monitor`. The new create/update commands take a flat body (`name`, `method`, `url`, `headers`, …) rather than `meta`/`request`; monitor IDs are the same, so `set-call-conditions`, `get-call-passfail-*`, and `list-results-by-call` keep working. Use whichever set your `--help` lists. +- **Create/update commands read their JSON body from stdin** via heredoc: ```bash apimetrics create-call <<'EOF' { "meta": { "name": "Health" }, "request": { "method": "GET", "url": "https://example.com/health" } } EOF ``` - CLI Shorthand is also accepted (`apimetrics create-call meta.name: "Health", request.method: GET, request.url: https://example.com/health`). -- **Output**: `-o json` for machine use; `-f` projects with a shorthand query; `-q` adds confirmed query params. The command spec is cached ~24h and refreshes automatically; `--rsh-no-cache` forces a refresh. -- **List envelopes are not uniform.** `list-calls`, `list-results`, `list-call-results`, and `list-auth-settings` return `{ "meta": ..., "results": [...] }`; `list-schedules` returns `{ "data": [...] }`; `list-slos`, `list-browser-monitors`, and `list-mcp-monitors` return a bare `{ "results": [...] }`. Inspect each command's own output before writing an `-f` path. -- **`get-result` is a summary** (`result`, `http_code`, `response_time` ms, `location_id`, `test`, `created`; `result` ∈ `PASS`/`FAIL`/`WARN`/`ERROR`/`TIMEOUT`/`QUEUED`). Deeper data comes from `get-result-content`, `get-result-screenshot`, the `query-*-performance`/`query-*-dns-diagnostics` commands, and `conformance-results`. -- **Percentiles are server-computed** by the `query-*-performance` commands (`measures`: `mean`/`p50`/`p90`/`p95`/`p99`; API `metrics`: `total`/`dns`/`connect`/`tls`/`ttfb`/`response`). Do not hand-average result summaries. -- **API assertions** are set with `set-call-conditions ` (read with `get-call-conditions`), not in the `create-call` body. -- **Bulk config-as-code** lives under the hidden `apimetrics bulk` group (`init ` e.g. `apimetrics:/monitors`, `status`, `diff [--remote]`, `pull`, `reset`, `push`). -- **Never print secrets.** Reference auth settings by ID (`auth_id`/`token_id`); never paste tokens, cookies, keys, or certificate contents. + CLI Shorthand is also accepted. Quote any value containing `//`, or the shorthand parser rejects it, and redirect stdin from `/dev/null` when running non-interactively — shorthand also reads stdin to merge with the arguments, so an open stdin pipe blocks: `apimetrics create-call meta.name: Health, request.method: GET, request.url: '"https://example.com/health"' ` while `meta.more` is true. Inspect each command's own output before writing an `-f` path. +- **Result rows** (`list-results`, `list-results-by-call`) are `{ id, meta, timing }`: `meta.result_category`, `meta.result`, `meta.http_code`, `meta.location_id`, `meta.call_id` (the monitor ID), `meta.start_time`, and `timing.total` in ms with `dns`/`tcp`/`handshake`/`processing`/`download`/`upload` components. `result_category` ∈ `PASS`/`SLOW`/`WARNING`/`FAIL` (HTTP 4xx lands in `WARNING`, connection errors in `FAIL`) and is empty while the run is queued; `result` carries the detail — `QUEUED` while in flight, then e.g. `COMPLETE`, `HTTP_CLIENT_ERROR`, `CONNECTION_ERROR`. `--result-category` takes those four values case-insensitively and silently ignores any other value, returning every category. `--from`/`--time` are the window start and end as full ISO-8601 date-times (a bare date is rejected) and are minute-granular; always pass `--from`, as the default window is short. +- **`get-result` detail depends on caller role.** A below-ANALYST caller gets a summary only — `meta` and `timing` as in a result row, with no `request`, `response`, or `context`. An ANALYST-or-above caller (most project members) gets the full object by default — response headers and body, request method/URL/headers, `meta.dns_details`, `meta.tls_version`, and the `timing` breakdown. Add `--mode full` for the request body, `messages`/`log_messages`, and certificate detail. An oversized response body is replaced by `"<< LARGE DATA >>"` with a `response.content_path`. Check `apimetrics get-result --help` and what actually comes back before reaching for `get-result-content`/`get-result-screenshot`/`query-*` commands. +- **Percentiles are server-computed** by the `query-*-performance` commands (`measures`: `mean`/`p50`/`p90`/`p95`/`p99`; API `metrics`: `total`/`dns`/`connect`/`tls`/`ttfb`/`response`/`upload`/`network`/`server`/`total_no_dns`; browser and MCP metric names differ — check `--help`). Without a `status` filter, failed runs are included; pass `"status": ["PASS"]` for the latency of successful runs. Do not hand-average result rows. +- **API assertions** are set with `set-call-conditions ` (read with `get-call-conditions`), as a separate step after `create-call`. +- **SLOs are one per project.** Use `get-project-slo`/`update-project-slo`/`delete-project-slo` (update-project-slo creates one if none exists); scope objectives with `include_tags`/`exclude_tags`. `get-project-slo` returns the definition — or platform defaults, with `created`/`last_update` null, when none is stored; measure attainment with the server-side aggregates — `get-call-passfail-range`/`get-call-passfail-total` per scoped monitor for availability (sum the per-category `stat..total` values from `get-call-passfail-total`; its top-level `stat.total` counts only uncategorised results), `query-*-performance` for latency. +- **Bulk config-as-code** lives under the hidden `apimetrics bulk` group (`init `, `list`, `status`, `diff [--remote]`, `pull`, `reset`, `push`). `init` needs a real, resolvable URL — a full host+path against the CLI's configured API server (e.g. `qc-client.apimetrics.io/api/2/calls/`). The checkout does not record its project, so pass the same `--project-id` (or keep the same selected project) on every `bulk` command. Files hold the full GET representation, which `PUT /api/2/calls/{id}` rejects (read-only `id`/`meta.*` fields), so reduce a changed file to just the writable fields being changed before `bulk push` — that endpoint applies partial updates. `bulk push` prints `Push complete.` and exits 0 even when an upload fails, and does not check for remote drift: treat any `Error uploading` line as a failure, and confirm `bulk status` is clean afterwards. +- **Never print secrets.** Reference auth settings by ID (`auth_id`/`token_id`); never paste tokens, cookies, keys, or certificate contents. `list-auth-settings` returns credential values in clear text, and monitor `request.headers`/`request.body` can hold literal tokens, so project those lists with `-f` to the fields you need rather than printing full JSON — e.g. `-f 'body.results.{id, name: meta.name, auth_type: settings.auth_type}'`. ## Workflows (skills) -Each entry names when to reach for it and the key commands. Full step-by-step workflows — including safety gates, evidence rules, and output templates — are in `skills/claude-skills//SKILL.md`; read that file before executing the workflow. Run `apimetrics onboard` to print every workflow to stdout for context injection. +Each entry names when to reach for it and the key commands. Full step-by-step workflows — including safety gates, evidence rules, and output templates — are in `skills/claude-skills//SKILL.md`; read that file before executing the workflow. Run `apimetrics onboard` to print the three monitor-setup guides and all eight workflows to stdout for context injection. | Workflow | Use when | Key commands | |---|---|---| -| **project-bootstrap** | setting up monitoring / onboarding a service | `project select`, `create-call`/`create-browser-monitor`/`create-mcp-monitor`, `set-call-conditions`, `create-schedule`, `add-call-to-schedule`, `run-call`/`run-monitor` | -| **weekly-health-review** | reliability report for the last 7 complete days | `list-results --since --before`, `query-api-performance`, `list-calls` | -| **failure-investigation** | root-causing one failing result or a series | `get-result`, `list-call-results --since --before`, `get-result-content`, `query-api-monitor-dns-diagnostics`, `get-call-conditions` | -| **incident-triage** | blast radius / common cause across many monitors | `list-results --since --before`, `query-api-dns-diagnostics`, `query-api-performance` | -| **monitoring-estate-audit** | coverage & hygiene governance review | `list-calls`, `list-schedules`, `list-slos`, `list-auth-settings`, `list-calls-by-auth` | -| **slo-review** | SLO design & attainment (no attainment endpoint — derive it) | `list-slos`, `get-slo`, `list-results`, `query-api-performance` | -| **config-as-code** | reviewed bulk edits with rollback | `bulk init apimetrics:/monitors`, `bulk status`, `bulk diff`, `bulk push`, `bulk reset` | +| **project-bootstrap** | setting up monitoring / onboarding a service | `project select`, `create-project-in-org` + `create-project-access`, `create-call`/`create-browser-monitor`/`create-mcp-monitor`, `set-call-conditions`, `create-schedule`, `add-call-to-schedule`, `run-monitor` | +| **weekly-health-review** | reliability report for the last 7 complete days | `get-call-passfail-range`, `list-results --from --time`, `query-api-performance`, `list-calls` | +| **failure-investigation** | root-causing one failing result or a series | `get-result`, `list-results-by-call --from --result-category`, `get-result-content`, `query-api-monitor-dns-diagnostics`, `get-call-conditions` | +| **incident-triage** | blast radius / common cause across many monitors | `list-results --from --time`, `query-api-dns-diagnostics`, `query-api-performance` | +| **monitoring-estate-audit** | coverage & hygiene governance review | `list-calls`, `list-schedules`, `get-project-slo`, `list-auth-settings`, `list-calls-by-auth` | +| **slo-review** | SLO design & attainment (one SLO per project; attainment from server-side aggregates) | `get-project-slo`, `update-project-slo`, `get-call-passfail-range`, `query-api-performance` | +| **config-as-code** | reviewed bulk edits with rollback | `bulk init `, `bulk status`, `bulk diff`, `bulk push`, `bulk reset` | | **performance-analytics** | latency percentiles, component timing, DNS drift | `query-api-performance`, `query-api-monitor-performance`, `query-api-dns-diagnostics`, `query-browser-performance`, `query-mcp-performance` | ## Safety @@ -55,4 +58,4 @@ The eight workflows above are shipped to Claude Code as native skills: apimetrics skills install --claude-skills # writes .claude/skills//SKILL.md ``` -The same content backs this `AGENTS.md` and `apimetrics onboard`, so Claude Code and other agents operate from one source of truth. +`apimetrics onboard` prints the same eight `SKILL.md` files that `skills install --claude-skills` installs, and this `AGENTS.md` indexes them, so Claude Code and other agents work from the same workflows. diff --git a/README.md b/README.md index 011e903d..70e048e2 100644 --- a/README.md +++ b/README.md @@ -160,9 +160,11 @@ EOF You can also use **CLI Shorthand** as a concise alternative to JSON: ```bash -apimetrics create-call meta.name: "My API Check", request.method: GET, request.url: https://api.example.com/health +apimetrics create-call meta.name: "My API Check", request.method: GET, request.url: '"https://api.example.com/health"' ``` +Quote any shorthand value that contains `//` (such as a URL), as above; unquoted, the shorthand parser rejects it. Shorthand also reads stdin and merges it with the arguments, so in scripts and other non-interactive shells redirect stdin from `/dev/null` (`... --help` before constructing a body or assuming an option name. -4. Commands are generally flat (`list-calls`, `create-call`), not noun/verb groups. -5. Create and update operations read JSON from stdin. Use a quoted heredoc: +4. Commands are generally flat (`list-calls`, `create-call`). A command the server has deprecated is hidden from `--help` and prints `Command "X" is deprecated, do not use` on stderr; use the replacement `--help` lists (for example `list-api-monitors`/`list-monitors` and `create-api-monitor` where `list-calls`/`create-call` are deprecated). +5. Create and update operations read JSON from stdin (CLI Shorthand arguments also work, but quote any value containing `//` and add ` ... <<'EOF' { @@ -30,9 +30,8 @@ Use the CLI's built-in hidden `bulk` workflow for local JSON checkout and synchr } EOF ``` - Do not invent `--body`, `--data`, or `-d`. -6. Use `-o json` for analysis. Use `-f` only after inspecting the response shape. The top-level response envelope includes status, headers, and `body`. List bodies are NOT uniform: `list-calls`, `list-results`, `list-call-results`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` returns `{"data":[...]}`; `list-slos`, `list-browser-monitors`, and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Inspect each command's own output before writing an `-f` path (e.g. `-f body.results[0]` vs `-f body.data[0]`). -7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation. +6. Use `-o json` for analysis; it prints the response body. Use `-f` only after inspecting the response shape: `-f` paths are rooted at the whole response (`status`, `headers`, `body`), so they start with `body.`, and a path that matches nothing prints nothing — empty `-f` output is not evidence of an empty list. List bodies are NOT uniform: `list-calls`, `list-api-monitors`, `list-results`, `list-results-by-call`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` and `list-monitors` return `{"data":[...], "meta":...}`; `list-browser-monitors` and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Single-quote `-f` paths so the shell doesn't expand `[]` (e.g. `-f 'body.results[0]'` vs `-f 'body.data[0]'`). +7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation; the CLI appends it without validation. 8. Preserve evidence. Record the active project, commands run, IDs, time window, and the smallest response excerpts needed to support conclusions. 9. Never print, store, or paste credentials into the report. Prefer existing auth-setting IDs. Do not include bearer tokens, cookies, API keys, client secrets, or private certificate contents. 10. Do not mutate project configuration unless the user explicitly authorized the change. Before any mutation, show the planned objects and rollback path. @@ -60,7 +59,7 @@ Confirm with: apimetrics bulk --help ``` -The bulk workflow keeps metadata and resource versions, detects local and remote changes, and can avoid overwriting local edits during pull. +The bulk workflow keeps metadata and resource versions, detects local and remote changes with `status` and `diff --remote`, and does not overwrite local edits during pull. `bulk push` does not check for remote drift (it sends no conditional request), and it prints `Push complete.` with exit status 0 even when individual uploads fail. ## Workflow @@ -77,20 +76,21 @@ Confirm: Do not initialize into a directory containing unrelated JSON files. +A checkout writes each resource verbatim to the working directory and `.rshbulk/`, including any literal credentials in request headers or bodies. Never check out auth settings with `bulk`. Keep the checkout directory out of version control and delete it when done. + +The checkout does not record which project it belongs to. Pass the same `--project-id ` (or keep the same selected project) on **every** `bulk` command — `init`, `list`, `status`, `diff`, `pull`, `reset`, `push`. Against a different project, `status` reports every file as remote-removed and that project's resources as remote-added, and `pull` replaces the checkout. + ### 2. Initialize the checkout -`bulk init` takes exactly one URL argument that returns a list of resources, each with a link and a version. Use the CLI's own API-name scheme (`apimetrics:/`) rather than a raw host — the repository's own example is: +`bulk init` takes exactly one URL argument that returns a list of resources, each with a link and a version. Use a real, resolvable URL — a full host+path against the CLI's configured API server (check the host with `apimetrics --version`) and the real resource path, shown on the `GET` line of `apimetrics -v` (help text does not include it). Use the `/api/2/calls/` collection, whose `PUT` applies partial updates: ```bash -apimetrics bulk init apimetrics:/monitors +apimetrics bulk init 'qc-client.apimetrics.io/api/2/calls/?limit=1000' -f 'body.results.{id, version: meta.last_update}' --url-template='/api/2/calls/{id}' --project-id ``` -`init` auto-detects the resource URL from `url`/`uri`/`self`/`link` and the version from `version`/`etag`/`last_modified`/`lastModified`/`modified`. If the collection response doesn't expose those directly, shape it with `-f` and/or build links from IDs with `--url-template`: +`init` checks out one page only, so set `?limit=` on the URL (the list defaults to 100; `1000` is the maximum). Split a larger project into several checkouts scoped with a `-f` `[?...]` filter. `init` fails on a list of exactly one resource (`slice bounds out of range`); widen the list, or make that single change with the monitor's own update command. If `init` fails, delete `.rshbulk/` before retrying. -```bash -apimetrics bulk init apimetrics:/monitors -f 'body.{url, version: last_update}' -apimetrics bulk init apimetrics:/monitors -f 'body.{id, version: last_update}' --url-template='/monitors/{id}' -``` +`init` auto-detects the resource URL from `url`/`uri`/`self`/`link` and the version from `version`/`etag`/`last_modified`/`lastModified`/`modified`. The `list-calls` response doesn't expose a direct link, so the example above shapes it with `-f` down to `id`/`version` and builds the link from the ID with `--url-template`. Always confirm the exact flags first: @@ -98,7 +98,7 @@ Always confirm the exact flags first: apimetrics bulk init --help ``` -Do not guess an undocumented host path; prefer the `apimetrics:` scheme, which resolves against the CLI's configured server. `bulk` is a hidden command group — it will not appear in the top-level `--help` list, but `apimetrics bulk --help` is authoritative. +`bulk` is a hidden command group — it will not appear in the top-level `--help` list, but `apimetrics bulk --help` is authoritative. ### 3. Baseline @@ -109,11 +109,11 @@ apimetrics bulk status apimetrics bulk list ``` -Read the metadata and checked-out JSON. Record a clean baseline before editing. If remote changes already exist, pull or reconcile first. +Read the metadata and checked-out JSON. If remote changes already exist, pull or reconcile first. Then record a clean baseline by copying the checkout aside before editing (e.g. `cp -R . ../-baseline`). `.rshbulk/` is overwritten by `pull` and by each successful `push`, so it is not a rollback source. ### 4. Edit and validate -Make only the authorized JSON changes. Preserve IDs, versions, references, and unrelated fields. +Make only the authorized JSON changes. Each file holds the full `GET` representation, and `PUT /api/2/calls/{id}` rejects its read-only fields (`id`, `meta.created`, `meta.last_update`, `meta.domain`, `meta.owner`, `meta.project_id`, `meta.monitor_type`, `meta.accept`, `meta.content_type`) with `extra_forbidden`. Because that `PUT` is a partial update, reduce each changed file to just the writable fields being changed — e.g. `{"meta": {"description": "…"}}` — and keep the full original in the baseline copy for review. After a successful push the CLI re-fetches the full resource into the file. Run: @@ -122,7 +122,7 @@ apimetrics bulk status apimetrics bulk diff ``` -Review every changed, added, and removed resource. Use `--match` only after testing the expression against `bulk list`. +Review every changed, added, and removed resource against the baseline copy. `--match` resolves top-level fields only (e.g. `id contains …`); a nested path such as `meta.name` is rejected with a warning and matches nothing. Test any expression against `bulk list` before using it with `diff` or `reset`, or pass explicit file names. ### 5. Reconcile remote drift @@ -152,7 +152,8 @@ Run `apimetrics bulk push` only after explicit approval in the current conversat After push: -- run `bulk status` and require clean local/remote state +- treat any `Error uploading …` line as a failed push, whatever the final message and exit status say +- run `bulk status` and require `No local changes` and a clean remote state - read changed resources through normal CLI commands - run affected monitors on demand - verify schedule targets and successful results @@ -166,7 +167,7 @@ For unpushed local edits: apimetrics bulk reset [file...] ``` -For pushed changes, restore the reviewed baseline JSON and push only with fresh approval. Do not treat `reset` as a remote rollback. +For pushed changes, write the baseline values of the changed writable fields (from the baseline copy) into the file as a partial body, show the diff, and push only with fresh approval. Pushing a full baseline file fails on the read-only fields. Do not treat `reset` as a remote rollback. ## Hard rules diff --git a/skills/claude-skills/apimetrics-failure-investigation/SKILL.md b/skills/claude-skills/apimetrics-failure-investigation/SKILL.md index 8080815d..b912625b 100644 --- a/skills/claude-skills/apimetrics-failure-investigation/SKILL.md +++ b/skills/claude-skills/apimetrics-failure-investigation/SKILL.md @@ -20,8 +20,8 @@ Find the failure boundary and the smallest evidence-backed remediation. Do not s ``` Run `apimetrics login` or `apimetrics project select` only when needed. 3. The CLI command tree is generated from the platform's current OpenAPI description. Inspect `apimetrics --help` before constructing a body or assuming an option name. -4. Commands are generally flat (`list-calls`, `create-call`), not noun/verb groups. -5. Create and update operations read JSON from stdin. Use a quoted heredoc: +4. Commands are generally flat (`list-calls`, `create-call`). A command the server has deprecated is hidden from `--help` and prints `Command "X" is deprecated, do not use` on stderr; use the replacement `--help` lists (for example `list-api-monitors`/`list-monitors` and `create-api-monitor` where `list-calls`/`create-call` are deprecated). +5. Create and update operations read JSON from stdin (CLI Shorthand arguments also work, but quote any value containing `//` and add ` ... <<'EOF' { @@ -29,9 +29,8 @@ Find the failure boundary and the smallest evidence-backed remediation. Do not s } EOF ``` - Do not invent `--body`, `--data`, or `-d`. -6. Use `-o json` for analysis. Use `-f` only after inspecting the response shape. The top-level response envelope includes status, headers, and `body`. List bodies are NOT uniform: `list-calls`, `list-results`, `list-call-results`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` returns `{"data":[...]}`; `list-slos`, `list-browser-monitors`, and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Inspect each command's own output before writing an `-f` path (e.g. `-f body.results[0]` vs `-f body.data[0]`). -7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation. +6. Use `-o json` for analysis; it prints the response body. Use `-f` only after inspecting the response shape: `-f` paths are rooted at the whole response (`status`, `headers`, `body`), so they start with `body.`, and a path that matches nothing prints nothing — empty `-f` output is not evidence of an empty list. List bodies are NOT uniform: `list-calls`, `list-api-monitors`, `list-results`, `list-results-by-call`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` and `list-monitors` return `{"data":[...], "meta":...}`; `list-browser-monitors` and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Single-quote `-f` paths so the shell doesn't expand `[]` (e.g. `-f 'body.results[0]'` vs `-f 'body.data[0]'`). +7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation; the CLI appends it without validation. 8. Preserve evidence. Record the active project, commands run, IDs, time window, and the smallest response excerpts needed to support conclusions. 9. Never print, store, or paste credentials into the report. Prefer existing auth-setting IDs. Do not include bearer tokens, cookies, API keys, client secrets, or private certificate contents. 10. Do not mutate project configuration unless the user explicitly authorized the change. Before any mutation, show the planned objects and rollback path. @@ -61,17 +60,17 @@ If given a result ID, retrieve the result summary: apimetrics get-result -o json ``` -**Know what this returns.** `get-result` (and each row of `list-call-results`/`list-results`) is a *summary*: `result` (`PASS`/`FAIL`/`WARN`/`ERROR`/`TIMEOUT`/`QUEUED`), `http_code`, `response_time` (ms), `location_id`, `test` (monitor ID), and `created`. It does **not** contain component timings, DNS/TLS detail, headers, body, or assertion breakdowns. Pull those from the dedicated commands in step 3 — do not claim a DNS or TLS cause from `get-result` alone. +**Know what this returns — it depends on your role.** Every caller gets `meta` and `timing`: `meta.result_category` (`PASS`/`SLOW`/`WARNING`/`FAIL` — HTTP 4xx lands in `WARNING`, connection errors in `FAIL`; empty while the run is queued), `meta.result` (the detail: `QUEUED` while in flight, then e.g. `COMPLETE`, `HTTP_CLIENT_ERROR`, `CONNECTION_ERROR`), `meta.http_code`, `meta.location_id`, `meta.call_id` (the monitor ID), `meta.start_time`, and `timing` (`dns`/`tcp`/`handshake`/`upload`/`processing`/`download`/`total`, ms). A below-ANALYST caller gets only that summary — no `request`, `response`, or `context`. Most project members (ANALYST or above) also get `request` (method, URL, headers), `response` (headers and body), `meta.dns_details`, and `meta.tls_version` by default. Add `--mode full` for the request body, `messages`/`log_messages`, and certificate detail, and `--header-style mapping` for name→value headers. An oversized response body is replaced by `"<< LARGE DATA >>"` with a `response.content_path`. Check `apimetrics get-result --help` and what actually comes back before assuming you need step 3's dedicated commands — but still don't claim a DNS or TLS cause from a single result alone. -For an API call series (the argument is the call/monitor ID; supports `--since`/`--before`/`--limit`): +For a series from one monitor of any type (API, browser, or MCP), use `list-results-by-call `, whose flags are `--from`, `--time` (window end; both full ISO-8601 date-times, as a bare date is rejected), `--result-category`, `--cursor`, `--limit` (default 100, one page per call; repeat with `--cursor ` while `meta.more` is true). `--result-category` takes `PASS`/`SLOW`/`WARNING`/`FAIL` and silently ignores any other value, returning every category: ```bash -apimetrics list-call-results --since 2026-07-01T00:00:00Z -o json +apimetrics list-results-by-call --from 2026-07-01T00:00:00Z -o json ``` -For a project-wide sweep across monitor types, use `list-results --since ... --before ...`. Discover browser/MCP result access from `apimetrics --help`. +For a project-wide sweep across all monitors, use `list-results --from ... --time ...`. -Also read the monitor configuration (`get-call`, `read-browser-monitor`, `read-mcp-monitor`), conditions (`get-call-conditions `), schedule, auth-setting metadata, and related SLO where those commands exist. +Also read the monitor configuration (`get-monitor` for any type, or `get-call`/`get-api-monitor`, `read-browser-monitor`, `read-mcp-monitor` — project the fields you need with `-f`, as `request.headers`/`request.body` can hold literal credentials), conditions (`get-call-conditions `), schedule, auth-setting metadata, and related SLO where those commands exist. ### 2. Establish the timeline @@ -91,13 +90,13 @@ Compare failing and passing results across the dimensions below. Each names the | Dimension | Where it comes from | |---|---| -| status / HTTP code / total response time / location | `get-result`, `list-call-results`, `list-results` | +| status / HTTP code / total response time / location | `get-result`, `list-results-by-call`, `list-results` | | component timings (`dns`, `connect`, `tls`, `ttfb`, `response`) and percentiles | `query-api-monitor-performance` (per monitor) or `query-api-performance` (project), body `{from,to,metrics,measures,group_by}` | | DNS provider / resolved IPs / CNAME chain / NS / mean lookup time | `query-api-monitor-dns-diagnostics` / `query-api-dns-diagnostics`, body `{from,to,group_by,locations}` | -| response body / headers / named content | `get-result-content ` (the second positional arg is the path into the captured content) | -| assertion / condition outcomes | `get-call-conditions ` plus the `result` category; `conformance-results` / `conformance-results-summary` for spec conformance | -| browser screenshot | `get-result-screenshot ` | -| MCP session step / tool response | the MCP result summary and `get-result-content` | +| response body / headers | `get-result` (`response.content`, `response.headers`; `--mode full` adds the request body). For an oversized body (`"<< LARGE DATA >>"`), `get-result-content ` is the retrieval endpoint — check its `--help`, and treat `Content not found` as not retrievable, not as an empty body | +| assertion / condition outcomes | `get-call-conditions ` plus `meta.result_category`/`meta.result`; `conformance-results` / `conformance-results-summary` for spec conformance | +| browser screenshot | `get-result-screenshot > shot.jpg` (binary JPEG on stdout; browser results only) | +| MCP session step / tool response | the MCP result from `get-result` | Example — is a slow/failing API monitor a DNS problem, and is it isolated to one location? @@ -129,12 +128,7 @@ For each candidate, list supporting and contradicting evidence. Run an on-demand check only when authorized and when it will not create harmful traffic. -API: -```bash -apimetrics run-call -``` - -Browser/MCP: +`run-monitor ` runs API, browser, and MCP monitors alike. It requires a body (at least `{}`) and returns the `result_id` to poll with `get-result` until `meta.result` is no longer `QUEUED`: ```bash apimetrics run-monitor <<'EOF' {} diff --git a/skills/claude-skills/apimetrics-incident-triage/SKILL.md b/skills/claude-skills/apimetrics-incident-triage/SKILL.md index f8cc1e76..ab76f1d4 100644 --- a/skills/claude-skills/apimetrics-incident-triage/SKILL.md +++ b/skills/claude-skills/apimetrics-incident-triage/SKILL.md @@ -20,8 +20,8 @@ Use project-wide evidence to distinguish an isolated monitor failure from a shar ``` Run `apimetrics login` or `apimetrics project select` only when needed. 3. The CLI command tree is generated from the platform's current OpenAPI description. Inspect `apimetrics --help` before constructing a body or assuming an option name. -4. Commands are generally flat (`list-calls`, `create-call`), not noun/verb groups. -5. Create and update operations read JSON from stdin. Use a quoted heredoc: +4. Commands are generally flat (`list-calls`, `create-call`). A command the server has deprecated is hidden from `--help` and prints `Command "X" is deprecated, do not use` on stderr; use the replacement `--help` lists (for example `list-api-monitors`/`list-monitors` and `create-api-monitor` where `list-calls`/`create-call` are deprecated). +5. Create and update operations read JSON from stdin (CLI Shorthand arguments also work, but quote any value containing `//` and add ` ... <<'EOF' { @@ -29,9 +29,8 @@ Use project-wide evidence to distinguish an isolated monitor failure from a shar } EOF ``` - Do not invent `--body`, `--data`, or `-d`. -6. Use `-o json` for analysis. Use `-f` only after inspecting the response shape. The top-level response envelope includes status, headers, and `body`. List bodies are NOT uniform: `list-calls`, `list-results`, `list-call-results`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` returns `{"data":[...]}`; `list-slos`, `list-browser-monitors`, and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Inspect each command's own output before writing an `-f` path (e.g. `-f body.results[0]` vs `-f body.data[0]`). -7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation. +6. Use `-o json` for analysis; it prints the response body. Use `-f` only after inspecting the response shape: `-f` paths are rooted at the whole response (`status`, `headers`, `body`), so they start with `body.`, and a path that matches nothing prints nothing — empty `-f` output is not evidence of an empty list. List bodies are NOT uniform: `list-calls`, `list-api-monitors`, `list-results`, `list-results-by-call`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` and `list-monitors` return `{"data":[...], "meta":...}`; `list-browser-monitors` and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Single-quote `-f` paths so the shell doesn't expand `[]` (e.g. `-f 'body.results[0]'` vs `-f 'body.data[0]'`). +7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation; the CLI appends it without validation. 8. Preserve evidence. Record the active project, commands run, IDs, time window, and the smallest response excerpts needed to support conclusions. 9. Never print, store, or paste credentials into the report. Prefer existing auth-setting IDs. Do not include bearer tokens, cookies, API keys, client secrets, or private certificate contents. 10. Do not mutate project configuration unless the user explicitly authorized the change. Before any mutation, show the planned objects and rollback path. @@ -43,17 +42,19 @@ Use project-wide evidence to distinguish an isolated monitor failure from a shar ### 1. Define the incident window -Set exact start/end timestamps and timezone. Default to a narrow window around the reported symptom, then expand only when necessary. +Set exact start/end timestamps and timezone, as full ISO-8601 date-times (the server buckets in UTC). Default to a narrow window around the reported symptom, then expand only when necessary. ### 2. Inventory and collect failures Collect the project-wide result stream for the window with server-side filtering, then segment: ```bash -apimetrics list-results --since --before -o json # {meta, results:[...]}, page via meta.next_cursor +apimetrics list-results --from --time -o json # {meta, results:[...]}, one page per call ``` -Each row gives `result`, `http_code`, `response_time`, `location_id`, and `test` (monitor ID) — enough to segment by monitor, status, and location. For shared-cause hypotheses, use `query-api-dns-diagnostics` (DNS provider/IP/CNAME drift by location) and `query-api-performance` (latency shift by location/interval) over the incident window. Map monitor IDs back to names/hosts/auth with `list-calls`, `list-browser-monitors`, `list-mcp-monitors`, and `list-auth-settings`. Identify: +(`list-results` takes `--from` (start) and `--time` (end), plus `--result-category`/`--cursor`/`--limit` (default 100). Always pass `--from` — without it only a short recent window is searched. `--time` is truncated to the minute, so round the window end up. Repeat with `--cursor ` while `meta.more` is true. `--result-category` takes `PASS`/`SLOW`/`WARNING`/`FAIL` and silently ignores any other value, returning every category; failures span both `FAIL` (e.g. connection errors) and `WARNING` (HTTP 4xx), so collect both.) + +Each row is `{id, meta, timing}`: `meta.result_category`, `meta.result` (failure detail such as `HTTP_CLIENT_ERROR` or `CONNECTION_ERROR`), `meta.http_code`, `meta.location_id`, `meta.call_id` (monitor ID), `meta.start_time`, and `timing.total` (ms, with `dns`/`tcp`/`handshake`/`processing`/`download` components) — enough to segment by monitor, status, and location. For shared-cause hypotheses, use `query-api-dns-diagnostics` (DNS provider/IP/CNAME drift by location) and `query-api-performance` (latency shift by location/interval) over the incident window. Map `meta.call_id` back to names/types/hosts with `list-monitors` (every type; `{data:[...], meta}`, type in `meta.monitor_type`) or `list-calls`/`list-browser-monitors`/`list-mcp-monitors`, and to auth with `list-auth-settings`. Project these with `-f` to the fields you need — monitor `request.headers`/`request.body` and auth settings can hold literal credentials. Identify: - affected monitors and unaffected controls - target hosts/services diff --git a/skills/claude-skills/apimetrics-monitoring-estate-audit/SKILL.md b/skills/claude-skills/apimetrics-monitoring-estate-audit/SKILL.md index 5c721862..95b4fad6 100644 --- a/skills/claude-skills/apimetrics-monitoring-estate-audit/SKILL.md +++ b/skills/claude-skills/apimetrics-monitoring-estate-audit/SKILL.md @@ -20,8 +20,8 @@ Perform a read-only audit by default. The deliverable is a prioritized cleanup a ``` Run `apimetrics login` or `apimetrics project select` only when needed. 3. The CLI command tree is generated from the platform's current OpenAPI description. Inspect `apimetrics --help` before constructing a body or assuming an option name. -4. Commands are generally flat (`list-calls`, `create-call`), not noun/verb groups. -5. Create and update operations read JSON from stdin. Use a quoted heredoc: +4. Commands are generally flat (`list-calls`, `create-call`). A command the server has deprecated is hidden from `--help` and prints `Command "X" is deprecated, do not use` on stderr; use the replacement `--help` lists (for example `list-api-monitors`/`list-monitors` and `create-api-monitor` where `list-calls`/`create-call` are deprecated). +5. Create and update operations read JSON from stdin (CLI Shorthand arguments also work, but quote any value containing `//` and add ` ... <<'EOF' { @@ -29,9 +29,8 @@ Perform a read-only audit by default. The deliverable is a prioritized cleanup a } EOF ``` - Do not invent `--body`, `--data`, or `-d`. -6. Use `-o json` for analysis. Use `-f` only after inspecting the response shape. The top-level response envelope includes status, headers, and `body`. List bodies are NOT uniform: `list-calls`, `list-results`, `list-call-results`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` returns `{"data":[...]}`; `list-slos`, `list-browser-monitors`, and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Inspect each command's own output before writing an `-f` path (e.g. `-f body.results[0]` vs `-f body.data[0]`). -7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation. +6. Use `-o json` for analysis; it prints the response body. Use `-f` only after inspecting the response shape: `-f` paths are rooted at the whole response (`status`, `headers`, `body`), so they start with `body.`, and a path that matches nothing prints nothing — empty `-f` output is not evidence of an empty list. List bodies are NOT uniform: `list-calls`, `list-api-monitors`, `list-results`, `list-results-by-call`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` and `list-monitors` return `{"data":[...], "meta":...}`; `list-browser-monitors` and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Single-quote `-f` paths so the shell doesn't expand `[]` (e.g. `-f 'body.results[0]'` vs `-f 'body.data[0]'`). +7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation; the CLI appends it without validation. 8. Preserve evidence. Record the active project, commands run, IDs, time window, and the smallest response excerpts needed to support conclusions. 9. Never print, store, or paste credentials into the report. Prefer existing auth-setting IDs. Do not include bearer tokens, cookies, API keys, client secrets, or private certificate contents. 10. Do not mutate project configuration unless the user explicitly authorized the change. Before any mutation, show the planned objects and rollback path. @@ -43,16 +42,19 @@ Perform a read-only audit by default. The deliverable is a prioritized cleanup a ### 1. Discover the estate -Run the available list/read commands (JSON output), noting each envelope shape: +Run the available list/read commands (JSON output), noting each envelope shape. Monitor `request.headers`/`request.body` and auth settings can hold literal credentials, so project each list with `-f` to the fields you need instead of printing full JSON: -- API calls — `list-calls` (`{meta, results}`) +- every monitor type — `list-monitors` (`{data, meta}`; type in `meta.monitor_type` = `api`/`www`/`mcp`), e.g. `-f 'body.data.{id, name: meta.name, type: meta.monitor_type, domain: meta.domain, tags: meta.tags}'` +- API calls — `list-calls` (`{meta, results}`), or `list-api-monitors` where `list-calls` is deprecated - browser monitors — `list-browser-monitors` (bare `{results}`) - MCP monitors — `list-mcp-monitors` (bare `{results}`) -- schedules — `list-schedules` (**`{data:[...]}`**), plus `list-schedule-downtimes` for maintenance windows -- SLOs — `list-slos` (bare `{results}`), detail via `get-slo ` -- auth settings/tokens as metadata only — `list-auth-settings`, and `list-calls-by-auth ` to size the blast radius of each shared credential -- recent results — `list-results --since ` for working-vs-configured coverage -- which schedules a call belongs to — `list-schedules-for-call ` +- schedules — `list-schedules` (**`{data:[...], meta}`**; schedule ID at `data[].meta.id`, targets at `data[].schedule.target_ids`), plus `list-schedule-downtimes ` per schedule for maintenance windows +- SLO — `get-project-slo` (a single object per project, scoped via `include_tags`/`exclude_tags`). `created: null` means no SLO is saved and the object holds platform defaults; report that as "no SLO defined" +- auth settings/tokens as metadata only — `list-auth-settings` returns credential values (`keys.client_secret`, `settings.custom_variables`) in clear text, so always project it: `-f 'body.results.{id, name: meta.name, auth_type: settings.auth_type, domain: meta.domain}'`. Use `list-calls-by-auth ` to size the blast radius of each shared credential +- assertions — `get-call-conditions ` per API call, and `get-project-conditions` for the project defaults +- locations — `list-agents-for-project`, plus each schedule's locations/regions +- recent results — `list-results --from ` for working-vs-configured coverage +- which schedules a call belongs to — `list-schedules-by-call ` (`{meta, results}`) ### 2. Normalize the inventory diff --git a/skills/claude-skills/apimetrics-performance-analytics/SKILL.md b/skills/claude-skills/apimetrics-performance-analytics/SKILL.md index 5b17cfa1..0dea6c4b 100644 --- a/skills/claude-skills/apimetrics-performance-analytics/SKILL.md +++ b/skills/claude-skills/apimetrics-performance-analytics/SKILL.md @@ -1,12 +1,12 @@ --- name: apimetrics-performance-analytics -description: Analyze APImetrics latency distribution, component timing breakdown (DNS/connect/TLS/TTFB/response), DNS diagnostics, and browser resource/session data using the server-side query-* commands. Use for latency deep-dives, "why is this slow", percentile trends, or DNS/CDN drift questions across API, browser, or MCP monitors. +description: Analyze APImetrics latency distribution, component timing breakdown (API: DNS/connect/TLS/TTFB/response; browser: Web Vitals plus DNS/TCP/TLS; MCP: initialize plus DNS/TCP/TLS), DNS diagnostics, and browser resource/session data using the server-side query-* commands. Use for latency deep-dives, "why is this slow", percentile trends, or DNS/CDN drift questions across API, browser, or MCP monitors. argument-hint: "[monitor, service, or metric focus]" --- # APImetrics performance analytics -Answer performance and latency questions with the platform's **server-computed** aggregate commands rather than by scraping raw result summaries. Result summaries carry only a single `response_time`; the `query-*` family returns percentiles, per-component timing, and DNS breakdowns over a window. +Answer performance and latency questions with the platform's **server-computed** aggregate commands rather than by scraping raw result summaries. Result rows carry one run's `timing` (`total` and components); the `query-*` family returns server-computed percentiles, per-component aggregates, and DNS breakdowns over a window. ## Operating rules @@ -19,15 +19,14 @@ Answer performance and latency questions with the platform's **server-computed** ``` Run `apimetrics login` or `apimetrics project select` only when needed. 3. The CLI command tree is generated from the platform's current OpenAPI description. Inspect `apimetrics --help` before constructing a body or assuming an option name. -4. Commands are generally flat (`query-api-performance`), not noun/verb groups. +4. Commands are generally flat (`query-api-performance`). 5. The `query-*` commands read a JSON body from stdin. Use a quoted heredoc: ```bash apimetrics query-api-performance <<'EOF' { "from": "...", "to": "..." } EOF ``` - Do not invent `--body`, `--data`, or `-d`. -6. Use `-o json` for analysis. Query responses are `{meta, results:[...]}`, where `meta` echoes the requested `from`/`to`/`measures`/`metrics`/`group_by` so you can confirm the CLI honored your request. +6. Use `-o json` for analysis; it prints the response body. `-f` paths are rooted at the whole response, so they start with `body.` (e.g. `-f 'body.results'`). Performance query responses are `{meta, results:[...]}`, where `meta` echoes the requested `from`/`to`/`measures`/`metrics`/`group_by`/`interval` so you can confirm the CLI honored your request; DNS diagnostics' `meta` echoes only the window, `group_by`, and `interval`. 7. Use `-q key=value` only for query parameters confirmed by command help. 8. Preserve evidence. Record the active project, commands run, IDs, exact window, and the smallest response excerpts needed to support conclusions. 9. Never print, store, or paste credentials. Do not include bearer tokens, cookies, API keys, client secrets, or private certificate contents. @@ -66,13 +65,19 @@ Baselines and org/project rollups also exist: `baseline-dailies`, `baseline-hour ``` - `from`/`to` are required ISO-8601 timestamps. -- `metrics` (API): `total`, `dns`, `connect`, `tls`, `ttfb`, `response`. -- `measures`: `mean`, `p50`, `p90`, `p95`, `p99`. -- `group_by`: `location`, `interval`, `monitor`. Add `interval` (ISO-8601 duration, e.g. `PT1H`, `P1D`) to produce a time series. -- `status` filters which result categories feed the aggregate. +- `metrics` differ by monitor type (omitted, API defaults to `total`, `dns`, `ttfb`, `response`): + - API: `total`, `dns`, `connect`, `tls`, `ttfb`, `response`, `upload`, `network`, `server`, `total_no_dns`. + - Browser: `lcp`, `fcp`, `cls`, `fid`, `ttfb`, `total`, `dns`, `tcp`, `tls`, `request`, `response`, `dom_processing`, `dom_content_loaded`, `load_event_end`, `resources_count`, `page_weight`, `network`, `total_no_dns`. + - MCP: `total`, `initialize`, `dns`, `tcp`, `tls`, `network`, `total_no_dns`. + A name from another type's list is rejected with 422; confirm with the command's `--help`. +- `measures`: `mean`, `p50`, `p90`, `p95`, `p99` (all five when omitted). Responses also carry `p75`, `p999`, `sample_count`, and `pass_count`. +- `group_by`: `location`, `interval`, `monitor`. `group_by: ["interval"]` requires `interval` (ISO-8601 duration, e.g. `PT15M`, `PT1H`, `P1D`); setting `interval` buckets results by time either way. Buckets are UTC. +- `status` (`PASS`/`SLOW`/`WARNING`/`FAIL`) filters which result categories feed the aggregate. Omitted, every category is included, failed runs too; use `["PASS"]` (or `["PASS", "SLOW"]`) for the latency of successful runs. - `locations` restricts to specific agents. -DNS diagnostics take the same window/`group_by`/`locations` shape and return observed DNS providers, resolved IPs, CNAME chains, NS servers, and mean lookup time. +DNS diagnostics take the same window/`group_by`/`locations`/`status`/`interval` shape and return observed DNS providers, resolved IPs, CNAME chains, NS servers, FQDNs, registrars, TLDs, and mean lookup time. Project-wide `query-api-dns-diagnostics` groups only by `location`/`interval` — `monitor` is accepted but ignored — so use `query-api-monitor-dns-diagnostics ` per monitor. + +Browser resource queries (`*-resources`) take only `from`, `to`, and `interval`, and return `{meta, results, third_party_timeseries}`. Browser session queries (`*-sessions`) take `from`, `to`, `limit` (1–200, default 10), `order` (`asc`/`desc`), and `sort_by` (e.g. `lcp`, `total`, `page_weight`, `timestamp`), and return `{meta, results}` rows that include `result_id`. ## Workflow @@ -80,13 +85,13 @@ DNS diagnostics take the same window/`group_by`/`locations` shape and return obs State the metric of interest (availability is a different skill — this is latency/timing/DNS), the monitors or tags in scope, the exact window, and the comparison window if any. ### 2. Choose scope -Project-wide first to spot the outliers (`group_by: ["monitor"]`), then drill into a single monitor with the `*-monitor-*` variant and `group_by: ["location","interval"]`. +Project-wide first to spot the outliers (`group_by: ["monitor"]` on the performance queries), then drill into a single monitor with the `*-monitor-*` variant and `group_by: ["location","interval"]`. ### 3. Query and read Run the query, confirm `meta` echoes your window/measures, and read `results`. Each result carries a `key` (the group_by dimension values) and the requested measures per metric. ### 4. Interpret -- Split `total` into components (`dns` + `connect` + `tls` + `ttfb` + `response`) to locate where latency accrues. +- Split API `total` into components (`dns` + `connect` + `tls` + `ttfb` + `response`) to locate where latency accrues. A failed run counts in a group's `sample_count` but may have no value for a metric. - Compare `p95`/`p99` against `mean` to detect tail latency and jitter. - Compare across `location` groups to separate a regional/agent effect from a global one. - For DNS drift, compare resolved IPs / providers / lookup time across locations and time buckets. diff --git a/skills/claude-skills/apimetrics-project-bootstrap/SKILL.md b/skills/claude-skills/apimetrics-project-bootstrap/SKILL.md index 7f21123d..6089cc6d 100644 --- a/skills/claude-skills/apimetrics-project-bootstrap/SKILL.md +++ b/skills/claude-skills/apimetrics-project-bootstrap/SKILL.md @@ -21,8 +21,8 @@ Build a usable monitoring project from an empty or selected APImetrics project. ``` Run `apimetrics login` or `apimetrics project select` only when needed. 3. The CLI command tree is generated from the platform's current OpenAPI description. Inspect `apimetrics --help` before constructing a body or assuming an option name. -4. Commands are generally flat (`list-calls`, `create-call`), not noun/verb groups. -5. Create and update operations read JSON from stdin. Use a quoted heredoc: +4. Commands are generally flat (`list-calls`, `create-call`). A command the server has deprecated is hidden from `--help` and prints `Command "X" is deprecated, do not use` on stderr; use the replacement `--help` lists (for example `list-api-monitors`/`list-monitors` and `create-api-monitor` where `list-calls`/`create-call` are deprecated). +5. Create and update operations read JSON from stdin (CLI Shorthand arguments also work, but quote any value containing `//` and add ` ... <<'EOF' { @@ -30,9 +30,8 @@ Build a usable monitoring project from an empty or selected APImetrics project. } EOF ``` - Do not invent `--body`, `--data`, or `-d`. -6. Use `-o json` for analysis. Use `-f` only after inspecting the response shape. The top-level response envelope includes status, headers, and `body`. List bodies are NOT uniform: `list-calls`, `list-results`, `list-call-results`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` returns `{"data":[...]}`; `list-slos`, `list-browser-monitors`, and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Inspect each command's own output before writing an `-f` path (e.g. `-f body.results[0]` vs `-f body.data[0]`). -7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation. +6. Use `-o json` for analysis; it prints the response body. Use `-f` only after inspecting the response shape: `-f` paths are rooted at the whole response (`status`, `headers`, `body`), so they start with `body.`, and a path that matches nothing prints nothing — empty `-f` output is not evidence of an empty list. List bodies are NOT uniform: `list-calls`, `list-api-monitors`, `list-results`, `list-results-by-call`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` and `list-monitors` return `{"data":[...], "meta":...}`; `list-browser-monitors` and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Single-quote `-f` paths so the shell doesn't expand `[]` (e.g. `-f 'body.results[0]'` vs `-f 'body.data[0]'`). +7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation; the CLI appends it without validation. 8. Preserve evidence. Record the active project, commands run, IDs, time window, and the smallest response excerpts needed to support conclusions. 9. Never print, store, or paste credentials into the report. Prefer existing auth-setting IDs. Do not include bearer tokens, cookies, API keys, client secrets, or private certificate contents. 10. Do not mutate project configuration unless the user explicitly authorized the change. Before any mutation, show the planned objects and rollback path. @@ -66,7 +65,11 @@ apimetrics --help apimetrics project --help ``` -The `project` group exposes only `select` and `show` — **the CLI cannot create a project.** Use `apimetrics project select` to choose an existing project (created in the APImetrics web app) and state this plainly if the user expected creation. Re-check `apimetrics project --help` in case a future build adds more. +The `project` noun-group exposes `select` and `show`. Project creation is the top-level command `create-project-in-org ` (listed under "Projects Commands" in `apimetrics --help`). It requires org-admin rights; for a user without them it returns `403 Forbidden`, in which case use `apimetrics project select` to choose an existing project (created by an org admin or in the APImetrics web app) and state the permission limitation plainly. + +**If you do create a project with `create-project-in-org`, it does not grant you access to it.** Immediately follow with `create-project-access ` for your own account, or every subsequent `create-call`/monitor command in that project will 403. With a body of just `{"access_level": "OWNER"}` it grants the calling account. The grantee is set only by `account_id`, which defaults to the caller; `email` is an informational label, so a body with `email` but no `account_id` still grants the caller. Check that `account_email` in the response is the intended account, then verify with `list-project-access ` before proceeding. Grants are not de-duplicated, so do not repeat the call. + +To work in a project without changing the saved selection (for example, one you just created), pass the global `--project-id ` flag on each command. Confirm the selected project after creation/selection: @@ -80,13 +83,14 @@ apimetrics project show --id Discover and run the available read-only inventory commands, normally including: ```bash +apimetrics list-monitors -o json apimetrics list-calls -o json apimetrics list-browser-monitors -o json apimetrics list-mcp-monitors -o json apimetrics list-schedules -o json ``` -Some builds may use different monitor names. Use `apimetrics --help` as authority. Identify duplicates before creating anything. +`list-monitors` covers every type (`{data, meta}`, type in `meta.monitor_type`: `api`, `www`, `mcp`) and pages with `--cursor`. Some builds may use different monitor names. Use `apimetrics --help` as authority. Identify duplicates before creating anything: the platform does not reject a second monitor with the same name. ### 3. Design the minimum useful monitoring set @@ -123,6 +127,13 @@ apimetrics create-call <<'EOF' EOF ``` +Where the server has deprecated `create-call`, use `create-api-monitor`, which takes a flat body (`name`, `method`, `url`, plus optional `headers`, `parameters`, `body`, `auth_id`, `token_id`, `file_id`, `description`, `tags`, `workspace`): +```bash +apimetrics create-api-monitor <<'EOF' +{ "name": "", "method": "GET", "url": "https://example.com/health" } +EOF +``` + Browser monitor: ```bash apimetrics create-browser-monitor <<'EOF' @@ -138,17 +149,20 @@ MCP monitor: apimetrics create-mcp-monitor <<'EOF' { "name": "", - "url": "https://mcp.example.com/sse", + "url": "https://mcp.example.com/mcp", "steps": [ { "step_type": "list_tools", - "timeout_ms": 10000 + "timeout_ms": 10000, + "wait_ms": 0 } ] } EOF ``` +The MCP `url` must be the server's Streamable HTTP endpoint (usually `/mcp`); an SSE-only endpoint (`/sse`) fails with HTTP 405. Every MCP step needs `timeout_ms` and `wait_ms`. + Treat these as starting patterns, not substitutes for current help. Capture every returned ID. If a create response does not contain an ID, stop that branch and diagnose it. If the user already has the request as a `curl` command, convert it instead of hand-writing the body: @@ -159,29 +173,31 @@ apimetrics build-call-from-curl <<'EOF' EOF ``` -`build-call-from-curl` returns a call definition (it does **not** persist it); pipe or paste the `meta`/`request` object into `create-call`. +`build-call-from-curl` returns a call definition (it does **not** persist it); pipe or paste the `meta`/`request` object into `create-call`. For `create-api-monitor`, flatten it: `name` from `meta.name`; `method`, `url`, `headers`, `parameters`, `body`, and `auth_id` from `request`. ### 5. Configure assertions, auth, and metadata -Use only fields confirmed by the command's option/body schema. The `create-call` body carries `meta` (`name`, `description`, `domain`, `tags`, `username`, `workspace`) and `request` (`method`, `url`, `headers[{key,value}]`, `parameters`, `body`, `auth_id`, `token_id`, `file_id`). +Use only fields confirmed by the command's option/body schema. The `create-call` body carries `meta` (`name`, `description`, `tags`, `workspace`) and `request` (`method`, `url`, `headers[{key,value}]`, `parameters`, `body`, `auth_id`, `token_id`, `file_id`). The monitor's domain is derived from `request.url` and cannot be set. - Prefer references to existing auth settings (`auth_id`/`token_id` from `list-auth-settings`) over embedding secrets. -- Add `meta.tags`/`meta.domain` when the schema supports them. +- Add `meta.tags` and `meta.description` when the schema supports them. - **Assertions for API calls are NOT part of the `create-call` body.** Post-test conditions (assertions) are set with a separate command: ```bash apimetrics get-call-conditions # read current conditions apimetrics set-call-conditions <<'EOF' # replace conditions { "meta": { "call_id": "", "break_on_fail": false }, - "conditions": [ { "source": "...", "condition": "...", "val": "..." } ] + "conditions": [ + { "source": "HTTP_STATUS", "condition": "EQUAL_TO", "val": "200", "test_result_on_false": "HTTP_CLIENT_ERROR" } + ] } EOF ``` - Inspect `apimetrics set-call-conditions --help` for the exact `condition`/`source` grammar before writing one. Without conditions, APImetrics grades a call on its HTTP status alone. + `meta` is optional. The help does not list the allowed values, and an unknown value returns 400 (`Unknown condition source` / `Unknown condition operator`). Sources include e.g. `HTTP_STATUS`, `HEADERS`, `RESPONSE_BODY`, `RESPONSE_TIME`, `RESPONSE_SIZE`, `NAMELOOKUP_TIME`, `TIME_TO_CONNECT`, `TIME_TO_FIRST_BYTE`, and for MCP monitors `INITIALIZE_TIME`, `STEP_RESULT`, `STEP_TIME`, `STEP_ERROR`. Operators include `EXISTS`, `DOES_NOT_EXIST`, `EQUAL_TO`, `NOT_EQUAL_TO`, `LESS_THAN`, `LESS_THAN_OR_EQUAL_TO`, `GREATER_THAN`, `GREATER_THAN_OR_EQUAL_TO`, `CONTAINS`, `DOES_NOT_CONTAIN`. `test_result_on_true`/`test_result_on_false` take result codes such as `HTTP_CLIENT_ERROR`, `CONTENT_ERROR`, `SLA_WARNING`, `SLOW_COMPLETE`. Without call conditions, a call is graded on its HTTP status plus the project-wide conditions (`get-project-conditions`), which by default mark slow responses `SLOW`. - For MCP, verify the transport/endpoint type and step schema. - For browser, configure a journey or performance expectations only when supported by this CLI build. -Read the created object back (`get-call ` / `get-call-conditions `) and compare it against the plan. +Read the created object back (`get-monitor ` for any type, or `get-call`/`get-api-monitor`, `read-browser-monitor`, `read-mcp-monitor`; plus `get-call-conditions `) and compare it against the plan. Project the output with `-f` — monitor configs can hold literal credentials in headers or bodies. ### 6. Create or reuse schedules @@ -197,6 +213,8 @@ The known positional pattern is: apimetrics add-call-to-schedule ``` +It rejects an invalid ID, another project's ID, swapped arguments, or a target already in the schedule with 400, but accepts a valid ID of a different monitor in this project without warning — confirm the ID first. + To create a schedule, inspect current help and use stdin JSON. A common body is: ```bash @@ -209,20 +227,11 @@ apimetrics create-schedule <<'EOF' EOF ``` -Frequency is in seconds. The body also accepts `locations` (agent IDs), `regions`, `tags`, and `backoff_method` (`fibo`, `expo`, `constant`, or `none`). Read the schedule back with `get-schedule ` and confirm every intended target is present. Note `list-schedules` returns its list under `data`, not `results`. +Frequency is in seconds. The body also accepts `locations` (agent IDs), `regions`, `tags`, and `backoff_method` (`fibo`, `expo`, `constant`, or `none`). The response carries the schedule ID at `meta.id` and its targets at `schedule.target_ids`. Read the schedule back with `get-schedule ` and confirm every intended target is present. Note `list-schedules` returns its list under `data`, not `results`. ### 7. Verify on demand -For API calls (`run-call` accepts an optional body of `location_id`, `run_delay` (0–86400s), and `context`; with no body it runs at the default location): - -```bash -apimetrics run-call -apimetrics list-call-results -o json -``` - -`run-call` returns `{call_id, location_id, result_id}`. `list-call-results` takes the call ID and supports `--since`/`--before` (ISO-8601) and `--limit` (max 100). - -For browser or MCP monitors (both use `run-monitor`; there is no `run-mcp-monitor` command): +One command, `run-monitor `, runs API calls, browser monitors, and MCP monitors alike. It requires a JSON body — `{}` for defaults — with optional `location_id`, `run_delay` (seconds), and `context`; without a `location_id` the platform picks a location suited to the monitor type: ```bash apimetrics run-monitor <<'EOF' @@ -231,7 +240,9 @@ EOF apimetrics get-result -o json ``` -Wait between polls. Match the exact returned `result_id`. The `result` field is one of `PASS`, `FAIL`, `WARN`, `ERROR`, `TIMEOUT`, or `QUEUED`; poll until it leaves `QUEUED`. Note that `get-result` returns only a summary (`result`, `http_code`, `response_time` ms, `location_id`, `test`, `created`); deeper forensics live in `get-result-content`, `get-result-screenshot`, and the `query-*` commands. A successful HTTP transport alone is not enough: verify the platform result status and any configured conditions. +`run-monitor` returns `{call_id, context, location_id, result_id, user}`. For a monitor's result history (any type) use `list-results-by-call ` — flags are `--from`, `--time` (full ISO-8601 date-times; a bare date is rejected), `--result-category` (`PASS`/`SLOW`/`WARNING`/`FAIL`; any other value is silently ignored and returns every category), `--cursor`, `--limit` (default 100). + +Wait between polls. Match the exact returned `result_id`. Poll until `meta.result` is no longer `QUEUED`; `meta.result_category` is empty until then, and afterwards one of `PASS`, `SLOW`, `WARNING`, or `FAIL`, with `meta.result` holding the detailed code (`COMPLETE`, `HTTP_CLIENT_ERROR`, `CONNECTION_ERROR`, `SLA_ERROR`, …). Read the outcome with `-f 'body.meta.{result_category, result, http_code}'`; total time is `timing.total` in ms. **`get-result`'s detail depends on your role**: a below-ANALYST caller gets a summary only (`meta` and `timing`, no `request`, `response`, or `context`), but most project members (ANALYST or above) get the full object back by default — response headers and body, request method/URL/headers, DNS/TLS detail, and the `timing` breakdown, no separate call needed; `--mode full` adds the request body and log messages. Check what actually comes back before reaching for `get-result-content`/`get-result-screenshot`/the `query-*` commands for deeper forensics. A successful HTTP transport alone is not enough: verify the platform result status and any configured conditions. ### 8. Final audit and handoff @@ -244,13 +255,14 @@ Re-list monitors and schedules. Produce: - frequencies and agents/locations - auth references without secrets - known gaps and next recommended monitors -- rollback commands or object IDs to remove. The reversal commands are: `remove-call-from-schedule ` to detach, then `delete-call`, `delete-browser-monitor`, `delete-mcp-monitor`, or `delete-schedule` to remove created objects +- rollback commands or object IDs to remove. The reversal commands are: `remove-call-from-schedule ` to detach, then `delete-call` (or `delete-api-monitor` where `delete-call` is deprecated), `delete-browser-monitor`, `delete-mcp-monitor`, or `delete-schedule` to remove created objects ## Failure handling -- `400`: inspect current command help and required body fields. -- `401/403`: re-authenticate and confirm the selected project; do not assume an unsupported `--api-key` flag. -- `409`: look for an existing duplicate and prefer reuse/update. -- `422`: report quota or validation details exactly as returned. +- `400`: a value was rejected; read `error_msg` and inspect current command help. +- `401/403`: re-authenticate with `apimetrics login` and confirm the selected project with `apimetrics project show`. +- duplicates: the platform mostly does not reject them (a same-name monitor or a repeated access grant is created again), so rely on the inventory step and prefer reuse/update. +- `422`: schema validation; `detail[].loc` names the field. Report it exactly as returned. +- out of quota: the error reads `Project is out of quota`; report it and stop creating runs. - no result: confirm the run was queued, use the exact result ID, and wait within the monitor timeout plus processing time. - failure after creation: do not broaden rollout. Diagnose or remove the failed object according to the authorized rollback plan. diff --git a/skills/claude-skills/apimetrics-slo-review/SKILL.md b/skills/claude-skills/apimetrics-slo-review/SKILL.md index 8c40ec4a..9d69408b 100644 --- a/skills/claude-skills/apimetrics-slo-review/SKILL.md +++ b/skills/claude-skills/apimetrics-slo-review/SKILL.md @@ -20,8 +20,8 @@ Review both the SLO configuration and the evidence behind it. Do not recommend c ``` Run `apimetrics login` or `apimetrics project select` only when needed. 3. The CLI command tree is generated from the platform's current OpenAPI description. Inspect `apimetrics --help` before constructing a body or assuming an option name. -4. Commands are generally flat (`list-calls`, `create-call`), not noun/verb groups. -5. Create and update operations read JSON from stdin. Use a quoted heredoc: +4. Commands are generally flat (`list-calls`, `create-call`). A command the server has deprecated is hidden from `--help` and prints `Command "X" is deprecated, do not use` on stderr; use the replacement `--help` lists (for example `list-api-monitors`/`list-monitors` and `create-api-monitor` where `list-calls`/`create-call` are deprecated). +5. Create and update operations read JSON from stdin (CLI Shorthand arguments also work, but quote any value containing `//` and add ` ... <<'EOF' { @@ -29,9 +29,8 @@ Review both the SLO configuration and the evidence behind it. Do not recommend c } EOF ``` - Do not invent `--body`, `--data`, or `-d`. -6. Use `-o json` for analysis. Use `-f` only after inspecting the response shape. The top-level response envelope includes status, headers, and `body`. List bodies are NOT uniform: `list-calls`, `list-results`, `list-call-results`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` returns `{"data":[...]}`; `list-slos`, `list-browser-monitors`, and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Inspect each command's own output before writing an `-f` path (e.g. `-f body.results[0]` vs `-f body.data[0]`). -7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation. +6. Use `-o json` for analysis; it prints the response body. Use `-f` only after inspecting the response shape: `-f` paths are rooted at the whole response (`status`, `headers`, `body`), so they start with `body.`, and a path that matches nothing prints nothing — empty `-f` output is not evidence of an empty list. List bodies are NOT uniform: `list-calls`, `list-api-monitors`, `list-results`, `list-results-by-call`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` and `list-monitors` return `{"data":[...], "meta":...}`; `list-browser-monitors` and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Single-quote `-f` paths so the shell doesn't expand `[]` (e.g. `-f 'body.results[0]'` vs `-f 'body.data[0]'`). +7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation; the CLI appends it without validation. 8. Preserve evidence. Record the active project, commands run, IDs, time window, and the smallest response excerpts needed to support conclusions. 9. Never print, store, or paste credentials into the report. Prefer existing auth-setting IDs. Do not include bearer tokens, cookies, API keys, client secrets, or private certificate contents. 10. Do not mutate project configuration unless the user explicitly authorized the change. Before any mutation, show the planned objects and rollback path. @@ -43,40 +42,38 @@ Review both the SLO configuration and the evidence behind it. Do not recommend c ### 1. Discover SLO operations -The SLO command set is CRUD only: `list-slos`, `get-slo ` (not `read-slo`), `create-slo`, `update-slo`, `delete-slo`. There is **no SLO attainment, status, or error-budget endpoint** — the CLI returns SLO *definitions*, not computed attainment. Plan to derive attainment yourself from result and performance data (step 4). +Each project has one SLO, managed with `get-project-slo`, `update-project-slo` (which creates it if none exists yet), and `delete-project-slo`. Use the global `--project-id` flag to target a project other than the active one. `get-project-slo` returns the SLO definition; attainment comes from the server-side pass/fail and performance aggregates in step 4. If `apimetrics --help` lists a project-level SLO attainment or error-budget command, use that instead. ```bash -apimetrics list-slos -o json # bare {results:[...]}, no pagination meta -apimetrics get-slo -o json +apimetrics get-project-slo -o json # single SLO object ``` -Each SLO has `include_tags`/`exclude_tags` (the monitor scope), `scope` (`project`/`workflow`/`monitor`) + `scope_id`, `objectives[]`, and `thresholds[]`. An objective has `metric` (e.g. `availability`, `total`, `dns`, `latency`-family, plus web-vitals like `largest_contentful_paint`), `measure` (`mean`/`value`), `comparator` (`<`/`>`), `value`, `unit` (`ms`/`percent`/`pp`/`value`), and `period` (`PT5M`…`PT1H`, `DAY`, `WEEK`, `MONTH`). +If `created` and `last_update` are null, no SLO is stored and the object holds platform defaults — report it as "no SLO defined" rather than reviewing the defaults as the project's own. `update-project-slo` merges top-level keys into the stored SLO, so send the full `objectives` array when replacing objectives. + +The object has `include_tags`/`exclude_tags` (the monitor scope, tag-based only), `objectives[]`, and `thresholds[]`. An objective has `metric` (observed values include `availability`, `slow`, `dns`, `tcp`, `casc`, plus web-vitals like `largest_contentful_paint`/`cumulative_layout_shift` — confirm the current set with `apimetrics update-project-slo --help`), `measure` (`mean`/`value`), `comparator` (`<`/`>`), `value`, `unit` (`ms`/`percent`/`pp`/`value`), a `description`, and `period` (`DAY`, `WEEK`, `MONTH`). Percent values are fractions (`0.995` = 99.5%). A threshold has `metric` (`grade`/`latency`), `period`, `unit`, `delta`, and `description`. ### 2. Define the review period -Use the user-specified period or the previous complete calendar month for formal review; use seven days for an operational pulse. State exact timestamps and timezone. +Use the user-specified period or the previous complete calendar month for formal review; use seven days for an operational pulse. State exact timestamps and timezone; the server's daily buckets are UTC days. ### 3. Inspect definitions -For each scoped SLO capture: +For each objective in the project's SLO (and, separately, each threshold) capture: -- name/ID -- target and objective -- evaluation window -- associated monitor(s) -- threshold and measurement -- enabled state -- ownership/tags -- alerting or reporting relationship when exposed +- objective target and metric +- evaluation window (`period`) +- monitor scope (`include_tags`/`exclude_tags`) +- comparator/value/unit and the `description` field +- ownership/tags on the scoped monitors themselves (the SLO object has no owner field) -Flag missing ownership, ambiguous scope, unattached SLOs, conflicting windows, and objectives unsupported by the monitored journey. +Flag ambiguous or overly broad tag scope, conflicting windows across objectives, and objectives unsupported by the monitored journey. ### 4. Derive attainment evidence -Because the CLI exposes no attainment endpoint, reconstruct it from the monitors the SLO scopes (via `include_tags`/`exclude_tags` or `scope_id`): +Use the server's aggregates for the monitors the SLO's `include_tags`/`exclude_tags` scope to (match tags against `meta.tags` in `list-monitors -o json`, which covers every monitor type). Keep `list-results`/`list-results-by-call` for drilling into specific violations in step 5. -- For availability/pass objectives, count `result` categories over the objective `period` using `list-results`/`list-call-results` with `--since`/`--before`. -- For latency objectives (`measure: mean`, metrics like `total`), use `query-api-performance` / `query-api-monitor-performance` with matching `metrics`/`measures` and window; align the query `interval` to the objective `period`. +- For availability/pass objectives, use `get-call-passfail-range --kind DAY|MONTH --start --end ` per scoped monitor; it returns server-counted `pass`/`warning`/`failure` per period as an object keyed by period, omitting periods with no runs. Sum across monitors for the project figure. `get-call-passfail-total --start --end` gives window totals as `stat..total` for `PASS`/`SLOW`/`WARNING`/`FAIL`, broken down by `result` (e.g. `stat.WARNING.HTTP_CLIENT_ERROR`); sum the category totals for the run count — the top-level `stat.total` counts only uncategorised results. +- For latency objectives, use `query-api-performance` (with `group_by: ["monitor"]`, keeping only the scoped monitors) or `query-api-monitor-performance` with matching `metrics`/`measures` and window. For `DAY` objectives use `interval: "P1D"`; for `WEEK`/`MONTH` objectives run one query per calendar period with `from`/`to` set to its bounds, because `P1W`/`P1M` intervals become epoch-aligned 7- and 30-day buckets. Confirm the objective's actual `metric` name maps to a real `query-*-performance` metric before querying — don't assume it's called `total`. SLO and timing names differ from the API query names: `tcp`→`connect`, `handshake`→`tls`, `processing`→`server`, `download`→`response`; `dns`, `total`, `upload`, `network`, and `total_no_dns` keep their names; `availability`/`pass`/`slow`/`warning`/`fail` come from the pass/fail counts; web vitals map to `query-browser-performance` `lcp`/`fcp`/`cls`/`fid`; `casc` and `apdex` have no query metric. Report: @@ -89,7 +86,7 @@ Report: - largest violating periods - excluded/missing data -If the CLI does not provide enough raw information for an exact error-budget calculation, say so and report the platform-provided value or a clearly labeled approximation. +If the aggregates do not support an exact error-budget calculation, say so and report a clearly labeled approximation. ### 5. Explain violations @@ -111,4 +108,4 @@ Prioritize: 3. coverage improvements 4. SLO definition changes only when the objective is structurally wrong -Deliver a table with SLO, attainment, objective, budget status, cause, owner, action, and verification. +Deliver a table with objective (metric/scope), attainment, target, budget status, cause, owner, action, and verification — one row per objective. diff --git a/skills/claude-skills/apimetrics-weekly-health-review/SKILL.md b/skills/claude-skills/apimetrics-weekly-health-review/SKILL.md index 7b5c73fa..8962a8fa 100644 --- a/skills/claude-skills/apimetrics-weekly-health-review/SKILL.md +++ b/skills/claude-skills/apimetrics-weekly-health-review/SKILL.md @@ -6,7 +6,7 @@ argument-hint: "[optional project or monitor scope]" # APImetrics weekly health review -Analyze the previous seven complete calendar days in the active project's timezone unless the user specifies a different window. Compare with the preceding seven days when the CLI exposes enough history. +Analyze the previous seven complete UTC calendar days unless the user specifies a different window or timezone; the server's daily buckets (`get-call-passfail-range --kind DAY`, `interval: "P1D"`) are UTC days. Compare with the preceding seven days when the CLI exposes enough history. ## Operating rules @@ -20,8 +20,8 @@ Analyze the previous seven complete calendar days in the active project's timezo ``` Run `apimetrics login` or `apimetrics project select` only when needed. 3. The CLI command tree is generated from the platform's current OpenAPI description. Inspect `apimetrics --help` before constructing a body or assuming an option name. -4. Commands are generally flat (`list-calls`, `create-call`), not noun/verb groups. -5. Create and update operations read JSON from stdin. Use a quoted heredoc: +4. Commands are generally flat (`list-calls`, `create-call`). A command the server has deprecated is hidden from `--help` and prints `Command "X" is deprecated, do not use` on stderr; use the replacement `--help` lists (for example `list-api-monitors`/`list-monitors` and `create-api-monitor` where `list-calls`/`create-call` are deprecated). +5. Create and update operations read JSON from stdin (CLI Shorthand arguments also work, but quote any value containing `//` and add ` ... <<'EOF' { @@ -29,9 +29,8 @@ Analyze the previous seven complete calendar days in the active project's timezo } EOF ``` - Do not invent `--body`, `--data`, or `-d`. -6. Use `-o json` for analysis. Use `-f` only after inspecting the response shape. The top-level response envelope includes status, headers, and `body`. List bodies are NOT uniform: `list-calls`, `list-results`, `list-call-results`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` returns `{"data":[...]}`; `list-slos`, `list-browser-monitors`, and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Inspect each command's own output before writing an `-f` path (e.g. `-f body.results[0]` vs `-f body.data[0]`). -7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation. +6. Use `-o json` for analysis; it prints the response body. Use `-f` only after inspecting the response shape: `-f` paths are rooted at the whole response (`status`, `headers`, `body`), so they start with `body.`, and a path that matches nothing prints nothing — empty `-f` output is not evidence of an empty list. List bodies are NOT uniform: `list-calls`, `list-api-monitors`, `list-results`, `list-results-by-call`, and `list-auth-settings` return `{"meta":..., "results":[...]}`; `list-schedules` and `list-monitors` return `{"data":[...], "meta":...}`; `list-browser-monitors` and `list-mcp-monitors` return a bare `{"results":[...]}` with no `meta`/pagination. Single-quote `-f` paths so the shell doesn't expand `[]` (e.g. `-f 'body.results[0]'` vs `-f 'body.data[0]'`). +7. Use `-q key=value` only for query parameters confirmed by command help or observed request documentation; the CLI appends it without validation. 8. Preserve evidence. Record the active project, commands run, IDs, time window, and the smallest response excerpts needed to support conclusions. 9. Never print, store, or paste credentials into the report. Prefer existing auth-setting IDs. Do not include bearer tokens, cookies, API keys, client secrets, or private certificate contents. 10. Do not mutate project configuration unless the user explicitly authorized the change. Before any mutation, show the planned objects and rollback path. @@ -59,16 +58,18 @@ Calculate dates in reasoning; do not require a date utility. Run `apimetrics --help` and inspect help for the available inventory, result, and analytics operations. The grounded commands are: ```bash -apimetrics list-calls -o json # {meta, results:[...]} -apimetrics list-schedules -o json # {data:[...]} (note: data, not results) -apimetrics list-results --since --before -o json # project-wide result summaries -apimetrics list-call-results --since --before -o json -apimetrics get-result -o json # single result summary +apimetrics list-monitors -o json # {data:[...], meta} — every monitor type; type in meta.monitor_type +apimetrics list-calls -o json # {meta, results:[...]} — API monitors (list-api-monitors where list-calls is deprecated) +apimetrics list-schedules -o json # {data:[...], meta} (note: data, not results) +apimetrics list-results --from --time -o json # project-wide result rows +apimetrics list-results-by-call --from --time -o json +apimetrics get-result -o json # summary for below-ANALYST callers; full detail otherwise — see below +apimetrics get-call-passfail-range --kind DAY --start --end -o json # server-counted pass/warning/failure per day ``` -`list-results` and `list-call-results` support **server-side** `--since`/`--before` (ISO-8601) plus `--limit` (max 100) and `--cursor` — prefer these over pulling everything and filtering by hand. `list-results.meta.more`/`next_cursor` drive pagination. +`list-results` and `list-results-by-call` support **server-side** `--from`/`--time` plus `--result-category`, `--limit` (default 100), and `--cursor` — prefer these over pulling everything and filtering by hand. `--from`/`--time` are the window start and end as full ISO-8601 date-times (a bare date is rejected) and are minute-granular, so round the end up. `--result-category` takes `PASS`/`SLOW`/`WARNING`/`FAIL`; any other value is silently ignored and returns every category. Each call returns one page: repeat with `--cursor ` while `meta.more` is true. `get-call-passfail-range --kind` takes `HOUR`/`DAY`/`MONTH`/`YEAR`. -**Do not hand-compute percentiles from result summaries.** Result rows carry only `result`, `http_code`, `response_time` (ms), and `location_id` — not component timings. For latency distribution use the analytics commands, which compute the statistics server-side over your window: +**Do not hand-compute percentiles from result rows.** Each row carries one run's `timing` (`total` ms plus `dns`/`tcp`/`handshake`/`processing`/`download`/`upload`); for latency distribution across many results, use the analytics commands, which compute the statistics server-side over your window: ```bash apimetrics query-api-performance -o json <<'EOF' @@ -77,11 +78,11 @@ EOF ``` - `measures`: `mean`, `p50`, `p90`, `p95`, `p99` -- `metrics`: `total`, `dns`, `connect`, `tls`, `ttfb`, `response` -- `group_by`: `location`, `interval`, `monitor` (add `interval` with an ISO-8601 `interval` like `PT1H` for a time series) -- optional `status` filter: `PASS`, `SLOW`, `WARNING`, `FAIL` +- `metrics`: `total`, `dns`, `connect`, `tls`, `ttfb`, `response`, `upload`, `network`, `server`, `total_no_dns` +- `group_by`: `location`, `interval`, `monitor` (`interval` requires an ISO-8601 `interval` such as `PT1H` or `P1D`) +- optional `status` filter: `PASS`, `SLOW`, `WARNING`, `FAIL` — omitted, every category is included -Browser and MCP have parallel commands: `query-browser-performance` / `query-browser-monitor-performance`, `query-mcp-performance` / `query-mcp-monitor-performance`. Use `apimetrics --help` as authority for names in the current build. +Browser and MCP have parallel commands: `query-browser-performance` / `query-browser-monitor-performance`, `query-mcp-performance` / `query-mcp-monitor-performance`. Their metric names differ (e.g. browser `lcp`/`fcp`/`cls`, MCP `initialize`). Use `apimetrics --help` as authority for names in the current build. ### 3. Build the monitor census @@ -101,7 +102,8 @@ Flag unscheduled, disabled, orphaned, and never-run monitors separately; do not Two complementary sources, both with `-o json`: -- **Availability / pass-fail** — `list-results --since --before` (project-wide) or `list-call-results --since --before` (per monitor). Each row gives `result`, `http_code`, `response_time`, `location_id`, `test`, `created`. Page with `--cursor` until `meta.more` is false. Count by `result` category; do not disable pagination unless intentionally sampling. +- **Availability / pass-fail** — `get-call-passfail-range --kind DAY --start --end` per monitor returns server-counted `pass`/`warning`/`failure` per UTC day; sum across monitors for the project figure. `body.results` is an object keyed by day, and days with no runs are omitted — treat a missing day as no data, not as passing. +- **Failure detail** — `list-results --from --time --result-category ` (project-wide) or `list-results-by-call --from --time` (per monitor, any type) for the individual failing runs behind those counts. Each row is `{id, meta, timing}`: `meta.result_category`, `meta.result`, `meta.http_code`, `meta.location_id`, `meta.call_id` (monitor ID), `meta.start_time`, and `timing.total` (ms); page with `--cursor` until `meta.more` is false. - **Latency distribution** — the `query-*-performance` commands from step 2, which return server-computed `mean`/`p50`/`p95`/`p99` per metric, optionally grouped by monitor, location, and interval. For conformance signal, `conformance-results-summary` is available but best-effort (it can return a 500 on projects with no conformance data — treat a failure as "no data", not a finding). @@ -111,15 +113,15 @@ For conformance signal, `conformance-results-summary` is available but best-effo For the project and each meaningful monitor/segment, report: - total runs and runs with usable data -- pass count, fail count, warning/error/timeout count (the `result` categories are `PASS`, `FAIL`, `WARN`, `ERROR`, `TIMEOUT`, `QUEUED`) +- pass, warning, and failure counts from `get-call-passfail-range`; split failures by `meta.result_category` (`PASS`, `SLOW`, `WARNING`, `FAIL`; HTTP 4xx lands in `WARNING`) and the finer `meta.result` (e.g. `CONNECTION_ERROR`, `HTTP_CLIENT_ERROR`, `HTTP_SERVER_ERROR`) from the failure detail when the breakdown matters - availability/pass rate - median (`p50`) and `p95` latency **from `query-*-performance`**, not estimated from summaries - slowest monitors and locations (`group_by: ["monitor"]` / `["location"]`) - first/last failure and longest visible failure streak -- recurring failure signatures (`http_code` + `result` patterns) +- recurring failure signatures (`http_code` + `result_category` + `result` patterns) - location-specific or time-of-day concentration (`group_by` with `interval`) - changes versus the preceding week (run the same query for the prior window) -- SLO status/error budget when directly available +- SLO attainment against the `get-project-slo` objectives, from the same server-side aggregates (see `apimetrics-slo-review`) If the `query-*` commands return empty for the window (no data), say so explicitly rather than reporting a rate over zero runs. Label any hand-derived figure and its denominator. diff --git a/skills/embed/setup-api-monitor.md b/skills/embed/setup-api-monitor.md index e5f930e4..8fb4e7e6 100644 --- a/skills/embed/setup-api-monitor.md +++ b/skills/embed/setup-api-monitor.md @@ -9,7 +9,10 @@ description: > ## Input format `apimetrics` create commands take their body from stdin or from CLI Shorthand -arguments. Prefer stdin with heredoc syntax — it keeps nested JSON unambiguous: +arguments. Prefer stdin with heredoc syntax — it keeps nested JSON unambiguous. +In shorthand, quote any value containing `//`, such as a URL: +`url: '"https://example.com/health"'`. Shorthand also reads stdin, so add +`", + "method": "GET", + "url": "", + "headers": [{"key": "Accept", "value": "application/json"}], + "auth_id": "" +} +EOF +``` + +**Validation gate:** Response must contain an `id` field. A 422 means a required field is missing — `detail[].loc` names it (`meta.name`, `request.method`, `request.url`; or `name`, `method`, `url` for `create-api-monitor`). A 400 means a value was rejected, e.g. `URL must have a hostname`. ### 2. Attach a schedule @@ -90,43 +106,54 @@ EOF `frequency` is in seconds. Common values: `60` (1 min), `300` (5 min), `3600` (1 hour). -**Validation gate:** Confirm the schedule lists the call ID in its targets before proceeding. +**Validation gate:** Confirm the schedule lists the call ID in `schedule.target_ids` before proceeding (the schedule's own ID is at `meta.id`). ### 3. Run the monitor on-demand -`run-call` takes the call ID as a positional argument: +`run-monitor` takes the call ID as a positional argument and a JSON body (`{}` for defaults): ```bash -apimetrics run-call +apimetrics run-monitor <<'EOF' +{} +EOF ``` The response contains a `result_id`. Save it for the next step. -**Validation gate:** Response must contain `result_id`. A 422 response means the project is out of quota. +**Validation gate:** Response must contain `result_id`. A 422 is a body validation error (for example, no body was sent); an out-of-quota project returns a separate `Project is out of quota` error. ### 4. Poll results to verify ```bash -apimetrics list-call-results +apimetrics get-result +``` + +Poll until `meta.result` is no longer `QUEUED` (match by `result_id`); `meta.result_category` is empty until then. A successful result has `meta.result_category: PASS` and a 2xx `meta.http_code`. `SLOW` means it passed but was slow; `WARNING` (e.g. HTTP 4xx) or `FAIL` (e.g. a connection error) indicate a problem, and `meta.result` gives the detail (`COMPLETE`, `HTTP_CLIENT_ERROR`, `CONNECTION_ERROR`, …). Total time is `timing.total` in ms. To read just the outcome: + +```bash +apimetrics get-result -f 'body.meta.{result_category, result, http_code}' ``` -A successful result has `result.success: true` and an HTTP status code in the 2xx range. Poll until the result from step 3 appears (match by `result_id`). Use `-f` to narrow output: +For the full run history of this monitor instead of a single result, use `list-results-by-call `: ```bash -apimetrics list-call-results -f body.results[0] +apimetrics list-results-by-call --from -f 'body.results[0]' ``` -**Validation gate:** Confirm `result.success` is `true`. If `false`, inspect `result.failure_reason` and the response body for details. +**Validation gate:** Confirm `meta.result_category` is `PASS`. If not, inspect the result (a below-ANALYST caller only gets a summary from `get-result`; most project members get the full request/response/timing detail by default) for the failure cause. ## Hard rules -- Always verify the call ID before attaching to a schedule — attaching the wrong ID silently succeeds. +- Always verify the call ID before attaching to a schedule. Invalid or other-project IDs are rejected with 400, but a valid ID of a different monitor in this project is accepted without warning. +- `run-monitor` starts a run, and `list-results-by-call` lists result history, for any monitor type — API, browser, or MCP. - Do not poll results in a tight loop. Wait 5–10 seconds between checks; on-demand runs typically complete within 30 seconds. - `frequency` on schedules is in seconds, not minutes. - `add-call-to-schedule` takes two positional args: schedule ID first, then target ID. ## Error recovery -- **400 on create:** Missing required fields. Check `meta.name`, `request.url`, and `request.method` are all present and non-empty. -- **401/403:** Confirm `--api-key` or project is configured. Run `apimetrics project show` to check the active project. -- **422 on run:** Project is out of quota. Check billing or reduce monitor frequency. +- **422 on create:** A required field is missing; `detail[].loc` names it. +- **400 on create:** A value was rejected (e.g. `URL must have a hostname`); the error message names it. +- **401/403:** Confirm login state and that a project is active with `apimetrics project show`. +- **422 on run:** The body failed validation — send at least `{}`. +- **Out of quota on run:** The error reads `Project is out of quota`. Check billing or reduce monitor frequency. - **No result after 60s:** The run may be queued behind other runs. Increase wait time or check monitor status. diff --git a/skills/embed/setup-browser-monitor.md b/skills/embed/setup-browser-monitor.md index 1508bdb9..d9646621 100644 --- a/skills/embed/setup-browser-monitor.md +++ b/skills/embed/setup-browser-monitor.md @@ -9,7 +9,8 @@ description: > ## Input format `apimetrics` create commands take their body from stdin or from CLI Shorthand -arguments. Prefer stdin with heredoc syntax — it keeps nested JSON unambiguous: +arguments (in shorthand, quote values containing `//` and add ` ``` -Poll until `result` is no longer `QUEUED`. Wait 10–15 seconds between checks (browser monitors typically complete within 60 seconds). +Poll until `meta.result` is no longer `QUEUED`; `meta.result_category` is empty until then. Wait 10–15 seconds between checks (browser runs typically complete in well under a minute). -**Validation gate:** `result` must be `PASS`. Values of `FAIL`, `WARN`, `ERROR`, or `TIMEOUT` indicate a problem — inspect the response for details. +**Validation gate:** `meta.result_category` must be `PASS`. `SLOW` means it passed but was slow; `WARNING` or `FAIL` indicate a problem — check `meta.result` and `meta.http_code` for the cause. -To get a screenshot of the page at the time of the run: +To save a screenshot of the page at the time of the run (a binary JPEG written to stdout): ```bash -apimetrics get-result-screenshot +apimetrics get-result-screenshot > screenshot.jpg ``` ## Hard rules -- Always verify the monitor ID before attaching to a schedule — attaching the wrong ID silently succeeds. +- Always verify the monitor ID before attaching to a schedule. Invalid or other-project IDs are rejected with 400, but a valid ID of a different monitor in this project is accepted without warning. - `run-monitor` and `get-result` take positional arguments — do not use `--monitor-id` or `--result-id` flags. - `add-call-to-schedule` takes two positional args: schedule ID first, then target ID. -- Browser monitors take longer to complete than API monitors — wait at least 60 seconds before polling results. +- Browser monitors take longer to complete than API monitors — make the first poll about 15 seconds after the run starts. - Do not poll results in a tight loop. Wait 10–15 seconds between checks. - `frequency` on schedules is in seconds, not minutes. ## Error recovery -- **400 on create:** Confirm `name` and `url` are both provided and that the URL includes the scheme (`https://`). -- **401/403:** Confirm `--api-key` or project is configured. Run `apimetrics project show` to check the active project. +- **422 on create:** `name` or `url` is missing; `detail[].loc` names it. Include the scheme (`https://`) in the URL. +- **401/403:** Confirm login state and that a project is active with `apimetrics project show`. - **No result after 120s:** Browser monitors may take longer in high-load periods. Check the monitor with `apimetrics read-browser-monitor ` and retry. -- **Screenshot unavailable:** Not all result types include screenshots. Fall back to `get-result-content`. +- **Screenshot unavailable:** Not all results include screenshots. Fall back to `get-result ` (most project members get the response detail inline) or `get-result-content `. diff --git a/skills/embed/setup-mcp-monitor.md b/skills/embed/setup-mcp-monitor.md index c08f4782..2bcf82af 100644 --- a/skills/embed/setup-mcp-monitor.md +++ b/skills/embed/setup-mcp-monitor.md @@ -9,7 +9,8 @@ description: > ## Input format `apimetrics` create commands take their body from stdin or from CLI Shorthand -arguments. Prefer stdin with heredoc syntax — it keeps nested JSON unambiguous: +arguments (in shorthand, quote values containing `//` and add ` ``` -Poll until `result` is no longer `QUEUED`. Allow up to the `overall_timeout_ms` value plus processing time. Wait 10–15 seconds between checks. +Poll until `meta.result` is no longer `QUEUED`; `meta.result_category` is empty until then. Allow up to the `overall_timeout_ms` value plus processing time. Wait 10–15 seconds between checks. -**Validation gate:** `result` must be `PASS`. Values of `FAIL`, `WARN`, `ERROR`, or `TIMEOUT` indicate a problem. Common failure causes: unreachable server, auth failure, or a step that did not return the expected tool response. +**Validation gate:** `meta.result_category` must be `PASS`. `SLOW` means it passed but was slow; `WARNING` or `FAIL` indicate a problem — check `meta.result` and `meta.http_code` for the cause. Common failure causes: unreachable server, auth failure, an SSE-only endpoint (`WARNING` / `HTTP_CLIENT_ERROR` / 405), or a step that did not return the expected tool response. ## Hard rules -- Always verify the monitor ID before attaching to a schedule — attaching the wrong ID silently succeeds. +- Always verify the monitor ID before attaching to a schedule. Invalid or other-project IDs are rejected with 400, but a valid ID of a different monitor in this project is accepted without warning. - `run-monitor` and `get-result` take positional arguments — do not use `--monitor-id` or `--result-id` flags. - `add-call-to-schedule` takes two positional args: schedule ID first, then target ID. - MCP monitor runs are bounded by `overall_timeout_ms`. Sessions exceeding this limit are terminated and reported as failures. - Do not poll results in a tight loop. Wait 10–15 seconds between checks. - `frequency` on schedules is in seconds, not minutes. -- The `url` must be an SSE endpoint, not a plain HTTP endpoint. +- The `url` must be the server's Streamable HTTP MCP endpoint (usually `/mcp`). SSE-only endpoints (`/sse`) fail with HTTP 405. ## Error recovery -- **400 on create:** Confirm `name` and `url` are both provided and that the URL is a valid SSE endpoint. -- **401/403:** Confirm `--api-key` or project is configured. Run `apimetrics project show` to check the active project. +- **422 on create:** `name` or `url` is missing, or a step lacks `timeout_ms`/`wait_ms`; `detail[].loc` names it. +- **401/403:** Confirm login state and that a project is active with `apimetrics project show`. - **Session timeout failures:** Increase `overall_timeout_ms` and re-run. -- **422 on run:** Project is out of quota. Check billing or reduce monitor frequency. +- **422 on run:** The body failed validation — send at least `{}`. +- **Out of quota on run:** The error reads `Project is out of quota`. Check billing or reduce monitor frequency. - **No result after 120s:** Verify the MCP server URL is reachable before retrying. diff --git a/skills/skills.go b/skills/skills.go index 092bf85a..ac3c3e72 100644 --- a/skills/skills.go +++ b/skills/skills.go @@ -71,7 +71,7 @@ func Init(cmd *cobra.Command) { onboard := cobra.Command{ Use: "onboard", Short: "Print all APImetrics skills to stdout for agent onboarding", - Long: "Prints all embedded skill workflows to stdout.\n" + + Long: "Prints the embedded monitor-setup guides and the eight workflow skills to stdout.\n" + "Run this command and include the output in your agent's context\n" + "to onboard it on the correct APImetrics CLI workflows.", RunE: func(cmd *cobra.Command, args []string) error { @@ -276,12 +276,12 @@ func buildAgentsMD(skillNames []string, bin string) string { b.WriteString("## Prerequisites\n\n") b.WriteString("Before running any command:\n\n") - b.WriteString(fmt.Sprintf("1. Run `%s project show` to confirm a project is active — all commands fail without one. If none is set, run `%s project select`.\n", bin, bin)) + b.WriteString(fmt.Sprintf("1. Run `%s project show` to confirm a project is active — project-scoped commands fail without one unless you pass `--project-id `. If none is set, run `%s project select`.\n", bin, bin)) b.WriteString("2. Invoke the relevant skill for the task.\n\n") b.WriteString("## Critical facts\n\n") - b.WriteString("- Commands are flat, not grouped: use `create-call`, not `calls create`\n") - b.WriteString("- Create commands take their body from stdin or from CLI Shorthand arguments — there is no `--body`, `--data`, or `-d` flag\n") + b.WriteString("- Commands are flat: `list-calls`, `create-call`, `run-monitor`\n") + b.WriteString("- Create and update commands take their JSON body from stdin or from CLI Shorthand arguments\n") b.WriteString("- Prefer stdin with heredoc syntax; it keeps nested JSON unambiguous\n") b.WriteString(fmt.Sprintf("- Correct pattern: `%s create-call <<'EOF' ... EOF`\n\n", bin)) @@ -294,6 +294,9 @@ func buildAgentsMD(skillNames []string, bin string) string { return b.String() } +// printSkills writes the three monitor-setup guides followed by the eight +// workflow skills, so `onboard` carries the same content that +// `skills install --claude-skills` installs. func printSkills() error { entries, err := fs.ReadDir(skillFiles, "embed") if err != nil { @@ -301,6 +304,14 @@ func printSkills() error { } first := true + emit := func(data []byte) { + if !first { + fmt.Fprintln(cli.Stdout) + } + first = false + fmt.Fprint(cli.Stdout, string(data)) + } + for _, entry := range entries { if entry.IsDir() { continue @@ -310,13 +321,26 @@ func printSkills() error { if err != nil { return fmt.Errorf("reading skill %s: %w", entry.Name(), err) } + emit(data) + } - if !first { - fmt.Fprintln(cli.Stdout) + err = fs.WalkDir(claudeSkillFiles, "claude-skills", func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || path.Base(p) != "SKILL.md" { + return nil } - first = false - fmt.Fprint(cli.Stdout, string(data)) + data, err := claudeSkillFiles.ReadFile(p) + if err != nil { + return fmt.Errorf("reading skill %s: %w", p, err) + } + emit(data) + return nil + }) + if err != nil { + return fmt.Errorf("reading embedded skills: %w", err) } return nil diff --git a/skills/skills_test.go b/skills/skills_test.go index b8033037..fb12edb9 100644 --- a/skills/skills_test.go +++ b/skills/skills_test.go @@ -1,8 +1,14 @@ package skills import ( + "bytes" + "io/fs" + "path" + "strings" "testing" + "apicontext.com/apimetrics/cli" + "github.com/stretchr/testify/assert" ) @@ -95,3 +101,37 @@ func TestResolveTarget(t *testing.T) { }) } } + +func TestPrintSkillsIncludesEveryWorkflow(t *testing.T) { + var buf bytes.Buffer + orig := cli.Stdout + cli.Stdout = &buf + defer func() { cli.Stdout = orig }() + + assert.NoError(t, printSkills()) + out := buf.String() + + var count int + err := fs.WalkDir(claudeSkillFiles, "claude-skills", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || path.Base(p) != "SKILL.md" { + return err + } + count++ + data, err := claudeSkillFiles.ReadFile(p) + if err != nil { + return err + } + assert.True(t, strings.Contains(out, string(data)), "onboard output is missing %s", p) + return nil + }) + assert.NoError(t, err) + assert.Equal(t, 8, count) + + entries, err := fs.ReadDir(skillFiles, "embed") + assert.NoError(t, err) + for _, e := range entries { + data, err := skillFiles.ReadFile("embed/" + e.Name()) + assert.NoError(t, err) + assert.True(t, strings.Contains(out, string(data)), "onboard output is missing embed/%s", e.Name()) + } +}