diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index fc937f4b..0955fbba 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -2115,22 +2115,6 @@ "lineCount": 1 } }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 46, - "endColumn": 53, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 46, - "endColumn": 53, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -8366,706 +8350,18 @@ } }, { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 40, - "endColumn": 53, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 32, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 32, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 35, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", + "code": "reportOptionalMemberAccess", "range": { - "startColumn": 21, - "endColumn": 34, + "startColumn": 29, + "endColumn": 36, "lineCount": 1 } }, { "code": "reportArgumentType", "range": { - "startColumn": 40, - "endColumn": 53, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 40, - "endColumn": 53, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 29, - "endColumn": 36, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 35, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 32, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 32, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 33, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 33, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 33, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 33, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 33, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 33, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 33, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 16, - "endColumn": 22, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 34, + "startColumn": 16, + "endColumn": 22, "lineCount": 1 } }, @@ -9077,30 +8373,6 @@ "lineCount": 1 } }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 63, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { diff --git a/.github/workflows/usage-reporting-retention.yml b/.github/workflows/usage-reporting-retention.yml new file mode 100644 index 00000000..b4829ff4 --- /dev/null +++ b/.github/workflows/usage-reporting-retention.yml @@ -0,0 +1,32 @@ +name: Usage-reporting retention + +# Weekly: delete PostHog usage reports older than the retention window (see docs/reference/usage-reporting.md). +# Manual run with an install ID: delete that install's reports now (erasure request). +on: + schedule: + - cron: "17 4 * * 1" + workflow_dispatch: + inputs: + install_id: + description: "Install ID to delete now (leave empty for the regular retention sweep)" + required: false + default: "" + +permissions: + contents: read + +jobs: + retention: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Delete expired or requested usage reports + env: + POSTHOG_PERSONAL_API_KEY: ${{ secrets.POSTHOG_PERSONAL_API_KEY }} + INSTALL_ID: ${{ inputs.install_id }} + run: python scripts/usage_reporting_retention.py diff --git a/README.md b/README.md index 28625c36..16168196 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,10 @@ Now SLayer MCP will be visible in Claude Code next time you start it. Make sure Read more on how to get started with [MCP](https://docs.motley.ai/slayer/getting-started/mcp/), [CLI](https://docs.motley.ai/slayer/getting-started/cli/), [REST API](https://docs.motley.ai/slayer/getting-started/rest-api/), [Python](https://docs.motley.ai/slayer/getting-started/python/) in the docs. +## Usage reporting + +The `slayer` command sends anonymous, aggregated usage statistics (command, tool and query-feature counts, versions) at most once a day — never queries, names, data or credentials; library use sends nothing. Opt out with `SLAYER_USAGE_REPORTING=off`, `DO_NOT_TRACK=1` or `slayer usage-reporting disable`; see [Usage reporting](https://docs.motley.ai/slayer/reference/usage-reporting/) for every field. + ## License MIT diff --git a/architecture/core.arc42.md b/architecture/core.arc42.md index cfb21884..3e807d57 100644 --- a/architecture/core.arc42.md +++ b/architecture/core.arc42.md @@ -37,6 +37,9 @@ flowchart TD search("Search & embeddings") memories("Agent memories") protocols("BI wire protocols") + subgraph usage_reporting["Usage reporting"] + usage_reporting__features["Features"] + end surfaces("User-facing surfaces") core__query -.-> engine__syntax core__models -.-> sql__dialects @@ -56,8 +59,9 @@ flowchart TD sql --> core storage --> core surfaces --> core + usage_reporting__features --> core classDef leaf fill:none; - class core__query,core__models,sql__dialects,sql__sql_predicate,sql__window_detect,engine__syntax,ir,storage__migrations,importers,search,memories,protocols,surfaces leaf; + class core__query,core__models,sql__dialects,sql__sql_predicate,sql__window_detect,engine__syntax,ir,storage__migrations,importers,search,memories,protocols,usage_reporting__features,surfaces leaf; ``` *Dashed arrows: legacy edges slated to die.* diff --git a/architecture/model.c4 b/architecture/model.c4 index 551f5e91..5b3a34ea 100644 --- a/architecture/model.c4 +++ b/architecture/model.c4 @@ -82,6 +82,14 @@ model { specs ['facade'] } } + usage_reporting = node 'Usage reporting' { + metadata { + package 'slayer.usage_reporting' + arc42 'architecture/usage_reporting.arc42.md' + specs ['usage-reporting'] + } + features = node 'Features' + } surfaces = bucket 'User-facing surfaces' { metadata { packages ['slayer.api', 'slayer.mcp', 'slayer.cli', 'slayer.client', 'slayer.inspect', 'slayer.demo', 'slayer.__main__'] @@ -146,5 +154,9 @@ model { surfaces -> search surfaces -> sql surfaces -> storage + protocols -> usage_reporting + surfaces -> usage_reporting + usage_reporting.features -> core + usage_reporting.features -> sql } } diff --git a/architecture/sql.arc42.md b/architecture/sql.arc42.md index a885184b..772a1af4 100644 --- a/architecture/sql.arc42.md +++ b/architecture/sql.arc42.md @@ -31,6 +31,9 @@ flowchart TD ir["Intermediate representation"] storage["Storage backends"] importers("Importers") + subgraph usage_reporting["Usage reporting"] + usage_reporting__features["Features"] + end surfaces("User-facing surfaces") core__models -.-> sql__dialects core__models -.-> sql__sql_predicate @@ -44,8 +47,9 @@ flowchart TD sql --> ir storage --> sql surfaces --> sql + usage_reporting__features --> sql classDef leaf fill:none; - class core__query,core__models,sql__render,sql__dialects,sql__sql_predicate,sql__window_detect,engine,ir,storage,importers,surfaces leaf; + class core__query,core__models,sql__render,sql__dialects,sql__sql_predicate,sql__window_detect,engine,ir,storage,importers,usage_reporting__features,surfaces leaf; ``` *Dashed arrows: legacy edges slated to die.* diff --git a/architecture/system.arc42.md b/architecture/system.arc42.md index e4928afc..50fa687c 100644 --- a/architecture/system.arc42.md +++ b/architecture/system.arc42.md @@ -26,6 +26,7 @@ flowchart TD search("Search & embeddings") memories("Agent memories") protocols("BI wire protocols") + usage_reporting["Usage reporting"] surfaces("User-facing surfaces") core -.-> engine core -.-> sql @@ -66,12 +67,16 @@ flowchart TD surfaces --> search surfaces --> sql surfaces --> storage + protocols --> usage_reporting + surfaces --> usage_reporting + usage_reporting --> core + usage_reporting --> sql ``` *Dashed arrows: legacy edges slated to die.* -Ten nodes: precise `core`, `sql`, `ir`, -`engine`, `storage` around the query pipeline; virtual buckets `importers`, +Eleven nodes: precise `core`, `sql`, `ir`, +`engine`, `storage` around the query pipeline, and `usage_reporting`; virtual buckets `importers`, `search`, `memories`, `protocols`, `surfaces` for the rest. Package claims and spec mapping are node metadata in [model.c4](model.c4); the legacy-arrow baseline is in [index.yaml](index.yaml). diff --git a/architecture/usage_reporting.arc42.md b/architecture/usage_reporting.arc42.md new file mode 100644 index 00000000..9dea275f --- /dev/null +++ b/architecture/usage_reporting.arc42.md @@ -0,0 +1,45 @@ +# usage_reporting — usage reporting + +## 1. Purpose & context + +`slayer/usage_reporting` records anonymous usage counters in processes started by the +`slayer` CLI, spools them per process, and sends them to PostHog EU at most once +per 24 h. Behaviour is specified in `openspec/specs/usage-reporting/`. + +## 2. Building blocks + +Child `features` turns the queries and datasources a surface has already loaded +into vocabulary tokens, and is the only part that imports other SLayer nodes +(`core`, `sql`). The rest (settings, payload vocabulary, recorder, spool, sender) +imports no SLayer node. + +## 3. Principles + +1. **Fail-silent**: no usage-reporting path raises into, writes to stdout of, or blocks + the host process beyond the one bounded send wait. + [enforced: test:tests/test_usage_reporting_isolation.py] +2. **Usage reports carry no user content**: every value a usage report sends is + drawn from a SLayer-defined vocabulary (versions, enums, booleans, buckets, + counts keyed by built-in names); anything else is reported as `other`. + Library use never reports. + [enforced: test:tests/test_usage_reporting_payload.py] +3. **No user-derived values**: no value supplied by a user, an MCP client or a + database leaves the process or reaches a usage-reporting file in any form: verbatim, + truncated, case-folded, hashed, encoded, or as its length. Such a value may + only pick a token from a fixed table; a miss is reported as `other` / `custom`. + The one exception is an MCP client's self-reported major version, bounded to + 0–999. + [enforced: test:tests/test_usage_reporting_payload.py] +4. **Errors by class only**: a failure is reduced to a token by its exception + class hierarchy; usage reporting never reads an exception's message, arguments, + traceback or `repr`. + [enforced: test:tests/test_usage_reporting_isolation.py] +5. **Sizes are bucketed**: a count of user-owned things (models, datasources) is + reported only as a bucket, never as the exact number. + [enforced: test:tests/test_usage_reporting_features.py] + +## 4. Rationale + +Usage reporting runs inside every CLI process, so its own defects must never reach the +user; principles 2–5 keep user content, and anything derived from it, from +leaving the machine. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index e5b4c52c..23b34fcb 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -467,3 +467,14 @@ slayer memory forget kb.brooklyn-pos | `save` | `--query JSON_OR_@FILE` | Inline SLayer query (or `@path.json`). Entities are auto-extracted and the query is persisted on the memory. Mutually exclusive with `--entities`. | | `save` | `--id ID` | User-pinned canonical memory id. Forbidden charset: `:`, `/`, `?`, `#`, whitespace, ASCII control. Omit to auto-allocate (`max(int-shaped id) + 1`). Duplicate id → unconditional upsert; `created_at` preserved. | | `forget` | `` (positional) | Memory id. Cascade-strips every `memory:` reference to it from every other memory's `entities` list. | + +### `slayer usage-reporting` + +Show or change [usage reporting](usage-reporting.md). + +```bash +slayer usage-reporting status # on/off, the rule that decided it, the install ID +slayer usage-reporting show # the report the next send would contain (sends nothing) +slayer usage-reporting disable # persist off; delete the install ID and unsent data +slayer usage-reporting enable # persist on +``` diff --git a/docs/reference/usage-reporting.md b/docs/reference/usage-reporting.md new file mode 100644 index 00000000..9eeb0f52 --- /dev/null +++ b/docs/reference/usage-reporting.md @@ -0,0 +1,94 @@ +# Usage reporting + +SLayer collects **anonymous, aggregated usage statistics** to learn which surfaces, databases and +query features people use. It never sends your queries, data, model / column / datasource names, +memory text, arguments, error messages, file paths or credentials. + +## When it is active + +Only in processes started by the `slayer` command (`slayer query`, `slayer mcp`, `slayer serve`, +`slayer flight-serve`, `slayer pg-serve`, …, and the Docker image, which runs `slayer serve`). +Using SLayer as a Python library — the query engine, the client, or an app or MCP server you +build in your own code — never records, prints a notice or sends anything. + +## Turning it off + +The first rule that matches decides: + +| # | Rule | Effect | +|---|------|--------| +| 1 | `DO_NOT_TRACK=1` (or `true` / `yes` / `on`) | off | +| 2 | `SLAYER_USAGE_REPORTING=on` / `SLAYER_USAGE_REPORTING=off` | on / off | +| 3 | `CI` is set; SLayer is an editable install; the config directory cannot be written | off | +| 4 | `slayer usage-reporting enable` / `slayer usage-reporting disable` | on / off (persisted) | +| 5 | Default | on | + +```bash +slayer usage-reporting status # on or off, which rule decided it, the install ID +slayer usage-reporting show # print exactly the report the next send would contain (sends nothing) +slayer usage-reporting disable # turn off for good; also deletes the install ID and unsent data +slayer usage-reporting enable +``` + +## The notice + +The first time usage reporting is on, SLayer prints a one-line notice on **stderr** (never stdout, so +stdio MCP stays clean) and records that it did. It is shown again only if the report format +changes in a major way. + +## What a report contains + +One JSON document per send; every value is a version, a SLayer-defined token, a boolean, a +bucket, a date, a random ID or a count. Anything SLayer does not recognise is reported as +`other` (or `custom` for user-defined aggregations). + +| Field | Content | +|-------|---------| +| `schema` | Report format version. | +| `install_id` | Random UUID created on first use, replaced every 11 months. | +| `batch_id` | Random UUID per send (lets the backend drop duplicates). | +| `period` | First and last **date** covered (no times). | +| `env` | SLayer version, Python `major.minor`, OS family, CPU architecture, whether it runs in a container, installed SLayer extras (read from package metadata). | +| `usage` | Per action — CLI command (except `slayer usage-reporting …`), MCP tool name, REST route template (e.g. `/models/{name}`), Flight SQL / PostgreSQL query or probe — the count of successes and of failures per error kind (a fixed list such as `value_error`, `schema_drift`, `http_4xx`). | +| `features` | Per query: counts of multistage queries, inline source models, time dimensions, filters, computed dimensions, saved-query runs, non-default `to_many_handling`; counts of each built-in aggregation and transform (`sum`, `time_shift`, …), user-defined ones as `custom`. | +| `dialects` | Queries per SQL dialect; datasources per dialect in buckets `0` / `1` / `2-5` / `6+`. | +| `context` | Per process: storage backend (`yaml` / `sqlite` / `other`), model count in buckets `0` / `1-10` / `11-50` / `51-200` / `200+`, whether the bundled demo database was queried. | +| `mcp_clients` | MCP sessions per client (Claude Code, Claude Desktop, Cursor, VS Code, Windsurf, Codex, Zed, Cline, Goose, Continue; anything else `other`) with its major version. | + +## How and when it is sent + +Each process keeps its counts in memory and writes them to its own spool file when it ends — no +network at that point. A report is sent at most **once per 24 hours**, and only while SLayer is +running anyway (no heartbeat): when a `slayer` process starts, or a running server records an +action, and the last successful send was 24 hours ago or more. The send merges all spool files +and runs in a background thread; at exit SLayer waits for it at most about one second. Unsent +data stays in the spool (capped at about 100 files / 1 MB) for a later try. + +Usage reporting never changes a command's output or exit code: every usage reporting failure is ignored. + +Files live in the platform config directory, not in SLayer's model storage: +`$XDG_CONFIG_HOME/slayer` (default `~/.config/slayer`) on Linux, +`~/Library/Application Support/slayer` on macOS, `%APPDATA%\slayer` on Windows — +`usage-reporting.json` (setting, install ID, notice and send bookkeeping), `usage-reporting-spool/` +and two small lock files that keep concurrent `slayer` processes from overwriting each other. + +## Destination + +Reports go to the PostHog EU capture API (`https://eu.i.posthog.com/i/v0/e/`) as one +`slayer_usage` event per send, keyed by the install ID, which PostHog keeps as one "person" so that +all of an install's reports can be deleted together. Your IP address is not in the report; +PostHog receives it at the network level, our project is configured to discard it, and every +event disables GeoIP lookup. `SLAYER_USAGE_REPORTING_ENDPOINT` overrides the URL. + +## Privacy notice + +- **Controller**: Motley AI, the maintainer of SLayer. +- **Purpose**: understanding how SLayer is used, to prioritise its development. +- **Processor**: PostHog (EU hosting), under a data processing agreement. +- **Retention**: raw reports (keyed by install ID) are deleted within 12 months: an install ID is used + for at most 11 months, and a weekly job deletes it, with all its reports, once its first report is + 11 months and one week old. Aggregates without install IDs are kept but not published; any future + summary suppresses every figure covering fewer than 10 installs. +- **Your rights**: `slayer usage-reporting status` prints your install ID; to have its data deleted, + email privacy@motley.ai with that ID, and its reports are deleted, normally within a few days. + `slayer usage-reporting disable` stops collection and deletes the local ID and unsent data. diff --git a/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/.openspec.yaml b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/.openspec.yaml new file mode 100644 index 00000000..01660bab --- /dev/null +++ b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-08 diff --git a/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/design.md b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/design.md new file mode 100644 index 00000000..fc48e79e --- /dev/null +++ b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/design.md @@ -0,0 +1,33 @@ +## Context + +See proposal.md for motivation; the comparable-tools matrix and the decision log (D1–D13) live on the Linear issue DEV-2035. SLayer's entry points all pass through `slayer.cli:main` (the Docker image runs `slayer serve`). Flight validates a query with a `LIMIT 0` run in `get_flight_info` and executes it again in `_execute_full`; PG executes every statement through `_run_query`; FastMCP routes every tool call through `FastMCP.call_tool` and exposes `clientInfo` on the session's `client_params`; the built-in aggregation and transform name sets live in `slayer/core/enums.py` and `slayer/core/formula.py`. + +## Goals / Non-Goals + +**Goals:** the specs under `specs/usage-reporting/`; zero new dependencies; no usage-reporting code path able to change user-visible behaviour. + +**Non-Goals:** reporting library use; error messages or stack traces; a Motley-hosted proxy; publishing aggregates (later, not v1); consent prompts. + +## Decisions + +1. **Architecture — a precise LikeC4 node `usage_reporting`** inside `python` (metadata: `package 'slayer.usage_reporting'`, `arc42 'architecture/usage_reporting.arc42.md'`, `specs ['usage-reporting']`) with one child `features`. Edges: `surfaces -> usage_reporting`, `protocols -> usage_reporting`, `usage_reporting.features -> core`, plus any other edge `la-arch-check` measures for the extractor; the rest of `usage_reporting` imports no SLayer node. Why: protocols (Flight/PG) and surfaces both record, so usage reporting cannot live inside `surfaces`; the child proves only the extractor reads queries. The exact `.c4` diff is shown for approval before it lands. +2. **arc42** (wording approved): new `usage_reporting.arc42.md` whose §3 holds **Fail-silent** (enforced by `tests/test_usage_reporting_isolation.py`) and the payload-vocabulary principle ("Usage reports carry no user content … Library use never reports.", enforced by `tests/test_usage_reporting_payload.py`), plus the other privacy principles approved with it; `system.arc42.md` changes only its node list and landscape diagram. The node file's §1/§2/§4 prose is shown for approval before it lands. +3. **Activation by construction.** A module-level recorder is a no-op until `slayer.cli.main` calls `usage_reporting.start()`; every hook calls the recorder unconditionally. Library use therefore never records without any per-site check. +4. **Vocabulary at the type level.** The payload is a Pydantic model whose every leaf is a `Literal`/enum, bool, bucket label, date, generated UUID or `int`. Tokens come from fixed tables (CLI leaf tokens via `set_defaults`, MCP tool names from the registered tool set, REST route templates from the matched `APIRoute`, exception-class→token table, MCP client list, built-in name sets); a lookup miss maps to `other`/`custom`. A schema-policy test walks the model to enforce this; sentinel fuzzing tests the hooks. +5. **Hook points.** CLI: `main` (start + leaf token; `argparse` errors exit before start). MCP: override/wrap `FastMCP.call_tool` once — tool schemas untouched — and read `client_params.clientInfo` on first call per session. REST: a middleware reading `request.scope["route"]` after the handler; the `/mcp` mount excluded. Flight: `_execute_full`. PG: `_run_query`. Features: on the normalized `SlayerQuery` immediately before engine execution in each query surface. +6. **Storage and sending.** Config dir via the platform convention (`$XDG_CONFIG_HOME/slayer`, `~/Library/Application Support/slayer`, `%APPDATA%\slayer`), never `SLAYER_STORAGE`. `usage-reporting.json` holds setting, install ID + creation date, notice schema, last-send time. Spool files: random names, temp-write + rename, mode 0600, capped. A lease file with expiry serialises senders; claim by rename, delete + record `last_sent` only after 2xx; stale claims recovered by age. Send: stdlib `urllib` POST in a daemon thread; `atexit` joins it for ≤ ~1 s only if in flight. +7. **Lifecycle.** Counts are written to the spool in a `try/finally` around each server's run function, with `atexit` as a fallback. Every server command (`serve`, `mcp`, `flight-serve`, `pg-serve`) installs a SIGTERM handler that first calls any handler already installed, then raises `SystemExit`. Stdio `mcp` handles SIGINT the same way (asyncio would only cancel) and exits hard after writing its spool, because its blocked stdin reader holds up a normal exit. `os.register_at_fork` resets the recorder in child processes. +8. **Own-usage exclusion.** Editable installs detected from PEP 610 `direct_url.json`; an autouse session fixture in `tests/conftest.py` sets `SLAYER_USAGE_REPORTING=off`, and usage-reporting tests re-enable it with a local endpoint. + +## Risks / Trade-offs + +- [Default-on usage reporting draws criticism] → strict vocabulary, stderr notice, `DO_NOT_TRACK`, CI auto-off, `show` for inspection, docs page with every field. +- [ePrivacy Art. 5(3) exposure for EU users with an on-by-default stored ID] → accepted for now (D1); revisit if Motley establishes in the EU. +- [Backend frozen into releases — PostHog EU URL ships in every version] → `SLAYER_USAGE_REPORTING_ENDPOINT` override; old versions keep posting to PostHog. +- [PostHog sees client IPs at transport] → project discards IPs, GeoIP disabled per event; documented exactly. +- [Short-lived stdio MCP sessions killed with SIGKILL lose counts] → accepted. +- [FastMCP internals change across `mcp` <2 versions] → interception tested against the pinned version; tool-schema identity test. + +## Migration Plan + +Additive. Release notes announce usage reporting, the notice, and the opt-outs. Rollback: ship `SLAYER_USAGE_REPORTING` default `off` in a patch release. diff --git a/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/proposal.md b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/proposal.md new file mode 100644 index 00000000..e520a16a --- /dev/null +++ b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/proposal.md @@ -0,0 +1,30 @@ +## Why + +We have no signal on how SLayer is used — active installs, which surfaces, dialects and query features, version adoption, failure rates. Comparable tools (dbt, Cube, Kaelio ktx, CrewAI) collect anonymous usage data; dbt-mcp's CVE-2026-44970 (raw MCP tool arguments sent in usage reports) shows the failure mode to design out from the start. + +## What Changes + +- New anonymous, aggregated, **opt-out** usage reporting for processes started by the `slayer` CLI (`slayer `, `mcp`, `serve`, `flight-serve`, `pg-serve`). Library use never records, notifies or sends. +- Payload carries only SLayer-defined vocabulary: versions, enums, booleans, buckets, counts keyed by built-in names; anything else is `other` / `custom`. No SQL, names, arguments, messages, paths, IPs. +- Per-process in-memory counters, spooled locally at exit; sent to PostHog EU at most once per 24 h per install, only when SLayer runs anyway (no heartbeat), from a background thread. +- Controls: `DO_NOT_TRACK=1`, `SLAYER_USAGE_REPORTING=on|off`, auto-off in `CI`, editable installs and unwritable config dirs, and `slayer usage-reporting status|enable|disable|show`. +- One-time stderr notice; docs page with every field and a privacy notice; random install ID rotated every 11 months; reports deleted at PostHog within 12 months, or on request. +- Architecture: new precise node `usage_reporting` in the LikeC4 model; new `usage_reporting.arc42.md` (fail-silent, payload vocabulary, privacy principles). + +## Capabilities + +### New Capabilities +- `usage-reporting/collection`: when usage reporting is active and exactly what a usage report may contain. +- `usage-reporting/controls`: default, opt-out precedence, the `slayer usage-reporting` command, and the first-run notice. +- `usage-reporting/delivery`: local spooling, the 24 h send gate, destination, install-ID rotation, and the guarantee never to affect the host process. + +### Modified Capabilities +(none) + +## Impact + +- New package `slayer/usage_reporting/` (stdlib `urllib`, no new dependency). +- Hooks in `slayer/cli.py`, `slayer/mcp/server.py` (`FastMCP.call_tool`), `slayer/api/server.py` (middleware), `slayer/flight/handlers.py` (`_execute_full`), `slayer/pg_facade/connection.py` (`_run_query`), and the query-executing surfaces (feature extraction). +- Architecture: `architecture/model.c4` (node + edges), `architecture/system.arc42.md` §2, new `architecture/usage_reporting.arc42.md`, regenerated landscape diagram. +- Docs: `docs/reference/usage-reporting.md` (+ `zensical.toml` nav), README, CLI reference, release notes. +- External prerequisite (release-blocking): PostHog EU project with "Discard client IP data", PostHog DPA, project API key. diff --git a/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/collection/spec.md b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/collection/spec.md new file mode 100644 index 00000000..2f0c7f96 --- /dev/null +++ b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/collection/spec.md @@ -0,0 +1,95 @@ +## Purpose + +Defines when SLayer records usage reporting and exactly what a usage report may contain, so that no user content can ever leave the machine. + +## ADDED Requirements + +### Requirement: Usage reporting is active only in processes started by the slayer CLI +SLayer SHALL record, notify and send usage reporting only in processes started through the `slayer` command-line entry point (`slayer `, `python -m slayer`). Using SLayer as a library — the query engine, the Python client, or an app/server object built from user code — SHALL record nothing, write no usage-reporting file, print no notice and make no usage-reporting network request. + +#### Scenario: Library use records nothing +- **WHEN** user code builds the query engine, the REST app or the MCP server object and runs queries without the `slayer` CLI +- **THEN** no usage-reporting file is created, no notice is printed, and no usage-reporting request is made + +#### Scenario: CLI use records +- **WHEN** `slayer query ...` runs with usage reporting enabled +- **THEN** the `cli` usage counter for the `query` command increments + +### Requirement: A usage report carries only SLayer-defined vocabulary +Every value in a usage report SHALL be a version string, a SLayer-defined enum token, a boolean, a bucket label, a date, a random identifier generated by usage reporting, or a count keyed by such a token. Any value not in SLayer's vocabulary SHALL be reported as `other` (or `custom` for user-defined aggregations and transforms). A report SHALL NOT contain SQL, model / column / datasource / memory names or text, filter or formula text, tool or command arguments, error messages, stack traces, file paths, hostnames or IP addresses, and SHALL NOT contain any value hashed or otherwise derived from them. + +#### Scenario: Hostile user strings never appear +- **WHEN** queries, models, datasources, memories, MCP arguments, CLI arguments, error messages and the MCP client name contain a sentinel string (including hostile Unicode and very long values) +- **THEN** the serialized report contains neither the sentinel nor any value derived from it + +#### Scenario: Unknown exception class +- **WHEN** a counted action fails with an exception class absent from the fixed error table +- **THEN** the failure is counted under the error token `other` + +#### Scenario: Error message is never sent +- **WHEN** a counted action fails with a message containing SQL or a table name +- **THEN** only the action's error token is counted + +### Requirement: Usage counters per surface +A usage report SHALL count actions per surface as `{ok, errors: {error_token: n}}`, keyed by a fixed token per action: +- CLI: one fixed token per leaf command, never built from argument values; argument-parse errors and help output are not counted; the `slayer usage-reporting` commands are not counted. +- MCP: one count per tool call, keyed by tool name. +- REST: one count per request, keyed by the matched route template; unmatched requests and mounts map to fixed tokens; requests to the mounted MCP endpoint are counted as MCP, not REST. +- Flight SQL: one count per executed query (data fetch); schema validation or prepared-statement creation alone is not counted. +- PostgreSQL wire: one count per executed statement, including each statement of a multi-statement message and each execution of a prepared statement; probe and catalog queries map to their own fixed token. + +#### Scenario: CLI parse error not counted +- **WHEN** `slayer` is invoked with arguments that fail to parse +- **THEN** no usage counter changes + +#### Scenario: Flight query counted once +- **WHEN** a Flight SQL client validates and then fetches one query +- **THEN** the `flight` query counter increments exactly once + +#### Scenario: PG multi-statement message +- **WHEN** one PostgreSQL simple-query message holds two statements that both execute +- **THEN** the `pg` query counter increments by two + +#### Scenario: Mounted MCP not counted as REST +- **WHEN** a tool call arrives over the MCP endpoint mounted on `slayer serve` +- **THEN** it is counted under `mcp` and not under any REST route + +### Requirement: Query feature counters +For each query about to execute, a usage report SHALL count structural flags — multistage, inline source model or model extension, time dimensions, filters, computed dimensions, saved-query reference, non-default `to_many_handling` — and the occurrences of each built-in transform and aggregation name. A name that is not built in SHALL be counted as `custom`. Features SHALL be counted per attempted query, independently of its success. + +#### Scenario: Built-in and custom names +- **WHEN** a query uses `time_shift`, `sum` and a user-defined aggregation +- **THEN** the report counts `time_shift` and `sum` by name and the user-defined one as `custom` + +#### Scenario: Multistage flag +- **WHEN** a query list of three stages executes +- **THEN** the `multistage` flag count increments once + +### Requirement: Environment and context fields +A usage report SHALL include the SLayer version, Python `major.minor`, OS family, CPU architecture, whether the process runs in a container, and which optional extras are installed (determined from package metadata without importing them). It SHALL include queries per dialect, datasources per dialect bucketed (0 / 1 / 2–5 / 6+), and per-process histograms of storage backend type, model-count bucket (0 / 1–10 / 11–50 / 51–200 / 200+) and whether the bundled demo datasource is in use. These values SHALL be taken only from objects the process has already loaded; usage reporting SHALL NOT itself open a connection, call storage or import an optional package. + +#### Scenario: Model count is bucketed +- **WHEN** the loaded storage holds 37 models +- **THEN** the report records the bucket `11-50`, not the number + +#### Scenario: No extra storage access +- **WHEN** a server starts with usage reporting enabled +- **THEN** usage reporting makes no storage call or database connection of its own + +### Requirement: MCP client identification +For each MCP session, a usage report SHALL count the connecting client, mapped from the client's self-reported name against a fixed list of known clients and reported with its major version only. Any other client name SHALL be reported as `other`. + +#### Scenario: Known client +- **WHEN** an MCP client identifies as a known client at version `2.3.1` +- **THEN** the session is counted under that client's token with major version `2` + +#### Scenario: Unknown client +- **WHEN** an MCP client identifies as `acme-finance-bot` +- **THEN** the session is counted under `other` + +### Requirement: Report identity and time resolution +A usage report SHALL carry the install ID, a random batch ID, a schema version, and the first and last calendar dates it covers. It SHALL NOT carry timestamps finer than a date. + +#### Scenario: Dates only +- **WHEN** a report covering activity on two days is built +- **THEN** its period holds two dates and no time-of-day values diff --git a/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/controls/spec.md b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/controls/spec.md new file mode 100644 index 00000000..767a9d1c --- /dev/null +++ b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/controls/spec.md @@ -0,0 +1,72 @@ +## Purpose + +Defines how usage reporting is switched on and off, how users inspect it, and how its existence is disclosed on first use. + +## ADDED Requirements + +### Requirement: Enabled by default with an opt-out precedence +Usage reporting SHALL be enabled by default. Its effective state SHALL be decided by the first matching rule, highest first: +1. `DO_NOT_TRACK` set to a truthy value → off. +2. `SLAYER_USAGE_REPORTING` set to `on` or `off` → that value. +3. Automatic off: the `CI` variable is set; SLayer is an editable install; the usage-reporting config directory cannot be written. +4. The persisted setting from `slayer usage-reporting enable|disable`. +5. Default → on. + +When off, usage reporting SHALL record nothing, write no usage-reporting file, print no notice and make no usage-reporting request. + +#### Scenario: DO_NOT_TRACK beats explicit enable +- **WHEN** `DO_NOT_TRACK=1` and `SLAYER_USAGE_REPORTING=on` are both set +- **THEN** usage reporting is off + +#### Scenario: Explicit enable beats CI +- **WHEN** `CI=true` and `SLAYER_USAGE_REPORTING=on` are set +- **THEN** usage reporting is on + +#### Scenario: CI auto-off +- **WHEN** `CI` is set and no higher rule applies +- **THEN** usage reporting is off + +#### Scenario: Editable install auto-off +- **WHEN** SLayer is installed in editable mode and no higher rule applies +- **THEN** usage reporting is off + +#### Scenario: Unwritable config directory +- **WHEN** the usage-reporting config directory cannot be created or written and no higher rule applies +- **THEN** usage reporting is off, with no notice and no network request + +#### Scenario: Persisted disable +- **WHEN** the user ran `slayer usage-reporting disable` and no environment rule applies +- **THEN** usage reporting is off in later processes + +### Requirement: The slayer usage-reporting command +`slayer usage-reporting` SHALL provide: +- `status`: the effective state, the rule that decided it, and the current install ID (if any). +- `enable` / `disable`: persist the setting; `disable` also deletes the spool and the install ID. +- `show`: print exactly the report the next send would contain, without sending it. + +#### Scenario: Status names the deciding rule +- **WHEN** `CI` is set and the user runs `slayer usage-reporting status` +- **THEN** the output says usage reporting is off because of `CI` + +#### Scenario: Disable removes local data +- **WHEN** the user runs `slayer usage-reporting disable` +- **THEN** the spool and the install ID are deleted and no later process sends + +#### Scenario: Show does not send +- **WHEN** the user runs `slayer usage-reporting show` +- **THEN** the pending report is printed and no request is made + +### Requirement: One-time disclosure notice +When usage reporting is on, SLayer SHALL print a notice once, on stderr only, naming what is collected, linking the usage-reporting docs page and giving the opt-out. The notice SHALL be printed again when the report schema's major version changes. It SHALL never be written to stdout. + +#### Scenario: Notice once +- **WHEN** two `slayer` commands run in sequence with usage reporting on for the first time +- **THEN** only the first prints the notice, on stderr + +#### Scenario: stdio MCP stays clean +- **WHEN** `slayer mcp` starts with usage reporting on for the first time +- **THEN** the notice goes to stderr and stdout carries only MCP protocol messages + +#### Scenario: Schema major bump +- **WHEN** the recorded notice was shown for an older report schema major version +- **THEN** the notice is printed again once diff --git a/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/delivery/spec.md b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/delivery/spec.md new file mode 100644 index 00000000..2bfbd702 --- /dev/null +++ b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/specs/usage-reporting/delivery/spec.md @@ -0,0 +1,101 @@ +## Purpose + +Defines how recorded usage is stored locally, when and where it is sent, and the guarantee that usage reporting never affects the process it runs in. + +## ADDED Requirements + +### Requirement: Usage reporting never affects the host process +No usage reporting failure — file I/O, network, timeout, non-2xx response, corrupt or foreign files — SHALL change a command's result, exit code or stdout, or raise into the host process. Usage reporting SHALL NOT write to stdout, and SHALL NOT block the process except for the bounded exit wait defined below. + +#### Scenario: Endpoint down +- **WHEN** the usage-reporting endpoint refuses connections, times out or returns 500 during a command +- **THEN** the command's output and exit code are identical to a run with usage reporting off + +#### Scenario: Corrupt spool file +- **WHEN** a spool file contains invalid JSON +- **THEN** it is skipped and the command runs normally + +### Requirement: Usage is spooled locally at process end +Each process SHALL keep its counters in memory and write them to its own spool file when it ends — normal exit, SIGINT, server shutdown, end of input or broken pipe on stdio MCP, and SIGTERM for stdio `slayer mcp` (chaining any handler already installed) — with no network request at that point. A process killed with SIGKILL loses its counts. A process that forks SHALL start the child with empty counters. + +#### Scenario: stdio MCP terminated +- **WHEN** an MCP client sends SIGTERM to a `slayer mcp` process that handled tool calls +- **THEN** its counters are written to a spool file before it exits + +#### Scenario: Concurrent processes +- **WHEN** several `slayer` processes end at the same time +- **THEN** each writes its own spool file and none overwrites another + +#### Scenario: Forked child +- **WHEN** a process with recorded counts forks +- **THEN** the child's report does not repeat the parent's counts + +### Requirement: Send at most once per 24 hours, only when SLayer runs +A report SHALL be sent only when a `slayer` process starts or records an action, at least 24 hours have passed since the last successful send, and usage reporting is on. The report SHALL merge all spool files and the sending process's live counters. An idle process SHALL send nothing. A last-send time in the future SHALL count as due. + +#### Scenario: Idle server +- **WHEN** a server records no action for 48 hours +- **THEN** it sends nothing + +#### Scenario: Due on activity +- **WHEN** a server records an action 25 hours after the last successful send +- **THEN** exactly one report is sent, merging the spool and its live counters + +#### Scenario: Not yet due +- **WHEN** a command starts 3 hours after the last successful send +- **THEN** nothing is sent and the command's counters are spooled at exit + +#### Scenario: Clock moved backwards +- **WHEN** the recorded last-send time lies in the future +- **THEN** a send is due + +### Requirement: Background send with a bounded exit wait +A due send SHALL run in a background thread started when the process starts (or, in a running server, when the action that made it due is recorded). At exit, the process SHALL wait for an in-flight send at most about one second; with no send in flight it SHALL not wait. + +#### Scenario: Slow network at exit +- **WHEN** a short command exits while its send has been stalled for 10 seconds +- **THEN** the process exits within about one second of finishing its work, and the unsent data stays in the spool + +### Requirement: Exactly-once-per-batch delivery bookkeeping +Only one process SHALL send at a time. Spool files included in a send SHALL be deleted, and the last-send time recorded, only after a 2xx response; otherwise they SHALL remain for a later send. Files claimed by a sender that died SHALL be recovered by a later send. Each report SHALL carry a random batch ID so the backend can drop duplicates. The spool SHALL be capped (about 100 files / 1 MB, oldest dropped first); spool files SHALL be readable only by the owner, and entries that are not regular files SHALL be ignored. + +#### Scenario: Two simultaneous senders +- **WHEN** two processes find a send due at the same moment +- **THEN** one report is sent + +#### Scenario: Failed send retained +- **WHEN** a send gets a 500 response +- **THEN** the spool files remain and the last-send time is unchanged + +#### Scenario: Sender crash recovered +- **WHEN** a sender dies after claiming spool files +- **THEN** a later send includes those files + +### Requirement: Destination and request shape +A report SHALL be sent as one event to the PostHog EU capture API (`https://eu.i.posthog.com/i/v0/e/`), with event name `slayer_usage`, the install ID as distinct ID, the batch ID as event UUID and GeoIP enrichment disabled; each install ID SHALL be a PostHog person, so that its reports can be deleted together. `SLAYER_USAGE_REPORTING_ENDPOINT` SHALL override the URL. + +#### Scenario: Endpoint override +- **WHEN** `SLAYER_USAGE_REPORTING_ENDPOINT` points to a local test server and a send is due +- **THEN** the report is posted to that server with the event name `slayer_usage` and GeoIP disabled + +### Requirement: Install identity and rotation +The install ID SHALL be a random UUID created on first use and stored in the platform config directory (not in SLayer's model storage). It SHALL be replaced by a new random UUID at the first start or send after it is 11 months old, so no report carries an install ID older than that. + +#### Scenario: Rotation +- **WHEN** a process starts and the install ID was created 12 months ago +- **THEN** a new install ID is created and used for subsequent reports + +#### Scenario: Rotation in a long-running process +- **WHEN** a server started while its install ID was 10 months old sends a report two months later +- **THEN** the report carries a new install ID + +### Requirement: Retention and deletion at the destination +Reports SHALL be deleted from the destination within 12 months of being sent: a weekly scheduled job SHALL delete every install ID whose first report is older than the install-ID lifetime plus one week, together with all its reports. The same job SHALL delete one given install ID and all its reports on request. + +#### Scenario: Expired install deleted +- **WHEN** the weekly job runs and an install ID's first report is 11 months and two weeks old +- **THEN** that install ID and all its reports are deleted + +#### Scenario: Deletion on request +- **WHEN** the job is run manually with an install ID +- **THEN** that install ID and all its reports are deleted, and no other install is touched diff --git a/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/tasks.md b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/tasks.md new file mode 100644 index 00000000..5f854c90 --- /dev/null +++ b/openspec/changes/archive/2026-10-17-dev-2035-usage-tracking/tasks.md @@ -0,0 +1,45 @@ +## 1. Tests first (pr-tests stage; all must fail before implementation) + +- [x] 1.1 `tests/conftest.py`: autouse session fixture sets `SLAYER_USAGE_REPORTING=off`; a `usage_reporting_env` fixture re-enables it with a tmp config dir (`XDG_CONFIG_HOME` / platform override), an injected clock, and a local HTTP capture server via `SLAYER_USAGE_REPORTING_ENDPOINT`. Verify: the existing suite still passes with the autouse fixture. +- [x] 1.2 `tests/test_usage_reporting_payload.py` (usage_reporting.arc42 payload-vocabulary principle): schema-policy walk — every payload leaf is a `Literal`/enum, bool, bucket label, date, generated UUID or `int`; sentinel fuzz (hostile Unicode, 10 kB strings) through model / column / datasource / memory names, SQL, filters, custom aggregations, MCP args, CLI args, error messages, `clientInfo`, paths — no sentinel or derivative in the serialized report; unknown exception → `other`; unknown MCP client → `other`; dates only. Verify: fails today (no `slayer.usage_reporting`). +- [x] 1.3 `tests/test_usage_reporting_scope.py`: library use (engine, `create_app`, `create_mcp_server`, client) records nothing, writes no file, prints nothing, makes no request; CLI use records. Verify: fails today. +- [x] 1.4 `tests/test_usage_reporting_controls.py`: full precedence matrix (`DO_NOT_TRACK`, `SLAYER_USAGE_REPORTING`, `CI`, editable via a fake `direct_url.json`, unwritable dir, persisted setting, default); `slayer usage-reporting status|enable|disable|show` per spec; notice once on stderr, re-shown on schema-major bump, none when off. Verify: fails today. +- [x] 1.5 `tests/test_usage_reporting_counting.py`: CLI leaf tokens, parse errors / `--help` not counted; MCP tool calls counted with ok / error token, `clientInfo` mapping, tool schemas byte-identical with usage reporting on vs off; REST route template counted, `/mcp` mount counted as MCP only; Flight validate+fetch counted once; PG multi-statement and repeated prepared execution counted per execution, probes under their own token. Verify: fails today. +- [x] 1.6 `tests/test_usage_reporting_features.py`: each structural flag; built-in transform / aggregation names counted by name, user-defined → `custom`; counted per attempt; dialect counts; datasource / model-count buckets; extras detected from metadata without importing (assert module not in `sys.modules`); no storage call or connection made by usage reporting (spy). Verify: fails today. +- [x] 1.7 `tests/test_usage_reporting_delivery.py`: spool written on normal exit, SIGINT, SIGTERM for stdio `slayer mcp` (handler chained), server shutdown, MCP EOF / broken pipe (subprocess tests); concurrent writers; fork resets counters; 24 h gate (idle → nothing, due → exactly one merged send, future `last_sent` → due); two simultaneous senders → one report; 500 / timeout → spool kept, `last_sent` unchanged; crash after claim recovered; size cap; non-regular files ignored; file mode 0600; `disable` racing a send; request shape (`slayer_usage`, distinct_id, uuid, `$geoip_disable`); install-ID rotation after 11 months, also in a running process. Verify: fails today. +- [x] 1.8 `tests/test_usage_reporting_isolation.py` (usage_reporting.arc42 P1): endpoint refused / timeout / 500, corrupt spool, unwritable files → command output and exit code identical to usage-reporting-off; usage reporting never writes stdout; a stalled send delays exit by ≤ ~1 s and only when in flight. Verify: fails today. + +## 2. Architecture (each exact edit shown to the user and approved before it lands) + +- [x] 2.1 `architecture/model.c4`: node `usage_reporting` in `python` (metadata `package 'slayer.usage_reporting'`, `arc42 'architecture/usage_reporting.arc42.md'`, `specs ['usage-reporting']`), child `features`; edges `surfaces -> usage_reporting`, `protocols -> usage_reporting`, `usage_reporting.features -> core` (+ any measured extractor edge). Verify: `poetry run la-arch-check` green after 3.x. +- [x] 2.2 `architecture/system.arc42.md` §2 node list; new `architecture/usage_reporting.arc42.md` (§1–§4; §3 principles approved in design.md D2: Fail-silent, the payload vocabulary, and the other privacy principles). Verify: `la-arch-check` arc42 checks green. +- [x] 2.3 Regenerate the landscape diagram: `poetry run la-arch-diagrams`. Verify: `diagrams-fresh` green; `npx -y likec4@1.47.0 validate architecture` green. + +## 3. Usage-reporting package + +- [x] 3.1 `slayer/usage_reporting/` settings: config-dir resolution, `usage-reporting.json` model, precedence + deciding rule, editable detection, unwritable → off, install ID + 11-month rotation. Verify: 1.4, 1.7 rotation. +- [x] 3.2 Payload model + fixed token tables (error classes, MCP clients, CLI leaves, buckets, extras list). Verify: 1.2. +- [x] 3.3 Recorder (no-op until `start()`), fork reset, merge semantics. Verify: 1.3, 1.7 fork. +- [x] 3.4 Spool (random names, temp + rename, 0600, cap, regular files only) and lease / claim / commit / stale-claim recovery. Verify: 1.7. +- [x] 3.5 Sender (stdlib `urllib`, PostHog EU request shape, endpoint override, daemon thread, ≤ ~1 s exit join only when in flight, 24 h gate). Verify: 1.7, 1.8. +- [x] 3.6 Notice (stderr, once per schema major). Verify: 1.4. +- [x] 3.7 `slayer/usage_reporting/features/`: structural flags + built-in names from `slayer.core` sets. Verify: 1.6. + +## 4. Hooks + +- [x] 4.1 `slayer/cli.py`: `usage_reporting.start()` in `main`, a `set_defaults` usage token on every leaf parser, `slayer usage-reporting status|enable|disable|show`, `try/finally` flush around each server run, SIGTERM handler for stdio `mcp`. Verify: 1.4, 1.5 CLI, 1.7 lifecycle. +- [x] 4.2 `slayer/mcp/server.py`: one `call_tool` interception + `clientInfo` capture; features before execution in `query`. Verify: 1.5 MCP, 1.6. +- [x] 4.3 `slayer/api/server.py`: route-template middleware excluding `/mcp`; features before execution. Verify: 1.5 REST. +- [x] 4.4 `slayer/flight/handlers.py` `_execute_full` and `slayer/pg_facade/connection.py` `_run_query`: count + features. Verify: 1.5 Flight / PG. + +## 5. Docs and release + +- [x] 5.1 `docs/reference/usage-reporting.md`: every field, controls, notice text, destination (IP received at transport, discarded, no GeoIP), privacy notice (Motley as controller, purpose, PostHog EU processor, 12-month raw retention, ID rotation, erasure by install ID + contact address); nav entry in `zensical.toml`. Verify: page renders in nav. +- [x] 5.2 README usage-reporting section (one paragraph + opt-out); CLI reference for `slayer usage-reporting`; release-notes item. Verify: grep. + +## 6. Gates + +- [x] 6.1 Full non-integration suite `poetry run pytest -m "not integration" -n auto`, `poetry run ruff check slayer/ tests/`, `poetry run basedpyright` (no new errors vs baseline), `poetry run la-arch-check`. Verify: all green. +- [x] 6.2 **Release-blocking:** PostHog EU project created with "Discard client IP data", DPA signed, project API key (`phc_…`) received from the user and set as the shipped default; a live smoke send verified in PostHog. Implementation and tests proceed without it via `SLAYER_USAGE_REPORTING_ENDPOINT`. +- [x] 6.3 Update the DEV-2035 issue body: Lifecycle section per design.md decision 7 (SIGTERM unwinding for every server; SIGINT + hard exit for stdio `mcp`), and D10: the payload-vocabulary principle and the other privacy principles live in `usage_reporting.arc42.md` §3, not `system.arc42.md`. +- [x] 6.4 Retention at PostHog: person profiles on, weekly `.github/workflows/usage-reporting-retention.yml` running `scripts/usage_reporting_retention.py` (deletes install IDs whose first report is older than 11 months + 1 week; manual run deletes one install ID). Verify: `tests/test_usage_reporting_retention.py`. diff --git a/openspec/specs/usage-reporting/collection/spec.md b/openspec/specs/usage-reporting/collection/spec.md new file mode 100644 index 00000000..ae47f901 --- /dev/null +++ b/openspec/specs/usage-reporting/collection/spec.md @@ -0,0 +1,96 @@ +# usage-reporting/collection Specification + +## Purpose +Defines when SLayer records usage reporting and exactly what a usage report may contain, so that no user content can ever leave the machine. + +## Requirements + +### Requirement: Usage reporting is active only in processes started by the slayer CLI +SLayer SHALL record, notify and send usage reporting only in processes started through the `slayer` command-line entry point (`slayer `, `python -m slayer`). Using SLayer as a library — the query engine, the Python client, or an app/server object built from user code — SHALL record nothing, write no usage-reporting file, print no notice and make no usage-reporting network request. + +#### Scenario: Library use records nothing +- **WHEN** user code builds the query engine, the REST app or the MCP server object and runs queries without the `slayer` CLI +- **THEN** no usage-reporting file is created, no notice is printed, and no usage-reporting request is made + +#### Scenario: CLI use records +- **WHEN** `slayer query ...` runs with usage reporting enabled +- **THEN** the `cli` usage counter for the `query` command increments + +### Requirement: A usage report carries only SLayer-defined vocabulary +Every value in a usage report SHALL be a version string, a SLayer-defined enum token, a boolean, a bucket label, a date, a random identifier generated by usage reporting, or a count keyed by such a token. Any value not in SLayer's vocabulary SHALL be reported as `other` (or `custom` for user-defined aggregations and transforms). A report SHALL NOT contain SQL, model / column / datasource / memory names or text, filter or formula text, tool or command arguments, error messages, stack traces, file paths, hostnames or IP addresses, and SHALL NOT contain any value hashed or otherwise derived from them. + +#### Scenario: Hostile user strings never appear +- **WHEN** queries, models, datasources, memories, MCP arguments, CLI arguments, error messages and the MCP client name contain a sentinel string (including hostile Unicode and very long values) +- **THEN** the serialized report contains neither the sentinel nor any value derived from it + +#### Scenario: Unknown exception class +- **WHEN** a counted action fails with an exception class absent from the fixed error table +- **THEN** the failure is counted under the error token `other` + +#### Scenario: Error message is never sent +- **WHEN** a counted action fails with a message containing SQL or a table name +- **THEN** only the action's error token is counted + +### Requirement: Usage counters per surface +A usage report SHALL count actions per surface as `{ok, errors: {error_token: n}}`, keyed by a fixed token per action: +- CLI: one fixed token per leaf command, never built from argument values; argument-parse errors and help output are not counted; the `slayer usage-reporting` commands are not counted. +- MCP: one count per tool call, keyed by tool name. +- REST: one count per request, keyed by the matched route template; unmatched requests and mounts map to fixed tokens; requests to the mounted MCP endpoint are counted as MCP, not REST. +- Flight SQL: one count per executed query (data fetch); schema validation or prepared-statement creation alone is not counted. +- PostgreSQL wire: one count per executed statement, including each statement of a multi-statement message and each execution of a prepared statement; probe and catalog queries map to their own fixed token. + +#### Scenario: CLI parse error not counted +- **WHEN** `slayer` is invoked with arguments that fail to parse +- **THEN** no usage counter changes + +#### Scenario: Flight query counted once +- **WHEN** a Flight SQL client validates and then fetches one query +- **THEN** the `flight` query counter increments exactly once + +#### Scenario: PG multi-statement message +- **WHEN** one PostgreSQL simple-query message holds two statements that both execute +- **THEN** the `pg` query counter increments by two + +#### Scenario: Mounted MCP not counted as REST +- **WHEN** a tool call arrives over the MCP endpoint mounted on `slayer serve` +- **THEN** it is counted under `mcp` and not under any REST route + +### Requirement: Query feature counters +For each query about to execute, a usage report SHALL count structural flags — multistage, inline source model or model extension, time dimensions, filters, computed dimensions, saved-query reference, non-default `to_many_handling` — and the occurrences of each built-in transform and aggregation name. A name that is not built in SHALL be counted as `custom`. Features SHALL be counted per attempted query, independently of its success. + +#### Scenario: Built-in and custom names +- **WHEN** a query uses `time_shift`, `sum` and a user-defined aggregation +- **THEN** the report counts `time_shift` and `sum` by name and the user-defined one as `custom` + +#### Scenario: Multistage flag +- **WHEN** a query list of three stages executes +- **THEN** the `multistage` flag count increments once + +### Requirement: Environment and context fields +A usage report SHALL include the SLayer version, Python `major.minor`, OS family, CPU architecture, whether the process runs in a container, and which optional extras are installed (determined from package metadata without importing them). It SHALL include queries per dialect, datasources per dialect bucketed (0 / 1 / 2–5 / 6+), and per-process histograms of storage backend type, model-count bucket (0 / 1–10 / 11–50 / 51–200 / 200+) and whether the bundled demo datasource is in use. These values SHALL be taken only from objects the process has already loaded; usage reporting SHALL NOT itself open a connection, call storage or import an optional package. + +#### Scenario: Model count is bucketed +- **WHEN** the loaded storage holds 37 models +- **THEN** the report records the bucket `11-50`, not the number + +#### Scenario: No extra storage access +- **WHEN** a server starts with usage reporting enabled +- **THEN** usage reporting makes no storage call or database connection of its own + +### Requirement: MCP client identification +For each MCP session, a usage report SHALL count the connecting client, mapped from the client's self-reported name against a fixed list of known clients and reported with its major version only. Any other client name SHALL be reported as `other`. + +#### Scenario: Known client +- **WHEN** an MCP client identifies as a known client at version `2.3.1` +- **THEN** the session is counted under that client's token with major version `2` + +#### Scenario: Unknown client +- **WHEN** an MCP client identifies as `acme-finance-bot` +- **THEN** the session is counted under `other` + +### Requirement: Report identity and time resolution +A usage report SHALL carry the install ID, a random batch ID, a schema version, and the first and last calendar dates it covers. It SHALL NOT carry timestamps finer than a date. + +#### Scenario: Dates only +- **WHEN** a report covering activity on two days is built +- **THEN** its period holds two dates and no time-of-day values diff --git a/openspec/specs/usage-reporting/controls/spec.md b/openspec/specs/usage-reporting/controls/spec.md new file mode 100644 index 00000000..101f91c6 --- /dev/null +++ b/openspec/specs/usage-reporting/controls/spec.md @@ -0,0 +1,73 @@ +# usage-reporting/controls Specification + +## Purpose +Defines how usage reporting is switched on and off, how users inspect it, and how its existence is disclosed on first use. + +## Requirements + +### Requirement: Enabled by default with an opt-out precedence +Usage reporting SHALL be enabled by default. Its effective state SHALL be decided by the first matching rule, highest first: +1. `DO_NOT_TRACK` set to a truthy value → off. +2. `SLAYER_USAGE_REPORTING` set to `on` or `off` → that value. +3. Automatic off: the `CI` variable is set; SLayer is an editable install; the usage-reporting config directory cannot be written. +4. The persisted setting from `slayer usage-reporting enable|disable`. +5. Default → on. + +When off, usage reporting SHALL record nothing, write no usage-reporting file, print no notice and make no usage-reporting request. + +#### Scenario: DO_NOT_TRACK beats explicit enable +- **WHEN** `DO_NOT_TRACK=1` and `SLAYER_USAGE_REPORTING=on` are both set +- **THEN** usage reporting is off + +#### Scenario: Explicit enable beats CI +- **WHEN** `CI=true` and `SLAYER_USAGE_REPORTING=on` are set +- **THEN** usage reporting is on + +#### Scenario: CI auto-off +- **WHEN** `CI` is set and no higher rule applies +- **THEN** usage reporting is off + +#### Scenario: Editable install auto-off +- **WHEN** SLayer is installed in editable mode and no higher rule applies +- **THEN** usage reporting is off + +#### Scenario: Unwritable config directory +- **WHEN** the usage-reporting config directory cannot be created or written and no higher rule applies +- **THEN** usage reporting is off, with no notice and no network request + +#### Scenario: Persisted disable +- **WHEN** the user ran `slayer usage-reporting disable` and no environment rule applies +- **THEN** usage reporting is off in later processes + +### Requirement: The slayer usage-reporting command +`slayer usage-reporting` SHALL provide: +- `status`: the effective state, the rule that decided it, and the current install ID (if any). +- `enable` / `disable`: persist the setting; `disable` also deletes the spool and the install ID. +- `show`: print exactly the report the next send would contain, without sending it. + +#### Scenario: Status names the deciding rule +- **WHEN** `CI` is set and the user runs `slayer usage-reporting status` +- **THEN** the output says usage reporting is off because of `CI` + +#### Scenario: Disable removes local data +- **WHEN** the user runs `slayer usage-reporting disable` +- **THEN** the spool and the install ID are deleted and no later process sends + +#### Scenario: Show does not send +- **WHEN** the user runs `slayer usage-reporting show` +- **THEN** the pending report is printed and no request is made + +### Requirement: One-time disclosure notice +When usage reporting is on, SLayer SHALL print a notice once, on stderr only, naming what is collected, linking the usage-reporting docs page and giving the opt-out. The notice SHALL be printed again when the report schema's major version changes. It SHALL never be written to stdout. + +#### Scenario: Notice once +- **WHEN** two `slayer` commands run in sequence with usage reporting on for the first time +- **THEN** only the first prints the notice, on stderr + +#### Scenario: stdio MCP stays clean +- **WHEN** `slayer mcp` starts with usage reporting on for the first time +- **THEN** the notice goes to stderr and stdout carries only MCP protocol messages + +#### Scenario: Schema major bump +- **WHEN** the recorded notice was shown for an older report schema major version +- **THEN** the notice is printed again once diff --git a/openspec/specs/usage-reporting/delivery/spec.md b/openspec/specs/usage-reporting/delivery/spec.md new file mode 100644 index 00000000..75c2be10 --- /dev/null +++ b/openspec/specs/usage-reporting/delivery/spec.md @@ -0,0 +1,102 @@ +# usage-reporting/delivery Specification + +## Purpose +Defines how recorded usage is stored locally, when and where it is sent, and the guarantee that usage reporting never affects the process it runs in. + +## Requirements + +### Requirement: Usage reporting never affects the host process +No usage reporting failure — file I/O, network, timeout, non-2xx response, corrupt or foreign files — SHALL change a command's result, exit code or stdout, or raise into the host process. Usage reporting SHALL NOT write to stdout, and SHALL NOT block the process except for the bounded exit wait defined below. + +#### Scenario: Endpoint down +- **WHEN** the usage-reporting endpoint refuses connections, times out or returns 500 during a command +- **THEN** the command's output and exit code are identical to a run with usage reporting off + +#### Scenario: Corrupt spool file +- **WHEN** a spool file contains invalid JSON +- **THEN** it is skipped and the command runs normally + +### Requirement: Usage is spooled locally at process end +Each process SHALL keep its counters in memory and write them to its own spool file when it ends — normal exit, SIGINT, server shutdown, end of input or broken pipe on stdio MCP, and SIGTERM for stdio `slayer mcp` (chaining any handler already installed) — with no network request at that point. A process killed with SIGKILL loses its counts. A process that forks SHALL start the child with empty counters. + +#### Scenario: stdio MCP terminated +- **WHEN** an MCP client sends SIGTERM to a `slayer mcp` process that handled tool calls +- **THEN** its counters are written to a spool file before it exits + +#### Scenario: Concurrent processes +- **WHEN** several `slayer` processes end at the same time +- **THEN** each writes its own spool file and none overwrites another + +#### Scenario: Forked child +- **WHEN** a process with recorded counts forks +- **THEN** the child's report does not repeat the parent's counts + +### Requirement: Send at most once per 24 hours, only when SLayer runs +A report SHALL be sent only when a `slayer` process starts or records an action, at least 24 hours have passed since the last successful send, and usage reporting is on. The report SHALL merge all spool files and the sending process's live counters. An idle process SHALL send nothing. A last-send time in the future SHALL count as due. + +#### Scenario: Idle server +- **WHEN** a server records no action for 48 hours +- **THEN** it sends nothing + +#### Scenario: Due on activity +- **WHEN** a server records an action 25 hours after the last successful send +- **THEN** exactly one report is sent, merging the spool and its live counters + +#### Scenario: Not yet due +- **WHEN** a command starts 3 hours after the last successful send +- **THEN** nothing is sent and the command's counters are spooled at exit + +#### Scenario: Clock moved backwards +- **WHEN** the recorded last-send time lies in the future +- **THEN** a send is due + +### Requirement: Background send with a bounded exit wait +A due send SHALL run in a background thread started when the process starts (or, in a running server, when the action that made it due is recorded). At exit, the process SHALL wait for an in-flight send at most about one second; with no send in flight it SHALL not wait. + +#### Scenario: Slow network at exit +- **WHEN** a short command exits while its send has been stalled for 10 seconds +- **THEN** the process exits within about one second of finishing its work, and the unsent data stays in the spool + +### Requirement: Exactly-once-per-batch delivery bookkeeping +Only one process SHALL send at a time. Spool files included in a send SHALL be deleted, and the last-send time recorded, only after a 2xx response; otherwise they SHALL remain for a later send. Files claimed by a sender that died SHALL be recovered by a later send. Each report SHALL carry a random batch ID so the backend can drop duplicates. The spool SHALL be capped (about 100 files / 1 MB, oldest dropped first); spool files SHALL be readable only by the owner, and entries that are not regular files SHALL be ignored. + +#### Scenario: Two simultaneous senders +- **WHEN** two processes find a send due at the same moment +- **THEN** one report is sent + +#### Scenario: Failed send retained +- **WHEN** a send gets a 500 response +- **THEN** the spool files remain and the last-send time is unchanged + +#### Scenario: Sender crash recovered +- **WHEN** a sender dies after claiming spool files +- **THEN** a later send includes those files + +### Requirement: Destination and request shape +A report SHALL be sent as one event to the PostHog EU capture API (`https://eu.i.posthog.com/i/v0/e/`), with event name `slayer_usage`, the install ID as distinct ID, the batch ID as event UUID and GeoIP enrichment disabled; each install ID SHALL be a PostHog person, so that its reports can be deleted together. `SLAYER_USAGE_REPORTING_ENDPOINT` SHALL override the URL. + +#### Scenario: Endpoint override +- **WHEN** `SLAYER_USAGE_REPORTING_ENDPOINT` points to a local test server and a send is due +- **THEN** the report is posted to that server with the event name `slayer_usage` and GeoIP disabled + +### Requirement: Install identity and rotation +The install ID SHALL be a random UUID created on first use and stored in the platform config directory (not in SLayer's model storage). It SHALL be replaced by a new random UUID at the first start or send after it is 11 months old, so no report carries an install ID older than that. + +#### Scenario: Rotation +- **WHEN** a process starts and the install ID was created 12 months ago +- **THEN** a new install ID is created and used for subsequent reports + +#### Scenario: Rotation in a long-running process +- **WHEN** a server started while its install ID was 10 months old sends a report two months later +- **THEN** the report carries a new install ID + +### Requirement: Retention and deletion at the destination +Reports SHALL be deleted from the destination within 12 months of being sent: a weekly scheduled job SHALL delete every install ID whose first report is older than the install-ID lifetime plus one week, together with all its reports. The same job SHALL delete one given install ID and all its reports on request. + +#### Scenario: Expired install deleted +- **WHEN** the weekly job runs and an install ID's first report is 11 months and two weeks old +- **THEN** that install ID and all its reports are deleted + +#### Scenario: Deletion on request +- **WHEN** the job is run manually with an install ID +- **THEN** that install ID and all its reports are deleted, and no other install is touched diff --git a/scripts/usage_reporting_retention.py b/scripts/usage_reporting_retention.py new file mode 100644 index 00000000..c9e15182 --- /dev/null +++ b/scripts/usage_reporting_retention.py @@ -0,0 +1,107 @@ +"""Delete SLayer usage reports from PostHog once past retention, or one install's on request. + +Run weekly by `.github/workflows/usage-reporting-retention.yml`; stdlib only. +""" + +import functools +import json +import os +import re +import sys +import urllib.error +import urllib.request +import uuid +from collections.abc import Callable, Mapping +from typing import Any + +HOST = "https://eu.posthog.com" +PROJECT_ID = "308099" +EVENT = "slayer_usage" +# An install ID sends nothing once this old (it is replaced), so its whole history can go. +ID_LIFETIME_MONTHS = 11 +GRACE_DAYS = 7 +BATCH = 1000 +QUERY_PAGE = 10_000 +TIMEOUT = 60.0 +_INSTALL_ID = re.compile(r"[0-9a-f-]{36}") + +Post = Callable[[str, dict[str, Any]], dict[str, Any]] + + +def _post(path: str, body: dict[str, Any], *, key: str) -> dict[str, Any]: + request = urllib.request.Request( + f"{HOST}/api/projects/{PROJECT_ID}/{path}", data=json.dumps(body).encode(), method="POST", + headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"}, + ) + with urllib.request.urlopen(request, timeout=TIMEOUT) as response: + return json.loads(response.read() or b"{}") + + +def expired_install_ids(post: Post) -> list[str]: + """Install IDs whose first report is older than their lifetime plus a grace period.""" + install_ids: list[str] = [] + while True: + # Keyset pagination (personal API keys may not use OFFSET); only UUID-shaped IDs are + # selected, and each is checked, so the next page's key is safe to inline. + after = install_ids[-1] if install_ids else "" + query = ( + f"SELECT distinct_id FROM events WHERE event = '{EVENT}' AND length(distinct_id) = 36 " + f"AND match(distinct_id, '^[0-9a-f-]+$') AND distinct_id > '{after}' GROUP BY distinct_id " + f"HAVING min(timestamp) < now() - INTERVAL {ID_LIFETIME_MONTHS} MONTH - INTERVAL {GRACE_DAYS} DAY " + f"ORDER BY distinct_id LIMIT {QUERY_PAGE}" + ) + response = post("query/", {"query": {"kind": "HogQLQuery", "query": query}, "name": "usage-reporting retention"}) + page = [_install_id(row[0]) for row in response["results"]] + install_ids.extend(page) + if len(page) < QUERY_PAGE: + return install_ids + + +def _install_id(value: object) -> str: + if not isinstance(value, str) or not _INSTALL_ID.fullmatch(value): + raise ValueError(f"unexpected distinct_id from PostHog: {value!r}") + return value + + +def delete_installs(install_ids: list[str], post: Post) -> tuple[int, list[Any]]: + """Delete each install's person and events; return how many persons PostHog found, and its errors.""" + found = 0 + errors: list[Any] = [] + for start in range(0, len(install_ids), BATCH): + response = post("persons/bulk_delete/", {"distinct_ids": install_ids[start:start + BATCH], "delete_events": True}) + if "persons_found" not in response: + errors.append({"batch_start": start, "unexpected_response": sorted(response)}) + found += response.get("persons_found", 0) + errors.extend(response.get("deletion_errors", [])) + return found, errors + + +def main(env: Mapping[str, str] = os.environ, post: Post | None = None) -> int: + key = env.get("POSTHOG_PERSONAL_API_KEY", "") + if not key: + print("POSTHOG_PERSONAL_API_KEY is not set", file=sys.stderr) + return 2 + post = post or functools.partial(_post, key=key) + requested = env.get("INSTALL_ID", "").strip() + try: + install_ids = [str(uuid.UUID(requested))] if requested else [] + except ValueError: + print(f"not an install ID: {requested!r}", file=sys.stderr) + return 2 + try: + if not requested: + install_ids = expired_install_ids(post) + found, errors = delete_installs(install_ids, post) + except urllib.error.HTTPError as exc: + print(f"PostHog returned {exc.code}: {exc.read().decode(errors='replace')[:1000]}", file=sys.stderr) + return 1 + # Fewer found than requested is expected: events sent without a person, or deletions still queued. + print(f"{len(install_ids)} install(s) requested, {found} found and queued for deletion; {len(errors)} error(s)") + if errors: + print(json.dumps(errors), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/slayer/api/server.py b/slayer/api/server.py index 86d16b76..e5e50660 100644 --- a/slayer/api/server.py +++ b/slayer/api/server.py @@ -8,8 +8,11 @@ from sqlalchemy.exc import SQLAlchemyError from fastapi import FastAPI, HTTPException +from fastapi.routing import APIRoute from pydantic import BaseModel, ConfigDict, Field, model_validator +from starlette.types import ASGIApp, Message, Receive, Scope, Send +from slayer import usage_reporting from slayer.mcp.server import create_mcp_server from slayer.core.errors import ( AmbiguousModelError, @@ -32,6 +35,8 @@ logger = logging.getLogger(__name__) +_MCP_MOUNT = "/mcp" + class QueryRequest(BaseModel): # Allow extra keys (e.g. forward-compat fields a newer client might send) @@ -231,6 +236,39 @@ class InspectRequest(BaseModel): descriptions_max_chars: int | None = None +class _UsageMiddleware: + """Counts each REST request by its matched route template; requests under ``mounted`` count themselves.""" + + def __init__(self, app: ASGIApp, *, mounted: str) -> None: + self.app = app + self.mounted = mounted + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + path = scope.get("path", "") + if scope["type"] != "http" or not usage_reporting.is_active() or path == self.mounted or path.startswith(f"{self.mounted}/"): + await self.app(scope, receive, send) + return + status: list[int] = [] + + async def capture(message: Message) -> None: + if message["type"] == "http.response.start": + status.append(message["status"]) + await send(message) + + try: + await self.app(scope, receive, capture) + except BaseException as exc: + self._record(scope, error=exc) + raise + self._record(scope, error=usage_reporting.HttpError(status[0]) if status and status[0] >= 400 else None) + + @staticmethod + def _record(scope: Scope, *, error: BaseException | None) -> None: + route = scope.get("route") + token = route.path if isinstance(route, APIRoute) else "other" + usage_reporting.record(surface="rest", token=token, error=error) + + def _slayer_version() -> str: try: return _pkg_version("motley-slayer") @@ -259,6 +297,7 @@ def create_app( # NOSONAR(S3776) — FastAPI route-handler factory; complexity ) ) app = FastAPI(title="SLayer", version=_slayer_version()) + app.add_middleware(_UsageMiddleware, mounted=_MCP_MOUNT) # Mount MCP server over SSE at /mcp. The embedded server intentionally # does NOT receive `ingest_on_startup` — orchestration happens once, @@ -266,7 +305,7 @@ def create_app( # NOSONAR(S3776) — FastAPI route-handler factory; complexity # the orchestrator twice. mcp = create_mcp_server(storage=storage, always_load_query=always_load_query, _seed_help=False) mcp_app = mcp.sse_app() - app.mount("/mcp", mcp_app) + app.mount(_MCP_MOUNT, mcp_app) @app.get("/health") async def health() -> dict[str, str]: @@ -295,6 +334,7 @@ async def query( # "dry_run": ..., "explain": ...}``. Mirrors the MCP ``query`` # tool's list form and ``engine.execute(query=[...])``. # Engine auto-sorts the list and validates DAG invariants. + run_kwargs: dict[str, Any] = {} if isinstance(request, QueryListRequest): if not request.queries: raise HTTPException( @@ -303,12 +343,8 @@ async def query( ) dry_run = bool(request.dry_run) explain = bool(request.explain) - result = await engine.execute( - query=list(request.queries), - variables=request.variables or {}, - dry_run=dry_run, - explain=explain, - ) + run_query: Any = list(request.queries) + variables = request.variables or {} # Run-by-name: ``{"name": "", "refine": {...}, "variables": {...}}`` # runs the model's stored backing query, ``refine`` merged into its # final stage. Any other query field (even ``null``) is rejected. @@ -324,13 +360,9 @@ async def query( ) dry_run = bool(request.dry_run) explain = bool(request.explain) - result = await engine.execute( - request.name, - variables=request.variables or {}, - dry_run=dry_run, - explain=explain, - refine=request.refine, - ) + run_query = request.name + variables = request.variables or {} + run_kwargs["refine"] = request.refine elif "refine" in request.model_fields_set: raise HTTPException( status_code=400, @@ -344,19 +376,18 @@ async def query( # ``variables`` is consumed at execute() level, not part of # SlayerQuery's filter-substitution variables (those merge # automatically via the kwarg path). - runtime_kwarg = payload.pop("variables", None) + variables = payload.pop("variables", None) # ``dry_run``/``explain`` are execution-mode flags only — pop # them here and pass as engine kwargs so v3 SlayerQuery # (extra="forbid") doesn't reject them. dry_run = bool(payload.pop("dry_run", False)) explain = bool(payload.pop("explain", False)) - slayer_query = SlayerQuery.model_validate(payload) - result = await engine.execute( - query=slayer_query, - variables=runtime_kwarg, - dry_run=dry_run, - explain=explain, - ) + run_query = SlayerQuery.model_validate(payload) + usage_reporting.observe_query(run_query) + result = await engine.execute( + query=run_query, variables=variables, dry_run=dry_run, explain=explain, **run_kwargs, + ) + usage_reporting.observe_executed(result) attrs = result.attributes def _convert_meta(d: dict) -> dict[str, FieldMetadataResponse]: diff --git a/slayer/cli.py b/slayer/cli.py index 468d3547..5ff97798 100644 --- a/slayer/cli.py +++ b/slayer/cli.py @@ -5,14 +5,16 @@ import copy import json import os +import signal import sys -from typing import Any +from typing import Any, NoReturn from urllib.parse import parse_qs, urlparse import sqlalchemy as sa import yaml from pydantic import BaseModel, Field +from slayer import usage_reporting from slayer.async_utils import run_sync from slayer.core.errors import ( AmbiguousModelError, @@ -67,6 +69,7 @@ has_sqlite_widenable_columns, refine_dict_with_live_schema, ) +from slayer.usage_reporting import control as usage_reporting_control _STORAGE_DEFAULT = default_storage_path() _DATASOURCE_NAME_HELP = "Datasource name" @@ -122,10 +125,63 @@ def _add_storage_arg(parser): def _resolve_storage(args): """Resolve storage backend from --storage or --models-dir flags.""" path = args.storage or args.models_dir or _STORAGE_DEFAULT - return resolve_storage(path) + storage = resolve_storage(path) + usage_reporting.observe_storage(storage) + return storage -def main(): # NOSONAR(S3776) — linear top-level CLI command dispatch (one elif per subcommand); splitting the dispatch chain would not improve readability +def _assign_usage_tokens(parser: argparse.ArgumentParser, path: tuple[str, ...] = ()) -> None: + """Give every leaf command except ``slayer usage-reporting …`` a fixed usage token: its subcommand path.""" + groups = [a for a in parser._actions if isinstance(a, argparse._SubParsersAction)] + for group in groups: + for name, sub in group.choices.items(): + if (*path, name) != ("usage-reporting",): + _assign_usage_tokens(sub, (*path, name)) + if path and not groups: + parser.set_defaults(usage_token=".".join(path)) + + +def main(): + parser = _build_parser() + _assign_usage_tokens(parser) + args = parser.parse_args() + + try: + host_clock() # Validates SLAYER_NOW before any command's side effects. + except SlayerError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + + if args.command == "usage-reporting": + _run_usage_reporting(args) + return + token = getattr(args, "usage_token", None) + if token is not None: + usage_reporting.start() + try: + with usage_reporting.counting(surface="cli", token=token): + _dispatch(parser, args) + except (SystemExit, KeyboardInterrupt) as exc: + if args.command == "mcp": + _exit_stdio_now(exc) + raise + finally: + usage_reporting.flush() + + +def _exit_stdio_now(stop: SystemExit | KeyboardInterrupt) -> NoReturn: + """Spool, then exit hard: stdio MCP's blocked stdin reader thread would hold up a normal exit.""" + code = stop.code if isinstance(stop, SystemExit) else 128 + signal.SIGINT + if code is None: + code = 0 + usage_reporting.flush() + usage_reporting.shutdown() + sys.stdout.flush() + sys.stderr.flush() + os._exit(code if isinstance(code, int) else 1) + + +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="slayer", description="SLayer — a lightweight semantic layer for AI agents", @@ -874,6 +930,7 @@ def main(): # NOSONAR(S3776) — linear top-level CLI command dispatch (one eli _add_storage_arg(search_parser) search_subparsers = search_parser.add_subparsers(dest="search_command") # ``slayer search`` (no subcommand) runs the search query directly. + search_parser.set_defaults(usage_token="search") search_parser.add_argument( "--entity", action="append", @@ -963,14 +1020,41 @@ def main(): # NOSONAR(S3776) — linear top-level CLI command dispatch (one eli # concepts ship as help memories — read them with # `slayer inspect memory:help.intro --type memory` (see the epilog above). - args = parser.parse_args() + # ── usage reporting ───────────────────────────────────────────────────── + usage_reporting_parser = subparsers.add_parser( + "usage-reporting", + help="Show or change anonymous usage reporting", + epilog="docs: https://docs.motley.ai/slayer/reference/usage-reporting/", + ) + usage_reporting_subparsers = usage_reporting_parser.add_subparsers(dest="usage_reporting_command", required=True) + usage_reporting_subparsers.add_parser("status", help="Whether usage reporting is on, why, and the install ID") + usage_reporting_subparsers.add_parser("enable", help="Turn usage reporting on (persisted)") + usage_reporting_subparsers.add_parser( + "disable", help="Turn usage reporting off (persisted) and delete the install ID and unsent data", + ) + usage_reporting_subparsers.add_parser("show", help="Print the report the next send would contain, without sending") + return parser + + +def _run_usage_reporting(args) -> None: + if args.usage_reporting_command == "status": + print(usage_reporting_control.status()) + elif args.usage_reporting_command == "enable": + usage_reporting_control.enable() + print("Usage reporting enabled.") + elif args.usage_reporting_command == "disable": + usage_reporting_control.disable() + print("Usage reporting disabled; the install ID and unsent data were deleted.") + else: + print(usage_reporting_control.pending_report().model_dump_json(indent=2)) - try: - host_clock() # Validates SLAYER_NOW before any command's side effects. - except SlayerError as exc: - print(f"Error: {exc}", file=sys.stderr) - sys.exit(1) +_SERVER_COMMANDS = frozenset({"serve", "flight-serve", "pg-serve", "mcp"}) + + +def _dispatch(parser: argparse.ArgumentParser, args) -> None: # NOSONAR(S3776) — linear top-level CLI command dispatch (one elif per subcommand); splitting the dispatch chain would not improve readability + if args.command in _SERVER_COMMANDS: + _unwind_on(signal.SIGTERM) if args.command == "serve": _run_serve(args) elif args.command == "flight-serve": @@ -1416,6 +1500,7 @@ def _run_query(args): # NOSONAR S3776 — argparse-driven dispatch; one straigh query = json.loads(query_input) if isinstance(query, list) and not query: raise SystemExit("Query list cannot be empty.") + usage_reporting.observe_query(query) result = engine.execute_sync( query=query, variables=runtime_kwarg or None, @@ -1423,6 +1508,7 @@ def _run_query(args): # NOSONAR S3776 — argparse-driven dispatch; one straigh explain=bool(args.explain), refine=refine, ) + usage_reporting.observe_executed(result) _print_query_warnings(result) @@ -1497,9 +1583,22 @@ def _run_mcp(args): if getattr(args, "demo", False): _prepare_demo(args, storage) mcp = create_mcp_server(storage=storage, **_server_flags(args)) + _unwind_on(signal.SIGINT) mcp.run() +def _unwind_on(signum: signal.Signals) -> None: + """Make ``signum`` raise ``SystemExit`` so a server unwinds and spools; chains any installed handler.""" + previous = signal.getsignal(signum) + + def handler(received, frame): + if callable(previous): + previous(received, frame) + raise SystemExit(128 + received) + + signal.signal(signum, handler) + + def _parse_csv_arg(value): if not value: return None @@ -2032,6 +2131,7 @@ def _run_import_osi(args): def _models_list(storage, args) -> None: names = run_sync(storage.list_models()) + usage_reporting.observe_model_count(len(names)) builtins = [m for ds in run_sync(storage.list_datasources()) for m in run_sync(storage.builtin_models(ds))] if not names and not builtins: print("No models found.") @@ -2093,12 +2193,17 @@ def _run_models(args): def _datasources_list(storage, args) -> None: names = run_sync(storage.list_datasources()) if not names: + usage_reporting.observe_datasources([]) print("No datasources found.") return + loaded = [] for name in names: ds = run_sync(storage.get_datasource(name)) ds_type = ds.type if ds and ds.type else "unknown" print(f"{name} ({ds_type})") + if ds is not None: + loaded.append(ds) + usage_reporting.observe_datasources(loaded) def _datasource_or_exit(storage, name: str): diff --git a/slayer/engine/query_engine.py b/slayer/engine/query_engine.py index 4ad8f984..3323eb22 100644 --- a/slayer/engine/query_engine.py +++ b/slayer/engine/query_engine.py @@ -572,6 +572,8 @@ class SlayerResponse(BaseModel): # The effective population model and whether it was inferred. population: Optional[str] = None population_inferred: bool = False + # The datasource the query ran on; never serialized. + datasource: Optional[DatasourceConfig] = PydanticField(default=None, exclude=True, repr=False) @model_validator(mode="after") def _populate_columns(self) -> "SlayerResponse": @@ -1368,6 +1370,7 @@ async def _execute_pipeline( warnings=[*prepared.slack_warnings, *explained.warnings], population=prepared.population, population_inferred=prepared.population_inferred, + datasource=prepared.datasource, ) # Capture refresh-key baselines before the data query (cached data then @@ -1386,6 +1389,7 @@ async def _execute_pipeline( attributes=prepared.attributes, warnings=[*prepared.slack_warnings, *result.warnings], population=prepared.population, population_inferred=prepared.population_inferred, + datasource=prepared.datasource, ) if use_cache: @@ -1574,6 +1578,7 @@ async def _reexecute_entry(self, entry: _CacheEntry, now: float) -> Optional[_Ca attributes=prepared.attributes, warnings=[*prepared.slack_warnings, *result.warnings], population=prepared.population, population_inferred=prepared.population_inferred, + datasource=prepared.datasource, ) return self._build_cache_entry( prepared=prepared, diff --git a/slayer/facade/translator.py b/slayer/facade/translator.py index 2a45f5d8..330c1dd0 100644 --- a/slayer/facade/translator.py +++ b/slayer/facade/translator.py @@ -1438,13 +1438,13 @@ def _parse_with_keyword_alias_fallback( with the original parse error if both attempts fail.""" try: return sqlglot.parse_one(sql, dialect=dialect) # pyright: ignore[reportReturnType] — parse_one's Expr TypeVar - except sqlglot.errors.ParseError as primary: + except sqlglot.errors.SqlglotError as primary: retry_sql = _quote_keyword_aliases(sql) if retry_sql == sql: raise TranslationError(f"SQL parse error: {primary}") from primary try: return sqlglot.parse_one(retry_sql, dialect=dialect) # pyright: ignore[reportReturnType] — parse_one's Expr TypeVar - except sqlglot.errors.ParseError: + except sqlglot.errors.SqlglotError: raise TranslationError(f"SQL parse error: {primary}") from primary diff --git a/slayer/flight/handlers.py b/slayer/flight/handlers.py index 6527639a..8f637654 100644 --- a/slayer/flight/handlers.py +++ b/slayer/flight/handlers.py @@ -18,6 +18,7 @@ import pyarrow.flight as fl from google.protobuf.any_pb2 import Any as PbAny +from slayer import usage_reporting from slayer.async_utils import run_sync from slayer.core.models import SlayerModel from slayer.engine.query_engine import SlayerQueryEngine @@ -300,9 +301,8 @@ def get_flight_info_for_sql( def do_get_for_sql(self, sql: str) -> fl.FlightDataStream: """Execute ``sql`` and return the record-batch stream.""" result = self._translate(sql) - if isinstance(result, ProbeResult): - return _table_to_record_batch_stream(result.table) - if isinstance(result, InfoSchemaResult): + if isinstance(result, (ProbeResult, InfoSchemaResult)): + usage_reporting.record(surface="flight", token="probe") return _table_to_record_batch_stream(result.table) if isinstance(result, NoOpResult): return _table_to_record_batch_stream(pa.Table.from_pylist([])) @@ -376,7 +376,10 @@ def _execute_full(self, result: "QueryResult") -> pa.Table: async def execute_full(): return await self._engine.execute(query=result.query) - response = run_sync(execute_full()) + usage_reporting.observe_query(result.query) + with usage_reporting.counting(surface="flight", token="query"): + response = run_sync(execute_full()) + usage_reporting.observe_executed(response) schema = self._build_schema(result) rows = [ self._rewrite_row(row, result.column_name_mapping) diff --git a/slayer/mcp/server.py b/slayer/mcp/server.py index e9063fd9..537dfb5d 100644 --- a/slayer/mcp/server.py +++ b/slayer/mcp/server.py @@ -3,17 +3,18 @@ import json import logging import sys +import weakref from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _pkg_version from collections.abc import Callable, Sequence from inspect import cleandoc -from typing import Annotated, Any +from typing import TYPE_CHECKING, Annotated, Any import sqlalchemy as sa from pydantic import Discriminator, Field, Tag from sqlalchemy.exc import DatabaseError -from slayer import __version__ +from slayer import __version__, usage_reporting from slayer.core.errors import ( AmbiguousModelError, @@ -74,6 +75,9 @@ from slayer.storage.base import StorageBackend from slayer.storage.document_loading import DocumentLoadFailures +if TYPE_CHECKING: + from mcp.server.fastmcp import FastMCP + logger = logging.getLogger(__name__) VALID_DIMENSION_TYPES = {"string", "time", "date", "boolean", "number"} @@ -171,6 +175,44 @@ def _import_fastmcp(): return FastMCP +def _counted(fastmcp_cls: "type[FastMCP]") -> "type[FastMCP]": + """``fastmcp_cls`` counting every tool call, and each session's client once.""" + + class CountedFastMCP(fastmcp_cls): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._usage_reporting_sessions: weakref.WeakSet = weakref.WeakSet() + + async def call_tool(self, name: str, arguments: dict[str, Any]): + self._observe_client() + try: + result = await super().call_tool(name, arguments) + except BaseException as exc: + # FastMCP re-raises a tool's exception as ToolError from the original. + usage_reporting.record(surface="mcp", token=name, error=exc.__cause__ or exc) + raise + usage_reporting.record(surface="mcp", token=name) + return result + + def _observe_client(self) -> None: + if not usage_reporting.is_active(): + return + try: + session = self.get_context().session + if session in self._usage_reporting_sessions: + return + self._usage_reporting_sessions.add(session) + info = session.client_params.clientInfo if session.client_params else None + except Exception: + logger.debug("usage reporting: no MCP session to identify", exc_info=True) + return + usage_reporting.observe_mcp_client( + name=getattr(info, "name", None), version=getattr(info, "version", None), + ) + + return CountedFastMCP + + def _set_server_version(mcp) -> None: """Stamp SLayer's version onto the lowlevel MCP server (best-effort; cosmetic).""" lowlevel = getattr(mcp, "_mcp_server", None) @@ -480,7 +522,7 @@ def create_mcp_server( # NOSONAR(S3776) — FastMCP tool-registration factory; ) FastMCP = _import_fastmcp() # NOSONAR(S117) — holds a class object; CapWords matches the class it aliases - mcp = FastMCP("SLayer", instructions=SERVER_INSTRUCTIONS) + mcp = _counted(FastMCP)("SLayer", instructions=SERVER_INSTRUCTIONS) _set_server_version(mcp) # Expose the closure engine so callers can dispose per-task pools via # mcp._slayer_engine.aclose() (idempotent; leaves the engine reusable). @@ -533,6 +575,7 @@ async def query( # opaque (its stored SQL can't take a pushed-down limit) — cap # response-side only. exec_query, capped, cap_hint = _apply_mcp_row_cap(query) + usage_reporting.observe_query(query) result = await engine.execute( query=exec_query, variables=variables, @@ -540,6 +583,7 @@ async def query( explain=explain, refine=refine, ) + usage_reporting.observe_executed(result) if dry_run: return f"SQL:\n{result.sql}" if capped: diff --git a/slayer/pg_facade/connection.py b/slayer/pg_facade/connection.py index 6cb830c0..29650254 100644 --- a/slayer/pg_facade/connection.py +++ b/slayer/pg_facade/connection.py @@ -28,6 +28,7 @@ from pydantic import BaseModel, ConfigDict from sqlglot.optimizer.scope import traverse_scope +from slayer import usage_reporting from slayer.core.enums import DataType from slayer.core.models import SlayerModel from slayer.storage.base import StorageBackend @@ -723,7 +724,7 @@ async def _read_message(self): async def _handle_simple_query(self, sql: str) -> None: try: statements = [s for s in sqlglot.parse(sql, dialect="postgres") if s is not None] - except sqlglot.errors.ParseError as exc: + except sqlglot.errors.SqlglotError as exc: # BI tools (Metabase) wrap reads in ``BEGIN READ ONLY`` etc., which # sqlglot can't parse. Strip the transaction characteristics and # retry once before surfacing a syntax error. @@ -732,7 +733,7 @@ async def _handle_simple_query(self, sql: str) -> None: statements = [ s for s in sqlglot.parse(stripped, dialect="postgres") if s is not None ] if stripped != sql else None - except sqlglot.errors.ParseError: + except sqlglot.errors.SqlglotError: statements = None if statements is None: logger.warning("pg facade: cannot parse simple query %r: %s", sql, exc) @@ -967,7 +968,7 @@ async def _handle_execute(self, msg: proto.ExecuteMessage) -> None: def _portal_is_tx_end(sql: str) -> bool: try: parsed = sqlglot.parse_one(sql, dialect="postgres") - except sqlglot.errors.ParseError: + except sqlglot.errors.SqlglotError: return False return _is_tx_end(parsed) @@ -1047,6 +1048,7 @@ async def _run_statement( return False if isinstance(result, (ProbeResult, InfoSchemaResult, PgCatalogResult)): + usage_reporting.record(surface="pg", token="probe") self._emit_row_batch(result.batch, result_formats, send_row_description) # DEV-1569: set_config(...) mutation hint surfaces on ProbeResult. # Apply ONLY in the Execute path (not Describe). Pushes @@ -1106,17 +1108,21 @@ def _emit_row_batch( async def _run_query( self, result: QueryResult, result_formats: list[int] | None, send_row_description: bool, ) -> bool: + usage_reporting.observe_query(result.query) try: # The translator resolves the per-query datasource from the # referenced model(s) and rejects cross-datasource joins. A model # query always carries one; guard the impossible None rather than # passing it to the engine. - if result.data_source is None: - raise ValueError("could not resolve a datasource for the query") - with timing.open_query_profile(): - response = await self._engine.execute( - query=result.query, data_source=result.data_source, - ) + with usage_reporting.counting(surface="pg", token="query"): + if result.data_source is None: + raise ValueError("could not resolve a datasource for the query") + engine = cast(SlayerQueryEngine, self._engine) + with timing.open_query_profile(): + response = await engine.execute( + query=result.query, data_source=result.data_source, + ) + usage_reporting.observe_executed(response) except Exception as exc: # noqa: BLE001 — surface any engine error to the client code, message = _engine_error_fields(exc) await self._send_error(code=code, message=message) diff --git a/slayer/usage_reporting/__init__.py b/slayer/usage_reporting/__init__.py new file mode 100644 index 00000000..7c635817 --- /dev/null +++ b/slayer/usage_reporting/__init__.py @@ -0,0 +1,35 @@ +"""Anonymous usage reporting for processes started by the ``slayer`` CLI.""" + +from slayer.usage_reporting.recorder import ( + HttpError, + counting, + flush, + is_active, + observe_datasources, + observe_executed, + observe_mcp_client, + observe_model_count, + observe_query, + observe_storage, + record, + reset, + shutdown, + start, +) + +__all__ = [ + "HttpError", + "counting", + "flush", + "is_active", + "observe_datasources", + "observe_executed", + "observe_mcp_client", + "observe_model_count", + "observe_query", + "observe_storage", + "record", + "reset", + "shutdown", + "start", +] diff --git a/slayer/usage_reporting/clock.py b/slayer/usage_reporting/clock.py new file mode 100644 index 00000000..5485b767 --- /dev/null +++ b/slayer/usage_reporting/clock.py @@ -0,0 +1,7 @@ +"""The usage reporter's clock (a seam: tests replace ``now``).""" + +import datetime + + +def now() -> datetime.datetime: + return datetime.datetime.now(datetime.timezone.utc) diff --git a/slayer/usage_reporting/control.py b/slayer/usage_reporting/control.py new file mode 100644 index 00000000..c119d2a7 --- /dev/null +++ b/slayer/usage_reporting/control.py @@ -0,0 +1,33 @@ +"""``slayer usage-reporting status|enable|disable|show``.""" + +from slayer.usage_reporting import recorder, sender, settings, spool +from slayer.usage_reporting.payload import UsageReport + + +def status() -> str: + state = settings.resolve_state() + install_id = settings.read_config().install_id + return "\n".join([ + f"Usage reporting is {'on' if state.enabled else 'off'} (decided by: {state.rule.value}).", + f"Install ID: {install_id or 'none'}", + f"Config: {settings.config_path()}", + ]) + + +def enable() -> None: + settings.update_config(lambda config: config.model_copy(update={"setting": "enabled"})) + + +def disable() -> None: + """Persist ``disabled`` and delete the install ID and every unsent batch.""" + recorder.reset() + settings.update_config(lambda config: config.model_copy(update={ + "setting": "disabled", "install_id": None, "install_id_created": None, + })) + spool.purge(settings.spool_dir()) + + +def pending_report() -> UsageReport: + """Exactly what the next send would contain (with a fresh batch ID).""" + directory = settings.spool_dir() + return sender.build_report(config=settings.read_config(), batches=spool.load(spool.pending_files(directory))) diff --git a/slayer/usage_reporting/environment.py b/slayer/usage_reporting/environment.py new file mode 100644 index 00000000..92f76eac --- /dev/null +++ b/slayer/usage_reporting/environment.py @@ -0,0 +1,73 @@ +"""The ``env`` part of a report, read from the interpreter and package metadata (never by importing).""" + +import functools +import os +import platform +import re +import sys +from importlib.metadata import PackageNotFoundError, distribution, version +from pathlib import Path + +from slayer.usage_reporting.payload import EXTRAS, VERSION_PATTERN, ArchToken, Env, ExtraToken, OsToken +from slayer.usage_reporting.settings import DIST_NAME + +_UNKNOWN_VERSION = "0.0.0+unknown" +_OS: dict[str, OsToken] = {"linux": "linux", "darwin": "darwin", "win32": "windows"} +_ARCH: dict[str, ArchToken] = {"x86_64": "x86_64", "amd64": "x86_64", "arm64": "arm64", "aarch64": "arm64"} +_REQUIREMENT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +_EXTRA_MARKER = re.compile(r"""extra\s*==\s*["']([^"']+)["']""") + + +def _slayer_version() -> str: + try: + found = version(DIST_NAME) + except PackageNotFoundError: + return _UNKNOWN_VERSION + return found if re.fullmatch(VERSION_PATTERN, found) and len(found) <= 64 else _UNKNOWN_VERSION + + +def _installed(name: str) -> bool: + try: + distribution(name) + except PackageNotFoundError: + return False + return True + + +def _extras() -> list[ExtraToken]: + """Extras whose every requirement is installed.""" + try: + requires = distribution(DIST_NAME).requires or [] + except PackageNotFoundError: + return [] + needs: dict[str, list[str]] = {} + for requirement in requires: + name = _REQUIREMENT_NAME.match(requirement) + for extra in _EXTRA_MARKER.findall(requirement): + if name is not None: + needs.setdefault(extra, []).append(name.group(0)) + return [extra for extra in EXTRAS if extra in needs and all(_installed(n) for n in needs[extra])] + + +def _in_container() -> bool: + if Path("/.dockerenv").exists() or Path("/run/.containerenv").exists(): + return True + if os.environ.get("KUBERNETES_SERVICE_HOST"): + return True + try: + cgroup = Path("/proc/1/cgroup").read_text() + except OSError: + return False + return any(marker in cgroup for marker in ("docker", "kubepods", "containerd", "libpod")) + + +@functools.cache +def current() -> Env: + return Env( + slayer_version=_slayer_version(), + python=f"{sys.version_info.major}.{sys.version_info.minor}", + os=_OS.get(sys.platform, "other"), + arch=_ARCH.get(platform.machine().lower(), "other"), + in_container=_in_container(), + extras=_extras(), + ) diff --git a/slayer/usage_reporting/features/__init__.py b/slayer/usage_reporting/features/__init__.py new file mode 100644 index 00000000..9602294c --- /dev/null +++ b/slayer/usage_reporting/features/__init__.py @@ -0,0 +1,135 @@ +"""Turn the queries and datasources SLayer has already loaded into usage tokens.""" + +import io +import tokenize +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, Field + +from slayer.core.enums import BUILTIN_AGGREGATIONS, normalize_aggregation_name +from slayer.core.formula import ALL_TRANSFORMS +from slayer.core.keys import SCALAR_FUNCTIONS +from slayer.core.models import DatasourceConfig +from slayer.core.query import ComputedDimension, SlayerQuery +from slayer.sql.dialects import dialect_for_ds_type + +CUSTOM = "custom" +OTHER = "other" +_DEMO_DB = ("demo", "jaffle_shop.duckdb") +_SIGNIFICANT = (tokenize.NAME, tokenize.OP, tokenize.NUMBER, tokenize.STRING) + + +class QueryFeatures(BaseModel): + """Structural flags and built-in (or ``custom``) function names of one query attempt.""" + + flags: set[str] = Field(default_factory=set) + transforms: list[str] = Field(default_factory=list) + aggregations: list[str] = Field(default_factory=list) + + +def _tokens(text: str) -> list[tokenize.TokenInfo]: + try: + return [t for t in tokenize.generate_tokens(io.StringIO(text).readline) if t.type in _SIGNIFICANT] + except (tokenize.TokenError, SyntaxError): + return [] + + +def _close_paren(tokens: list[tokenize.TokenInfo], open_at: int) -> int: + depth = 0 + for i in range(open_at, len(tokens)): + if tokens[i].string == "(": + depth += 1 + elif tokens[i].string == ")": + depth -= 1 + if depth == 0: + return i + return len(tokens) + + +def _wraps_aggregate(inner: list[tokenize.TokenInfo]) -> bool: + return any( + t.string == ":" or (t.type == tokenize.NAME and i + 1 < len(inner) and inner[i + 1].string == "(") + for i, t in enumerate(inner) + ) + + +def _aggregation_token(name: str) -> str: + aggregation = normalize_aggregation_name(name) + return aggregation if aggregation in BUILTIN_AGGREGATIONS else CUSTOM + + +def _is_call(tokens: list[tokenize.TokenInfo], i: int) -> bool: + return i + 1 < len(tokens) and tokens[i + 1].string == "(" and (i == 0 or tokens[i - 1].string != ".") + + +def _count_call(tokens: list[tokenize.TokenInfo], i: int, out: QueryFeatures) -> None: + """Count the call at ``tokens[i]`` as a transform or an aggregation; scalar functions are skipped.""" + name = tokens[i].string.lower() + if name in SCALAR_FUNCTIONS: + return + wraps = _wraps_aggregate(tokens[i + 2:_close_paren(tokens, i + 1)]) + aggregation = _aggregation_token(tokens[i].string) + if name in ALL_TRANSFORMS and (wraps or aggregation == CUSTOM): + out.transforms.append(name) + else: + out.aggregations.append(aggregation) + + +def _count_calls(text: str, out: QueryFeatures) -> None: + tokens = _tokens(text) + for i, tok in enumerate(tokens): + if tok.type != tokenize.NAME: + continue + if i and tokens[i - 1].string == ":": + out.aggregations.append(_aggregation_token(tok.string)) + elif _is_call(tokens, i): + _count_call(tokens, i, out) + + +def _stage(raw: Any) -> SlayerQuery | None: + if isinstance(raw, SlayerQuery): + return raw + try: + return SlayerQuery.model_validate(raw) + except Exception: + return None + + +def _stage_flags(stage: SlayerQuery) -> set[str]: + present = { + "inline_source": stage.source_model is not None and not isinstance(stage.source_model, str), + "time_dimensions": bool(stage.time_dimensions), + "filters": bool(stage.filters), + "computed_dimensions": any(isinstance(d, ComputedDimension) for d in stage.dimensions or []), + "to_many_handling": stage.to_many_handling != SlayerQuery.model_fields["to_many_handling"].default, + } + return {flag for flag, on in present.items() if on} + + +def query_features(query: Any) -> QueryFeatures: + """Features of a query as handed to the engine (a query, a stage list, or a saved query's name).""" + out = QueryFeatures() + if isinstance(query, str): + out.flags.add("saved_query") + return out + raw_stages = query if isinstance(query, list) else [query] + if len(raw_stages) > 1: + out.flags.add("multistage") + for stage in filter(None, map(_stage, raw_stages)): + out.flags |= _stage_flags(stage) + for text in [m.formula for m in stage.measures or [] if m.formula] + list(stage.filters or []): + _count_calls(text, out) + return out + + +def dialect(datasource: DatasourceConfig) -> str: + """The datasource's SQL dialect name, or ``other`` for an unrecognised type.""" + ds_type = (datasource.type or "").lower() + found = dialect_for_ds_type(ds_type) + return found.sqlglot_name if ds_type in found.ds_type_aliases else OTHER + + +def is_demo(datasource: DatasourceConfig) -> bool: + """The bundled Jaffle Shop demo database.""" + return datasource.type == "duckdb" and Path(datasource.database or "").parts[-2:] == _DEMO_DB diff --git a/slayer/usage_reporting/payload.py b/slayer/usage_reporting/payload.py new file mode 100644 index 00000000..cc11951a --- /dev/null +++ b/slayer/usage_reporting/payload.py @@ -0,0 +1,210 @@ +"""The usage report: every leaf is SLayer vocabulary (enum token, bool, bucket, date, UUID, count).""" + +import datetime +import uuid +from typing import Annotated, Literal, get_args + +from pydantic import BaseModel, ConfigDict, Field + +SCHEMA_VERSION = 1 +VERSION_PATTERN = r"^\d+(\.\d+)*((a|b|rc)\d+)?(\.post\d+)?(\.dev\d+)?(\+[0-9A-Za-z]+(\.[0-9A-Za-z]+)*)?$" + +Count = Annotated[int, Field(ge=0)] +Version = Annotated[str, Field(pattern=VERSION_PATTERN, max_length=64)] + +UsageKey = Literal[ + # CLI leaf commands + "cli:serve", "cli:flight-serve", "cli:pg-serve", "cli:mcp", "cli:query", "cli:ingest", + "cli:validate-models", "cli:recommend-root-model", + "cli:import-dbt", "cli:import-cube", "cli:import-osi", + "cli:models.list", "cli:models.show", "cli:models.create", "cli:models.delete", + "cli:datasources.list", "cli:datasources.show", "cli:datasources.create", + "cli:datasources.delete", "cli:datasources.test", + "cli:memory.save", "cli:memory.forget", "cli:storage.migrate-types", + "cli:inspect", "cli:search", "cli:search.refresh-samples", + "cli:other", + # MCP tools + "mcp:create_datasource", "mcp:create_model", "mcp:delete_datasource", "mcp:delete_model", + "mcp:describe_datasource", "mcp:edit_datasource", "mcp:edit_model", "mcp:forget_memory", + "mcp:get_datasource_priority", "mcp:ingest_datasource_models", "mcp:inspect", + "mcp:inspect_model", "mcp:list_datasources", "mcp:models_summary", "mcp:query", + "mcp:recommend_root_model", "mcp:save_memory", "mcp:search", "mcp:set_datasource_priority", + "mcp:validate_models", "mcp:other", + # REST route templates + "rest:/datasources", "rest:/datasources/priority", "rest:/datasources/{name}", "rest:/health", + "rest:/ingest", "rest:/inspect", "rest:/memories", "rest:/memories/{memory_id}", "rest:/models", + "rest:/models/{name}", "rest:/query", "rest:/recommend-root-model", "rest:/search", + "rest:/validate-models", "rest:other", + # wire protocols + "flight:query", "flight:probe", "pg:query", "pg:probe", +] + +ErrorToken = Literal[ + "other", "exit", "interrupted", "cancelled", "timeout", "connection", "file_not_found", + "permission", "os_error", "import_error", "not_implemented", "key_error", "type_error", + "value_error", "json_error", "validation_error", "db_operational", "db_programming", + "db_error", "slayer_error", "query_type", "ambiguous_model", "entity_resolution", + "memory_not_found", "schema_drift", "unknown_reference", "population_inference", + "missing_driver", "translation", "tool_error", "http_4xx", "http_5xx", +] + +FlagToken = Literal[ + "multistage", "inline_source", "time_dimensions", "filters", + "computed_dimensions", "saved_query", "to_many_handling", +] + +TransformToken = Literal[ + "cumsum", "change", "change_pct", "time_shift", "first", "last", "lag", "lead", + "consecutive_periods", "rank", "percent_rank", "dense_rank", "ntile", "custom", +] + +AggregationToken = Literal[ + "sum", "avg", "min", "max", "count", "count_distinct", "count_distinct_approx", + "first", "last", "weighted_avg", "median", "percentile", "stddev_samp", "stddev_pop", + "var_samp", "var_pop", "corr", "covar_samp", "covar_pop", "custom", +] + +DialectToken = Literal[ + "sqlite", "postgres", "duckdb", "mysql", "clickhouse", "tsql", "snowflake", "bigquery", + "redshift", "trino", "presto", "databricks", "spark", "oracle", "other", +] + +DatasourceBucket = Literal["0", "1", "2-5", "6+"] +ModelCountBucket = Literal["0", "1-10", "11-50", "51-200", "200+"] +StorageToken = Literal["yaml", "sqlite", "other"] + +ClientToken = Literal[ + "claude-code", "claude-desktop", "cursor", "vscode", "windsurf", "codex", "zed", "cline", + "goose", "continue", "other", +] + +OsToken = Literal["linux", "darwin", "windows", "other"] +ArchToken = Literal["x86_64", "arm64", "other"] +ExtraToken = Literal[ + "client", "postgres", "mysql", "clickhouse", "sqlserver", "snowflake", "bigquery", "dbt", + "flight", "advanced-search", "trino", +] + +USAGE_KEYS: frozenset[str] = frozenset(get_args(UsageKey)) +FLAGS: frozenset[str] = frozenset(get_args(FlagToken)) +TRANSFORMS: frozenset[str] = frozenset(get_args(TransformToken)) +AGGREGATIONS: frozenset[str] = frozenset(get_args(AggregationToken)) +DIALECTS: frozenset[str] = frozenset(get_args(DialectToken)) +CLIENTS: frozenset[str] = frozenset(get_args(ClientToken)) +EXTRAS: tuple[ExtraToken, ...] = get_args(ExtraToken) + + +class _Strict(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class Period(_Strict): + first: datetime.date + last: datetime.date + + +class UsageEntry(_Strict): + ok: Count = 0 + errors: dict[ErrorToken, Count] = Field(default_factory=dict) + + +class Features(_Strict): + flags: dict[FlagToken, Count] = Field(default_factory=dict) + transforms: dict[TransformToken, Count] = Field(default_factory=dict) + aggregations: dict[AggregationToken, Count] = Field(default_factory=dict) + + +class Dialects(_Strict): + queries: dict[DialectToken, Count] = Field(default_factory=dict) + datasources: dict[DialectToken, dict[DatasourceBucket, Count]] = Field(default_factory=dict) + + +class Context(_Strict): + """Histograms over processes.""" + + storage: dict[StorageToken, Count] = Field(default_factory=dict) + model_count: dict[ModelCountBucket, Count] = Field(default_factory=dict) + demo: Count = 0 + + +class McpClientCount(_Strict): + client: ClientToken + major: Annotated[int, Field(ge=0, le=999)] | None = None + sessions: Count = 0 + + +class Batch(_Strict): + """Counters of one or more processes; the content of a spool file.""" + + period: Period | None = None + usage: dict[UsageKey, UsageEntry] = Field(default_factory=dict) + features: Features = Field(default_factory=Features) + dialects: Dialects = Field(default_factory=Dialects) + context: Context = Field(default_factory=Context) + mcp_clients: list[McpClientCount] = Field(default_factory=list) + + +class Env(_Strict): + slayer_version: Version + python: Version + os: OsToken + arch: ArchToken + in_container: bool + extras: list[ExtraToken] + + +class UsageReport(Batch): + """One send: a merged batch plus identity and environment.""" + + model_config = ConfigDict(extra="forbid", populate_by_name=True, serialize_by_alias=True) + + schema_version: Literal[1] = Field(default=SCHEMA_VERSION, alias="schema") + install_id: uuid.UUID | None + batch_id: uuid.UUID + env: Env + + +def _add(target: dict, source: dict) -> None: + for key, value in source.items(): + target[key] = target.get(key, 0) + value + + +def merge(batches: list[Batch]) -> Batch: + """Sum the counters of ``batches`` into one.""" + usage: dict = {} + features = Features() + dialects = Dialects() + context = Context() + clients: dict[tuple, int] = {} + firsts, lasts = [], [] + for batch in batches: + if batch.period is not None: + firsts.append(batch.period.first) + lasts.append(batch.period.last) + for key, entry in batch.usage.items(): + merged = usage.setdefault(key, UsageEntry()) + merged.ok += entry.ok + _add(merged.errors, entry.errors) + _add(features.flags, batch.features.flags) + _add(features.transforms, batch.features.transforms) + _add(features.aggregations, batch.features.aggregations) + _add(dialects.queries, batch.dialects.queries) + for dialect, buckets in batch.dialects.datasources.items(): + _add(dialects.datasources.setdefault(dialect, {}), buckets) + _add(context.storage, batch.context.storage) + _add(context.model_count, batch.context.model_count) + context.demo += batch.context.demo + for client in batch.mcp_clients: + key = (client.client, client.major) + clients[key] = clients.get(key, 0) + client.sessions + return Batch( + period=Period(first=min(firsts), last=max(lasts)) if firsts else None, + usage=usage, + features=features, + dialects=dialects, + context=context, + mcp_clients=[ + McpClientCount(client=client, major=major, sessions=sessions) + for (client, major), sessions in sorted(clients.items(), key=lambda kv: (kv[0][0], kv[0][1] or 0)) + ], + ) diff --git a/slayer/usage_reporting/recorder.py b/slayer/usage_reporting/recorder.py new file mode 100644 index 00000000..bb7da86b --- /dev/null +++ b/slayer/usage_reporting/recorder.py @@ -0,0 +1,427 @@ +"""The per-process recorder: a no-op until ``start()``, and silent on every failure.""" + +import atexit +import datetime +import functools +import logging +import os +import sys +import threading +from collections import Counter +from collections.abc import Callable, Generator +from contextlib import contextmanager +from typing import Any, ParamSpec, TypeVar + +from slayer.usage_reporting import clock, features, sender, settings, spool +from slayer.usage_reporting.payload import ( + AGGREGATIONS, + CLIENTS, + DIALECTS, + FLAGS, + SCHEMA_VERSION, + TRANSFORMS, + USAGE_KEYS, + Batch, +) + +logger = logging.getLogger(__name__) + +EXIT_WAIT = 1.0 +NOTICE = ( + "SLayer collects anonymous, aggregated usage statistics (command, tool and query-feature counts, " + "versions, OS) to guide its development - never queries, names, data or credentials. " + "Details: https://docs.motley.ai/slayer/reference/usage-reporting/ - " + "opt out with SLAYER_USAGE_REPORTING=off, DO_NOT_TRACK=1 or `slayer usage-reporting disable`." +) + +_ERROR_TOKENS = { + "builtins.SystemExit": "exit", + "builtins.KeyboardInterrupt": "interrupted", + "asyncio.exceptions.CancelledError": "cancelled", + "builtins.TimeoutError": "timeout", + "builtins.ConnectionError": "connection", + "builtins.FileNotFoundError": "file_not_found", + "builtins.PermissionError": "permission", + "builtins.OSError": "os_error", + "builtins.ImportError": "import_error", + "builtins.NotImplementedError": "not_implemented", + "builtins.KeyError": "key_error", + "builtins.TypeError": "type_error", + "builtins.ValueError": "value_error", + "json.decoder.JSONDecodeError": "json_error", + "pydantic_core._pydantic_core.ValidationError": "validation_error", + "sqlalchemy.exc.OperationalError": "db_operational", + "sqlalchemy.exc.ProgrammingError": "db_programming", + "sqlalchemy.exc.SQLAlchemyError": "db_error", + "slayer.core.errors.SlayerError": "slayer_error", + "slayer.core.errors.QueryTypeError": "query_type", + "slayer.core.errors.AmbiguousModelError": "ambiguous_model", + "slayer.core.errors.EntityResolutionError": "entity_resolution", + "slayer.core.errors.MemoryNotFoundError": "memory_not_found", + "slayer.core.errors.SchemaDriftError": "schema_drift", + "slayer.core.errors.UnknownReferenceError": "unknown_reference", + "slayer.core.errors.PopulationInferenceError": "population_inference", + "slayer.core.errors.MissingDriverError": "missing_driver", + "slayer.facade.translator.TranslationError": "translation", + "mcp.server.fastmcp.exceptions.ToolError": "tool_error", +} +_STORAGE_TOKENS = { + "slayer.storage.yaml_storage.YAMLStorage": "yaml", + "slayer.storage.sqlite_storage.SQLiteStorage": "sqlite", +} +_CLIENT_NAMES = { + "claude-code": "claude-code", + "claude-ai": "claude-desktop", + "claude-desktop": "claude-desktop", + "cursor": "cursor", + "cursor-vscode": "cursor", + "visual studio code": "vscode", + "visual-studio-code": "vscode", + "vscode": "vscode", + "windsurf": "windsurf", + "windsurf-client": "windsurf", + "codex": "codex", + "codex-mcp-client": "codex", + "zed": "zed", + "cline": "cline", + "goose": "goose", + "continue": "continue", + "continue-client": "continue", +} +_DATASOURCE_BUCKETS = ((0, "0"), (1, "1"), (5, "2-5")) +_MODEL_BUCKETS = ((0, "0"), (10, "1-10"), (50, "11-50"), (200, "51-200")) + +P = ParamSpec("P") +R = TypeVar("R") + + +def _silent(fn: Callable[P, R]) -> Callable[P, R | None]: + @functools.wraps(fn) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R | None: + try: + return fn(*args, **kwargs) + except Exception: + logger.debug("usage reporting: %s failed", fn.__name__, exc_info=True) + return None + + return wrapper + + +def _class_token(cls: type, table: dict[str, str]) -> str | None: + for base in cls.__mro__: + token = table.get(f"{base.__module__}.{base.__qualname__}") + if token is not None: + return token + return None + + +class HttpError(Exception): + """An HTTP error response, counted by its status class.""" + + def __init__(self, status: int) -> None: + super().__init__(status) + self.status = status + + +def error_token(error: BaseException) -> str: + if isinstance(error, HttpError): + return "http_5xx" if error.status >= 500 else "http_4xx" + return _class_token(type(error), _ERROR_TOKENS) or "other" + + +def _bucket(n: int, buckets: tuple[tuple[int, str], ...], top: str) -> str: + return next((label for bound, label in buckets if n <= bound), top) + + +class _Recorder: + """This process's counters; ``paths`` is ``None`` while inactive.""" + + def __init__(self) -> None: + self.lock = threading.RLock() + self.paths: sender.Paths | None = None + self.last_sent: datetime.datetime | None = None + self.sending: threading.Thread | None = None + self.clear() + + def clear(self) -> None: + self.ok: Counter[str] = Counter() + self.errors: dict[str, Counter[str]] = {} + self.flags: Counter[str] = Counter() + self.transforms: Counter[str] = Counter() + self.aggregations: Counter[str] = Counter() + self.queries: Counter[str] = Counter() + self.datasources: dict[str, str] = {} + self.storage: str | None = None + self.model_count: str | None = None + self.demo = False + self.clients: Counter[tuple[str, int | None]] = Counter() + self.first: datetime.date | None = None + self.last: datetime.date | None = None + + def touch(self) -> None: + today = clock.now().date() + self.first = min(self.first or today, today) + self.last = max(self.last or today, today) + + def take(self) -> Batch | None: + """The live counters as a batch (``None`` if empty), clearing them.""" + if self.first is None or self.last is None: + return None + batch = Batch.model_validate({ + "period": {"first": self.first, "last": self.last}, + "usage": { + key: {"ok": self.ok[key], "errors": dict(self.errors.get(key, {}))} + for key in self.ok.keys() | self.errors.keys() + }, + "features": { + "flags": dict(self.flags), + "transforms": dict(self.transforms), + "aggregations": dict(self.aggregations), + }, + "dialects": { + "queries": dict(self.queries), + "datasources": {d: {bucket: 1} for d, bucket in self.datasources.items()}, + }, + "context": { + "storage": {self.storage: 1} if self.storage else {}, + "model_count": {self.model_count: 1} if self.model_count else {}, + "demo": int(self.demo), + }, + "mcp_clients": [ + {"client": c, "major": m, "sessions": n} for (c, m), n in self.clients.items() + ], + }) + self.clear() + return batch + + +_recorder = _Recorder() + + +def _after_fork_in_child() -> None: + _recorder.lock = threading.RLock() + _recorder.sending = None + _recorder.clear() + + +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=_after_fork_in_child) + + +def is_active() -> bool: + return _recorder.paths is not None + + +def _send_in_background(paths: sender.Paths, now: datetime.datetime) -> None: + thread = threading.Thread( + target=sender.send, args=(paths,), kwargs={"now": now, "url": sender.endpoint()}, + name="slayer-usage-reporting-send", daemon=True, + ) + _recorder.sending = thread + thread.start() + + +@functools.cache +def _register_exit_hook() -> None: + atexit.register(_at_exit) + + +def _at_exit() -> None: + flush() + shutdown() + + +@_silent +def start() -> None: + """Activate recording in this process (called by the ``slayer`` CLI only).""" + if _recorder.paths is not None or not settings.resolve_state().enabled: + return + now = clock.now() + paths = sender.Paths(config=settings.config_path(), spool=settings.spool_dir(), lease=settings.lease_path()) + + def prepare(config: settings.UsageReportingConfig) -> settings.UsageReportingConfig: + if not settings.resolve_state(config).enabled: # disabled meanwhile by another process + return config + config = settings.with_install_id(config, today=now.date()) + if config.last_sent is None: + config = config.model_copy(update={"last_sent": now}) + return config.model_copy(update={"notice_shown_schema": SCHEMA_VERSION}) + + before, config = settings.update_config(prepare, paths.config) + if config.install_id is None: + return + if before.notice_shown_schema != SCHEMA_VERSION: + print(NOTICE, file=sys.stderr) + with _recorder.lock: + _recorder.paths = paths + _recorder.last_sent = config.last_sent + _recorder.clear() + _register_exit_hook() + if sender.is_due(config.last_sent, now=now): + with _recorder.lock: + _recorder.last_sent = now + _send_in_background(paths, now) + + +def _maybe_send() -> None: + """Send from a running process once 24 h have passed: spool the live counters, then send.""" + now = clock.now() + with _recorder.lock: + paths = _recorder.paths + in_flight = _recorder.sending is not None and _recorder.sending.is_alive() + if paths is None or in_flight or not sender.is_due(_recorder.last_sent, now=now): + return + _recorder.last_sent = now + batch = _recorder.take() + if batch is not None: + spool.write(paths.spool, batch) + _send_in_background(paths, now) + + +def _usage_key(surface: object, token: object) -> str | None: + if not isinstance(surface, str): + return None + key = f"{surface}:{token}" if isinstance(token, str) else "" + if key in USAGE_KEYS: + return key + other = f"{surface}:other" + return other if other in USAGE_KEYS else None + + +def _succeeded(error: BaseException) -> bool: + return isinstance(error, SystemExit) and error.code in (None, 0) + + +@_silent +def record(*, surface: str, token: str | None, error: BaseException | None = None) -> None: + """Count one action on ``surface``, failed if ``error`` is given.""" + if _recorder.paths is None: + return + key = _usage_key(surface, token) + if key is None: + return + with _recorder.lock: + if error is None or _succeeded(error): + _recorder.ok[key] += 1 + else: + _recorder.errors.setdefault(key, Counter())[error_token(error)] += 1 + _recorder.touch() + _maybe_send() + + +@contextmanager +def counting(*, surface: str, token: str | None) -> Generator[None]: + """Count the enclosed action as ok, or as failed with the exception it raises.""" + error: BaseException | None = None + try: + yield + except BaseException as exc: + error = exc + raise + finally: + record(surface=surface, token=token, error=error) + + +@_silent +def observe_query(query: Any) -> None: + """Count the features of a query about to execute.""" + if _recorder.paths is None: + return + found = features.query_features(query) + with _recorder.lock: + _recorder.flags.update(f for f in found.flags if f in FLAGS) + _recorder.transforms.update(n if n in TRANSFORMS else features.CUSTOM for n in found.transforms) + _recorder.aggregations.update(n if n in AGGREGATIONS else features.CUSTOM for n in found.aggregations) + _recorder.touch() + + +def _dialect(datasource: Any) -> str: + name = features.dialect(datasource) + return name if name in DIALECTS else features.OTHER + + +@_silent +def observe_executed(response: Any) -> None: + """Count a query that ran, by the datasource on its engine response.""" + datasource = getattr(response, "datasource", None) + if _recorder.paths is None or datasource is None: + return + dialect, demo = _dialect(datasource), features.is_demo(datasource) + with _recorder.lock: + _recorder.queries[dialect] += 1 + _recorder.demo = _recorder.demo or demo + _recorder.touch() + + +@_silent +def observe_storage(storage: object) -> None: + if _recorder.paths is None: + return + with _recorder.lock: + _recorder.storage = _class_token(type(storage), _STORAGE_TOKENS) or "other" + _recorder.touch() + + +@_silent +def observe_model_count(count: int) -> None: + if _recorder.paths is None: + return + with _recorder.lock: + _recorder.model_count = _bucket(count, _MODEL_BUCKETS, "200+") + _recorder.touch() + + +@_silent +def observe_datasources(datasources: list[Any]) -> None: + """Bucket the loaded datasources per dialect.""" + if _recorder.paths is None: + return + per_dialect = Counter(_dialect(ds) for ds in datasources) + with _recorder.lock: + _recorder.datasources = {d: _bucket(n, _DATASOURCE_BUCKETS, "6+") for d, n in per_dialect.items()} + _recorder.touch() + + +def _major(version: object) -> int | None: + head = version.split(".", 1)[0].strip() if isinstance(version, str) else "" + return int(head) if head.isdigit() and head.isascii() and int(head) <= 999 else None + + +@_silent +def observe_mcp_client(*, name: object, version: object) -> None: + """Count one MCP session of a client, by fixed client token and major version.""" + if _recorder.paths is None: + return + client = _CLIENT_NAMES.get(name.strip().lower(), "other") if isinstance(name, str) else "other" + if client not in CLIENTS: + client = "other" + with _recorder.lock: + _recorder.clients[(client, _major(version) if client != "other" else None)] += 1 + _recorder.touch() + + +@_silent +def flush() -> None: + """Write this process's counters to its own spool file (no network).""" + with _recorder.lock: + paths = _recorder.paths + batch = _recorder.take() if paths is not None else None + if paths is not None and batch is not None: + spool.write(paths.spool, batch) + + +@_silent +def shutdown() -> None: + """Wait (bounded) for an in-flight send.""" + thread = _recorder.sending + if thread is not None and thread.is_alive(): + thread.join(EXIT_WAIT) + + +def reset() -> None: + """Deactivate and forget all live counters.""" + with _recorder.lock: + _recorder.paths = None + _recorder.last_sent = None + _recorder.sending = None + _recorder.clear() diff --git a/slayer/usage_reporting/sender.py b/slayer/usage_reporting/sender.py new file mode 100644 index 00000000..dcfd082d --- /dev/null +++ b/slayer/usage_reporting/sender.py @@ -0,0 +1,97 @@ +"""Building a report from the spool and posting it to PostHog.""" + +import datetime +import json +import logging +import os +import urllib.request +import uuid +from pathlib import Path + +from pydantic import BaseModel + +from slayer.usage_reporting import environment, spool +from slayer.usage_reporting.payload import Batch, UsageReport, merge +from slayer.usage_reporting.settings import UsageReportingConfig, update_config, with_install_id + +logger = logging.getLogger(__name__) + +DEFAULT_ENDPOINT = "https://eu.i.posthog.com/i/v0/e/" +# PostHog EU project token: write-only, meant to ship in clients. +API_KEY = "phc_ttaNssx8sGSD9dY5TYXqnnDuEkxp7fgpZvspyKpSY2EK" # NOSONAR(S6418) — public write-only PostHog project token, meant to ship in clients +EVENT = "slayer_usage" +HTTP_TIMEOUT = 10.0 +SEND_INTERVAL = datetime.timedelta(hours=24) + + +class Paths(BaseModel): + """Usage-reporting file locations, fixed when a process starts.""" + + config: Path + spool: Path + lease: Path + + +def endpoint() -> str: + return os.environ.get("SLAYER_USAGE_REPORTING_ENDPOINT") or DEFAULT_ENDPOINT + + +def is_due(last_sent: datetime.datetime | None, *, now: datetime.datetime) -> bool: + """24 h since the last successful send; a last send in the future counts as due.""" + return last_sent is not None and (last_sent > now or now - last_sent >= SEND_INTERVAL) + + +def build_report(*, config: UsageReportingConfig, batches: list[Batch]) -> UsageReport: + merged = merge(batches) + return UsageReport( + install_id=config.install_id, batch_id=uuid.uuid4(), env=environment.current(), + **{name: getattr(merged, name) for name in Batch.model_fields}, + ) + + +def _post(report: UsageReport, *, url: str) -> bool: + properties = report.model_dump(mode="json") + properties["$geoip_disable"] = True + body = { + "api_key": API_KEY, + "event": EVENT, + "distinct_id": properties["install_id"], + "uuid": properties["batch_id"], + "properties": properties, + } + request = urllib.request.Request( + url, data=json.dumps(body).encode(), method="POST", + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT) as response: # NOSONAR(S5332) — the endpoint is https unless overridden + return 200 <= response.status < 300 + + +def send(paths: Paths, *, now: datetime.datetime, url: str) -> None: + """Send every spooled batch once, if still due and no other process is sending.""" + try: + token = spool.acquire_lease(paths.lease, now=now) + if token is None: + return + try: + _send_leased(paths, now=now, url=url) + finally: + spool.release_lease(paths.lease, token) + except Exception: + logger.debug("usage reporting: send failed", exc_info=True) + + +def _send_leased(paths: Paths, *, now: datetime.datetime, url: str) -> None: + _, config = update_config( + lambda stored: stored if stored.install_id is None else with_install_id(stored, today=now.date()), paths.config, + ) + if config.install_id is None or not is_due(config.last_sent, now=now): + return + claimed = spool.claim(paths.spool) + batches = spool.load(claimed) + if batches and not _post(build_report(config=config, batches=batches), url=url): + return + for path in claimed: + path.unlink(missing_ok=True) + if batches: + update_config(lambda stored: stored.model_copy(update={"last_sent": now}), paths.config) diff --git a/slayer/usage_reporting/settings.py b/slayer/usage_reporting/settings.py new file mode 100644 index 00000000..51d4a5c0 --- /dev/null +++ b/slayer/usage_reporting/settings.py @@ -0,0 +1,186 @@ +"""Where usage-reporting state lives and whether usage reporting is on.""" + +import calendar +import datetime +import json +import os +import sys +import uuid +from collections.abc import Callable, Generator +from contextlib import contextmanager +from enum import StrEnum +from importlib.metadata import Distribution, distribution +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel + +if sys.platform == "win32": + import msvcrt +else: + import fcntl + +DIST_NAME = "motley-slayer" +_TRUTHY = frozenset({"1", "true", "yes", "on"}) +_FALSY = frozenset({"", "0", "false", "no", "off"}) +_ID_LIFETIME_MONTHS = 11 + + +class DecidingRule(StrEnum): + DO_NOT_TRACK = "DO_NOT_TRACK" + ENV = "SLAYER_USAGE_REPORTING" + CI = "CI" + EDITABLE = "editable install" + UNWRITABLE = "unwritable config directory" + PERSISTED = "slayer usage-reporting enable/disable" + DEFAULT = "default" + + +class UsageReportingState(BaseModel): + enabled: bool + rule: DecidingRule + + +class UsageReportingConfig(BaseModel): + setting: Literal["enabled", "disabled"] | None = None + install_id: uuid.UUID | None = None + install_id_created: datetime.date | None = None + notice_shown_schema: int | None = None + last_sent: datetime.datetime | None = None + + +def config_dir() -> Path: + if sys.platform == "win32": + appdata = os.environ.get("APPDATA") + return (Path(appdata) if appdata else Path.home() / "AppData" / "Roaming") / "slayer" + if sys.platform == "darwin": + return Path.home() / "Library" / "Application Support" / "slayer" + xdg = os.environ.get("XDG_CONFIG_HOME") + return (Path(xdg) if xdg else Path.home() / ".config") / "slayer" + + +def config_path() -> Path: + return config_dir() / "usage-reporting.json" + + +def spool_dir() -> Path: + return config_dir() / "usage-reporting-spool" + + +def lease_path() -> Path: + return config_dir() / "usage-reporting-send.lease" + + +def is_editable_install(dist: Distribution | None = None) -> bool: + """PEP 610: SLayer was installed with ``pip install -e`` (or ``poetry install``).""" + try: + text = (dist or distribution(DIST_NAME)).read_text("direct_url.json") + return bool(text) and json.loads(text).get("dir_info", {}).get("editable") is True + except Exception: + return False + + +def _writable(directory: Path) -> bool: + try: + directory.mkdir(parents=True, exist_ok=True) + return directory.is_dir() and os.access(directory, os.W_OK) + except OSError: + return False + + +def _set(value: str | None) -> bool: + return value is not None and value.strip().lower() not in _FALSY + + +def resolve_state(config: UsageReportingConfig | None = None) -> UsageReportingState: + """The effective state and the first rule (highest precedence first) that decided it.""" + if (os.environ.get("DO_NOT_TRACK") or "").strip().lower() in _TRUTHY: + return UsageReportingState(enabled=False, rule=DecidingRule.DO_NOT_TRACK) + explicit = (os.environ.get("SLAYER_USAGE_REPORTING") or "").strip().lower() + if explicit in ("on", "off"): + return UsageReportingState(enabled=explicit == "on", rule=DecidingRule.ENV) + if _set(os.environ.get("CI")): + return UsageReportingState(enabled=False, rule=DecidingRule.CI) + if is_editable_install(): + return UsageReportingState(enabled=False, rule=DecidingRule.EDITABLE) + if not _writable(config_dir()): + return UsageReportingState(enabled=False, rule=DecidingRule.UNWRITABLE) + setting = (config or read_config()).setting + if setting is not None: + return UsageReportingState(enabled=setting == "enabled", rule=DecidingRule.PERSISTED) + return UsageReportingState(enabled=True, rule=DecidingRule.DEFAULT) + + +def read_config(path: Path | None = None) -> UsageReportingConfig: + """The stored config; missing or corrupt reads as empty.""" + try: + config = UsageReportingConfig.model_validate_json((path or config_path()).read_bytes()) + except Exception: + return UsageReportingConfig() + if config.last_sent is not None and config.last_sent.tzinfo is None: + config.last_sent = config.last_sent.replace(tzinfo=datetime.timezone.utc) + return config + + +def update_config( + change: Callable[[UsageReportingConfig], UsageReportingConfig], path: Path | None = None, +) -> tuple[UsageReportingConfig, UsageReportingConfig]: + """Apply ``change`` to the stored config under a cross-process lock; returns (before, after).""" + target = path or config_path() + with exclusive_lock(target.with_name(target.name + ".lock"), wait=True): + before = read_config(target) + after = change(before) + if after != before: + write_private(target, after.model_dump_json().encode()) + return before, after + + +@contextmanager +def exclusive_lock(path: Path, *, wait: bool) -> Generator[bool]: + """Hold an OS lock on ``path`` (freed on close or process death); without ``wait``, yields ``False`` if held.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600) + try: + try: + if sys.platform == "win32": + msvcrt.locking(fd, msvcrt.LK_LOCK if wait else msvcrt.LK_NBLCK, 1) + else: + fcntl.flock(fd, fcntl.LOCK_EX if wait else fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + if wait: + raise + yield False + else: + yield True + finally: + os.close(fd) + + +def write_private(path: Path, data: bytes) -> None: + """Atomically replace ``path`` with ``data``, readable by the owner only.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".tmp-{uuid.uuid4().hex}") + fd = os.open(tmp, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + try: + with os.fdopen(fd, "wb") as fh: + fh.write(data) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +def _add_months(day: datetime.date, months: int) -> datetime.date: + year, month = divmod(day.month - 1 + months, 12) + year, month = day.year + year, month + 1 + return day.replace(year=year, month=month, day=min(day.day, calendar.monthrange(year, month)[1])) + + +def with_install_id(config: UsageReportingConfig, *, today: datetime.date) -> UsageReportingConfig: + """``config`` with a current install ID: created on first use, replaced once 11 months old.""" + created = config.install_id_created + if config.install_id is not None and created is None: + return config.model_copy(update={"install_id_created": today}) + if config.install_id is not None and created is not None and _add_months(created, _ID_LIFETIME_MONTHS) > today: + return config + return config.model_copy(update={"install_id": uuid.uuid4(), "install_id_created": today}) diff --git a/slayer/usage_reporting/spool.py b/slayer/usage_reporting/spool.py new file mode 100644 index 00000000..35dacae4 --- /dev/null +++ b/slayer/usage_reporting/spool.py @@ -0,0 +1,128 @@ +"""Local spool of unsent batches, and the lease that lets one process send at a time.""" + +import datetime +import json +import logging +import os +import stat +import time +import uuid +from contextlib import AbstractContextManager +from pathlib import Path + +from slayer.usage_reporting.payload import Batch +from slayer.usage_reporting.settings import exclusive_lock, write_private + +logger = logging.getLogger(__name__) + +_PENDING = "spool-" +_CLAIMED = "claim-" +_SUFFIX = ".json" +MAX_FILES = 100 +MAX_BYTES = 1_000_000 +LEASE_TTL = datetime.timedelta(minutes=5) + + +def _regular(path: Path) -> int | None: + """Size of ``path`` if it is a regular file (never following a link), else ``None``.""" + try: + st = path.lstat() + except OSError: + return None + return st.st_size if stat.S_ISREG(st.st_mode) else None + + +def _entries(directory: Path, prefix: str) -> list[Path]: + try: + names = os.listdir(directory) + except OSError: + return [] + return [directory / n for n in names if n.startswith(prefix) and n.endswith(_SUFFIX)] + + +def _age_key(path: Path) -> str: + return path.name.removeprefix(_PENDING).removeprefix(_CLAIMED) + + +def write(directory: Path, batch: Batch) -> None: + """Add ``batch`` as a new spool file, then enforce the cap.""" + name = f"{_PENDING}{time.time_ns():020d}-{uuid.uuid4().hex[:12]}{_SUFFIX}" + write_private(directory / name, batch.model_dump_json().encode()) + _enforce_cap(directory) + + +def _enforce_cap(directory: Path) -> None: + files = [(p, size) for p in pending_files(directory) if (size := _regular(p)) is not None] + files.sort(key=lambda item: _age_key(item[0])) + total = sum(size for _, size in files) + while files and (len(files) > MAX_FILES or total > MAX_BYTES): + oldest, size = files.pop(0) + oldest.unlink(missing_ok=True) + total -= size + + +def pending_files(directory: Path) -> list[Path]: + """Every regular spool file, claimed or not.""" + paths = _entries(directory, _PENDING) + _entries(directory, _CLAIMED) + return [p for p in paths if _regular(p) is not None] + + +def load(paths: list[Path]) -> list[Batch]: + batches = [] + for path in paths: + try: + batches.append(Batch.model_validate_json(path.read_bytes())) + except Exception: + logger.debug("usage reporting: skipping unreadable spool file", exc_info=True) + return batches + + +def claim(directory: Path) -> list[Path]: + """Mark every pending file as claimed; return all claimed files (including a dead sender's).""" + for path in _entries(directory, _PENDING): + if _regular(path) is not None: + try: + path.rename(directory / (_CLAIMED + path.name.removeprefix(_PENDING))) + except OSError: + continue + return [p for p in _entries(directory, _CLAIMED) if _regular(p) is not None] + + +def purge(directory: Path) -> None: + """Delete every spool file.""" + for path in pending_files(directory): + path.unlink(missing_ok=True) + + +def _guard(lease: Path) -> AbstractContextManager[bool]: + return exclusive_lock(lease.with_name(lease.name + ".lock"), wait=False) + + +def acquire_lease(path: Path, *, now: datetime.datetime) -> str | None: + """A token if this process may send now, else ``None`` (another sender's lease is live).""" + with _guard(path) as held: + if not held or not _lease_expired(path, now=now): + return None + token = uuid.uuid4().hex + write_private(path, json.dumps({"token": token, "expires": (now + LEASE_TTL).isoformat()}).encode()) + return token + + +def _lease_token(path: Path) -> str | None: + try: + return json.loads(path.read_bytes()).get("token") + except Exception: + return None + + +def _lease_expired(path: Path, *, now: datetime.datetime) -> bool: + try: + return datetime.datetime.fromisoformat(json.loads(path.read_bytes())["expires"]) <= now + except Exception: + return True + + +def release_lease(path: Path, token: str) -> None: + with _guard(path) as held: + if held and _lease_token(path) == token: + path.unlink(missing_ok=True) diff --git a/tests/_cli_inprocess.py b/tests/_cli_inprocess.py index 400416fb..10c93fed 100644 --- a/tests/_cli_inprocess.py +++ b/tests/_cli_inprocess.py @@ -14,6 +14,7 @@ import io import os import sys +import traceback from contextlib import redirect_stderr, redirect_stdout from pydantic import BaseModel @@ -28,13 +29,15 @@ class CliResult(BaseModel): def run_cli_in_process( - args: list[str], *, stdin: str = "", env: dict[str, str] | None = None + args: list[str], *, stdin: str = "", env: dict[str, str] | None = None, uncaught_exit: bool = False, ) -> CliResult: """Invoke ``slayer.cli.main()`` in-process and capture its result. ``sys.argv`` becomes ``["slayer", *args]``; ``sys.stdin`` is fed ``stdin``. A normal return maps to exit code 0; ``SystemExit`` maps to its code - (``None`` → 0, non-int → 1). argv/stdin/stdout/stderr/environ are restored. + (``None`` → 0, non-int → 1); with ``uncaught_exit`` any other exception + maps to 1 with its traceback on stderr, as in a real process. + argv/stdin/stdout/stderr/environ are restored. ``env`` replaces ``os.environ`` for the call (runtime-read vars only); it does not affect import-time constants like ``slayer.cli._STORAGE_DEFAULT`` — pass @@ -54,6 +57,11 @@ def run_cli_in_process( main() except SystemExit as exc: code = 0 if exc.code is None else exc.code if isinstance(exc.code, int) else 1 + except Exception: + if not uncaught_exit: + raise + code = 1 + err.write(traceback.format_exc()) finally: sys.argv, sys.stdin = old_argv, old_stdin os.environ.clear() diff --git a/tests/_usage_reporting_capture.py b/tests/_usage_reporting_capture.py new file mode 100644 index 00000000..5dc63648 --- /dev/null +++ b/tests/_usage_reporting_capture.py @@ -0,0 +1,108 @@ +"""Local HTTP server standing in for the usage-reporting endpoint (stdlib only).""" + +from __future__ import annotations + +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class CapturedRequest(BaseModel): + path: str + headers: dict[str, str] + body: bytes + + def json(self) -> Any: + return json.loads(self.body) + + +class CaptureBehaviour(BaseModel): + """Mutable response behaviour; ``stall_first`` delays only the first request.""" + + model_config = ConfigDict(validate_assignment=True) + + status: int = 200 + delay: float = 0.0 + stall_first: float = 0.0 + + +class CaptureServer: + """Records every POST; ``behaviour`` sets status and delays.""" + + def __init__(self) -> None: + self.behaviour = CaptureBehaviour() + self.requests: list[CapturedRequest] = [] + self.arrivals = 0 + self._lock = threading.Lock() + self._server = ThreadingHTTPServer(("127.0.0.1", 0), self._handler_class()) + self._server.daemon_threads = True + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + @property + def url(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host}:{port}/i/v0/e/" + + def start(self) -> "CaptureServer": + self._thread.start() + return self + + def stop(self) -> None: + self._server.shutdown() + self._server.server_close() + + def wait_for(self, *, count: int, timeout: float = 10.0) -> list[CapturedRequest]: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with self._lock: + if len(self.requests) >= count: + return list(self.requests) + time.sleep(0.02) + with self._lock: + return list(self.requests) + + def wait_for_arrival(self, *, timeout: float = 10.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with self._lock: + if self.arrivals: + return True + time.sleep(0.02) + return False + + def _handler_class(self) -> type[BaseHTTPRequestHandler]: + server = self + + class _Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 — http.server naming + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) + with server._lock: + server.arrivals += 1 + first = server.arrivals == 1 + delay = server.behaviour.delay + (server.behaviour.stall_first if first else 0.0) + if delay: + time.sleep(delay) + with server._lock: + server.requests.append(CapturedRequest( + path=self.path, headers=dict(self.headers.items()), body=body, + )) + self.send_response(server.behaviour.status) + self.send_header("Content-Length", "2") + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 — http.server signature + return None + + return _Handler + + +class RefusedEndpoint(BaseModel): + """A URL nothing listens on.""" + + url: str = Field(default="http://127.0.0.1:9/i/v0/e/") diff --git a/tests/_usage_reporting_helpers.py b/tests/_usage_reporting_helpers.py new file mode 100644 index 00000000..5e3ac1ac --- /dev/null +++ b/tests/_usage_reporting_helpers.py @@ -0,0 +1,338 @@ +"""Shared helpers for the usage-reporting tests.""" + +from __future__ import annotations + +import asyncio +import base64 +import datetime +import hashlib +import json +import os +import queue +import subprocess +import sys +import threading +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from slayer import usage_reporting +from slayer.core.models import Aggregation, Column, DatasourceConfig, SlayerModel +from slayer.core.enums import DataType +from slayer.storage.sqlite_conn import transaction +from slayer.storage.yaml_storage import YAMLStorage +from slayer.usage_reporting import settings +from slayer.usage_reporting.payload import SCHEMA_VERSION, UsageReport +from tests._cli_inprocess import CliResult, run_cli_in_process +from tests._saved_query_refinement_fixtures import build_refine_storage, orders_model + +SENT = "zqxjsentinel" +HOSTILE = "zqxj\u202e\u03a9\u0301\U0001f608'\";--\u200b" + "x" * 10_000 +_NEEDLES = (SENT, HOSTILE) + +SUBPROCESS_TIMEOUT = 120 + + +# --- time --------------------------------------------------------------------- + + +def now_utc() -> datetime.datetime: + return datetime.datetime.now(datetime.timezone.utc) + + +def wait_until(predicate: Callable[[], bool], *, timeout: float = 10.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.02) + return predicate() + + +# --- config + spool ------------------------------------------------------------- + + +def read_config() -> settings.UsageReportingConfig | None: + path = settings.config_path() + if not path.is_file(): + return None + return settings.UsageReportingConfig.model_validate_json(path.read_text()) + + +def config() -> settings.UsageReportingConfig: + current = read_config() + assert current is not None, "no usage-reporting config written" + return current + + +def write_config(**fields: Any) -> settings.UsageReportingConfig: + current = read_config() or settings.UsageReportingConfig() + config = current.model_copy(update=fields) + path = settings.config_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(config.model_dump_json()) + return config + + +def quiet_config(*, last_sent: datetime.datetime | None = None) -> settings.UsageReportingConfig: + """Notice already shown, last send just now (nothing due).""" + return write_config( + notice_shown_schema=SCHEMA_VERSION, + last_sent=last_sent or now_utc(), + ) + + +def make_due() -> settings.UsageReportingConfig: + return write_config(last_sent=now_utc() - datetime.timedelta(hours=25)) + + +def spool_files() -> list[Path]: + directory = settings.spool_dir() + if not directory.is_dir(): + return [] + return sorted(p for p in directory.iterdir() if p.is_file() and not p.is_symlink()) + + +def spool_bytes() -> bytes: + return b"\n".join(p.read_bytes() for p in spool_files()) + + +# --- reports ------------------------------------------------------------------------ + + +def usage(report: dict, key: str) -> dict: + return report.get("usage", {}).get(key, {"ok": 0, "errors": {}}) + + +def total(entry: dict) -> int: + return entry["ok"] + sum(entry["errors"].values()) + + +def usage_keys(report: dict, *, surface: str) -> set[str]: + return {k for k in report.get("usage", {}) if k.startswith(f"{surface}:")} + + +def _resolve(schema: dict, node: dict) -> dict: + while "$ref" in node: + node = schema["$defs"][node["$ref"].rsplit("/", 1)[-1]] + return node + + +def enum_values(schema: dict, node: dict) -> set[str]: + node = _resolve(schema, node) + if "const" in node: + return {node["const"]} + if "enum" in node: + return set(node["enum"]) + out: set[str] = set() + for branch in node.get("anyOf", []) + node.get("oneOf", []): + out |= enum_values(schema, branch) + return out + + +def usage_key_vocabulary() -> set[str]: + schema = UsageReport.model_json_schema() + usage_node = _resolve(schema, schema["properties"]["usage"]) + return enum_values(schema, usage_node["propertyNames"]) + + +# --- CLI ------------------------------------------------------------------------------ + + +def cli(args: list[str], *, stdin: str = "") -> CliResult: + """One in-process ``slayer`` run, ended like a process exit (flush, then reset).""" + try: + return run_cli_in_process(args, stdin=stdin, uncaught_exit=True) + finally: + usage_reporting.flush() + usage_reporting.reset() + + +def show() -> dict: + result = cli(["usage-reporting", "show"]) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +def end_process() -> None: + usage_reporting.flush() + usage_reporting.reset() + + +def run_slayer(args: list[str], *, env: dict[str, str], timeout: float = SUBPROCESS_TIMEOUT, + stdin: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "slayer", *args], + env=env, capture_output=True, text=True, timeout=timeout, input=stdin, + ) + + +def run_python(code: str, *, env: dict[str, str], timeout: float = SUBPROCESS_TIMEOUT) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-c", code], + env=env, capture_output=True, text=True, timeout=timeout, + ) + + +# --- sentinels -------------------------------------------------------------------------- + + +def _derivatives(needle: str) -> set[str]: + raw = needle.encode("utf-8") + out = {needle[:8], needle[:8].lower(), needle[:8].upper()} + for algo in ("md5", "sha1", "sha256"): + digest = hashlib.new(algo, raw).hexdigest() + out |= {digest[:12], digest[:12].upper()} + out.add(base64.b64encode(raw).decode()[:12]) + return out + + +def assert_no_sentinel(blob: bytes | str) -> None: + text = blob.decode("utf-8", errors="replace") if isinstance(blob, bytes) else blob + folded = text.casefold() + for needle in _NEEDLES: + for derived in _derivatives(needle): + assert derived.casefold() not in folded, f"sentinel derivative {derived!r} leaked" + assert "zqxj" not in folded + + +def assert_clean_everywhere() -> None: + """Spool, config and the pending report carry no sentinel.""" + assert_no_sentinel(spool_bytes()) + if settings.config_path().is_file(): + assert_no_sentinel(settings.config_path().read_bytes()) + assert_no_sentinel(json.dumps(show())) + + +# --- storage ------------------------------------------------------------------------------- + + +def build_storage(base_dir: Path) -> str: + """YAML storage over a seeded SQLite ``orders`` (+ custom ``dsum``) with saved queries; returns its path.""" + + async def build() -> YAMLStorage: + storage = await build_refine_storage(str(base_dir)) + model = orders_model().model_copy(update={ + "aggregations": [Aggregation(name="dsum", formula="SUM({value})")], + }) + await storage.save_model(model) + return storage + + asyncio.run(build()) + return str(base_dir / "store") + + +def build_sentinel_storage(base_dir: Path) -> str: + """A storage whose datasource, model, column, aggregation and description carry the sentinel.""" + db_path = base_dir / f"{SENT}.db" + + async def build() -> None: + storage = YAMLStorage(base_dir=str(base_dir / "store")) + await storage.save_datasource(DatasourceConfig( + name=f"{SENT}_ds", type="sqlite", database=str(db_path), description=HOSTILE, + )) + await storage.save_model(SlayerModel( + name=f"{SENT}_model", data_source=f"{SENT}_ds", sql_table=f"{SENT}_table", + description=HOSTILE, + columns=[ + Column(name="id", type=DataType.INT, primary_key=True), + Column(name=f"{SENT}_col", type=DataType.DOUBLE, description=HOSTILE), + ], + aggregations=[Aggregation(name=f"{SENT}_agg", formula="SUM({value})")], + )) + + with transaction(str(db_path)) as con: + con.execute(f"CREATE TABLE {SENT}_table (id INTEGER PRIMARY KEY, {SENT}_col REAL)") + con.execute(f"INSERT INTO {SENT}_table VALUES (1, 2.5)") + asyncio.run(build()) + return str(base_dir / "store") + + +def save_models(storage_dir: str, *, count: int) -> None: + """Add ``count`` trivial models over the ``orders`` table of datasource ``test``.""" + + async def build() -> None: + storage = YAMLStorage(base_dir=storage_dir) + for i in range(count): + await storage.save_model(SlayerModel( + name=f"extra_{i}", data_source="test", sql_table="orders", + columns=[Column(name="id", type=DataType.INT, primary_key=True)], + )) + + asyncio.run(build()) + + +# --- stdio MCP subprocess ------------------------------------------------------------------------ + + +class McpStdio: + """``slayer mcp`` over stdio, driven with raw newline-delimited JSON-RPC.""" + + def __init__(self, *, args: list[str], env: dict[str, str], argv: list[str] | None = None) -> None: + cmd = argv or [sys.executable, "-m", "slayer", "mcp", *args] + self.proc = subprocess.Popen( + cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + self.stdout_lines: list[bytes] = [] + self.stderr = bytearray() + self._lines: queue.Queue[bytes] = queue.Queue() + self._next_id = 1 + threading.Thread(target=self._pump_stdout, daemon=True).start() + threading.Thread(target=self._pump_stderr, daemon=True).start() + + def _pump_stdout(self) -> None: + assert self.proc.stdout is not None + for line in self.proc.stdout: + self.stdout_lines.append(line) + self._lines.put(line) + + def _pump_stderr(self) -> None: + assert self.proc.stderr is not None + for chunk in iter(lambda: self.proc.stderr.read(1024), b""): # type: ignore[union-attr] + self.stderr.extend(chunk) + + def send(self, message: dict) -> None: + assert self.proc.stdin is not None + self.proc.stdin.write((json.dumps(message) + "\n").encode()) + self.proc.stdin.flush() + + def request(self, method: str, params: dict, *, timeout: float = 60.0) -> dict: + request_id = self._next_id + self._next_id += 1 + self.send({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + assert remaining > 0, f"no MCP response to {method}; stderr: {bytes(self.stderr)!r}" + message = json.loads(self._lines.get(timeout=remaining)) + if message.get("id") == request_id: + return message + + def initialize(self, *, client_name: str = "claude-code", client_version: str = "2.3.1") -> dict: + response = self.request("initialize", { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": client_name, "version": client_version}, + }) + self.send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + return response + + def call_tool(self, name: str, arguments: dict) -> dict: + return self.request("tools/call", {"name": name, "arguments": arguments}) + + def close_stdin(self) -> None: + assert self.proc.stdin is not None + self.proc.stdin.close() + + def wait(self, *, timeout: float = 30.0) -> int: + return self.proc.wait(timeout=timeout) + + def kill(self) -> None: + if self.proc.poll() is None: + self.proc.kill() + self.proc.wait(timeout=10) + + +def is_posix() -> bool: + return os.name == "posix" diff --git a/tests/conftest.py b/tests/conftest.py index 84f62c53..beecd614 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,15 @@ """Shared test fixtures.""" +import datetime import gc import os +import sys import tempfile from collections.abc import AsyncIterator, Iterator +from pathlib import Path import pytest +from pydantic import BaseModel, ConfigDict from slayer.core.enums import DataType from slayer.core.models import Column, DatasourceConfig, SlayerModel @@ -15,6 +19,61 @@ from tests import _statement_render_law as statement_render_law from tests._dev1824_fixtures import make_exec_engine +from tests._usage_reporting_capture import CaptureServer + + +@pytest.fixture(scope="session", autouse=True) +def _usage_reporting_off_by_default() -> Iterator[None]: + """Our own test runs never report usage; ``usage_reporting_env`` re-enables it.""" + with pytest.MonkeyPatch.context() as mp: + mp.setenv("SLAYER_USAGE_REPORTING", "off") + yield + + +class UsageReportingEnv(BaseModel): + """Isolated usage-reporting environment: tmp config dirs, capture endpoint, injectable clock.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + root: Path + storage_dir: Path + capture: CaptureServer + monkeypatch: pytest.MonkeyPatch + + def set_now(self, now: datetime.datetime) -> None: + self.monkeypatch.setattr("slayer.usage_reporting.clock.now", lambda: now) + + def subprocess_env(self, **overrides: str | None) -> dict[str, str]: + env = dict(os.environ) + for key, value in overrides.items(): + if value is None: + env.pop(key, None) + else: + env[key] = value + return env + + +@pytest.fixture +def usage_reporting_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[UsageReportingEnv]: + """Usage reporting on, config under ``tmp_path``, reports posted to a local capture server.""" + capture = CaptureServer().start() + for var in ("DO_NOT_TRACK", "CI", "SLAYER_STORAGE", "SLAYER_MODELS_DIR"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("SLAYER_USAGE_REPORTING", "on") + monkeypatch.setenv("SLAYER_USAGE_REPORTING_ENDPOINT", capture.url) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("APPDATA", str(tmp_path / "appdata")) + storage_dir = tmp_path / "store" + storage_dir.mkdir() + env = UsageReportingEnv(root=tmp_path, storage_dir=storage_dir, capture=capture, monkeypatch=monkeypatch) + try: + yield env + finally: + usage_reporting = sys.modules.get("slayer.usage_reporting") + if usage_reporting is not None: + usage_reporting.reset() + capture.stop() @pytest.fixture(scope="session", autouse=True) diff --git a/tests/flight/test_handlers.py b/tests/flight/test_handlers.py index dc3c1ade..850fcb12 100644 --- a/tests/flight/test_handlers.py +++ b/tests/flight/test_handlers.py @@ -355,3 +355,9 @@ def test_do_get_for_dml_raises_translation_error_propagating() -> None: handlers = _make_handlers() with pytest.raises(TranslationError): handlers.do_get_for_sql("INSERT INTO orders VALUES (1)") + + +def test_do_get_for_untokenizable_sql_raises_translation_error() -> None: + handlers = _make_handlers() + with pytest.raises(TranslationError): + handlers.do_get_for_sql("SELECT revenue_sum FROM jaffle.orders WHERE status = 'x'\";--'") diff --git a/tests/pg_facade/test_connection.py b/tests/pg_facade/test_connection.py index 4aedd80f..41de7d8b 100644 --- a/tests/pg_facade/test_connection.py +++ b/tests/pg_facade/test_connection.py @@ -231,7 +231,7 @@ async def _run( return writer -def _messages(buf: bytes, *, leading_raw: int = 0) -> list[tuple[str, bytes]]: +def _messages(buf: bytes | bytearray, *, leading_raw: int = 0) -> list[tuple[str, bytes]]: return proto.split_messages(bytes(buf[leading_raw:])) @@ -2722,3 +2722,25 @@ async def test_describe_pins_execute_to_same_catalog_within_window() -> None: await conn._maybe_refresh_catalog() assert conn._catalog is pinned assert conn._catalog_fingerprint == "v2" + + +_UNTOKENIZABLE = "SELECT revenue_sum FROM orders WHERE status = 'x'\";--'" + + +async def test_untokenizable_simple_query_is_a_syntax_error() -> None: + writer = await _run(_startup(user="u", database="jaffle") + _query(_UNTOKENIZABLE) + _query("SELECT 1") + _terminate()) + msgs = _messages(writer.buffer) + err = next(body for t, body in msgs if t == "E") + assert _error_sqlstate(err) == proto.SQLSTATE_SYNTAX_ERROR + assert _types(msgs).count("Z") >= 3 + + +async def test_untokenizable_extended_query_errors_and_session_continues() -> None: + writer = await _run( + _startup(user="u", database="jaffle") + + _parse("", _UNTOKENIZABLE) + _bind("", "") + _execute("") + _sync() + + _query("SELECT 1") + _terminate(), + ) + msgs = _messages(writer.buffer) + assert any(t == "E" for t, _ in msgs) + assert _types(msgs)[-1] == "Z" diff --git a/tests/test_usage_reporting_controls.py b/tests/test_usage_reporting_controls.py new file mode 100644 index 00000000..e3562361 --- /dev/null +++ b/tests/test_usage_reporting_controls.py @@ -0,0 +1,293 @@ +"""Opt-out precedence, the ``slayer usage-reporting`` command and the one-time notice.""" + +from __future__ import annotations + +import json +import time +import uuid +from importlib.metadata import PathDistribution +from pathlib import Path + +import pytest + +from slayer import usage_reporting +from slayer.usage_reporting import settings +from slayer.usage_reporting.payload import SCHEMA_VERSION +from slayer.usage_reporting.settings import DecidingRule +from tests._usage_reporting_helpers import ( + McpStdio, + build_storage, + cli, + end_process, + make_due, + quiet_config, + config, + show, + spool_files, + usage, + write_config, +) + +_NOTICE_URL = "docs.motley.ai/slayer/reference/usage-reporting" +_ENV_VARS = ("DO_NOT_TRACK", "SLAYER_USAGE_REPORTING", "CI") + + +@pytest.fixture +def clean_env(usage_reporting_env, monkeypatch: pytest.MonkeyPatch): + """No control variable set, not an editable install, writable config dir.""" + for var in _ENV_VARS: + monkeypatch.delenv(var, raising=False) + monkeypatch.setattr(settings, "is_editable_install", lambda *args, **kwargs: False) + return usage_reporting_env + + +def _make_unwritable(env, monkeypatch: pytest.MonkeyPatch) -> None: + blocker = env.root / "blocker" + blocker.write_text("a file, not a directory") + for var in ("XDG_CONFIG_HOME", "HOME", "APPDATA"): + monkeypatch.setenv(var, str(blocker / "below")) + + +# --- precedence ---------------------------------------------------------------------- + + +_MATRIX = [ + # env, editable, unwritable, persisted -> enabled, rule + ({"DO_NOT_TRACK": "1", "SLAYER_USAGE_REPORTING": "on"}, False, False, None, False, DecidingRule.DO_NOT_TRACK), + ({"DO_NOT_TRACK": "true"}, False, False, "enabled", False, DecidingRule.DO_NOT_TRACK), + ({"DO_NOT_TRACK": "0"}, False, False, None, True, DecidingRule.DEFAULT), + ({"SLAYER_USAGE_REPORTING": "on", "CI": "true"}, False, False, None, True, DecidingRule.ENV), + ({"SLAYER_USAGE_REPORTING": "on"}, True, False, "disabled", True, DecidingRule.ENV), + ({"SLAYER_USAGE_REPORTING": "off"}, False, False, "enabled", False, DecidingRule.ENV), + ({"SLAYER_USAGE_REPORTING": "maybe"}, False, False, None, True, DecidingRule.DEFAULT), + ({"CI": "true"}, False, False, None, False, DecidingRule.CI), + ({"CI": "true"}, False, False, "enabled", False, DecidingRule.CI), + ({}, True, False, "enabled", False, DecidingRule.EDITABLE), + ({}, False, True, None, False, DecidingRule.UNWRITABLE), + ({}, False, False, "disabled", False, DecidingRule.PERSISTED), + ({}, False, False, "enabled", True, DecidingRule.PERSISTED), + ({}, False, False, None, True, DecidingRule.DEFAULT), +] + + +@pytest.mark.parametrize(("env", "editable", "unwritable", "persisted", "enabled", "rule"), _MATRIX) +def test_precedence( + clean_env, monkeypatch: pytest.MonkeyPatch, + env: dict[str, str], editable: bool, unwritable: bool, persisted: str | None, enabled: bool, rule: DecidingRule, +) -> None: + if persisted is not None: + write_config(setting=persisted) + if unwritable: + _make_unwritable(clean_env, monkeypatch) + for var, value in env.items(): + monkeypatch.setenv(var, value) + monkeypatch.setattr(settings, "is_editable_install", lambda *args, **kwargs: editable) + state = settings.resolve_state() + assert (state.enabled, state.rule) == (enabled, rule) + + +def _dist(tmp_path: Path, direct_url: str | None) -> PathDistribution: + info = tmp_path / "motley_slayer-9.9.9.dist-info" + info.mkdir() + (info / "METADATA").write_text("Metadata-Version: 2.1\nName: motley-slayer\nVersion: 9.9.9\n") + if direct_url is not None: + (info / "direct_url.json").write_text(direct_url) + return PathDistribution(info) + + +@pytest.mark.parametrize(("direct_url", "editable"), [ + ('{"url": "file:///src", "dir_info": {"editable": true}}', True), + ('{"url": "file:///src", "dir_info": {"editable": false}}', False), + ('{"url": "https://files.example/x.whl", "archive_info": {}}', False), + (None, False), + ("{not json", False), +]) +def test_editable_install_detection(tmp_path: Path, direct_url: str | None, editable: bool) -> None: + assert settings.is_editable_install(_dist(tmp_path, direct_url)) is editable + + +def test_config_dir_follows_platform_convention(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("SLAYER_STORAGE", str(tmp_path / "storage")) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + monkeypatch.setattr("sys.platform", "linux") + assert settings.config_dir() == tmp_path / "xdg" / "slayer" + monkeypatch.delenv("XDG_CONFIG_HOME") + assert settings.config_dir() == tmp_path / "home" / ".config" / "slayer" + monkeypatch.setattr("sys.platform", "darwin") + assert settings.config_dir() == tmp_path / "home" / "Library" / "Application Support" / "slayer" + monkeypatch.setattr("sys.platform", "win32") + monkeypatch.setenv("APPDATA", str(tmp_path / "appdata")) + assert settings.config_dir() == tmp_path / "appdata" / "slayer" + + +# --- off means nothing ------------------------------------------------------------------ + + +@pytest.fixture +def store(usage_reporting_env) -> str: + return build_storage(usage_reporting_env.root) + + +@pytest.mark.parametrize("env", [{"SLAYER_USAGE_REPORTING": "off"}, {"DO_NOT_TRACK": "1"}, {"CI": "true"}]) +def test_off_records_writes_prints_and_sends_nothing( + clean_env, store: str, monkeypatch: pytest.MonkeyPatch, env: dict[str, str], +) -> None: + for var, value in env.items(): + monkeypatch.setenv(var, value) + result = cli(["models", "--storage", store, "list"]) + assert result.returncode == 0, result.stderr + assert _NOTICE_URL not in result.stderr + assert not settings.config_dir().exists() + time.sleep(0.3) + assert clean_env.capture.arrivals == 0 + + +def test_unwritable_config_dir_is_silent(clean_env, store: str, monkeypatch: pytest.MonkeyPatch) -> None: + _make_unwritable(clean_env, monkeypatch) + result = cli(["models", "--storage", store, "list"]) + assert result.returncode == 0, result.stderr + assert _NOTICE_URL not in result.stderr + time.sleep(0.3) + assert clean_env.capture.arrivals == 0 + + +# --- slayer usage-reporting status|enable|disable|show ---------------------------------------------- + + +def test_status_names_ci_as_the_deciding_rule(clean_env, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CI", "true") + result = cli(["usage-reporting", "status"]) + assert result.returncode == 0, result.stderr + assert "off" in result.stdout.lower() + assert "CI" in result.stdout + + +def test_status_reports_default_on_and_install_id(clean_env) -> None: + install_id = uuid.uuid4() + write_config(install_id=install_id) + result = cli(["usage-reporting", "status"]) + assert result.returncode == 0, result.stderr + assert "on" in result.stdout.lower() + assert str(install_id) in result.stdout + + +def test_enable_and_disable_persist(clean_env) -> None: + assert cli(["usage-reporting", "disable"]).returncode == 0 + assert config().setting == "disabled" + assert (settings.resolve_state().enabled, settings.resolve_state().rule) == (False, DecidingRule.PERSISTED) + assert cli(["usage-reporting", "enable"]).returncode == 0 + assert config().setting == "enabled" + assert settings.resolve_state().enabled + + +def test_disable_removes_spool_and_install_id_and_stops_sending(clean_env) -> None: + quiet_config() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + end_process() + assert spool_files() + assert config().install_id is not None + + assert cli(["usage-reporting", "disable"]).returncode == 0 + assert spool_files() == [] + assert config().install_id is None + + make_due() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + end_process() + time.sleep(0.3) + assert clean_env.capture.arrivals == 0 + assert spool_files() == [] + + +def test_show_prints_pending_report_and_does_not_send(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + end_process() + make_due() + result = cli(["usage-reporting", "show"]) + assert result.returncode == 0, result.stderr + assert usage(json.loads(result.stdout), "mcp:query")["ok"] == 1 + time.sleep(0.5) + assert usage_reporting_env.capture.arrivals == 0 + + +def test_show_matches_the_next_send(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + usage_reporting.record(surface="mcp", token="search", error=ValueError("x")) + end_process() + shown = show() + make_due() + usage_reporting.start() + body = usage_reporting_env.capture.wait_for(count=1)[0].json() + end_process() + for key in ("mcp:query", "mcp:search"): + assert body["properties"]["usage"][key] == shown["usage"][key] + assert body["properties"]["install_id"] == shown["install_id"] + + +# --- the notice -------------------------------------------------------------------------------- + + +def _assert_notice(stderr: str) -> None: + assert _NOTICE_URL in stderr + assert "anonymous" in stderr.lower() + assert "usage" in stderr.lower() + assert "SLAYER_USAGE_REPORTING=off" in stderr or "slayer usage-reporting disable" in stderr + + +def test_notice_once_on_stderr(usage_reporting_env, store: str) -> None: + write_config(last_sent=None) + first = cli(["models", "--storage", store, "list"]) + second = cli(["models", "--storage", store, "list"]) + _assert_notice(first.stderr) + assert _NOTICE_URL not in first.stdout + assert _NOTICE_URL not in second.stderr + assert config().notice_shown_schema == SCHEMA_VERSION + + +def test_notice_again_after_schema_major_bump(usage_reporting_env, store: str) -> None: + quiet_config() + write_config(notice_shown_schema=SCHEMA_VERSION - 1) + first = cli(["models", "--storage", store, "list"]) + second = cli(["models", "--storage", store, "list"]) + _assert_notice(first.stderr) + assert _NOTICE_URL not in second.stderr + + +def test_stdio_mcp_notice_goes_to_stderr_only(usage_reporting_env, store: str) -> None: + session = McpStdio(args=["--storage", store], env=usage_reporting_env.subprocess_env()) + try: + session.initialize() + session.request("tools/list", {}) + session.close_stdin() + session.wait() + finally: + session.kill() + for line in session.stdout_lines: + assert json.loads(line)["jsonrpc"] == "2.0" + _assert_notice(session.stderr.decode()) + + +def _mcp_transcript(*, env: dict[str, str], store: str) -> list[bytes]: + session = McpStdio(args=["--storage", store], env=env) + try: + session.initialize() + session.request("tools/list", {}) + session.call_tool("list_datasources", {}) + session.close_stdin() + session.wait() + finally: + session.kill() + return session.stdout_lines + + +def test_stdio_mcp_stdout_identical_with_usage_reporting_on_and_off(usage_reporting_env, store: str) -> None: + on = _mcp_transcript(env=usage_reporting_env.subprocess_env(), store=store) + off = _mcp_transcript(env=usage_reporting_env.subprocess_env(SLAYER_USAGE_REPORTING="off"), store=store) + assert on == off diff --git a/tests/test_usage_reporting_counting.py b/tests/test_usage_reporting_counting.py new file mode 100644 index 00000000..ecd16270 --- /dev/null +++ b/tests/test_usage_reporting_counting.py @@ -0,0 +1,348 @@ +"""Usage counters per surface: CLI, MCP, REST, Flight SQL, PostgreSQL wire.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import socket +import threading +from collections.abc import Iterator +from types import SimpleNamespace + +import pyarrow.flight as fl +import pytest +import uvicorn +from fastapi.testclient import TestClient +from mcp import ClientSession +from mcp.client.sse import sse_client +from mcp.shared.memory import create_connected_server_and_client_session +from mcp.types import Implementation + +from slayer import usage_reporting +from slayer.api.server import create_app +from slayer.engine.query_engine import SlayerQueryEngine +from slayer.mcp.server import create_mcp_server +from slayer.storage.yaml_storage import YAMLStorage +from slayer.usage_reporting import settings +from slayer.usage_reporting.payload import USAGE_KEYS +from tests._usage_reporting_helpers import ( + build_storage, + cli, + end_process, + quiet_config, + show, + total, + usage, + usage_key_vocabulary, + usage_keys, + wait_until, +) +from tests.flight.test_handlers import _make_handlers +from tests.pg_facade.test_connection import ( + _bind, + _describe, + _execute, + _parse, + _query, + _run, + _startup, + _sync, + _terminate, +) + +_QUERY = {"source_model": "orders", "measures": ["sum(amount)"]} +_PG_QUERY = "SELECT revenue_sum FROM orders" +_FLIGHT_QUERY = "SELECT revenue_sum FROM jaffle.orders" + + +@pytest.fixture +def store(usage_reporting_env) -> str: + quiet_config() + return build_storage(usage_reporting_env.root) + + +@pytest.fixture +def started(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + + +# --- CLI --------------------------------------------------------------------------------------- + + +def test_cli_leaf_tokens(store: str) -> None: + cli(["query", json.dumps(_QUERY), "--storage", store]) + cli(["models", "--storage", store, "list"]) + cli(["datasources", "--storage", store, "list"]) + cli(["datasources", "--storage", store, "list"]) + report = show() + assert usage(report, "cli:query") == {"ok": 1, "errors": {}} + assert usage(report, "cli:models.list") == {"ok": 1, "errors": {}} + assert usage(report, "cli:datasources.list") == {"ok": 2, "errors": {}} + + +@pytest.mark.parametrize("args", [ + ["query"], + ["--help"], + ["query", "--help"], + ["models", "--bogus"], + ["inspect", "x", "--type", "not-a-kind"], + [], +]) +def test_parse_errors_and_help_not_counted(store: str, args: list[str]) -> None: + cli(args) + assert usage_keys(show(), surface="cli") <= {"cli:usage-reporting.show"} + + +def test_usage_reporting_commands_not_counted(store: str) -> None: + assert not {key for key in USAGE_KEYS if key.startswith("cli:usage-reporting")} + cli(["usage-reporting", "status"]) + cli(["usage-reporting", "show"]) + assert usage_keys(show(), surface="cli") == set() + + +def test_parse_error_exits_before_any_usage_reporting(usage_reporting_env) -> None: + cli(["query"]) + assert not settings.config_dir().exists() + + +def test_cli_failure_counted_as_error(store: str) -> None: + cli(["query", json.dumps({"source_model": "no_such_model", "measures": ["count(*)"]}), "--storage", store]) + cli(["models", "--storage", store, "show", "no_such_model"]) + report = show() + for key in ("cli:query", "cli:models.show"): + entry = usage(report, key) + assert entry["ok"] == 0 + assert sum(entry["errors"].values()) == 1 + + +# --- MCP ------------------------------------------------------------------------------------------ + + +class _UnlistedError(Exception): + pass + + +async def test_mcp_tool_calls_counted_once_each(store: str, started) -> None: + server = create_mcp_server(storage=YAMLStorage(base_dir=store), _seed_help=False) + async with create_connected_server_and_client_session(server) as session: + await session.call_tool("list_datasources", {}) + await session.call_tool("list_datasources", {}) + await session.call_tool("inspect_model", {"model_name": "no_such_model"}) + end_process() + report = show() + assert usage(report, "mcp:list_datasources") == {"ok": 2, "errors": {}} + assert total(usage(report, "mcp:inspect_model")) == 1 + + +async def test_mcp_unlisted_exception_counted_as_other( + store: str, started, monkeypatch: pytest.MonkeyPatch, +) -> None: + async def explode(self, *args, **kwargs): + raise _UnlistedError("boom") + + monkeypatch.setattr(SlayerQueryEngine, "execute", explode) + server = create_mcp_server(storage=YAMLStorage(base_dir=store), _seed_help=False) + async with create_connected_server_and_client_session(server) as session: + await session.call_tool("query", {"query": _QUERY}) + end_process() + assert usage(show(), "mcp:query") == {"ok": 0, "errors": {"other": 1}} + + +async def test_mcp_cancelled_tool_call_counted( + store: str, started, monkeypatch: pytest.MonkeyPatch, +) -> None: + async def cancelled(self, *args, **kwargs): + raise asyncio.CancelledError + + monkeypatch.setattr(SlayerQueryEngine, "execute", cancelled) + server = create_mcp_server(storage=YAMLStorage(base_dir=store), _seed_help=False) + with pytest.raises(asyncio.CancelledError): + await server.call_tool("query", {"query": _QUERY}) + end_process() + assert usage(show(), "mcp:query") == {"ok": 0, "errors": {"cancelled": 1}} + + +@pytest.mark.parametrize(("name", "version", "client", "major"), [ + ("claude-code", "2.3.1", "claude-code", 2), + ("acme-finance-bot", "2.3.1", "other", None), +]) +async def test_mcp_client_counted_per_session( + store: str, started, name: str, version: str, client: str, major: int | None, +) -> None: + server = create_mcp_server(storage=YAMLStorage(base_dir=store), _seed_help=False) + info = Implementation(name=name, version=version) + async with create_connected_server_and_client_session(server, client_info=info) as session: + await session.call_tool("list_datasources", {}) + await session.call_tool("list_datasources", {}) + end_process() + clients = show()["mcp_clients"] + assert clients == [{"client": client, "major": major, "sessions": 1}] + + +async def test_mcp_client_unparseable_version(store: str, started) -> None: + server = create_mcp_server(storage=YAMLStorage(base_dir=store), _seed_help=False) + info = Implementation(name="claude-code", version="nightly") + async with create_connected_server_and_client_session(server, client_info=info) as session: + await session.call_tool("list_datasources", {}) + end_process() + assert show()["mcp_clients"] == [{"client": "claude-code", "major": None, "sessions": 1}] + + +def _tool_schemas(store: str) -> str: + server = create_mcp_server(storage=YAMLStorage(base_dir=store), _seed_help=False) + tools = asyncio.run(server.list_tools()) + return json.dumps([t.model_dump(mode="json") for t in tools]) + + +def test_mcp_tool_schemas_identical_with_usage_reporting_on_and_off(store: str) -> None: + off = _tool_schemas(store) + usage_reporting.start() + on = _tool_schemas(store) + end_process() + assert on == off + + +# --- REST ------------------------------------------------------------------------------------------- + + +def test_rest_counts_route_templates(store: str, started) -> None: + client = TestClient(create_app(storage=YAMLStorage(base_dir=store))) + assert client.get("/health").status_code == 200 + client.get("/models/orders") + client.get("/models/no_such_model") + assert client.post("/query", json=_QUERY).status_code == 200 + end_process() + report = show() + assert usage(report, "rest:/health") == {"ok": 1, "errors": {}} + assert total(usage(report, "rest:/models/{name}")) == 2 + assert usage(report, "rest:/query")["ok"] == 1 + + +def test_rest_unmatched_maps_to_a_fixed_token(store: str, started) -> None: + client = TestClient(create_app(storage=YAMLStorage(base_dir=store))) + client.get("/health") + end_process() + before = usage_keys(show(), surface="rest") + usage_reporting.start() + client.get("/no/such/path/42") + client.get("/another/unknown") + end_process() + report = show() + new = usage_keys(report, surface="rest") - before + assert len(new) == 1 + (key,) = new + assert key in usage_key_vocabulary() + assert "path" not in key + assert "unknown" not in key + assert total(usage(report, key)) == 2 + + +def test_rest_mount_requests_not_counted_as_rest(store: str, started) -> None: + client = TestClient(create_app(storage=YAMLStorage(base_dir=store))) + client.post("/mcp/messages/?session_id=00000000000000000000000000000000", json={}) + end_process() + assert usage_keys(show(), surface="rest") == set() + + +@contextlib.contextmanager +def _serve(app) -> Iterator[str]: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error")) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + assert wait_until(lambda: server.started, timeout=30) + try: + yield f"http://127.0.0.1:{port}" + finally: + server.should_exit = True + thread.join(timeout=10) + + +async def test_mounted_mcp_tool_call_counted_as_mcp_not_rest(store: str, started) -> None: + app = create_app(storage=YAMLStorage(base_dir=store)) + with _serve(app) as url: + async with sse_client(f"{url}/mcp/sse") as (read, write): + async with ClientSession(read, write, client_info=Implementation(name="cursor", version="1.4.0")) as session: + await session.initialize() + await session.call_tool("list_datasources", {}) + end_process() + report = show() + assert usage(report, "mcp:list_datasources")["ok"] == 1 + assert usage_keys(report, surface="rest") == set() + + +# --- Flight SQL ------------------------------------------------------------------------------------- + + +def test_flight_validate_then_fetch_counted_once(started) -> None: + handlers = _make_handlers() + descriptor = fl.FlightDescriptor.for_command(b"") # pyright: ignore[reportPrivateImportUsage] — pyarrow re-export + handlers.get_flight_info_for_sql(descriptor, _FLIGHT_QUERY) + handlers.do_get_for_sql(_FLIGHT_QUERY) + end_process() + assert usage(show(), "flight:query") == {"ok": 1, "errors": {}} + + +def test_flight_prepared_statement_counted_per_fetch(started) -> None: + handlers = _make_handlers() + handlers.handle_create_prepared_statement(SimpleNamespace(query=_FLIGHT_QUERY)) + handlers.do_get_for_sql(_FLIGHT_QUERY) + handlers.do_get_for_sql(_FLIGHT_QUERY) + end_process() + assert usage(show(), "flight:query")["ok"] == 2 + + +def test_flight_probe_not_counted_as_query(started) -> None: + _make_handlers().do_get_for_sql("SELECT 1") + end_process() + assert total(usage(show(), "flight:query")) == 0 + + +# --- PostgreSQL wire -------------------------------------------------------------------------------- + + +async def test_pg_multi_statement_counted_per_statement(started) -> None: + await _run(_startup(user="u", database="jaffle") + _query(f"{_PG_QUERY}; {_PG_QUERY}") + _terminate()) + end_process() + assert usage(show(), "pg:query") == {"ok": 2, "errors": {}} + + +async def test_pg_prepared_statement_counted_per_execution(started) -> None: + executions = (_bind("", "s1") + _execute("") + _sync()) * 2 + await _run( + _startup(user="u", database="jaffle") + _parse("s1", _PG_QUERY) + _sync() + executions + _terminate(), + ) + end_process() + assert usage(show(), "pg:query")["ok"] == 2 + + +async def test_pg_describe_alone_not_counted(started) -> None: + await _run( + _startup(user="u", database="jaffle") + _parse("s1", _PG_QUERY) + _describe("S", "s1") + _sync() + _terminate(), + ) + end_process() + assert total(usage(show(), "pg:query")) == 0 + + +async def test_pg_probe_has_its_own_token(started) -> None: + await _run(_startup(user="u", database="jaffle") + _query("SELECT 1") + _terminate()) + end_process() + report = show() + assert total(usage(report, "pg:query")) == 0 + others = usage_keys(report, surface="pg") - {"pg:query"} + assert sum(total(usage(report, key)) for key in others) == 1 + + +class _FailingEngine: + async def execute(self, *, query=None, data_source=None): # NOSONAR(S7503) — awaited interface + raise _UnlistedError("engine down") + + +async def test_pg_engine_error_counted(started) -> None: + await _run(_startup(user="u", database="jaffle") + _query(_PG_QUERY) + _terminate(), engine=_FailingEngine()) + end_process() + assert usage(show(), "pg:query") == {"ok": 0, "errors": {"other": 1}} diff --git a/tests/test_usage_reporting_delivery.py b/tests/test_usage_reporting_delivery.py new file mode 100644 index 00000000..b9cb4572 --- /dev/null +++ b/tests/test_usage_reporting_delivery.py @@ -0,0 +1,612 @@ +"""Local spooling, the 24 h send gate, delivery bookkeeping, destination and install identity.""" + +from __future__ import annotations + +import datetime +import json +import os +import signal +import socket +import stat +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import pytest + +from slayer import usage_reporting +from slayer.usage_reporting import control, sender, settings, spool +from tests._usage_reporting_helpers import ( + McpStdio, + build_storage, + cli, + end_process, + is_posix, + make_due, + now_utc, + quiet_config, + config, + run_python, + run_slayer, + show, + spool_files, + total, + usage, + wait_until, + write_config, +) + +posix_only = pytest.mark.skipif(not is_posix(), reason="POSIX signals / file modes") + + +@pytest.fixture +def store(usage_reporting_env) -> str: + quiet_config() + return build_storage(usage_reporting_env.root) + + +def _spool_one(token: str = "query") -> None: + usage_reporting.start() + usage_reporting.record(surface="mcp", token=token) + end_process() + + +# --- spooling at process end --------------------------------------------------------------------- + + +def test_normal_exit_spools_without_a_request(store: str, usage_reporting_env) -> None: + result = run_slayer(["models", "--storage", store, "list"], env=usage_reporting_env.subprocess_env()) + assert result.returncode == 0, result.stderr + assert len(spool_files()) == 1 + assert usage_reporting_env.capture.arrivals == 0 + + +def test_concurrent_processes_each_write_their_own_spool_file(store: str, usage_reporting_env) -> None: + procs = [ + subprocess.Popen( + [sys.executable, "-m", "slayer", "models", "--storage", store, "list"], + env=usage_reporting_env.subprocess_env(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + for _ in range(5) + ] + assert [p.wait(timeout=120) for p in procs] == [0] * 5 + assert len(spool_files()) == 5 + assert usage(show(), "cli:models.list")["ok"] == 5 + + +def _mcp_with_one_call(store: str, env: dict[str, str], *, argv: list[str] | None = None) -> McpStdio: + session = McpStdio(args=["--storage", store], env=env, argv=argv) + session.initialize() + response = session.call_tool("list_datasources", {}) + assert "result" in response, response + return session + + +@posix_only +def test_stdio_mcp_sigterm_spools(store: str, usage_reporting_env) -> None: + session = _mcp_with_one_call(store, usage_reporting_env.subprocess_env()) + try: + session.proc.send_signal(signal.SIGTERM) + session.wait() + finally: + session.kill() + assert usage(show(), "mcp:list_datasources")["ok"] == 1 + + +_CHAINED_HANDLER_SCRIPT = """ +import pathlib, signal, sys +def handler(signum, frame): + pathlib.Path({marker!r}).write_text("called") +signal.signal(signal.SIGTERM, handler) +sys.argv = ["slayer", "mcp", "--storage", {store!r}] +from slayer.cli import main +main() +""" + + +@posix_only +def test_stdio_mcp_sigterm_chains_existing_handler(store: str, usage_reporting_env, tmp_path: Path) -> None: + marker = tmp_path / "marker" + argv = [sys.executable, "-c", _CHAINED_HANDLER_SCRIPT.format(marker=str(marker), store=store)] + session = _mcp_with_one_call(store, usage_reporting_env.subprocess_env(), argv=argv) + try: + session.proc.send_signal(signal.SIGTERM) + session.wait() + finally: + session.kill() + assert marker.read_text() == "called" + assert usage(show(), "mcp:list_datasources")["ok"] == 1 + + +@posix_only +def test_stdio_mcp_sigint_spools(store: str, usage_reporting_env) -> None: + session = _mcp_with_one_call(store, usage_reporting_env.subprocess_env()) + try: + session.proc.send_signal(signal.SIGINT) + session.wait() + finally: + session.kill() + assert usage(show(), "mcp:list_datasources")["ok"] == 1 + + +def test_stdio_mcp_end_of_input_spools(store: str, usage_reporting_env) -> None: + session = _mcp_with_one_call(store, usage_reporting_env.subprocess_env()) + try: + session.close_stdin() + session.wait() + finally: + session.kill() + assert usage(show(), "mcp:list_datasources")["ok"] == 1 + + +@posix_only +def test_stdio_mcp_broken_pipe_spools(store: str, usage_reporting_env) -> None: + proc = subprocess.Popen( + [sys.executable, "-m", "slayer", "mcp", "--storage", store], + env=usage_reporting_env.subprocess_env(), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + ) + stdin, stdout = proc.stdin, proc.stdout + assert stdin is not None + assert stdout is not None + + def send(message: dict) -> None: + stdin.write((json.dumps(message) + "\n").encode()) + stdin.flush() + + try: + send({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { + "protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "claude-code", "version": "2.3.1"}}}) + assert json.loads(stdout.readline())["id"] == 1 + send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + send({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "list_datasources", "arguments": {}}}) + assert json.loads(stdout.readline())["id"] == 2 + stdout.close() + send({"jsonrpc": "2.0", "id": 99, "method": "tools/call", "params": {"name": "list_datasources", "arguments": {}}}) + stdin.close() + proc.wait(timeout=30) + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=10) + assert total(usage(show(), "mcp:list_datasources")) >= 1 + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _wait_for_port(port: int, *, timeout: float = 60.0) -> bool: + def listening() -> bool: + with socket.socket() as sock: + return sock.connect_ex(("127.0.0.1", port)) == 0 + return wait_until(listening, timeout=timeout) + + +@posix_only +@pytest.mark.parametrize(("command", "sig"), [ + ("serve", signal.SIGTERM), + ("serve", signal.SIGINT), + ("pg-serve", signal.SIGINT), + ("flight-serve", signal.SIGINT), +]) +def test_server_shutdown_spools(store: str, usage_reporting_env, command: str, sig: signal.Signals) -> None: + port = _free_port() + proc = subprocess.Popen( + [sys.executable, "-m", "slayer", command, "--host", "127.0.0.1", "--port", str(port), "--storage", store], + env=usage_reporting_env.subprocess_env(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + assert _wait_for_port(port), f"{command} did not start (exit {proc.poll()})" + proc.send_signal(sig) + proc.wait(timeout=30) + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=10) + cli_key = f"cli:{command}" + assert total(usage(show(), cli_key)) == 1 + + +@posix_only +def test_serve_sigterm_spools_rest_counts(store: str, usage_reporting_env) -> None: + port = _free_port() + proc = subprocess.Popen( + [sys.executable, "-m", "slayer", "serve", "--host", "127.0.0.1", "--port", str(port), "--storage", store], + env=usage_reporting_env.subprocess_env(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + assert _wait_for_port(port) + result = run_python( + f"import urllib.request; urllib.request.urlopen('http://127.0.0.1:{port}/health').read()", + env=usage_reporting_env.subprocess_env(), + ) + assert result.returncode == 0, result.stderr + proc.send_signal(signal.SIGTERM) + proc.wait(timeout=30) + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=10) + assert usage(show(), "rest:/health")["ok"] == 1 + + +_FORK_SCRIPT = """ +import os, sys +from slayer import usage_reporting +usage_reporting.start() +usage_reporting.record(surface="mcp", token="query") +pid = os.fork() +if pid == 0: + usage_reporting.flush() + os._exit(0) +os.waitpid(pid, 0) +""" + + +@posix_only +def test_forked_child_does_not_repeat_parent_counts(usage_reporting_env) -> None: + quiet_config() + result = run_python(_FORK_SCRIPT, env=usage_reporting_env.subprocess_env()) + assert result.returncode == 0, result.stderr + assert usage(show(), "mcp:query")["ok"] == 1 + + +# --- spool files ---------------------------------------------------------------------------------------- + + +@posix_only +def test_spool_files_are_owner_only(usage_reporting_env) -> None: + quiet_config() + _spool_one() + files = spool_files() + assert files + assert all(stat.S_IMODE(p.stat().st_mode) == 0o600 for p in files) + + +@posix_only +def test_non_regular_spool_entries_ignored(usage_reporting_env) -> None: + quiet_config() + _spool_one() + (real,) = spool_files() + directory = settings.spool_dir() + (directory / f"link-{real.name}").symlink_to(real) + (directory / f"dir-{real.name}").mkdir() + os.mkfifo(directory / f"fifo-{real.name}") + assert usage(show(), "mcp:query")["ok"] == 1 + + +def test_corrupt_spool_file_skipped(usage_reporting_env) -> None: + quiet_config() + _spool_one() + (corrupt,) = spool_files() + corrupt.write_text("{not json") + _spool_one() + assert usage(show(), "mcp:query")["ok"] == 1 + + +def test_spool_is_capped_dropping_oldest(usage_reporting_env) -> None: + quiet_config() + for _ in range(140): + _spool_one("query") + for _ in range(10): + _spool_one("search") + files = spool_files() + assert len(files) <= 100 + assert sum(p.stat().st_size for p in files) <= 1_000_000 + report = show() + assert usage(report, "mcp:search")["ok"] == 10 + assert usage(report, "mcp:query")["ok"] <= 90 + + +# --- the 24 h gate (injected clock) ------------------------------------------------------------------------ + + +_T0 = datetime.datetime(2026, 5, 4, 12, 0, tzinfo=datetime.timezone.utc) + + +def test_not_yet_due_sends_nothing_and_spools(usage_reporting_env) -> None: + quiet_config(last_sent=_T0) + usage_reporting_env.set_now(_T0 + datetime.timedelta(hours=3)) + _spool_one() + time.sleep(0.5) + assert usage_reporting_env.capture.arrivals == 0 + assert usage(show(), "mcp:query")["ok"] == 1 + + +def test_idle_server_sends_nothing(usage_reporting_env) -> None: + quiet_config(last_sent=_T0) + usage_reporting_env.set_now(_T0 + datetime.timedelta(hours=1)) + usage_reporting.start() + usage_reporting_env.set_now(_T0 + datetime.timedelta(hours=49)) + time.sleep(0.5) + end_process() + assert usage_reporting_env.capture.arrivals == 0 + + +def test_due_on_activity_sends_once_merging_spool_and_live(usage_reporting_env) -> None: + quiet_config(last_sent=_T0) + usage_reporting_env.set_now(_T0 + datetime.timedelta(hours=1)) + _spool_one("search") + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + usage_reporting_env.set_now(_T0 + datetime.timedelta(hours=25)) + usage_reporting.record(surface="mcp", token="query") + requests = usage_reporting_env.capture.wait_for(count=1) + usage_reporting.record(surface="mcp", token="query") + time.sleep(0.3) + end_process() + assert len(usage_reporting_env.capture.requests) == 1 + sent = requests[0].json()["properties"]["usage"] + assert sent["mcp:search"]["ok"] == 1 + assert sent["mcp:query"]["ok"] == 2 + assert wait_until(lambda: config().last_sent == _T0 + datetime.timedelta(hours=25)) + remaining = show() + assert usage(remaining, "mcp:search")["ok"] == 0 + assert usage(remaining, "mcp:query")["ok"] == 1 + + +def test_due_at_start_sends_spool(usage_reporting_env) -> None: + quiet_config(last_sent=_T0) + usage_reporting_env.set_now(_T0 + datetime.timedelta(hours=1)) + _spool_one() + usage_reporting_env.set_now(_T0 + datetime.timedelta(hours=26)) + usage_reporting.start() + requests = usage_reporting_env.capture.wait_for(count=1) + end_process() + assert len(requests) == 1 + assert requests[0].json()["properties"]["usage"]["mcp:query"]["ok"] == 1 + + +def test_last_send_in_the_future_counts_as_due(usage_reporting_env) -> None: + quiet_config(last_sent=_T0 + datetime.timedelta(days=30)) + usage_reporting_env.set_now(_T0) + _spool_one() + usage_reporting.start() + assert len(usage_reporting_env.capture.wait_for(count=1)) == 1 + end_process() + + +def test_clock_jump_forward_is_due(usage_reporting_env) -> None: + quiet_config(last_sent=_T0) + usage_reporting_env.set_now(_T0) + _spool_one() + usage_reporting_env.set_now(_T0 + datetime.timedelta(days=400)) + usage_reporting.start() + assert len(usage_reporting_env.capture.wait_for(count=1)) == 1 + end_process() + + +# --- bookkeeping ----------------------------------------------------------------------------------------------- + + +def test_failed_send_keeps_spool_and_last_send(usage_reporting_env) -> None: + usage_reporting_env.capture.behaviour.status = 500 + quiet_config() + _spool_one() + before = make_due().last_sent + usage_reporting.start() + assert usage_reporting_env.capture.wait_for(count=1) + usage_reporting.shutdown() + end_process() + assert config().last_sent == before + assert usage(show(), "mcp:query")["ok"] == 1 + + +def test_timed_out_send_keeps_spool(usage_reporting_env) -> None: + usage_reporting_env.capture.behaviour.delay = 30 + quiet_config() + _spool_one() + before = make_due().last_sent + usage_reporting.start() + assert usage_reporting_env.capture.wait_for_arrival() + started = time.monotonic() + usage_reporting.shutdown() + assert time.monotonic() - started < 1.5 + end_process() + assert config().last_sent == before + usage_reporting_env.set_now(now_utc() + datetime.timedelta(days=1)) + assert usage(show(), "mcp:query")["ok"] == 1 + + +def test_no_wait_at_exit_without_send_in_flight(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + started = time.monotonic() + usage_reporting.shutdown() + assert time.monotonic() - started < 0.1 + end_process() + + +def test_two_simultaneous_senders_send_one_report(store: str, usage_reporting_env) -> None: + usage_reporting_env.capture.behaviour.delay = 0.5 + run_slayer(["models", "--storage", store, "list"], env=usage_reporting_env.subprocess_env()) + make_due() + procs = [ + subprocess.Popen( + [sys.executable, "-m", "slayer", "datasources", "--storage", store, "list"], + env=usage_reporting_env.subprocess_env(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + for _ in range(2) + ] + assert [p.wait(timeout=120) for p in procs] == [0, 0] + time.sleep(1.5) + assert usage_reporting_env.capture.arrivals == 1 + assert len(usage_reporting_env.capture.requests) == 1 + + +def test_stale_lease_taken_over_by_one_sender_only(tmp_path: Path, monkeypatch) -> None: + lease = tmp_path / "send.lease" + now = now_utc() + assert spool.acquire_lease(lease, now=now - datetime.timedelta(hours=1)) is not None + real_expired = spool._lease_expired + rival: list[str | None] = [] + + def rival_checks_meanwhile(path: Path, *, now: datetime.datetime) -> bool: + expired = real_expired(path, now=now) + if not rival: + rival.append(None) + rival[0] = spool.acquire_lease(path, now=now) + return expired + + monkeypatch.setattr(spool, "_lease_expired", rival_checks_meanwhile) + mine = spool.acquire_lease(lease, now=now) + assert sum(token is not None for token in (mine, *rival)) == 1 + + +def test_sender_crash_recovered_by_a_later_send(store: str, usage_reporting_env) -> None: + usage_reporting_env.capture.behaviour.stall_first = 60 + result = run_slayer(["models", "--storage", store, "list"], env=usage_reporting_env.subprocess_env()) + assert result.returncode == 0, result.stderr + make_due() + session = McpStdio(args=["--storage", store], env=usage_reporting_env.subprocess_env()) + try: + assert usage_reporting_env.capture.wait_for_arrival(timeout=60) + finally: + session.kill() + usage_reporting_env.set_now(now_utc() + datetime.timedelta(days=2)) + usage_reporting.start() + requests = usage_reporting_env.capture.wait_for(count=1, timeout=20) + end_process() + assert len(requests) == 1 + assert requests[0].json()["properties"]["usage"]["cli:models.list"]["ok"] == 1 + + +def test_disable_racing_a_send_leaves_nothing_behind(store: str, usage_reporting_env) -> None: + usage_reporting_env.capture.behaviour.delay = 3.0 + quiet_config() + _spool_one() + make_due() + usage_reporting.start() + assert usage_reporting_env.capture.wait_for_arrival() + disabled = run_slayer(["usage-reporting", "disable"], env=usage_reporting_env.subprocess_env()) + assert disabled.returncode == 0, disabled.stderr + assert not usage_reporting_env.capture.requests + assert usage_reporting_env.capture.wait_for(count=1, timeout=15) + usage_reporting.shutdown() + end_process() + current = config() + assert current.setting == "disabled" + assert current.install_id is None + assert spool_files() == [] + + +# --- destination and request shape ---------------------------------------------------------------------------- + + +def test_default_endpoint_is_posthog_eu() -> None: + assert sender.DEFAULT_ENDPOINT == "https://eu.i.posthog.com/i/v0/e/" + + +def test_request_shape(usage_reporting_env) -> None: + quiet_config() + _spool_one() + make_due() + usage_reporting.start() + (request,) = usage_reporting_env.capture.wait_for(count=1) + end_process() + assert request.path == "/i/v0/e/" + assert request.headers.get("Content-Type", "").startswith("application/json") + body = request.json() + properties = body["properties"] + assert body["event"] == "slayer_usage" + assert body["distinct_id"] == properties["install_id"] + assert body["uuid"] == properties["batch_id"] + assert properties["$geoip_disable"] is True + assert "$process_person_profile" not in properties + assert "api_key" in body + assert "timestamp" not in body + + +# --- install identity ------------------------------------------------------------------------------------------------- + + +def test_install_id_created_on_first_use(usage_reporting_env) -> None: + quiet_config() + _spool_one() + current = config() + assert isinstance(current.install_id, uuid.UUID) + assert current.install_id_created == now_utc().date() + + +@pytest.mark.parametrize(("age_days", "rotated"), [(11 * 31, True), (10 * 30, False)]) +def test_install_id_rotation(usage_reporting_env, age_days: int, rotated: bool) -> None: + old = uuid.uuid4() + write_config(install_id=old, install_id_created=_T0.date()) + quiet_config(last_sent=_T0) + usage_reporting_env.set_now(_T0 + datetime.timedelta(days=age_days)) + _spool_one() + current = config() + assert (current.install_id != old) is rotated + if rotated: + assert current.install_id_created == (_T0 + datetime.timedelta(days=age_days)).date() + assert show()["install_id"] == str(current.install_id) + + +def test_running_process_rotates_install_id_before_sending(usage_reporting_env) -> None: + old = uuid.uuid4() + write_config(install_id=old, install_id_created=(_T0 - datetime.timedelta(days=300)).date()) + quiet_config(last_sent=_T0) + usage_reporting_env.set_now(_T0 + datetime.timedelta(hours=1)) + usage_reporting.start() + usage_reporting_env.set_now(_T0 + datetime.timedelta(days=60)) + usage_reporting.record(surface="mcp", token="query") + (request,) = usage_reporting_env.capture.wait_for(count=1) + end_process() + assert request.json()["distinct_id"] != str(old) + assert request.json()["distinct_id"] == str(config().install_id) + + +def _disable_during_rotation(monkeypatch: pytest.MonkeyPatch) -> threading.Thread: + """Run ``slayer usage-reporting disable`` (as if from another process) while the install ID rotates.""" + real = settings.with_install_id + racer = threading.Thread(target=control.disable) + + def rotate_while_disabling(config: settings.UsageReportingConfig, *, today: datetime.date): + racer.start() + racer.join(timeout=0.5) + return real(config, today=today) + + monkeypatch.setattr(settings, "with_install_id", rotate_while_disabling) + monkeypatch.setattr(sender, "with_install_id", rotate_while_disabling, raising=False) + return racer + + +def _assert_stays_disabled(racer: threading.Thread) -> None: + racer.join(timeout=10) + assert not racer.is_alive() + final = config() + assert final.setting == "disabled" + assert final.install_id is None + + +def test_disable_during_send_rotation_is_never_undone(usage_reporting_env, monkeypatch) -> None: + write_config(install_id=uuid.uuid4(), install_id_created=(_T0 - datetime.timedelta(days=400)).date(), + last_sent=_T0 - datetime.timedelta(days=2)) + racer = _disable_during_rotation(monkeypatch) + paths = sender.Paths(config=settings.config_path(), spool=settings.spool_dir(), lease=settings.lease_path()) + sender.send(paths, now=_T0, url=sender.endpoint()) + _assert_stays_disabled(racer) + + +def test_disable_during_start_is_never_undone(usage_reporting_env, monkeypatch) -> None: + quiet_config() + racer = _disable_during_rotation(monkeypatch) + usage_reporting.start() + _assert_stays_disabled(racer) + end_process() + assert not spool_files() + + +def test_install_id_stored_in_config_dir_not_model_storage(store: str, usage_reporting_env) -> None: + cli(["models", "--storage", store, "list"]) + install_id = str(config().install_id) + for path in Path(store).rglob("*"): + if path.is_file(): + assert install_id not in path.read_text(errors="ignore") + assert settings.config_path().is_relative_to(settings.config_dir()) + assert json.loads(settings.config_path().read_text()) diff --git a/tests/test_usage_reporting_features.py b/tests/test_usage_reporting_features.py new file mode 100644 index 00000000..33757ea5 --- /dev/null +++ b/tests/test_usage_reporting_features.py @@ -0,0 +1,270 @@ +"""Query feature counters, environment and context fields.""" + +from __future__ import annotations + +import asyncio +import json +import platform +import sys +from collections import Counter +from importlib.metadata import version as pkg_version +from typing import Any + +import pytest + +import slayer.cli +from slayer.core.models import DatasourceConfig +from slayer.sql import engine_factory +from slayer.storage.yaml_storage import YAMLStorage +from tests._saved_query_refinement_fixtures import AVG_CUSTOMER_REVENUE, MONTH_TD +from tests._usage_reporting_helpers import ( + build_storage, + cli, + quiet_config, + run_python, + save_models, + show, + usage, +) + +_PLAIN = {"source_model": "orders", "measures": ["sum(amount)"]} +_FLAGS = ( + "multistage", "inline_source", "time_dimensions", "filters", + "computed_dimensions", "saved_query", "to_many_handling", +) + + +@pytest.fixture +def store(usage_reporting_env) -> str: + quiet_config() + return build_storage(usage_reporting_env.root) + + +def _run_query(store: str, query: Any) -> dict: + arg = query if isinstance(query, str) else json.dumps(query) + cli(["query", arg, "--storage", store]) + return show() + + +# --- structural flags ------------------------------------------------------------------------ + + +def test_plain_query_sets_no_flag(store: str) -> None: + flags = _run_query(store, _PLAIN)["features"]["flags"] + assert all(flags.get(flag, 0) == 0 for flag in _FLAGS) + + +@pytest.mark.parametrize(("query", "flag"), [ + (AVG_CUSTOMER_REVENUE, "multistage"), + ({"source_model": {"source_name": "orders", "columns": [{"name": "double_amount", "sql": "amount * 2", "type": "DOUBLE"}]}, + "measures": ["sum(double_amount)"]}, "inline_source"), + ({"source_model": {"name": "orders_inline", "data_source": "test", "sql_table": "orders", + "columns": [{"name": "id", "type": "INT", "primary_key": True}, + {"name": "amount", "type": "DOUBLE"}]}, + "measures": ["sum(amount)"]}, "inline_source"), + ({**_PLAIN, "time_dimensions": [MONTH_TD]}, "time_dimensions"), + ({**_PLAIN, "filters": ["status = 'paid'"]}, "filters"), + ({**_PLAIN, "dimensions": [{"expression": "amount > 50", "name": "big"}]}, "computed_dimensions"), + ("monthly_revenue", "saved_query"), + ({**_PLAIN, "dimensions": ["region"], "to_many_handling": "associate"}, "to_many_handling"), +]) +def test_structural_flag_counted_once(store: str, query: Any, flag: str) -> None: + assert _run_query(store, query)["features"]["flags"][flag] == 1 + + +def test_three_stage_list_counts_multistage_once(store: str) -> None: + stages = [ + {"name": "s1", "source_model": "orders", "dimensions": ["customer_id"], "measures": [{"formula": "sum(amount)", "name": "rev"}]}, + {"name": "s2", "source_model": "s1", "dimensions": ["customer_id"], "measures": [{"formula": "max(rev)", "name": "top"}]}, + {"source_model": "s2", "measures": ["avg(top)"]}, + ] + features = _run_query(store, stages)["features"] + assert features["flags"]["multistage"] == 1 + assert features["aggregations"]["sum"] == 1 + assert features["aggregations"]["max"] == 1 + assert features["aggregations"]["avg"] == 1 + + +# --- built-in and custom names --------------------------------------------------------------------- + + +def test_builtin_names_counted_by_name_and_custom_as_custom(store: str) -> None: + features = _run_query(store, { + "source_model": "orders", + "time_dimensions": [MONTH_TD], + "measures": ["time_shift(sum(amount), -1)", "dsum(amount)"], + })["features"] + assert features["transforms"]["time_shift"] == 1 + assert features["aggregations"]["sum"] == 1 + assert features["aggregations"]["custom"] == 1 + assert "dsum" not in features["aggregations"] + + +def test_features_counted_per_attempt_even_on_failure(store: str) -> None: + report = _run_query(store, {"source_model": "orders", "measures": ["sum(no_such_column)"]}) + assert report["features"]["aggregations"]["sum"] == 1 + assert usage(report, "cli:query")["ok"] == 0 + + +def test_features_counted_per_query(store: str) -> None: + cli(["query", json.dumps(_PLAIN), "--storage", store]) + report = _run_query(store, _PLAIN) + assert report["features"]["aggregations"]["sum"] == 2 + + +# --- dialects -------------------------------------------------------------------------------------------- + + +def test_queries_counted_per_dialect(store: str) -> None: + assert _run_query(store, _PLAIN)["dialects"]["queries"] == {"sqlite": 1} + + +def test_dry_run_not_counted_as_executed(store: str) -> None: + cli(["query", json.dumps(_PLAIN), "--storage", store, "--dry-run"]) + assert show()["dialects"]["queries"] == {} + + +@pytest.mark.parametrize(("extra", "bucket"), [(0, "1"), (1, "2-5"), (4, "2-5"), (5, "6+")]) +def test_datasources_per_dialect_bucketed(store: str, usage_reporting_env, extra: int, bucket: str) -> None: + async def add() -> None: + storage = YAMLStorage(base_dir=store) + for i in range(extra): + await storage.save_datasource(DatasourceConfig( + name=f"extra_{i}", type="sqlite", database=str(usage_reporting_env.root / f"extra_{i}.db"), + )) + + asyncio.run(add()) + cli(["datasources", "--storage", store, "list"]) + assert show()["dialects"]["datasources"] == {"sqlite": {bucket: 1}} + + +# --- context ------------------------------------------------------------------------------------------------ + + +def test_model_count_is_bucketed(store: str) -> None: + save_models(store, count=32) + cli(["models", "--storage", store, "list"]) + context = show()["context"] + assert context["model_count"] == {"11-50": 1} + assert context["storage"] == {"yaml": 1} + + +def test_demo_datasource_in_use(usage_reporting_env) -> None: + quiet_config() + store = str(usage_reporting_env.storage_dir) + cli(["datasources", "--storage", store, "list"]) + assert show()["context"]["demo"] == 0 + cli(["datasources", "--storage", store, "create", "demo", "--years", "1", "--ingest", "-y"]) + cli(["query", json.dumps({"source_model": "orders", "measures": ["count(*)"]}), "--storage", store]) + assert show()["context"]["demo"] >= 1 + + +class _CountingStorage: + def __init__(self, inner: Any) -> None: + self._inner = inner + self.calls: Counter[str] = Counter() + + def __getattr__(self, name: str) -> Any: + attr = getattr(self._inner, name) + if not callable(attr): + return attr + + def counted(*args: Any, **kwargs: Any) -> Any: + self.calls[name] += 1 + return attr(*args, **kwargs) + + return counted + + +def _storage_and_connection_calls(args: list[str], monkeypatch: pytest.MonkeyPatch) -> tuple[Counter, int]: + spies: list[_CountingStorage] = [] + connections = 0 + real_resolve = slayer.cli.resolve_storage + real_get_engine = engine_factory.get_engine + + def resolve(path: str) -> _CountingStorage: + spy = _CountingStorage(real_resolve(path)) + spies.append(spy) + return spy + + def get_engine(*a: Any, **k: Any) -> Any: + nonlocal connections + connections += 1 + return real_get_engine(*a, **k) + + with monkeypatch.context() as mp: + mp.setattr(slayer.cli, "resolve_storage", resolve) + mp.setattr(engine_factory, "get_engine", get_engine) + cli(args) + return sum((s.calls for s in spies), Counter()), connections + + +@pytest.mark.parametrize("command", [ + ["query", json.dumps(_PLAIN)], + ["models", "list"], + ["datasources", "list"], + ["serve"], +]) +def test_usage_reporting_makes_no_storage_call_or_connection( + store: str, monkeypatch: pytest.MonkeyPatch, command: list[str], +) -> None: + fake_uvicorn = type(sys)("uvicorn") + fake_uvicorn.run = lambda **kwargs: None # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "uvicorn", fake_uvicorn) + args = [*command[:1], "--storage", store, *command[1:]] if command[0] in ("models", "datasources") \ + else [*command, "--storage", store] + _storage_and_connection_calls(args, monkeypatch) # warm-up: the first run seeds help memories + on = _storage_and_connection_calls(args, monkeypatch) + monkeypatch.setenv("SLAYER_USAGE_REPORTING", "off") + off = _storage_and_connection_calls(args, monkeypatch) + assert on == off + + +# --- environment ------------------------------------------------------------------------------------------------- + + +def test_environment_fields(store: str) -> None: + env = _run_query(store, _PLAIN)["env"] + assert env["slayer_version"] == pkg_version("motley-slayer") + assert env["python"] == f"{sys.version_info.major}.{sys.version_info.minor}" + assert env["os"] == {"linux": "linux", "darwin": "darwin", "win32": "windows"}.get(sys.platform, "other") + assert env["arch"] in ("x86_64", "arm64", "other") + if platform.machine().lower() in ("x86_64", "amd64"): + assert env["arch"] == "x86_64" + assert isinstance(env["in_container"], bool) + assert "flight" in env["extras"] + + +_IMPORTS_SCRIPT = """ +import contextlib, io, json, sys +import slayer.cli + +def run(*args): + sys.argv = ["slayer", *args] + out = io.StringIO() + with contextlib.redirect_stdout(out): + try: + slayer.cli.main() + except SystemExit: + pass + return out.getvalue() + +run("models", "--storage", {store!r}, "list") +shown = run("usage-reporting", "show") +print(json.dumps({{"modules": sorted(sys.modules), "shown": shown}})) +""" + + +def _imports_and_show(store: str, env: dict[str, str]) -> dict: + result = run_python(_IMPORTS_SCRIPT.format(store=store), env=env) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_extras_detected_without_importing_them(store: str, usage_reporting_env) -> None: + on = _imports_and_show(store, usage_reporting_env.subprocess_env()) + off = _imports_and_show(store, usage_reporting_env.subprocess_env(SLAYER_USAGE_REPORTING="off")) + added = set(on["modules"]) - set(off["modules"]) + foreign = sorted(m for m in added if m.split(".")[0] not in sys.stdlib_module_names | {"slayer"}) + assert foreign == [] + assert "flight" in json.loads(on["shown"])["env"]["extras"] diff --git a/tests/test_usage_reporting_isolation.py b/tests/test_usage_reporting_isolation.py new file mode 100644 index 00000000..5ccc1a06 --- /dev/null +++ b/tests/test_usage_reporting_isolation.py @@ -0,0 +1,189 @@ +"""usage_reporting.arc42.md principle 1: fail-silent — usage reporting never affects the host process.""" + +from __future__ import annotations + +import datetime +import json +import shutil +import time +from collections.abc import Callable + +import pytest + +from slayer import usage_reporting +from slayer.usage_reporting import sender, settings +from tests._usage_reporting_capture import RefusedEndpoint +from tests._usage_reporting_helpers import ( + SENT, + assert_no_sentinel, + build_storage, + cli, + end_process, + make_due, + now_utc, + quiet_config, + run_slayer, + show, + spool_bytes, + usage, +) + +_QUERY = {"source_model": "orders", "measures": ["sum(amount)"]} + + +@pytest.fixture +def store(usage_reporting_env) -> str: + quiet_config() + return build_storage(usage_reporting_env.root) + + +def _commands(store: str) -> list[list[str]]: + return [ + ["models", "--storage", store, "list"], + ["query", json.dumps(_QUERY), "--storage", store], + ["query", json.dumps({"source_model": "no_such_model", "measures": ["count(*)"]}), "--storage", store], + ["models", "--storage", store, "show", "no_such_model"], + ] + + +def _outcomes(commands: list[list[str]]) -> list[tuple[int, str, str]]: + results = [] + for args in commands: + result = cli(args) + results.append((result.returncode, result.stdout, result.stderr)) + return results + + +def _baseline(store: str, monkeypatch: pytest.MonkeyPatch) -> list[tuple[int, str, str]]: + with monkeypatch.context() as mp: + mp.setenv("SLAYER_USAGE_REPORTING", "off") + return _outcomes(_commands(store)) + + +def _refused(env, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SLAYER_USAGE_REPORTING_ENDPOINT", RefusedEndpoint().url) + + +def _timeout(env, monkeypatch: pytest.MonkeyPatch) -> None: + env.capture.behaviour.delay = 30 + + +def _server_error(env, monkeypatch: pytest.MonkeyPatch) -> None: + env.capture.behaviour.status = 500 + + +def _corrupt_spool(env, monkeypatch: pytest.MonkeyPatch) -> None: + directory = settings.spool_dir() + directory.mkdir(parents=True, exist_ok=True) + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + end_process() + for path in directory.iterdir(): + path.write_bytes(b"\x00{not json") + + +def _spool_dir_is_a_file(env, monkeypatch: pytest.MonkeyPatch) -> None: + directory = settings.spool_dir() + directory.parent.mkdir(parents=True, exist_ok=True) + directory.write_text("not a directory") + + +def _clock_raises(env, monkeypatch: pytest.MonkeyPatch) -> None: + def broken() -> datetime.datetime: + raise RuntimeError("clock broken") + + monkeypatch.setattr("slayer.usage_reporting.clock.now", broken) + + +_FAILURES: dict[str, Callable] = { + "endpoint_refused": _refused, + "endpoint_timeout": _timeout, + "endpoint_500": _server_error, + "corrupt_spool": _corrupt_spool, + "spool_dir_is_a_file": _spool_dir_is_a_file, + "clock_raises": _clock_raises, +} + + +@pytest.mark.parametrize("failure", sorted(_FAILURES)) +def test_failures_leave_output_and_exit_code_unchanged( + store: str, usage_reporting_env, monkeypatch: pytest.MonkeyPatch, failure: str, +) -> None: + expected = _baseline(store, monkeypatch) + make_due() + _FAILURES[failure](usage_reporting_env, monkeypatch) + assert _outcomes(_commands(store)) == expected + + +def test_corrupt_config_leaves_stdout_and_exit_code_unchanged( + store: str, usage_reporting_env, monkeypatch: pytest.MonkeyPatch, +) -> None: + expected = [(code, out) for code, out, _ in _baseline(store, monkeypatch)] + settings.config_path().write_text('{"install_id": 42, "last_sent": "yesterday-ish", "setting": [1]}') + assert [(code, out) for code, out, _ in _outcomes(_commands(store))] == expected + + +class _EvilError(Exception): + def __str__(self) -> str: + raise RuntimeError(SENT) + + def __repr__(self) -> str: + raise RuntimeError(SENT) + + +def test_record_never_raises_on_hostile_input(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query", error=_EvilError()) + usage_reporting.record(surface="no-such-surface", token=SENT) + usage_reporting.record(surface="mcp", token=None) # type: ignore[arg-type] + usage_reporting.record(surface="mcp", token=SENT * 1000, error=KeyboardInterrupt()) + end_process() + report = show() + assert usage(report, "mcp:query")["errors"] == {"other": 1} + assert_no_sentinel(json.dumps(report)) + assert_no_sentinel(spool_bytes()) + + +def test_flush_and_shutdown_never_raise_when_storage_vanishes(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + shutil.rmtree(settings.config_dir()) + usage_reporting.shutdown() + end_process() + + +def test_send_never_raises_when_config_dir_vanishes(tmp_path) -> None: + gone = tmp_path / "gone" + paths = sender.Paths(config=gone / "usage-reporting.json", spool=gone / "spool", lease=gone / "send.lease") + sender.send(paths, now=datetime.datetime.now(datetime.timezone.utc), url="http://127.0.0.1:9/") + + +def test_stalled_send_delays_exit_by_at_most_about_a_second(store: str, usage_reporting_env) -> None: + args = ["models", "--storage", store, "list"] + off_env = usage_reporting_env.subprocess_env(SLAYER_USAGE_REPORTING="off") + started = time.monotonic() + assert run_slayer(args, env=off_env).returncode == 0 + off_duration = time.monotonic() - started + + run_slayer(args, env=usage_reporting_env.subprocess_env()) + make_due() + usage_reporting_env.capture.behaviour.delay = 10 + started = time.monotonic() + result = run_slayer(args, env=usage_reporting_env.subprocess_env()) + on_duration = time.monotonic() - started + assert result.returncode == 0, result.stderr + assert usage_reporting_env.capture.arrivals == 1 + assert on_duration - off_duration < 2.0 + + usage_reporting_env.set_now(now_utc() + datetime.timedelta(days=1)) + assert usage(show(), "cli:models.list")["ok"] == 2 + + +def test_first_run_notice_and_send_never_touch_stdout(store: str, usage_reporting_env) -> None: + settings.config_path().unlink() + args = ["models", "--storage", store, "list"] + first_run = run_slayer(args, env=usage_reporting_env.subprocess_env()) + off = run_slayer(args, env=usage_reporting_env.subprocess_env(SLAYER_USAGE_REPORTING="off")) + assert (first_run.returncode, first_run.stdout) == (off.returncode, off.stdout) diff --git a/tests/test_usage_reporting_payload.py b/tests/test_usage_reporting_payload.py new file mode 100644 index 00000000..f9dac843 --- /dev/null +++ b/tests/test_usage_reporting_payload.py @@ -0,0 +1,347 @@ +"""usage_reporting.arc42.md principle 2: a usage report carries only SLayer-defined vocabulary.""" + +from __future__ import annotations + +import argparse +import asyncio +import datetime +import json +import re +from pathlib import Path + +import pytest +from fastapi.routing import APIRoute +from fastapi.testclient import TestClient +from mcp.shared.memory import create_connected_server_and_client_session +from mcp.types import Implementation + +import slayer.cli +from slayer import usage_reporting +from slayer.api.server import create_app +from slayer.core.enums import BUILTIN_AGGREGATIONS +from slayer.core.formula import ALL_TRANSFORMS +from slayer.flight.translator import TranslationError +from slayer.mcp.server import create_mcp_server +from slayer.storage.yaml_storage import YAMLStorage +from slayer.usage_reporting.payload import SCHEMA_VERSION, VERSION_PATTERN, UsageReport +from tests._usage_reporting_helpers import ( + HOSTILE, + SENT, + assert_clean_everywhere, + assert_no_sentinel, + build_sentinel_storage, + cli, + end_process, + enum_values, + make_due, + quiet_config, + show, + usage, + usage_key_vocabulary, +) +from tests.flight.test_handlers import _make_handlers +from tests.pg_facade.test_connection import _parse, _bind, _execute, _query, _run, _startup, _sync, _terminate + +# --- the schema policy ---------------------------------------------------------------- + + +def _resolve(schema: dict, node: dict) -> dict: + while "$ref" in node: + node = schema["$defs"][node["$ref"].rsplit("/", 1)[-1]] + return node + + +def _check_leaf(schema: dict, node: dict, path: str) -> None: + node = _resolve(schema, node) + if "enum" in node or "const" in node: + return + branches = node.get("anyOf") or node.get("oneOf") + if branches: + for i, branch in enumerate(branches): + _check_leaf(schema, branch, f"{path}|{i}") + return + kind = node.get("type") + if kind in ("null", "boolean"): + return + if kind == "integer": + assert node.get("minimum", -1) >= 0, f"{path}: integer may be negative" + return + if kind == "string": + allowed = node.get("format") in ("date", "uuid") or node.get("pattern") == VERSION_PATTERN + assert allowed, f"{path}: free-form string" + return + if kind == "array": + _check_leaf(schema, node["items"], f"{path}[]") + return + if kind == "object": + for name, sub in node.get("properties", {}).items(): + _check_leaf(schema, sub, f"{path}.{name}") + extra = node.get("additionalProperties") + assert extra is not True, f"{path}: open object" + if isinstance(extra, dict): + assert enum_values(schema, node.get("propertyNames", {})), f"{path}: dict keys are not vocabulary" + _check_leaf(schema, extra, f"{path}[*]") + return + raise AssertionError(f"{path}: disallowed type {kind!r}") + + +def test_every_report_leaf_is_vocabulary() -> None: + schema = UsageReport.model_json_schema() + _check_leaf(schema, schema, "report") + + +@pytest.mark.parametrize("value", ["1.1.2", "3.12", "0.0.0+unknown", "1.2.0rc1", "1.1.2.dev3+g1a2b3c"]) +def test_version_pattern_accepts_versions(value: str) -> None: + assert re.fullmatch(VERSION_PATTERN, value) + + +@pytest.mark.parametrize("value", [SENT, HOSTILE, "1.2.3 extra", "x1.2", "1.2;DROP", "", "../1.2"]) +def test_version_pattern_rejects_free_text(value: str) -> None: + assert not re.fullmatch(VERSION_PATTERN, value) + + +# --- vocabulary covers every surface's fixed tokens ---------------------------------------- + + +class _Captured(Exception): + def __init__(self, parser: argparse.ArgumentParser) -> None: + super().__init__() + self.parser = parser + + +def _cli_parser(monkeypatch: pytest.MonkeyPatch) -> argparse.ArgumentParser: + def capture(self: argparse.ArgumentParser, *args, **kwargs): + raise _Captured(self) + + monkeypatch.setattr(argparse.ArgumentParser, "parse_args", capture) + with pytest.raises(_Captured) as info: + slayer.cli.main() + return info.value.parser + + +def _sub_parsers(parser: argparse.ArgumentParser, path: tuple[str, ...] = ()): + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + for name, sub in action.choices.items(): + yield path + (name,), sub + yield from _sub_parsers(sub, path + (name,)) + + +def _runs_bare(parser: argparse.ArgumentParser) -> bool: + """A leaf, or a group that runs without a subcommand (``slayer search``).""" + has_subcommands = any(isinstance(a, argparse._SubParsersAction) for a in parser._actions) + return not has_subcommands or parser.prog.endswith(" search") + + +def test_every_cli_command_has_a_vocabulary_token(monkeypatch: pytest.MonkeyPatch) -> None: + vocabulary = usage_key_vocabulary() + tokens = {} + for path, parser in _sub_parsers(_cli_parser(monkeypatch)): + if not _runs_bare(parser) or path[0] == "usage-reporting": + continue + token = parser.get_default("usage_token") + assert isinstance(token, str), f"slayer {' '.join(path)} has no usage token" + assert token + assert f"cli:{token}" in vocabulary, token + tokens[path] = token + assert len(set(tokens.values())) == len(tokens) + assert {("search",), ("search", "refresh-samples")} <= set(tokens) + + +def test_every_mcp_tool_has_a_vocabulary_token(tmp_path: Path) -> None: + server = create_mcp_server(storage=YAMLStorage(base_dir=str(tmp_path)), _seed_help=False) + names = {tool.name for tool in asyncio.run(server.list_tools())} + vocabulary = usage_key_vocabulary() + assert names + assert {f"mcp:{name}" for name in names} <= vocabulary + assert "mcp:other" in vocabulary + + +def test_every_rest_route_has_a_vocabulary_token(tmp_path: Path) -> None: + app = create_app(storage=YAMLStorage(base_dir=str(tmp_path))) + paths = {route.path for route in app.routes if isinstance(route, APIRoute)} + assert {f"rest:{path}" for path in paths} <= usage_key_vocabulary() + + +def test_protocol_tokens_in_vocabulary() -> None: + assert {"flight:query", "pg:query"} <= usage_key_vocabulary() + + +def test_feature_vocabulary_covers_builtins() -> None: + schema = UsageReport.model_json_schema() + features = _resolve(schema, schema["properties"]["features"]) + transforms = _resolve(schema, features["properties"]["transforms"]) + aggregations = _resolve(schema, features["properties"]["aggregations"]) + assert set(ALL_TRANSFORMS) | {"custom"} <= enum_values(schema, transforms["propertyNames"]) + assert set(BUILTIN_AGGREGATIONS) | {"custom"} <= enum_values(schema, aggregations["propertyNames"]) + + +# --- sentinel fuzz: hostile user strings never leave ---------------------------------------- + + +@pytest.fixture +def sentinel_store(usage_reporting_env) -> str: + quiet_config() + return build_sentinel_storage(usage_reporting_env.root) + + +def test_cli_arguments_and_names_never_leave(sentinel_store: str, usage_reporting_env) -> None: + query = { + "source_model": f"{SENT}_model", + "dimensions": [{"expression": f"{SENT}_col * 2", "name": SENT}], + "measures": [f"{SENT}_agg({SENT}_col)", {"formula": f"sum({SENT}_col)", "name": f"{SENT}_m"}], + "filters": [f"{SENT}_col > 0 or '{HOSTILE}' == 'x'"], + } + runs = [ + ["query", json.dumps(query), "--storage", sentinel_store], + ["query", json.dumps({**query, "source_model": HOSTILE}), "--storage", sentinel_store], + ["query", HOSTILE, "--storage", sentinel_store], + ["models", "--storage", sentinel_store, "show", HOSTILE], + ["models", "--storage", sentinel_store, "list"], + ["datasources", "--storage", sentinel_store, "list"], + ["datasources", "--storage", sentinel_store, "show", f"{SENT}_ds"], + ["memory", "--storage", sentinel_store, "save", "--learning", HOSTILE, "--entities", f"{SENT}_ds.{SENT}_model"], + ["inspect", f"{SENT}_model", "--type", "model", "--storage", sentinel_store], + ["search", "--question", HOSTILE, "--storage", sentinel_store], + ["recommend-root-model", f"{SENT}_model.{SENT}_col", "--storage", sentinel_store], + ] + for args in runs: + cli(args) + assert_clean_everywhere() + + +async def test_mcp_arguments_errors_and_client_never_leave(sentinel_store: str) -> None: + usage_reporting.start() + server = create_mcp_server(storage=YAMLStorage(base_dir=sentinel_store), _seed_help=False) + client = Implementation(name=HOSTILE, version=HOSTILE) + async with create_connected_server_and_client_session(server, client_info=client) as session: + await session.call_tool("query", {"query": { + "source_model": f"{SENT}_model", + "measures": [f"{SENT}_agg({SENT}_col)"], + "filters": [f"'{HOSTILE}' == '{HOSTILE}'"], + }}) + await session.call_tool("inspect_model", {"model_name": HOSTILE}) + await session.call_tool("save_memory", {"learning": HOSTILE, "linked_entities": [f"{SENT}_ds.{SENT}_model"]}) + await session.call_tool("search", {"question": HOSTILE}) + await session.call_tool("describe_datasource", {"name": HOSTILE}) + await session.call_tool(SENT, {"x": HOSTILE}) + end_process() + assert_clean_everywhere() + + +def test_rest_paths_bodies_and_errors_never_leave(sentinel_store: str) -> None: + usage_reporting.start() + client = TestClient(create_app(storage=YAMLStorage(base_dir=sentinel_store))) + client.get(f"/models/{SENT}_model") + client.get(f"/models/{SENT}missing") + client.get(f"/{SENT}/unmatched") + client.post("/query", json={"source_model": f"{SENT}_model", "measures": [f"{SENT}_agg({SENT}_col)"]}) + client.post("/query", json={"source_model": HOSTILE, "measures": ["count(*)"]}) + client.post(f"/mcp/messages/?session_id={SENT}", json={"x": HOSTILE}) + end_process() + assert_clean_everywhere() + + +class _HostileError(Exception): + pass + + +class _HostileEngine: + async def execute(self, *, query=None, data_source=None): # NOSONAR(S7503) — awaited interface + raise _HostileError(f"SELECT {HOSTILE} FROM {SENT}_table") + + +async def test_pg_sql_and_engine_errors_never_leave(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + await _run( + _startup(user=SENT, database=SENT) + + _query(f"SELECT revenue_sum FROM orders WHERE status = '{HOSTILE}'") + + _query(f"SELECT {SENT} FROM {SENT}") + + _parse("", f"SELECT revenue_sum FROM orders WHERE status = '{SENT}'") + + _bind("", "") + + _execute("") + + _sync() + + _terminate(), + engine=_HostileEngine(), + ) + end_process() + assert_clean_everywhere() + + +def test_flight_sql_never_leaves(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + handlers = _make_handlers() + handlers.do_get_for_sql(f"SELECT revenue_sum FROM jaffle.orders WHERE status = '{HOSTILE.replace(chr(39), chr(39) * 2)}'") + with pytest.raises(TranslationError): + handlers.do_get_for_sql(f"SELECT {SENT} FROM {SENT}") + end_process() + assert_clean_everywhere() + + +def test_sent_report_never_carries_a_sentinel(sentinel_store: str, usage_reporting_env) -> None: + cli(["query", json.dumps({"source_model": HOSTILE, "measures": ["count(*)"]}), "--storage", sentinel_store]) + cli(["models", "--storage", sentinel_store, "show", HOSTILE]) + make_due() + cli(["models", "--storage", sentinel_store, "list"]) + requests = usage_reporting_env.capture.wait_for(count=1) + assert len(requests) == 1 + assert_no_sentinel(requests[0].body) + + +# --- error tokens -------------------------------------------------------------------------- + + +class _UnlistedError(Exception): + pass + + +def test_unknown_exception_class_counts_as_other(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query", error=_UnlistedError("boom")) + end_process() + assert usage(show(), "mcp:query")["errors"] == {"other": 1} + + +def test_error_message_is_never_sent(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query", error=ValueError(f"no such table {SENT}_table: SELECT {HOSTILE}")) + end_process() + report = show() + entry = usage(report, "mcp:query") + assert entry["ok"] == 0 + assert sum(entry["errors"].values()) == 1 + assert_no_sentinel(json.dumps(report)) + + +# --- identity and time resolution ---------------------------------------------------------------- + + +def test_report_identity_fields(usage_reporting_env) -> None: + quiet_config() + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + end_process() + report = show() + assert report["schema"] == SCHEMA_VERSION + assert report["install_id"] + assert report["batch_id"] + assert report["install_id"] != report["batch_id"] + assert show()["batch_id"] != report["batch_id"] + + +def test_period_holds_dates_only(usage_reporting_env) -> None: + day_one = datetime.datetime(2026, 3, 1, 9, 30, tzinfo=datetime.timezone.utc) + day_two = datetime.datetime(2026, 3, 2, 8, 45, tzinfo=datetime.timezone.utc) + quiet_config(last_sent=day_one) + for day in (day_one, day_two): + usage_reporting_env.set_now(day) + usage_reporting.start() + usage_reporting.record(surface="mcp", token="query") + end_process() + report = show() + assert report["period"] == {"first": "2026-03-01", "last": "2026-03-02"} + assert not re.search(r"\d{2}:\d{2}", json.dumps(report)) diff --git a/tests/test_usage_reporting_retention.py b/tests/test_usage_reporting_retention.py new file mode 100644 index 00000000..313b3860 --- /dev/null +++ b/tests/test_usage_reporting_retention.py @@ -0,0 +1,158 @@ +"""The PostHog retention sweep: expired installs (or one requested install) deleted with their events.""" + +from __future__ import annotations + +import email.message +import io +import re +import urllib.error +import uuid + +import pytest + +from scripts import usage_reporting_retention as retention +from slayer.usage_reporting import settings + +_ENV = {"POSTHOG_PERSONAL_API_KEY": "phx_test"} + + +class _FakePostHog: + def __init__(self, *, expired: list[str] | None = None, errors: list[dict] | None = None) -> None: + self.calls: list[tuple[str, dict]] = [] + self.expired = sorted(expired or []) + self.errors = errors or [] + + def __call__(self, path: str, body: dict) -> dict: + self.calls.append((path, body)) + if path == "query/": + sql = body["query"]["query"] + after = re.search(r"distinct_id > '([^']*)'", sql) + limit = re.search(r"LIMIT (\d+)$", sql) + assert after is not None + assert limit is not None + page = [i for i in self.expired if i > after.group(1)][:int(limit.group(1))] + return {"results": [[install_id] for install_id in page]} + return {"persons_found": len(body["distinct_ids"]), "deletion_errors": self.errors} + + +class _Response: + def __enter__(self) -> _Response: + return self + + def __exit__(self, *exc: object) -> bool: + return False + + def read(self) -> bytes: + return b'{"results": []}' + + +def test_lifetime_matches_install_id_rotation() -> None: + assert retention.ID_LIFETIME_MONTHS == settings._ID_LIFETIME_MONTHS + + +def test_sweep_deletes_installs_past_lifetime_plus_grace() -> None: + ids = [str(uuid.uuid4()) for _ in range(2500)] + posthog = _FakePostHog(expired=ids) + assert retention.main(env=_ENV, post=posthog) == 0 + (query_path, query_body), *deletes = posthog.calls + assert query_path == "query/" + sql = query_body["query"]["query"] + assert "event = 'slayer_usage'" in sql + assert "min(timestamp) < now() - INTERVAL 11 MONTH - INTERVAL 7 DAY" in sql + assert "OFFSET" not in sql + assert [path for path, _ in deletes] == ["persons/bulk_delete/"] * 3 + assert [len(body["distinct_ids"]) for _, body in deletes] == [1000, 1000, 500] + assert all(body["delete_events"] is True for _, body in deletes) + assert [i for _, body in deletes for i in body["distinct_ids"]] == sorted(ids) + + +def test_sweep_pages_through_every_expired_install(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(retention, "QUERY_PAGE", 4) + ids = [str(uuid.uuid4()) for _ in range(10)] + posthog = _FakePostHog(expired=ids) + assert retention.main(env=_ENV, post=posthog) == 0 + queries = [body["query"]["query"] for path, body in posthog.calls if path == "query/"] + ordered = sorted(ids) + assert [re.findall(r"distinct_id > '([^']*)'", q) for q in queries] == [[""], [ordered[3]], [ordered[7]]] + assert all("ORDER BY distinct_id" in q for q in queries) + deleted = [i for path, body in posthog.calls if path != "query/" for i in body["distinct_ids"]] + assert deleted == ordered + + +def test_sweep_refuses_a_non_install_id_from_posthog() -> None: + def posthog(path: str, body: dict) -> dict: + return {"results": [["x' OR 1=1 --"]]} + + with pytest.raises(ValueError, match="unexpected distinct_id"): + retention.expired_install_ids(posthog) + + +def test_unexpected_delete_response_fails_the_run() -> None: + def posthog(path: str, body: dict) -> dict: + return {"results": [[str(uuid.uuid4())]]} if path == "query/" else {"detail": "something else"} + + assert retention.main(env=_ENV, post=posthog) == 1 + + +def test_nothing_expired_deletes_nothing() -> None: + posthog = _FakePostHog() + assert retention.main(env=_ENV, post=posthog) == 0 + assert [path for path, _ in posthog.calls] == ["query/"] + + +def test_run_reports_how_many_requested_installs_were_found(capsys) -> None: + def posthog(path: str, body: dict) -> dict: + return {"persons_found": 0, "deletion_errors": []} + + assert retention.main(env={**_ENV, "INSTALL_ID": str(uuid.uuid4())}, post=posthog) == 0 + assert "1 install(s) requested, 0 found" in capsys.readouterr().out + + +def test_requested_install_deleted_without_sweep() -> None: + install_id = str(uuid.uuid4()) + posthog = _FakePostHog(expired=[str(uuid.uuid4())]) + assert retention.main(env={**_ENV, "INSTALL_ID": f" {install_id} "}, post=posthog) == 0 + assert posthog.calls == [("persons/bulk_delete/", {"distinct_ids": [install_id], "delete_events": True})] + + +def test_malformed_install_id_rejected() -> None: + posthog = _FakePostHog() + assert retention.main(env={**_ENV, "INSTALL_ID": "x' OR 1=1"}, post=posthog) == 2 + assert posthog.calls == [] + + +def test_deletion_errors_fail_the_run() -> None: + posthog = _FakePostHog(expired=[str(uuid.uuid4())], errors=[{"person_uuid": "p", "step": "events"}]) + assert retention.main(env=_ENV, post=posthog) == 1 + + +def test_missing_api_key_fails_without_requests() -> None: + posthog = _FakePostHog() + assert retention.main(env={}, post=posthog) == 2 + assert posthog.calls == [] + + +def test_http_error_reports_status_and_posthog_detail(monkeypatch, capsys) -> None: + def refuse(request, timeout: float) -> _Response: + raise urllib.error.HTTPError(request.full_url, 403, "Forbidden", email.message.Message(), + io.BytesIO(b'{"detail": "API key missing required scope \'query:read\'"}')) + + monkeypatch.setattr(retention.urllib.request, "urlopen", refuse) + assert retention.main(env=_ENV) == 1 + err = capsys.readouterr().err + assert "403" in err + assert "query:read" in err + + +def test_requests_go_to_the_eu_project_with_the_personal_key(monkeypatch) -> None: + seen = [] + + def fake_urlopen(request, timeout: float) -> _Response: + seen.append(request) + return _Response() + + monkeypatch.setattr(retention.urllib.request, "urlopen", fake_urlopen) + assert retention.main(env=_ENV) == 0 + (request,) = seen + assert request.full_url == "https://eu.posthog.com/api/projects/308099/query/" + assert request.get_header("Authorization") == "Bearer phx_test" diff --git a/tests/test_usage_reporting_scope.py b/tests/test_usage_reporting_scope.py new file mode 100644 index 00000000..7f608e11 --- /dev/null +++ b/tests/test_usage_reporting_scope.py @@ -0,0 +1,96 @@ +"""Usage reporting is active only in processes started by the slayer CLI.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from slayer import usage_reporting +from slayer.usage_reporting import settings +from tests._usage_reporting_helpers import ( + SUBPROCESS_TIMEOUT, + build_storage, + cli, + quiet_config, + run_python, + run_slayer, + show, + usage, +) + +_NOTICE_URL = "docs.motley.ai/slayer/reference/usage-reporting" +_QUERY = {"source_model": "orders", "measures": ["sum(amount)"]} + +_LIBRARY_SCRIPT = """ +import asyncio +from fastapi.testclient import TestClient +from slayer.api.server import create_app +from slayer.client.slayer_client import SlayerClient +from slayer.engine.query_engine import SlayerQueryEngine +from slayer.mcp.server import create_mcp_server +from slayer.storage.yaml_storage import YAMLStorage + +query = {query!r} +storage = YAMLStorage(base_dir={store!r}) +SlayerQueryEngine(storage=storage).execute_sync(query=query) +client = TestClient(create_app(storage=storage)) +assert client.get("/health").status_code == 200 +assert client.post("/query", json=query).status_code == 200 +asyncio.run(create_mcp_server(storage=storage).call_tool("query", {{"query": query}})) +SlayerClient(storage=storage).query_sync(query) +print("done") +""" + + +@pytest.fixture +def store(usage_reporting_env) -> str: + return build_storage(usage_reporting_env.root) + + +def test_library_use_records_nothing(store: str, usage_reporting_env) -> None: + result = run_python(_LIBRARY_SCRIPT.format(query=_QUERY, store=store), env=usage_reporting_env.subprocess_env()) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "done" + assert not settings.config_dir().exists() + assert _NOTICE_URL not in result.stderr + time.sleep(0.5) + assert usage_reporting_env.capture.arrivals == 0 + + +def test_record_without_start_is_a_no_op(usage_reporting_env) -> None: + usage_reporting.record(surface="mcp", token="query") + usage_reporting.flush() + assert not usage_reporting.is_active() + assert not settings.config_dir().exists() + + +def test_cli_query_records(store: str, usage_reporting_env) -> None: + quiet_config() + result = cli(["query", json.dumps(_QUERY), "--storage", store]) + assert result.returncode == 0, result.stderr + assert usage(show(), "cli:query") == {"ok": 1, "errors": {}} + + +def test_python_m_slayer_records(store: str, usage_reporting_env) -> None: + quiet_config() + result = run_slayer(["models", "--storage", store, "list"], env=usage_reporting_env.subprocess_env()) + assert result.returncode == 0, result.stderr + assert usage(show(), "cli:models.list")["ok"] == 1 + + +def test_console_script_records(store: str, usage_reporting_env) -> None: + script = Path(sys.executable).parent / "slayer" + if not script.exists(): + pytest.skip("no slayer console script next to the interpreter") + quiet_config() + result = subprocess.run( + [str(script), "models", "--storage", store, "list"], + env=usage_reporting_env.subprocess_env(), capture_output=True, text=True, timeout=SUBPROCESS_TIMEOUT, + ) + assert result.returncode == 0, result.stderr + assert usage(show(), "cli:models.list")["ok"] == 1 diff --git a/tests/test_usage_reporting_vocabulary.py b/tests/test_usage_reporting_vocabulary.py new file mode 100644 index 00000000..90f681f6 --- /dev/null +++ b/tests/test_usage_reporting_vocabulary.py @@ -0,0 +1,82 @@ +"""The usage reporter's fixed lookup tables stay within the report vocabulary and name real things.""" + +from __future__ import annotations + +import importlib +from importlib.metadata import distribution +from typing import get_args + +import pytest + +from slayer.core.models import DatasourceConfig +from slayer.sql.dialects import SQLGLOT_NAMES +from slayer.usage_reporting import features, recorder +from slayer.usage_reporting.payload import CLIENTS, DIALECTS, EXTRAS, ErrorToken + + +@pytest.mark.parametrize("dotted", sorted(recorder._ERROR_TOKENS)) +def test_error_table_names_real_exception_classes(dotted: str) -> None: + module, _, name = dotted.rpartition(".") + cls = getattr(importlib.import_module(module), name) + assert issubclass(cls, BaseException) + assert recorder._class_token(cls, recorder._ERROR_TOKENS) == recorder._ERROR_TOKENS[dotted] + + +@pytest.mark.parametrize("dotted", sorted(recorder._STORAGE_TOKENS)) +def test_storage_table_names_real_classes(dotted: str) -> None: + module, _, name = dotted.rpartition(".") + assert isinstance(getattr(importlib.import_module(module), name), type) + + +def test_error_tokens_are_vocabulary() -> None: + assert set(recorder._ERROR_TOKENS.values()) | {"http_4xx", "http_5xx", "other"} <= set(get_args(ErrorToken)) + + +@pytest.mark.parametrize(("status", "token"), [(404, "http_4xx"), (422, "http_4xx"), (500, "http_5xx")]) +def test_http_errors_count_by_status_class(status: int, token: str) -> None: + assert recorder.error_token(recorder.HttpError(status)) == token + + +def test_client_table_maps_into_vocabulary() -> None: + assert set(recorder._CLIENT_NAMES.values()) <= CLIENTS + + +def test_every_package_extra_is_vocabulary() -> None: + declared = set(distribution("motley-slayer").metadata.get_all("Provides-Extra") or []) + assert declared - {"all", "embedding-search"} <= set(EXTRAS) + + +def test_every_dialect_is_vocabulary() -> None: + assert set(SQLGLOT_NAMES) <= DIALECTS + + +@pytest.mark.parametrize(("ds_type", "dialect"), [ + ("sqlite", "sqlite"), ("postgres", "postgres"), ("postgresql", "postgres"), + ("duckdb", "duckdb"), ("no-such-db", "other"), (None, "other"), +]) +def test_dialect_of_datasource_type(ds_type: str | None, dialect: str) -> None: + assert features.dialect(DatasourceConfig(name="d", type=ds_type)) == dialect + + +@pytest.mark.parametrize(("database", "demo"), [ + ("/srv/store/demo/jaffle_shop.duckdb", True), + ("/srv/store/jaffle_shop.duckdb", False), + ("/srv/store/demo/other.duckdb", False), +]) +def test_demo_database_detected_by_bundled_layout(database: str, demo: bool) -> None: + assert features.is_demo(DatasourceConfig(name="d", type="duckdb", database=database)) is demo + + +@pytest.mark.parametrize(("formula", "transforms", "aggregations"), [ + ("count(*)", [], ["count"]), + ("amount:sum", [], ["sum"]), + ("last(sum(amount))", ["last"], ["sum"]), + ("last(amount, ordered_at)", [], ["last"]), + ("cumsum(sum(amount)) / count(*)", ["cumsum"], ["sum", "count"]), + ("sum(iif(status = 'x(', 1, 0))", [], ["sum"]), + ("my_agg(amount)", [], ["custom"]), + ("sum(", [], []), +]) +def test_function_names_classified(formula: str, transforms: list[str], aggregations: list[str]) -> None: + found = features.query_features({"source_model": "m", "measures": [formula]}) + assert (sorted(found.transforms), sorted(found.aggregations)) == (sorted(transforms), sorted(aggregations)) diff --git a/zensical.toml b/zensical.toml index 74eecbcd..4c6d8150 100644 --- a/zensical.toml +++ b/zensical.toml @@ -43,6 +43,7 @@ nav = [ { "REST API" = "reference/rest-api.md" }, { "Python Client" = "reference/python-client.md" }, { "CLI" = "reference/cli.md" }, + { "Usage reporting" = "reference/usage-reporting.md" }, { "Database Support" = "database-support.md" }, ]}, { "Wire Protocols (BI Tools)" = [