diff --git a/.gitignore b/.gitignore index 02336c54..4935c519 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,8 @@ __pycache__/ node_modules +graphify-out + # Local scratch task notes — never tracked task.md TASK.md diff --git a/README.md b/README.md index 4570fcab..2a66d1dc 100644 --- a/README.md +++ b/README.md @@ -220,11 +220,11 @@ link for the current keg, a node (`42`, `@acme/engineering/42`), a flight browser. Agents get the same link from the MCP `url` tool. `tap explore kegs|flights|people QUERY` searches the hub's public directory; your own kegs come from `tap keg list`. -Export a keg for backup or migration: +Export a keg for backup or migration, then restore it on the keg's hub (see +[Backups And Archives](docs/backups-and-archives.md#restore)): ```bash tap archive export --keg @acme/engineering -o engineering.keg.tar.gz -tap archive import engineering.keg.tar.gz --keg @acme/engineering ``` ## Concepts diff --git a/docs/README.md b/docs/README.md index 60b59f4a..a117e03c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -113,8 +113,9 @@ writing KEG files directly. `-exclusions`. Each hit prints reference, score, title, and a highlighted snippet; `--limit` caps a page (the hub chooses by default), the next page's cursor is printed to stderr for `--cursor`, `--id-only` prints references, - and `--json` prints the hub's response. MCP's `node_search` takes the same - parameters; under a flight it never searches outside the cover. + and `--json` prints the hub's response. MCP's `node_search` takes `kegs`, + `cursor`, and `limit`: an empty `kegs` searches the call's whole flight + cover, and naming a KEG outside the cover is an error that names it. - `tap node grep PATTERN` - match node content lines in one keg against a regex (MCP `node_grep`), for exact identifiers and audits. - `tap tag list [EXPR]` - list tags or query tagged nodes. @@ -159,7 +160,7 @@ writing KEG files directly. - `tap snapshot list NODE_ID` - list node revisions. - `tap snapshot restore NODE_ID REV --yes` - restore a revision. - `tap archive export -o out.keg.tar.gz` - export a keg archive. -- `tap archive import out.keg.tar.gz` - import a keg archive. +- Restore an archive on the keg's hub; see [Backups And Archives](backups-and-archives.md#restore). - `tap keg check` - check keg health. - `tap index rebuild` - rebuild indexes. diff --git a/docs/ai-coding-agents/README.md b/docs/ai-coding-agents/README.md index 720d53df..70b9cdd1 100644 --- a/docs/ai-coding-agents/README.md +++ b/docs/ai-coding-agents/README.md @@ -31,7 +31,7 @@ native Tapper plugin. - [Relay](relay.md) — experimental `tap relay`, which offers this machine's model providers, such as a local Ollama, to your Hub account. - [Agent Conventions](agent-conventions.md) — tapper invariants every agent - should follow: MCP-first, never edit node files directly, never mix CLI + should follow: MCP-first, never bypass the tools, never mix CLI writes with a live MCP session, snapshot before destructive edits. ## Quick pick diff --git a/docs/ai-coding-agents/agent-conventions.md b/docs/ai-coding-agents/agent-conventions.md index 59d5ee06..b3299806 100644 --- a/docs/ai-coding-agents/agent-conventions.md +++ b/docs/ai-coding-agents/agent-conventions.md @@ -23,20 +23,14 @@ The CLI remains the right tool for interactive editor sessions, diagnostics that have no MCP equivalent, and shell-integration tasks. See the safety rules in the bundled [tapper skill](../../integrations/rendered/claude/tapper/skills/tapper/SKILL.md). -## Never read or write node files directly +## Never bypass the tools -A tapper node is a numbered directory: - -``` -/42/ - README.md # node content - meta.yaml # user-facing metadata - stats.json # programmatic stats: hash, timestamps, access count -``` - -These files are tapper's internal storage format, not an external API. -Reading them bypasses the index. Writing them bypasses per-node -locking and snapshot history. +Every KEG is hosted on a Tapper Hub, so there are no node files on the +agent's machine to read or edit. A node's content (markdown), metadata +(YAML), and stats (hash, timestamps, access count) are reached only through +the Hub, which enforces access, per-node locking, indexing, and snapshot +history on every call. An archive or export written to disk is a copy, not +the live KEG; editing it changes nothing. Always route through tapper's interfaces: @@ -45,7 +39,17 @@ Always route through tapper's interfaces: - `mcp__tapper__node_edit` to write a node — `content` is the markdown body and `meta` is the metadata document. They are separate fields, so `content` must not carry a frontmatter block; send either one alone to - leave the other untouched, or both to replace the whole node. + leave the other untouched, or both to replace the whole node. Each part + needs its token from `node_read`: `content_hash` as + `expected_content_hash`, `meta_hash` as `expected_meta_hash`. A refusal for + a missing token names the exact `node_read` call to make; make it rather + than resending the same arguments. + `schema` is a type name from `schema_list`, such as `task`, never a + description of the note. + +A tool error's `action` is the next call to make. A failure carrying +`repeat_count` means the same call has already failed that many times; do not +send it again unchanged. ## CLI and MCP share Hub concurrency rules diff --git a/docs/ai-coding-agents/mcp-setup.md b/docs/ai-coding-agents/mcp-setup.md index d102de2e..395f2ea6 100644 --- a/docs/ai-coding-agents/mcp-setup.md +++ b/docs/ai-coding-agents/mcp-setup.md @@ -3,7 +3,7 @@ The `tap mcp` command starts a Model Context Protocol server on stdio, exposing the same agent-safe tools and resources as Tapper Hub's authenticated `/mcp` endpoint — the one difference being attachment transfers, where `tap mcp` can -also read and write local paths because it runs on your machine. Both publish +also transfer files inside its working directory because it runs on your machine. Both publish connection-pinned authority at initialization. Orientation is a read-only view; `session_refresh` retries only a broken explicit selection. Without a flight, the server uses normal identity-authorized full access for the connection @@ -92,14 +92,98 @@ least-privilege flight, pin it with `tap use --flight @namespace/+slug` (or narrow the existing connection. Seeing only the six recovery tools means an explicitly configured root failed to initialize. +### Tool Schemas + +Each tool description opens with its category (`Session:`, `Memory:`, +`Explore:`, or `Manage:`). Every authority-bearing tool also takes an +optional `flight` argument, added to its input schema; the property's own +description is the one per-tool explanation of it ("Flight to run this call +under, such as +work or @ns/+work. Omit it to use the connection's flight."), +and the operating rules explain it once per session. Tool descriptions do not +repeat it. + +Every tool declares an `outputSchema` for its `structuredContent`: the paging +fields of the listing tools (`lines`, `offset`, `limit`, `next_offset`, +`has_more`, `pagination`), `node_read`'s `nodes`, `node_search`'s `hits` and +`next_cursor`, the write tools' new hashes, and so on. A tool whose result is +only text declares `{message}`, the object every result carries its text in. +The schemas leave additional properties open, so a newer server's extra field +never fails a client that validates results. + +An `outputSchema` describes a tool's success. A failed call (`isError: true`) +carries no `structuredContent`: its payload (`code`, `message`, `action`, +`operationPerformed`, and any error-specific fields such as `currentHash` or +`missing`) is the result's first text block, as a JSON object. Clients that +validate `structuredContent` against the schema even on errors, as the +TypeScript SDK does, therefore see the tool's error rather than a JSON-RPC +`-32602`. Relayed tools follow the same rule. + +### Failure Results + +Every failed tool call has one shape, whichever layer refused it (input +validation, flight authority, the tool, or the Hub). The result has +`isError: true`, no `structuredContent`, and a first text block holding one +JSON object. Non-text attachments, if any, follow that block; no second text +block repeats the message. + +```json +{ + "code": "CONFLICT", + "message": "flight @me/+work already exists\n\nThe request conflicts with current state. ...", + "action": "The request conflicts with current state. ...", + "operationPerformed": false, + "retryAfterSeconds": 2, + "repeat_count": 3 +} +``` + +| Field | Type | Presence | Meaning | +| --- | --- | --- | --- | +| `code` | string | always | Wire code: the Hub REST code for the same failure (`INVALID`, `NOT_FOUND`, `CONFLICT`, `RATE_LIMITED`, `ORIENTATION_DENIED`, ...), `INVALID_ARGUMENT` for input-schema validation, or `INTERNAL` when the cause is unknown. | +| `message` | string | always | The error text followed by the action, for a reader that shows only this field. | +| `action` | string | always | The next step, naming the tool or field to change. It never restates the error. | +| `operationPerformed` | `false`, `true`, or `null` | always | `false`: refused before anything was written. `true`: it ran, as `action` describes. `null`: unknown, reported only for `INTERNAL` (or a code this version does not know); read the target before retrying. | +| `retryAfterSeconds` | integer | rate limits only | Whole seconds to wait before retrying, rounded up, when the Hub sent one. | +| `repeat_count` | integer | third identical failure on | How many times in a row this session sent this exact call and it failed. `action` then starts with an instruction to stop. | + +Some codes add their own fields: `reorientRequired` for authority and +orientation failures, `currentHash`, `currentContentHash`, `currentMetaHash`, +and `currentContent` for a stale-hash `CONFLICT`, `node_id` and `missing` for +`PRECONDITION_REQUIRED`, and `kegv` for a KEG created at another version. +Clients ignore fields they do not know. + +Input the caller can fix (a malformed query expression, node id, settings +document, base64 payload, or guide topic) is `INVALID` with +`operationPerformed: false`. A failure in stored state the caller did not send, +such as a KEG settings index query saved before edits were validated, stays +`INTERNAL`. A Hub `CONFLICT`, such as a flight or agent name already taken, is +a refusal with `operationPerformed: false`. A KEG tool called while the +session's configured flight could not be activated is `ORIENTATION_DENIED`, and +its action says to repair the selection and call `session_refresh`, since +naming another flight per call cannot unlock it. + +A host that wraps this server, such as a Hub serving MCP, reports its own +classification by setting these fields on the same object, keeping +`repeat_count` and `message`, rather than adding a second block. + +Two arguments left the schemas because another one says the same thing. +Calls that still send them keep working; neither is advertised again: + +- `id_only` on `node_list`, `node_grep`, `tag_list`, `node_backlinks`, and + `node_links` is `format: "%i"`, which prints the same lines. A true + `id_only` overrides any `format` sent with it, as it always did. The CLI + keeps `--id-only` as shorthand for `--format %i`. +- `keg` on `node_search` is searched as the one KEG named, bare names + included. New calls name KEGs in `kegs`. + ### Read | Tool | Description | | --- | --- | | `node_read` | Read content of one or more nodes | | `node_list` | List nodes with optional query filtering | -| `node_search` | Rank nodes by relevance to a query across every readable KEG, one `keg`, or listed `kegs`; pages with `next_cursor` | -| `node_grep` | Match node content lines against a regex in one KEG | +| `node_search` | Rank nodes by relevance to a query across every KEG the call's flight covers, or only the listed `kegs` (each inside that cover); pages with `next_cursor` | +| `node_grep` | Match node content lines against a PostgreSQL regular expression in one KEG | | `tag_list` | List tags or find nodes by query | | `node_backlinks` | Find nodes linking to a node | | `node_links` | List outgoing links from a node | @@ -112,8 +196,8 @@ explicitly configured root failed to initialize. | Tool | Description | | --- | --- | -| `node_create` | Atomically create 1–100 nodes from `nodes[]`, each a markdown `content` document plus an optional YAML `meta` document; unique keys support forward/backward `{{node:key}}` content references | -| `node_edit` | Read with `node_read`, then atomically replace `content`, `meta`, or both for 1–100 nodes from `nodes[]`; supplied `content` requires `expected_content_hash` (the read's `content_hash`) and supplied `meta` requires `expected_meta_hash` (its `meta_hash`); editing one part never invalidates the other's hash | +| `node_create` | Atomically create 1–100 nodes from `nodes[]`, each a markdown `content` document plus an optional YAML `meta` document; unique keys support forward/backward `{{node:key}}` content references, each replaced with the bare id, so a link is `[Title](../{{node:key}})` (a placeholder that is a whole link target gets the `../` added) | +| `node_edit` | Read with `node_read`, then atomically replace `content` (or apply a `patch`), `meta`, or both for 1–100 nodes from `nodes[]`; supplied `content` or `patch` requires `expected_content_hash` (the read's `content_hash`) and supplied `meta` requires `expected_meta_hash` (its `meta_hash`); editing one part never invalidates the other's hash | | `node_delete` | Read with `node_read`, then atomically delete 1–100 `nodes[]`; every item requires its own `expected_hash` (the combined `hash` from `node_read`) | | `node_move` | Read with `node_read`, then move a node using its required `expected_hash` (the combined `hash` from `node_read`) | | `keg_settings_edit` | Read the full document with `keg_settings_read`, then replace it using its required `expected_hash`; requires admin flight authority and editor/admin KEG access | @@ -160,6 +244,32 @@ and, when practical, current content; node conflicts also return `currentContentHash` and `currentMetaHash`. Merge or refetch, then retry with the current hash for what you are writing. +A protected mutation sent without its token fails with `PRECONDITION_REQUIRED` +and writes nothing. The message names the missing field and the read that +fills it. For `node_edit`, `node_merge`'s target included, the error lists every +missing token at once in `missing`, carries the `node_id`, and its `action` is +the call to make, such as +`node_read(keg="notes", node_id="12")` then `content_hash as expected_content_hash`. +A `node_edit` item that supplies none of `content`, `patch`, or `meta` fails +with `INVALID` and an action naming the field to add; new nodes go through +`node_create`. + +`schema` on `node_create`, `node_edit`, and `node_merge` is a schema type name +from `schema_list`, such as `task`: one word of at most 64 characters, with no +spaces or path separators. The input schema carries that rule as a `pattern` +and `maxLength`, so a sentence is refused with `INVALID_ARGUMENT` before the +call runs, and every backend applies the same rule. A well-formed name the keg +does not define fails with `SCHEMA_INVALID`, and the message lists the types the +keg does define. + +A tool call that fails three times in a row with identical arguments in the +same session keeps its original `code` and gains `repeat_count` and an +`action` that says to stop retrying it, followed by the original action: read +the target or change the arguments the error names. Arguments are compared +as the client sent them, `flight` included, independent of key order and +whitespace: the same call under another flight is a different call. A success, +or a call with different arguments, starts that tool's count again. + ### Index, Diagnostics, And Safety | Tool | Description | @@ -167,7 +277,7 @@ the current hash for what you are writing. | `index_rebuild`, `index_list`, `index_read` | Rebuild or inspect indexes | | `keg_check` | Check only the selected keg's health (not local Tapper configuration) | | `snapshot_list`, `snapshot_create`, `snapshot_read`, `snapshot_restore` | Manage node snapshots; `snapshot_create` accepts 1–100 nodes atomically | -| `lock_acquire`, `lock_release`, `lock_status`, `lock_force_release` | Coordinate cross-process node locks | +| `lock_acquire`, `lock_release`, `lock_status`, `lock_force_release` | Coordinate Hub-held node locks; `lock_acquire` waits up to `wait_seconds` (default 10, max 60) for a held lock | ### Files And Images @@ -240,7 +350,7 @@ can read; an agent with no flight has no KEG access. | `flight_create`, `flight_edit`, `flight_delete` | Manage Hub-backed flights when the session agent holds the tool and the identity owns/administers the target namespace; edits/deletes require the hash from `flight_read` | MCP does not expose Tapper configuration, config templates, repository setup, -archive import/export, raw auth status, license text, keg visibility, or +archive export, raw auth status, license text, keg visibility, or namespace administration. Those remain external CLI, configuration, or Hub UI operations. To put a node's content in another KEG, read it with `node_read` and `node_create` it there. @@ -377,3 +487,19 @@ Device-login verification URLs must be HTTP(S) and have the same origin credentials in URLs, and cross-origin verification addresses are refused before the browser callback. Windows displays the URL for manual opening instead of passing it through `cmd.exe`. + +## Local attachment transfer boundary + +Local MCP `source_path` and `file:` upload sources must resolve to regular files +inside the server's working directory. Parent traversal and symlink escapes are +rejected. Download destinations must be explicit paths inside that directory +with an existing parent, and must not already exist; files are created with +exclusive creation and owner-only permissions. Environment expansion cannot +change a validated path on a later filesystem call. The local `image_download` +tool is marked as mutating because a destination writes a file. Hosted tools +continue to exchange attachment bytes without access to host paths. + +Start the MCP process in a project directory that contains only the files you +intend it to transfer. The boundary is its working directory, not the selected +KEG or a host-reported roots list. The CLI's explicitly requested local file +operations retain their existing path and overwrite behavior. diff --git a/docs/ai-coding-agents/orient.md b/docs/ai-coding-agents/orient.md index e42243ae..2b0ce6d6 100644 --- a/docs/ai-coding-agents/orient.md +++ b/docs/ai-coding-agents/orient.md @@ -22,6 +22,13 @@ The payload always starts with KEG system context: 5. Canonical Tapper agent guidance: linking, snapshots, tool inventory, and troubleshooting. +When `TAP_AGENT` names a Hub agent (a `tap launch --agent` session's +`tap mcp`, or `tap orient` run in that environment), the payload names the +agent and says its tool allowlist and flight govern the session. It never +claims the identity's full authority, even with no flight: an agent holds no +credentials of its own, and Hub narrows its requests to the flight `tap mcp` +sends. + Orient is best-effort. If a configured hub is unreachable or a selected flight cannot be resolved, the payload still renders the KEG system basics and includes a warning about what was skipped. diff --git a/docs/ai-coding-agents/relay.md b/docs/ai-coding-agents/relay.md index 1424f2e6..5d6d847a 100644 --- a/docs/ai-coding-agents/relay.md +++ b/docs/ai-coding-agents/relay.md @@ -29,6 +29,23 @@ wherever those commands are. - It runs in the foreground until interrupted and reconnects if the connection drops. Disconnecting the relay from a Hub's Account → Relay page stops it for that Hub; run it again to reconnect. +- On relay protocol 4, Hub tells the relay how long its login token has + left, and the relay refreshes the token and presents it on the live + connection (a `reauth` frame) at 80% of that time. The timing counts from + when the relay heard it, not from either machine's clock, so clock skew does + not matter, and requests in flight are not cut at the token's expiry. A Hub + without protocol 4 never sees a `reauth`. +- If the token does expire, Hub ends the connection with + `credential_expired`. The relay forces a refresh and reconnects at once; if + Hub refuses a dial's token, the relay forces one more refresh before giving + up. A failed refresh (the Hub unreachable, say) is retried with backoff. +- When Hub restarts it ends the connection with `going_away`, and the relay + reconnects within a second. +- A revoked sign-in is final: Hub ends the connection with + `credential_revoked` (`disconnected` before protocol 4), or refuses the + next dial, and the relay stops serving that Hub and says to run + `tap auth login`. An end code the relay does not know reconnects with + backoff. ## Several hubs diff --git a/docs/api-compatibility.md b/docs/api-compatibility.md index 43503ad7..707d0412 100644 --- a/docs/api-compatibility.md +++ b/docs/api-compatibility.md @@ -1,20 +1,32 @@ # REST API contract compatibility Tapper and Tapper Hub select a REST contract independently of their release -numbers. The sole supported revision is `2026-10-12`, which replaced `2026-09-28` when `POST /api/v1/namespaces/search` was removed in favor of `POST /api/v1/explore`, the public directory of KEGs, flights, and people. `2026-09-28` had replaced `2026-09-11` when grep, listing, and relationship responses gained server-side projection. +numbers. The current revision is `2026-10-17`, which replaced `2026-10-12` +when `POST /api/v1/@{ns}/kegs/{alias}/archive/restore` started answering +`202 Accepted` with a restore job and a `Location` to poll instead of +`200 {imported, removed}`. `2026-10-12` had replaced `2026-09-28` when `POST /api/v1/namespaces/search` was removed in favor of `POST /api/v1/explore`, the public directory of KEGs, flights, and people. `2026-09-28` had replaced `2026-09-11` when grep, listing, and relationship responses gained server-side projection. Change the revision for incompatible API behavior. Compatible additions and fixes retain the revision; release numbers remain diagnostic metadata. +A Hub also keeps accepting the previous revision, `2026-10-12` +(`apicontract.SupportedRevisions`), so released clients keep working through +the upgrade. A request made under it gets that revision's behavior: the Hub +reads the negotiated revision with `apicontract.RequestRevision` and does not +answer such a request with the asynchronous restore. A client always sends the +newest revision it knows (`apicontract.Revision`) and never falls back, so a +new client still needs a Hub that serves `2026-10-17`. + `GET /api/version` is public, requires no database access or credentials, and returns `Cache-Control: no-store` with the actual server build version: ```json -{"server_version":"0.24.0","api_versions":["2026-10-12"]} +{"server_version":"0.24.0","api_versions":["2026-10-17","2026-10-12"]} ``` Every request under `/api/v1`, including event-stream handshakes and telemetry, -must send exactly one `Tapper-API-Version: 2026-10-12` header. Accepted requests -echo it in the response. Clients send their release in `Tapper-Client-Version`. +must send exactly one `Tapper-API-Version` header naming a supported revision, +for example `Tapper-API-Version: 2026-10-17`. Accepted requests echo the +revision they sent in the response. Clients send their release in `Tapper-Client-Version`. The Hub validates the contract before authentication, flight authority, or application handlers. Discovery, health, OAuth, browser routes, and hosted `/mcp` are outside this REST gate. Browser code calling REST still sends it. @@ -26,11 +38,15 @@ duplicate, comma-joined, or unsupported values return HTTP 400 with `operationPerformed: false`, for example: ```json -{"error":"unsupported REST contract; upgrade Tapper and Hub together","code":"API_VERSION_UNSUPPORTED","requested_api_versions":["2099-01-01"],"supported_api_versions":["2026-10-12"],"server_version":"0.24.0","operationPerformed":false} +{"error":"unsupported REST contract; upgrade Tapper and Hub together","code":"API_VERSION_UNSUPPORTED","requested_api_versions":["2099-01-01"],"supported_api_versions":["2026-10-17","2026-10-12"],"server_version":"0.24.0","operationPerformed":false} ``` CLI invocations and MCP connections discover each selected Hub once, before -REST operations or flight-authority loading. Discovery sends no credentials or +REST operations or flight-authority loading. `tap` and `tap mcp` also keep a +successful discovery in `api-discovery.json` under the cache directory, keyed +by Hub URL and contract revision, and trust it for an hour, so most +invocations skip `GET /api/version` entirely. A stale entry is safe: the Hub +still validates every request, and its typed rejection evicts the entry. Discovery sends no credentials or cookies, refuses redirects, and is bounded by the caller's deadline and five seconds. Library callers can share `apicontract.Session` through their context. Standalone clients check discovery too. Legacy Hubs with no discovery endpoint @@ -43,7 +59,8 @@ The Hub validates every request even after deployment during an existing MCP connection. Rejections preserve typed compatibility errors and supported revisions in MCP results. Initialization failures leave recovery diagnostics available. Upgrade both sides to a common contract and start a new invocation -or connection; discovery results, including failures, are connection-scoped. +or connection; discovery failures are connection-scoped and never cached, and +the rejection that ends a connection also evicts any cached success. Clients never switch contracts or replay mutations automatically. Compatibility failures emit structured `api.compatibility_failure` logs: @@ -52,12 +69,12 @@ Fields include client/server releases, requested/supported revisions, `operationPerformed`, and a request identifier when available. Credentials, arguments, and request content are excluded. -This is a coordinated upgrade: old clients and scripts without the header stop -working against the new Hub, and new clients reject legacy Hubs. Update custom +Clients and scripts without the header do not work against a Hub, and a +client rejects a Hub whose discovery does not list the client's revision. Update custom scripts explicitly, for example: ```sh -curl -H 'Tapper-API-Version: 2026-10-12' \ +curl -H 'Tapper-API-Version: 2026-10-17' \ -H "Authorization: Bearer ${TAPPER_TOKEN}" \ https://hub.example.com/api/v1/whoami ``` @@ -81,6 +98,32 @@ than 50 KEGs still searches in one request. Without the proof, Tapper names the flight's cover itself and refuses a cover wider than 50 KEGs with `INVALID`, asking for `kegs` to narrow it. +## Caller context route + +`GET /api/v1/info` is an additive route, so it did not change the +revision. It returns, in one request, what `tap info` otherwise asks the +selected Hub in three: the `whoami` user, the flight named by the optional +`flight` query parameter (`@namespace/+slug`, or a bare `+slug` the Hub +resolves in the caller's personal namespace), and the identity KEG catalog. +A flight the Hub cannot read comes back as `flight: null` with the reason in +`flight_error` and HTTP 200, so the rest of the context still arrives. The +flight's `cover` is the Hub-composed cover, the rows a flight read reports as +`effective_cover`: + +```json +{"user":{"user_id":1,"username":"jared","default_namespace":"jared"}, + "flight":{"name":"@jared/+work","title":"Work","cover":[{"ref":"@jared/dev","role":"editor","source":"@jared/dev","distance":0}]}, + "flight_error":null, + "kegs":[{"ref":"@jared/dev","namespace":"jared","alias":"dev","title":"Dev","role":"admin","flight_cap":"editor","effective_role":"editor"}]} +``` + +Tapper ignores fields it does not know and projects the catalog through the +composed cover itself; `tap info` lists the composed cover on either path, +the authority MCP enforces. A Hub that predates the route answers 404; +Tapper then sends `whoami`, the flight read, and `GET /api/v1/kegs` +concurrently, and skips the route on that Hub for the rest of the invocation +or connection. + Publish the Tapper commit first; a hub that embeds Tapper then pins that published SHA on its own side, by its own procedure. This development change does not authorize tags, releases, merges, or deployment. diff --git a/docs/architecture/runtime-and-concurrency.md b/docs/architecture/runtime-and-concurrency.md index 781307f2..3fd9240b 100644 --- a/docs/architecture/runtime-and-concurrency.md +++ b/docs/architecture/runtime-and-concurrency.md @@ -69,9 +69,18 @@ use of `time.Now()` that cannot be driven by a fake clock: - `pkg/keg/node_id.go` (crypto/rand fallback uses `time.Now().UnixNano()`): used as an entropy source for a short random code when `crypto/rand` fails, not as a time measurement. -- `pkg/tapper/invocation_telemetry.go` (flush ticker and request/shutdown - deadlines): coalesces best-effort network uploads and bounds their wall-clock - impact independently of the frozen domain clock. +- `pkg/tapper/invocation_telemetry.go` (flush ticker, request deadline, and + the in-flight wait at `Close`): coalesces best-effort network uploads and + bounds their wall-clock impact independently of the frozen domain clock. + `Close` never starts a send; it waits at most + `InvocationTelemetryInFlightWait` for one already running and spools the + rest through `rt.AtomicWriteFile`, into a new file in the spool directory + for its destination (`telemetry-spool//`, keyed by Hub URL and the + login's stable `AuthEntry.AccountKey`, never by a rotating credential), + so events are never sent to another Hub or account. Concurrent processes + never share a file: each writes its own, and a starting process claims a + file with `rt.Rename` before reading it, so each spooled event is sent by + exactly one process. ## Concurrency Model @@ -80,6 +89,26 @@ use of `time.Now()` that cannot be driven by a fake clock: concurrency-safe in-memory implementation exists only in tests. - **Lock context propagation**: `contextWithNodeLock`/`contextHasNodeLock` allow re-entrant locking within the same call chain. +- **Bounded keg lock wait**: `keg.WithLockWait(ctx, d)` asks the repository to + give up acquiring the keg write lock after `d` with `ErrLockTimeout`, and + `keg.LockWaitFrom(ctx)` reads it back; zero means wait as long as it takes. + A touched `ReadNodes` or `OpenNode` waits about two seconds. When the lock + stays busy the touch is skipped and the nodes are read untouched, so a read + never queues behind a long writer just to record an access. The bound is + advisory and in-process only: it travels in `ctx`, is not part of the Hub + API, and so never reaches a Hub through `RemoteKeg`. A `Repository` that + does not read it (the in-memory test repository does not) waits as before, + and its touched reads wait for the lock rather than skip the touch. Nothing + may depend on the bound for correctness. +- **Bounded node lock wait**: `keg.WithNodeLockWait(ctx, d)` bounds how long + `Keg.Lock` waits for a node lock another holder has, and + `keg.NodeLockWaitFrom(ctx)` reads it back. Unlike the keg write lock bound it + crosses the wire: `RemoteKeg.Lock` sends it as `?wait=` on + `POST /nodes/{id}/lock`, and `LocalKeg.Lock` stops acquiring after it. A lock + still held when it passes fails with `ErrLockTimeout` (`LOCK_TIMEOUT`). + `Tap.Lock` always sets it through `keg.ClampNodeLockWait` from MCP + `lock_acquire`'s `wait_seconds` or `tap lock acquire --wait`: 10 seconds by + default, at most 60. - **Dex mutex**: `Dex.mu sync.RWMutex` guards index data; `LocalKeg.dexMu` guards lazy initialization. - **Node allocation**: Production allocation is a hub operation; @@ -87,6 +116,39 @@ use of `time.Now()` that cannot be driven by a fake clock: repository. - **KegService cache**: `cacheMu sync.Mutex` guards the shared keg resolution cache. +- **Auth store**: `AuthStore.mu sync.RWMutex` guards the in-memory credential + map, because the token resolver reads it from concurrent hub requests while + a refresh writes the rotated entry. +- **Auth refresh across processes**: hubs rotate refresh tokens single-use, so + two `tap` processes sharing one auth store must not both spend the stored + token. A refresh takes an exclusive OS lock on `.lock` (`flock` + on Unix, `LockFileEx` on Windows, opened through the runtime so the jail + applies), re-reads the store under it, and adopts a pair another process + persisted while it waited; only otherwise does it call the hub and persist + the result before unlocking. The wait is bounded by the refresh timeout plus + five seconds. A process that times out never refreshes unlocked: it adopts a + fresher on-disk entry or keeps its cached token. A token that is still fresh + never touches the lock. Under the lock the store file is authoritative: the + refresh spends the refresh token stored there rather than a stale in-memory + one, does not run once a logout removed the entry, and persists + compare-and-set, only while the stored entry still holds the refresh token + it started from. `tap auth login` and `tap auth logout` write the store + under the same lock, so a running process can neither undo a logout nor + overwrite a newer login. +- **Forced refresh**: the relay refreshes when the hub, not the local clock, + says a token is spent or soon will be (a reauth at 80% of the lifetime the + hub announced, a redial after `credential_expired`, a refused dial). That + refresh ignores the stored expiry, so clock skew cannot keep a dead token in + use, and otherwise follows the same lock discipline. Under the lock, a + stored access token that differs from the one the process held means another + process already rotated it, and that pair is adopted instead. +- **Refused versus failed refresh**: only an OAuth error body naming + `invalid_grant` or `invalid_client`, at any HTTP status, counts as the hub + refusing the refresh token (`ErrRefreshRejected`); the sign-in is gone and + only `tap auth login` restores it. A 5xx, a `server_error` or + `invalid_request` answer, or a body that is not an OAuth error is a failure + that may pass, so callers such as the relay retry with backoff instead of + giving up on a credential that is still good. - **Remote operations are single-request**: each `RemoteKeg` method is one HTTP round trip, and the hub serializes per-node writes server-side. There is no client-side lock lease or dex write over HTTP. @@ -96,6 +158,13 @@ use of `time.Now()` that cannot be driven by a fake clock: `orient`, `session_refresh`, flight mutations, and any failed call discard it. Reuse cannot widen access, because the Hub rechecks the orientation header and KEG authority on every request. Hosted servers leave it off. +- **Orientation requests overlap**: `tap info`, `tap orient`, and the MCP + orientation provider issue the Hub requests that do not depend on each other + (identity, flight, flight graph, KEG catalog) from separate goroutines and + read the results in the order the sequential code checked them, so errors + and warnings are reported exactly as before. `tap info` asks for all of it in + one `GET /api/v1/info` where the Hub supports it (see + [REST API contract compatibility](../api-compatibility.md#caller-context-route)). - **Advisory locks are session primitives**: `Keg.Lock`/`Unlock`/ `LockStatus`/`ForceUnlock` (used by `tap lock` / `tap node edit`) are opt-in advisory locks backed by the Hub's `/nodes/{id}/lock` endpoints. Leases carry a TTL diff --git a/docs/backups-and-archives.md b/docs/backups-and-archives.md index 1e3e568f..8f203070 100644 --- a/docs/backups-and-archives.md +++ b/docs/backups-and-archives.md @@ -1,16 +1,13 @@ # Backups And Archives -Export and import kegs as `.tar.gz` archives for backup, migration, and -sharing. +Export kegs as `.tar.gz` archives for backup, migration, and sharing, and +restore a whole keg from one on its Tapper Hub. ## Quick Start ```bash # Export a team keg to an archive tap archive export --keg @acme/engineering -o ~/backups/engineering-2026-03-14.tar.gz - -# Import an archive into a keg -tap archive import ~/backups/engineering-2026-03-14.tar.gz --keg @acme/engineering ``` ## Export @@ -49,36 +46,14 @@ tap archive export -o lightweight.tar.gz --no-history tap archive export -o engineering.tar.gz --keg @acme/engineering ``` -## Import - -```bash -tap archive import ARCHIVE [flags] -``` - -Imports nodes from an archive into the target keg. Prints the path of each -imported node to stdout. - -### Flags - -| Flag | Description | -|------|-------------| -| `--keg @namespace/name` | Target a specific keg | -| `--namespace NAME` | Resolve a bare keg name in a namespace | -| `--hub NAME` | Override namespace-to-hub resolution | - -### Sources +## Restore -Archives can be loaded from local files or remote URLs: - -```bash -# Local file -tap archive import ./backup.tar.gz - -# Remote URL -tap archive import https://example.com/shared-keg.tar.gz -``` - -HTTP and HTTPS URLs are downloaded automatically before import. +`tap` does not load archives into a keg. A full-keg export is restored on the +Tapper Hub that holds the keg, through its archive restore endpoint +(`POST /api/v1/@{namespace}/kegs/{alias}/archive/restore`) or the keg's +settings page. A restore replaces the keg's contents with the archive's and +runs as a background job on the hub; it needs an archive exported without +`--nodes`. ## Archive Format @@ -116,7 +91,7 @@ keg-archive/ `stats.json` is included because it is derived/internal node state (for example hashes, timestamps, access counters, and link summaries). It is carried -so imports can restore or remap node state consistently; authors normally edit +so a restore can bring node state back consistently; authors normally edit `README.md`, `meta.yaml`, and schema files instead. ### manifest.json @@ -145,13 +120,11 @@ revisions included. Full-keg exports include `keg.yaml`, the keg settings document, and any available keg-level schemas under `schemas/.schema.yaml`. Exports that -select explicit nodes with `--nodes` omit `keg.yaml` and schemas; imported -node-filtered archives do not overwrite the target keg's settings or schemas. +select explicit nodes with `--nodes` omit `keg.yaml` and schemas, so they +cannot be restored over a keg. -When an archive contains schemas, import validates nodes against those archived -schemas and then replaces matching schema types in the target keg. Schema types -that exist only in the target keg are preserved. Archives without schema -entries continue to validate against the target keg's current schemas. +When an archive contains schemas, a restore validates nodes against those +archived schemas and then replaces matching schema types in the keg. ## Snapshot History @@ -167,7 +140,7 @@ tap archive export -o slim.tar.gz --no-history ``` When history is included, each node's `snapshots/` directory is preserved in -the archive and replayed on import with original timestamps intact. +the archive and replayed on restore with original timestamps intact. For details on how snapshots work, see [Node Snapshots](node-snapshots.md). @@ -210,20 +183,15 @@ tap archive export -o ~/backups/engineering-$DATE.tar.gz --keg @acme/engineering ## Migration And Sharing -### Move a keg between machines +### Move a keg between hubs -On the source machine: +Export the keg from the source hub: ```bash tap archive export -o engineering.tar.gz --keg @acme/engineering ``` -Copy the archive to the target machine (scp, USB drive, cloud storage, etc.), -then import: - -```bash -tap archive import engineering.tar.gz --keg @acme/engineering -``` +Then restore the archive into a keg on the target hub (see [Restore](#restore)). ### Share nodes with a collaborator @@ -233,35 +201,26 @@ Export specific nodes and host the archive or share the file directly: tap archive export -o shared-notes.tar.gz --nodes 10,20,30 ``` -The recipient imports from a local file or a URL: - -```bash -tap archive import https://example.com/shared-notes.tar.gz -``` +The archive is a plain gzip tar, so its nodes can be read without `tap`. ## Behavior Details -Understanding these behaviors helps avoid surprises during import: - -- **Preserved IDs**: Archive imports restore nodes at their archive IDs, - replacing existing nodes at matching IDs. The hub API can request fresh IDs - with `assign_new_ids=1`; the CLI archive import keeps archive IDs. - -- **Link rewriting**: Relative links (`../N`) in node content are - automatically rewritten to point to the newly allocated IDs. Links in - `stats.json` are also remapped. - -- **Asset preservation**: If an imported node replaces an existing node at the - same target ID, any file and image attachments on the existing node are - preserved. Archive-carried `assets/` and `images/` entries land last and win - when names collide. - -- **Atomic failure**: If any node fails to import, the entire import operation - fails. Partially imported nodes are not rolled back, but the dex is only - rebuilt on complete success. - -- **Dex rebuild**: Keg indices are automatically rebuilt after a successful - import. - -- **Compression fallback**: The importer tries gzip decompression first. If - that fails, it falls back to reading the archive as an uncompressed tar. +- **Preserved IDs**: A restore lands nodes at their archive IDs and removes + every node the archive does not carry; node 0 is never removed. + +- **Compression detection**: An archive that starts with the gzip magic bytes + is decompressed; anything else is read as an uncompressed tar. + +- **Streaming read**: A restore decompresses an archive once into a plain tar + and reads each entry by offset, so it never holds every payload in memory + at once. Directory entries are ignored, and when a name appears more than + once the last entry wins. An entry name that is absolute or contains `..` + rejects the archive, as does an attachment entry under `assets/`, `images/`, + or `videos/` whose name is not a single plain filename. + +- **Size limits**: A Tapper Hub bounds the archives it exports and restores by + entry count, entry size, and compressed and uncompressed size. The + uncompressed size counts every byte of the tar stream, extended (PAX and GNU + long-name) headers included, and an archive is refused as soon as it crosses + a limit rather than after it has been read in full. The hub checks the same + limits on both sides, so an archive the hub exports is one it will restore. diff --git a/docs/configuration/flights.md b/docs/configuration/flights.md index efaf6059..27919251 100644 --- a/docs/configuration/flights.md +++ b/docs/configuration/flights.md @@ -9,7 +9,7 @@ live graph and the requested flight before every authority-bearing tool call. ## What A Flight Does -A flight carries five details: +A flight carries six details: 1. **A keg cover** (`cover`). MCP tools reject kegs outside the cover. Each cover entry has a `viewer`, `editor`, or `admin` cap. Reads require @@ -33,6 +33,22 @@ A flight carries five details: cover and may be broader or different from its ancestors. Cross-Hub relations and duplicate canonical children are rejected; referenced children cannot be deleted. +5. **Namespace management** (`manageKegs`, `manageFlights`, default + `false`). A request rooted on a flight writes no flight and creates or + deletes no KEG without them. `manageKegs` lets it create and delete the + namespace's KEGs and administer any of them (settings, rename, + visibility, grants and grant invitations), even outside its cover, though + their notes stay out of reach. `manageFlights` lets it create, edit, and + delete the namespace's flights, widening them past its cover. Each + switch covers the flight's own namespace; a subflight in another + namespace that sets one adds that namespace for every root whose graph + reaches it, which is how one flight manages several. Neither ever gives + more than the user's own role. Namespace membership, membership + invitations, policies, tokens, and relay shares stay refused to every + rooted request. Turning a switch on from a rooted request takes a root + that already holds it for the namespace; attaching a subflight that sets + one takes an owner or admin of that subflight's namespace. The old + `manage` flag sets both. A flight is memory only: it carries no tools and no capabilities. What a session may do is its agent's tool list (`tap agent`, `TAP_AGENT`), and an @@ -46,7 +62,7 @@ subsequent KEG operations. `tap orient` is flight-scoped and rejects ## Manifest Format Flights are stored by Tapper Hub and addressed as `@namespace/+slug`. Each -manifest has five optional fields: +manifest has six optional fields: ```yaml title: Release 42 cut @@ -54,6 +70,8 @@ visibility: private subflights: - "@acme/+release-notes" - "@acme/+verification" +manageKegs: false +manageFlights: false cover: - namespace: acme keg: release-notes @@ -86,7 +104,7 @@ with an explicit `=viewer` suffix keeps its viewer cap. | Preview orientation for a flight | `tap orient --flight @namespace/+slug` | | Persist the project flight | `tap use --flight @namespace/+slug` or `tap use +slug` | | Persist the user bootstrap baseline | `tap bootstrap --flight @namespace/+slug` | -| Create a Hub-backed flight | `tap flight create @namespace/+slug --cover @namespace/keg=viewer` | +| Create a Hub-backed flight | `tap flight create @namespace/+slug --cover @namespace/keg=viewer` (add `--manage-kegs` and `--manage-flights` for a flight that manages its namespace's KEGs and flights) | | Edit a Hub-backed flight in $EDITOR | `tap flight edit @namespace/+slug` (the manifest opens as YAML; every save is applied) | | Apply a manifest from a script | `cat manifest.yaml \| tap flight edit @namespace/+slug` | | Delete a Hub-backed flight | `tap flight delete @namespace/+slug` | @@ -95,8 +113,11 @@ with an explicit `=viewer` suffix keeps its viewer cap. first line is a `yaml-language-server` schema modeline for `schemas/flight-manifest.json`, followed by a short comment that the `@namespace/+slug` ref is immutable. The editable fields are `title`, -`visibility`, `subflights`, `cover`, and `instructions`; comments and the -modeline are ignored when deciding whether the manifest changed. +`visibility`, `subflights`, `cover`, `manageKegs`, `manageFlights`, and +`instructions`; comments and the modeline are ignored when deciding whether +the manifest changed. Each switch is sent to the Hub only when the edit +changes it, so editing anything else never touches it; a piped manifest +without one clears it. MCP always exposes `flight_list` and `flight_read`. In an active session the rest of the inventory is the session agent's tools: `flight_create`, @@ -104,7 +125,8 @@ rest of the inventory is the session agent's tools: `flight_create`, them, and dispatch then applies normal Hub authorization (namespace owner or admin for flights, membership for `keg_create`). `flight_edit` is a partial update where omitted fields retain their current -values. Call `flight_read` first and pass its manifest hash as the required +values; `flight_create` and `flight_edit` take `manage`, and `flight_read` +reports it. Call `flight_read` first and pass its manifest hash as the required `expected_hash` for both edits and deletes. On conflict, merge or refetch and retry with the returned current hash. Graph and authority edits are adopted on the next call automatically. Mutations are never replayed. A referenced @@ -119,9 +141,16 @@ A Hub agent is a model, instructions, a tool allowlist, and a flight. Its tools decide what a session may do; its flight is the memory it works in. An agent with no flight has no KEG access: a Hub session running it comes up in recovery mode with KEG tools locked until the agent is given a flight, after -which `session_refresh` activates it. Hosted apps and API tokens run as an -agent, and `tap launch --agent @ns/name` roots the session on the agent's -flight unless `--flight` or `TAP_FLIGHT` names another. Set an agent's flight +which `session_refresh` activates it. Hosted apps run as an agent, and +`tap launch --agent @ns/name` roots the session on the agent's flight unless +`--flight` or `TAP_FLIGHT` names another. + +An agent holds no credentials of its own. A local agent session reaches Hub +only through `tap mcp`, which applies the agent's tool allowlist (`TAP_AGENT`) +and sends its flight as `?flight=`, and Hub narrows each request to that +flight. Its orientation names the agent and says the allowlist and flight +govern the session, never the identity's full authority. An API token is never +bound to an agent: a token handed to one carries its owner's full access. Set an agent's flight with `tap agent edit @ns/name --flight @ns/+slug` (`--flight ""` clears it) or on the agent's page on Hub. diff --git a/docs/configuration/user-config.md b/docs/configuration/user-config.md index ce7f7efd..0d3e17e4 100644 --- a/docs/configuration/user-config.md +++ b/docs/configuration/user-config.md @@ -230,5 +230,32 @@ Invocation telemetry goes to `/api/v1/telemetry/invocations` on the active Hub, using an existing login. It reports the surface, command/tool name, duration, success, optional CLI interactivity, version, and agent identity. Arguments, content, paths, errors, credentials, and KEG/node identifiers are not uploaded. -Reporting is best effort. Disable it with `disableTelemetry: true` or -`TAP_DISABLE_TELEMETRY=1`. +Reporting is best effort and never delays exit: a CLI invocation's own event, +and anything else not yet delivered, is written to a spool under the state +directory (beside `auth.yaml`) and sent by the next invocation while its +command runs. Each destination -- the Hub plus the login on it -- has its own +directory, `telemetry-spool//`, where the key is a short hash of the Hub +URL and the login's `account_key` in `auth.yaml`. That key is random, minted +at `tap auth login`, never sent to the Hub, and kept when the access and +refresh tokens rotate, so a refresh never strands what was spooled before it. +Spooled events are only ever sent to the Hub and account that produced them: +never to another Hub, and not after a new login. Another login's directory is +kept for 7 days after its last write in case that login returns, and at most +8 directories are kept in all. A login written by an earlier release gets its +`account_key` the first time telemetry runs or its token is refreshed, +whichever comes first, and claims the directory those releases keyed by its +credential. Each process writes its own file and claims files by renaming them, so +concurrent invocations neither overwrite each other's events nor send one +twice. The spool holds events only, never credentials, and keeps the newest +500 per destination. The single `telemetry-spool.json` file earlier releases +wrote is deleted unsent. When the Hub refuses this client's events outright +(an older Hub, or a status that says it never will accept them), the client +records that in a `rejected` marker in the destination's directory, and +later invocations of the same version skip telemetry for 24 hours instead of +each sending a request bound to be refused. Another version tries again at +once. A refused credential (`401` or `403`, such as an expired access token) +is not a refusal of the client: it stops reporting for that invocation only +and writes no marker, so the next invocation with a valid token reports +again. Disable reporting with +`disableTelemetry: true` or `TAP_DISABLE_TELEMETRY=1`; a disabled client +neither sends nor writes the spool. diff --git a/docs/development/conventions.md b/docs/development/conventions.md index 312da214..5fcbcbbd 100644 --- a/docs/development/conventions.md +++ b/docs/development/conventions.md @@ -36,6 +36,13 @@ Branching, commits, versioning, and error handling. - Typed errors: `BackendError` (with Retryable), `RateLimitError`, `TransientError`. - Check with `errors.Is()` for sentinels, `errors.As()` for typed errors. +- A sentinel decides the wire code and the MCP failure result (see + [Failure Results](../ai-coding-agents/mcp-setup.md#failure-results)); an + unwrapped error is `INTERNAL` with an unknown outcome. Wrap a failure the + caller can fix with `keg.ErrInvalid` (`fmt.Errorf("...: %w", keg.ErrInvalid)`) + where the caller's input is parsed. When the same parser also reads stored + data, the stored-data caller drops that classification (as the settings + index loader does), because a bad stored value is not the caller's mistake. ## Read Next diff --git a/go.mod b/go.mod index 8bc529f8..f3803f66 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/yuin/goldmark v1.8.6 github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc golang.org/x/image v0.46.0 + golang.org/x/sys v0.48.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -55,7 +56,6 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.23.0 // indirect - golang.org/x/sys v0.48.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.15.0 // indirect diff --git a/integrations/content/tool-inventory.md b/integrations/content/tool-inventory.md index ef24e1a7..a2166a59 100644 --- a/integrations/content/tool-inventory.md +++ b/integrations/content/tool-inventory.md @@ -79,8 +79,8 @@ accessible transitive descendants. | Tool | Purpose | | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `mcp__tapper__node_search` | Find the nodes most relevant to a query, ranked by the Hub across every KEG the call may read, one `keg`, or the listed `kegs`. Takes plain terms, `"quoted phrases"`, and `-exclusions`. Hits carry `ref`, `score`, `title`, matched `fields`, and a `snippet` with highlights; follow `next_cursor`. | -| `mcp__tapper__node_grep` | Regex match over node content lines in one KEG, in node id order, for exact identifiers, symbols, and audits. Supports `ignore_case`, `limit`, `max_lines`, and `id_only`. | +| `mcp__tapper__node_search` | Find the nodes most relevant to a query, ranked by the Hub across every KEG the call's flight covers, or only the KEGs listed in `kegs` (each inside that cover). Takes plain terms, `"quoted phrases"`, and `-exclusions`. Hits carry `ref`, `score`, `title`, matched `fields`, and a `snippet` with highlights; follow `next_cursor`. | +| `mcp__tapper__node_grep` | PostgreSQL regular-expression match over node content lines in one KEG (`\y` is a word boundary), in node id order, for exact identifiers, symbols, and audits. Supports `ignore_case`, `limit`, `max_lines`, and `format`. | | `mcp__tapper__tag_list` | List tags or filter nodes by a boolean expression over tags, attributes, and dot-prefix stats fields (for example `tapper and .created>2026-01-01`). | | `mcp__tapper__node_list` | List nodes in a keg with optional filters. | | `mcp__tapper__node_read` | Read one or more nodes. Each structured row pairs `node_id` and `hash` with that node's `content` and `meta`, so a read feeds straight into `node_edit`. Supports `meta_only`, `content_only`, `stats_only`, and `query` expression selection as an alternative to explicit node IDs. | @@ -90,7 +90,7 @@ accessible transitive descendants. | `mcp__tapper__keg_settings_read` | Read targeted title, description, updated metadata, and instructions for one or more selected KEGs; batches accept up to 100 canonical references. | Start topic lookups with `node_search`; reach for `node_grep` when you need -every line matching an exact pattern. Pass `id_only: true` to `node_grep` and +every line matching an exact pattern. Pass `format: "%i"` to `node_grep` and `tag_list` when you only need IDs for follow-up reads — it keeps token consumption bounded on large result sets. @@ -124,7 +124,7 @@ code; the index does the work in O(matches) rather than O(total). | Tool | Purpose | | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | -| `mcp__tapper__node_create` | Atomically create 1–100 nodes. Each is a markdown `content` document plus an optional YAML `meta` document; the title is the content's H1. Nodes in one batch reference each other with `{{node:KEY}}`. | +| `mcp__tapper__node_create` | Atomically create 1–100 nodes. Each is a markdown `content` document plus an optional YAML `meta` document; the title is the content's H1. Nodes in one batch reference each other with `{{node:KEY}}`, which becomes the bare id: link with `[Title](../{{node:KEY}})`. | | `mcp__tapper__node_edit` | Call `node_read`, then atomically replace `content`, `meta`, or both for 1–100 `nodes[]`. Each part has its own hash: supplied `content` requires `expected_content_hash` (the read's `content_hash`) and supplied `meta` requires `expected_meta_hash` (its `meta_hash`). Editing one part never invalidates the other's hash, and results return the new `content_hash` and `meta_hash`. For a small change to a long note, send `patch` (a unified diff of the changed lines with a little context) instead of `content`; it needs `expected_content_hash` too, and a patch that no longer fits is refused with the lines it expected. A patch keeps the body's line endings, and `\ No newline at end of file` may only end a hunk that reaches the end of the body. | | `mcp__tapper__node_move` | Call `node_read`, then relocate a node using the combined `hash` from `node_read` as `expected_hash`. A trashed node's id stays reserved until it is restored or purged. | | `mcp__tapper__node_delete` | Call `node_read`, then atomically remove 1–100 `nodes[]`, each carrying the combined `hash` from `node_read` as `expected_hash`. | @@ -240,7 +240,9 @@ listing resources. Text content is JSON rendered from the same public object as structuredContent. The message field preserves prose, diagnostics, and legacy formatted output. -Images remain in MCP image content blocks alongside JSON metadata. +Images remain in MCP image content blocks alongside JSON metadata. Every tool +declares the object's shape as its `outputSchema`; a tool whose result is only +text declares `{message}`. For node_list, node_grep, tag_list, node_links, and node_backlinks, follow next_offset until null, @@ -254,9 +256,10 @@ node_search pages with an opaque cursor instead: repeat the same query with next_cursor as cursor until next_cursor is absent (text output ends with a `next cursor:` line while more remain). An omitted or 0 limit lets the Hub choose the page size; -1 is not accepted. Hits are ordered by score, then KEG, then node id; -mode reports `keyword` when the Hub ranked without semantic matching. A KEG -outside the flight's cover is never searched, and a Hub without ranked search -fails with a message pointing at node_grep. +mode reports `keyword` when the Hub ranked without semantic matching. An empty +or omitted `kegs` searches every KEG the call's flight covers; naming a KEG +outside that cover fails with an error that names it, and a Hub without ranked +search fails with a message pointing at node_grep. The explore tools return truncated and have no cursor; refine the query. @@ -273,7 +276,7 @@ does not activate it or change the connection's pinned root. | `mcp__tapper__schema_validate` | Schema validation findings. | | `mcp__tapper__keg_info`, `mcp__tapper__node_stats` | KEG diagnostics and node statistics. | | `mcp__tapper__url` | The Hub web link for a KEG, note, flight, or namespace (`ref` as `42`, `@ns/keg/42`, `keg:~alias/42`, `+slug`, `@ns`), returned as `url`, `kind`, and canonical `ref`. Use it to hand the reader a clickable link; it never reads the note. | -| `mcp__tapper__lock_acquire`, `mcp__tapper__lock_status`, `mcp__tapper__lock_release` | Advisory locks; acquisition returns a private token for release. node_edit does not accept a lock token. | +| `mcp__tapper__lock_acquire`, `mcp__tapper__lock_status`, `mcp__tapper__lock_release` | Advisory locks; acquisition returns a private token for release. A held lock is waited for up to `wait_seconds` (default 10, max 60), then fails with `LOCK_TIMEOUT`. node_edit does not accept a lock token. | | `mcp__tapper__file_list`, `mcp__tapper__image_list` | Stored attachment names. | | `mcp__tapper__file_upload`, `mcp__tapper__image_upload` | Inline base64/data URI/embedded resource uploads; local stdio also accepts source_path. Link the returned stored filename. | | `mcp__tapper__image_download` | Image content block and metadata; local stdio optionally accepts dest_path. | diff --git a/integrations/rendered/claude/tapper/skills/tapper/SKILL.md b/integrations/rendered/claude/tapper/skills/tapper/SKILL.md index fdc91912..25c306bb 100644 --- a/integrations/rendered/claude/tapper/skills/tapper/SKILL.md +++ b/integrations/rendered/claude/tapper/skills/tapper/SKILL.md @@ -124,8 +124,8 @@ accessible transitive descendants. | Tool | Purpose | | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `mcp__tapper__node_search` | Find the nodes most relevant to a query, ranked by the Hub across every KEG the call may read, one `keg`, or the listed `kegs`. Takes plain terms, `"quoted phrases"`, and `-exclusions`. Hits carry `ref`, `score`, `title`, matched `fields`, and a `snippet` with highlights; follow `next_cursor`. | -| `mcp__tapper__node_grep` | Regex match over node content lines in one KEG, in node id order, for exact identifiers, symbols, and audits. Supports `ignore_case`, `limit`, `max_lines`, and `id_only`. | +| `mcp__tapper__node_search` | Find the nodes most relevant to a query, ranked by the Hub across every KEG the call's flight covers, or only the KEGs listed in `kegs` (each inside that cover). Takes plain terms, `"quoted phrases"`, and `-exclusions`. Hits carry `ref`, `score`, `title`, matched `fields`, and a `snippet` with highlights; follow `next_cursor`. | +| `mcp__tapper__node_grep` | PostgreSQL regular-expression match over node content lines in one KEG (`\y` is a word boundary), in node id order, for exact identifiers, symbols, and audits. Supports `ignore_case`, `limit`, `max_lines`, and `format`. | | `mcp__tapper__tag_list` | List tags or filter nodes by a boolean expression over tags, attributes, and dot-prefix stats fields (for example `tapper and .created>2026-01-01`). | | `mcp__tapper__node_list` | List nodes in a keg with optional filters. | | `mcp__tapper__node_read` | Read one or more nodes. Each structured row pairs `node_id` and `hash` with that node's `content` and `meta`, so a read feeds straight into `node_edit`. Supports `meta_only`, `content_only`, `stats_only`, and `query` expression selection as an alternative to explicit node IDs. | @@ -135,7 +135,7 @@ accessible transitive descendants. | `mcp__tapper__keg_settings_read` | Read targeted title, description, updated metadata, and instructions for one or more selected KEGs; batches accept up to 100 canonical references. | Start topic lookups with `node_search`; reach for `node_grep` when you need -every line matching an exact pattern. Pass `id_only: true` to `node_grep` and +every line matching an exact pattern. Pass `format: "%i"` to `node_grep` and `tag_list` when you only need IDs for follow-up reads — it keeps token consumption bounded on large result sets. @@ -169,7 +169,7 @@ code; the index does the work in O(matches) rather than O(total). | Tool | Purpose | | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | -| `mcp__tapper__node_create` | Atomically create 1–100 nodes. Each is a markdown `content` document plus an optional YAML `meta` document; the title is the content's H1. Nodes in one batch reference each other with `{{node:KEY}}`. | +| `mcp__tapper__node_create` | Atomically create 1–100 nodes. Each is a markdown `content` document plus an optional YAML `meta` document; the title is the content's H1. Nodes in one batch reference each other with `{{node:KEY}}`, which becomes the bare id: link with `[Title](../{{node:KEY}})`. | | `mcp__tapper__node_edit` | Call `node_read`, then atomically replace `content`, `meta`, or both for 1–100 `nodes[]`. Each part has its own hash: supplied `content` requires `expected_content_hash` (the read's `content_hash`) and supplied `meta` requires `expected_meta_hash` (its `meta_hash`). Editing one part never invalidates the other's hash, and results return the new `content_hash` and `meta_hash`. For a small change to a long note, send `patch` (a unified diff of the changed lines with a little context) instead of `content`; it needs `expected_content_hash` too, and a patch that no longer fits is refused with the lines it expected. A patch keeps the body's line endings, and `\ No newline at end of file` may only end a hunk that reaches the end of the body. | | `mcp__tapper__node_move` | Call `node_read`, then relocate a node using the combined `hash` from `node_read` as `expected_hash`. A trashed node's id stays reserved until it is restored or purged. | | `mcp__tapper__node_delete` | Call `node_read`, then atomically remove 1–100 `nodes[]`, each carrying the combined `hash` from `node_read` as `expected_hash`. | @@ -285,7 +285,9 @@ listing resources. Text content is JSON rendered from the same public object as structuredContent. The message field preserves prose, diagnostics, and legacy formatted output. -Images remain in MCP image content blocks alongside JSON metadata. +Images remain in MCP image content blocks alongside JSON metadata. Every tool +declares the object's shape as its `outputSchema`; a tool whose result is only +text declares `{message}`. For node_list, node_grep, tag_list, node_links, and node_backlinks, follow next_offset until null, @@ -299,9 +301,10 @@ node_search pages with an opaque cursor instead: repeat the same query with next_cursor as cursor until next_cursor is absent (text output ends with a `next cursor:` line while more remain). An omitted or 0 limit lets the Hub choose the page size; -1 is not accepted. Hits are ordered by score, then KEG, then node id; -mode reports `keyword` when the Hub ranked without semantic matching. A KEG -outside the flight's cover is never searched, and a Hub without ranked search -fails with a message pointing at node_grep. +mode reports `keyword` when the Hub ranked without semantic matching. An empty +or omitted `kegs` searches every KEG the call's flight covers; naming a KEG +outside that cover fails with an error that names it, and a Hub without ranked +search fails with a message pointing at node_grep. The explore tools return truncated and have no cursor; refine the query. @@ -318,7 +321,7 @@ does not activate it or change the connection's pinned root. | `mcp__tapper__schema_validate` | Schema validation findings. | | `mcp__tapper__keg_info`, `mcp__tapper__node_stats` | KEG diagnostics and node statistics. | | `mcp__tapper__url` | The Hub web link for a KEG, note, flight, or namespace (`ref` as `42`, `@ns/keg/42`, `keg:~alias/42`, `+slug`, `@ns`), returned as `url`, `kind`, and canonical `ref`. Use it to hand the reader a clickable link; it never reads the note. | -| `mcp__tapper__lock_acquire`, `mcp__tapper__lock_status`, `mcp__tapper__lock_release` | Advisory locks; acquisition returns a private token for release. node_edit does not accept a lock token. | +| `mcp__tapper__lock_acquire`, `mcp__tapper__lock_status`, `mcp__tapper__lock_release` | Advisory locks; acquisition returns a private token for release. A held lock is waited for up to `wait_seconds` (default 10, max 60), then fails with `LOCK_TIMEOUT`. node_edit does not accept a lock token. | | `mcp__tapper__file_list`, `mcp__tapper__image_list` | Stored attachment names. | | `mcp__tapper__file_upload`, `mcp__tapper__image_upload` | Inline base64/data URI/embedded resource uploads; local stdio also accepts source_path. Link the returned stored filename. | | `mcp__tapper__image_download` | Image content block and metadata; local stdio optionally accepts dest_path. | diff --git a/integrations/rendered/codex/tapper/skills/tapper/SKILL.md b/integrations/rendered/codex/tapper/skills/tapper/SKILL.md index fdc91912..25c306bb 100644 --- a/integrations/rendered/codex/tapper/skills/tapper/SKILL.md +++ b/integrations/rendered/codex/tapper/skills/tapper/SKILL.md @@ -124,8 +124,8 @@ accessible transitive descendants. | Tool | Purpose | | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `mcp__tapper__node_search` | Find the nodes most relevant to a query, ranked by the Hub across every KEG the call may read, one `keg`, or the listed `kegs`. Takes plain terms, `"quoted phrases"`, and `-exclusions`. Hits carry `ref`, `score`, `title`, matched `fields`, and a `snippet` with highlights; follow `next_cursor`. | -| `mcp__tapper__node_grep` | Regex match over node content lines in one KEG, in node id order, for exact identifiers, symbols, and audits. Supports `ignore_case`, `limit`, `max_lines`, and `id_only`. | +| `mcp__tapper__node_search` | Find the nodes most relevant to a query, ranked by the Hub across every KEG the call's flight covers, or only the KEGs listed in `kegs` (each inside that cover). Takes plain terms, `"quoted phrases"`, and `-exclusions`. Hits carry `ref`, `score`, `title`, matched `fields`, and a `snippet` with highlights; follow `next_cursor`. | +| `mcp__tapper__node_grep` | PostgreSQL regular-expression match over node content lines in one KEG (`\y` is a word boundary), in node id order, for exact identifiers, symbols, and audits. Supports `ignore_case`, `limit`, `max_lines`, and `format`. | | `mcp__tapper__tag_list` | List tags or filter nodes by a boolean expression over tags, attributes, and dot-prefix stats fields (for example `tapper and .created>2026-01-01`). | | `mcp__tapper__node_list` | List nodes in a keg with optional filters. | | `mcp__tapper__node_read` | Read one or more nodes. Each structured row pairs `node_id` and `hash` with that node's `content` and `meta`, so a read feeds straight into `node_edit`. Supports `meta_only`, `content_only`, `stats_only`, and `query` expression selection as an alternative to explicit node IDs. | @@ -135,7 +135,7 @@ accessible transitive descendants. | `mcp__tapper__keg_settings_read` | Read targeted title, description, updated metadata, and instructions for one or more selected KEGs; batches accept up to 100 canonical references. | Start topic lookups with `node_search`; reach for `node_grep` when you need -every line matching an exact pattern. Pass `id_only: true` to `node_grep` and +every line matching an exact pattern. Pass `format: "%i"` to `node_grep` and `tag_list` when you only need IDs for follow-up reads — it keeps token consumption bounded on large result sets. @@ -169,7 +169,7 @@ code; the index does the work in O(matches) rather than O(total). | Tool | Purpose | | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | -| `mcp__tapper__node_create` | Atomically create 1–100 nodes. Each is a markdown `content` document plus an optional YAML `meta` document; the title is the content's H1. Nodes in one batch reference each other with `{{node:KEY}}`. | +| `mcp__tapper__node_create` | Atomically create 1–100 nodes. Each is a markdown `content` document plus an optional YAML `meta` document; the title is the content's H1. Nodes in one batch reference each other with `{{node:KEY}}`, which becomes the bare id: link with `[Title](../{{node:KEY}})`. | | `mcp__tapper__node_edit` | Call `node_read`, then atomically replace `content`, `meta`, or both for 1–100 `nodes[]`. Each part has its own hash: supplied `content` requires `expected_content_hash` (the read's `content_hash`) and supplied `meta` requires `expected_meta_hash` (its `meta_hash`). Editing one part never invalidates the other's hash, and results return the new `content_hash` and `meta_hash`. For a small change to a long note, send `patch` (a unified diff of the changed lines with a little context) instead of `content`; it needs `expected_content_hash` too, and a patch that no longer fits is refused with the lines it expected. A patch keeps the body's line endings, and `\ No newline at end of file` may only end a hunk that reaches the end of the body. | | `mcp__tapper__node_move` | Call `node_read`, then relocate a node using the combined `hash` from `node_read` as `expected_hash`. A trashed node's id stays reserved until it is restored or purged. | | `mcp__tapper__node_delete` | Call `node_read`, then atomically remove 1–100 `nodes[]`, each carrying the combined `hash` from `node_read` as `expected_hash`. | @@ -285,7 +285,9 @@ listing resources. Text content is JSON rendered from the same public object as structuredContent. The message field preserves prose, diagnostics, and legacy formatted output. -Images remain in MCP image content blocks alongside JSON metadata. +Images remain in MCP image content blocks alongside JSON metadata. Every tool +declares the object's shape as its `outputSchema`; a tool whose result is only +text declares `{message}`. For node_list, node_grep, tag_list, node_links, and node_backlinks, follow next_offset until null, @@ -299,9 +301,10 @@ node_search pages with an opaque cursor instead: repeat the same query with next_cursor as cursor until next_cursor is absent (text output ends with a `next cursor:` line while more remain). An omitted or 0 limit lets the Hub choose the page size; -1 is not accepted. Hits are ordered by score, then KEG, then node id; -mode reports `keyword` when the Hub ranked without semantic matching. A KEG -outside the flight's cover is never searched, and a Hub without ranked search -fails with a message pointing at node_grep. +mode reports `keyword` when the Hub ranked without semantic matching. An empty +or omitted `kegs` searches every KEG the call's flight covers; naming a KEG +outside that cover fails with an error that names it, and a Hub without ranked +search fails with a message pointing at node_grep. The explore tools return truncated and have no cursor; refine the query. @@ -318,7 +321,7 @@ does not activate it or change the connection's pinned root. | `mcp__tapper__schema_validate` | Schema validation findings. | | `mcp__tapper__keg_info`, `mcp__tapper__node_stats` | KEG diagnostics and node statistics. | | `mcp__tapper__url` | The Hub web link for a KEG, note, flight, or namespace (`ref` as `42`, `@ns/keg/42`, `keg:~alias/42`, `+slug`, `@ns`), returned as `url`, `kind`, and canonical `ref`. Use it to hand the reader a clickable link; it never reads the note. | -| `mcp__tapper__lock_acquire`, `mcp__tapper__lock_status`, `mcp__tapper__lock_release` | Advisory locks; acquisition returns a private token for release. node_edit does not accept a lock token. | +| `mcp__tapper__lock_acquire`, `mcp__tapper__lock_status`, `mcp__tapper__lock_release` | Advisory locks; acquisition returns a private token for release. A held lock is waited for up to `wait_seconds` (default 10, max 60), then fails with `LOCK_TIMEOUT`. node_edit does not accept a lock token. | | `mcp__tapper__file_list`, `mcp__tapper__image_list` | Stored attachment names. | | `mcp__tapper__file_upload`, `mcp__tapper__image_upload` | Inline base64/data URI/embedded resource uploads; local stdio also accepts source_path. Link the returned stored filename. | | `mcp__tapper__image_download` | Image content block and metadata; local stdio optionally accepts dest_path. | diff --git a/integrations/rendered/opencode/tapper/skills/tapper/SKILL.md b/integrations/rendered/opencode/tapper/skills/tapper/SKILL.md index fdc91912..25c306bb 100644 --- a/integrations/rendered/opencode/tapper/skills/tapper/SKILL.md +++ b/integrations/rendered/opencode/tapper/skills/tapper/SKILL.md @@ -124,8 +124,8 @@ accessible transitive descendants. | Tool | Purpose | | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `mcp__tapper__node_search` | Find the nodes most relevant to a query, ranked by the Hub across every KEG the call may read, one `keg`, or the listed `kegs`. Takes plain terms, `"quoted phrases"`, and `-exclusions`. Hits carry `ref`, `score`, `title`, matched `fields`, and a `snippet` with highlights; follow `next_cursor`. | -| `mcp__tapper__node_grep` | Regex match over node content lines in one KEG, in node id order, for exact identifiers, symbols, and audits. Supports `ignore_case`, `limit`, `max_lines`, and `id_only`. | +| `mcp__tapper__node_search` | Find the nodes most relevant to a query, ranked by the Hub across every KEG the call's flight covers, or only the KEGs listed in `kegs` (each inside that cover). Takes plain terms, `"quoted phrases"`, and `-exclusions`. Hits carry `ref`, `score`, `title`, matched `fields`, and a `snippet` with highlights; follow `next_cursor`. | +| `mcp__tapper__node_grep` | PostgreSQL regular-expression match over node content lines in one KEG (`\y` is a word boundary), in node id order, for exact identifiers, symbols, and audits. Supports `ignore_case`, `limit`, `max_lines`, and `format`. | | `mcp__tapper__tag_list` | List tags or filter nodes by a boolean expression over tags, attributes, and dot-prefix stats fields (for example `tapper and .created>2026-01-01`). | | `mcp__tapper__node_list` | List nodes in a keg with optional filters. | | `mcp__tapper__node_read` | Read one or more nodes. Each structured row pairs `node_id` and `hash` with that node's `content` and `meta`, so a read feeds straight into `node_edit`. Supports `meta_only`, `content_only`, `stats_only`, and `query` expression selection as an alternative to explicit node IDs. | @@ -135,7 +135,7 @@ accessible transitive descendants. | `mcp__tapper__keg_settings_read` | Read targeted title, description, updated metadata, and instructions for one or more selected KEGs; batches accept up to 100 canonical references. | Start topic lookups with `node_search`; reach for `node_grep` when you need -every line matching an exact pattern. Pass `id_only: true` to `node_grep` and +every line matching an exact pattern. Pass `format: "%i"` to `node_grep` and `tag_list` when you only need IDs for follow-up reads — it keeps token consumption bounded on large result sets. @@ -169,7 +169,7 @@ code; the index does the work in O(matches) rather than O(total). | Tool | Purpose | | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | -| `mcp__tapper__node_create` | Atomically create 1–100 nodes. Each is a markdown `content` document plus an optional YAML `meta` document; the title is the content's H1. Nodes in one batch reference each other with `{{node:KEY}}`. | +| `mcp__tapper__node_create` | Atomically create 1–100 nodes. Each is a markdown `content` document plus an optional YAML `meta` document; the title is the content's H1. Nodes in one batch reference each other with `{{node:KEY}}`, which becomes the bare id: link with `[Title](../{{node:KEY}})`. | | `mcp__tapper__node_edit` | Call `node_read`, then atomically replace `content`, `meta`, or both for 1–100 `nodes[]`. Each part has its own hash: supplied `content` requires `expected_content_hash` (the read's `content_hash`) and supplied `meta` requires `expected_meta_hash` (its `meta_hash`). Editing one part never invalidates the other's hash, and results return the new `content_hash` and `meta_hash`. For a small change to a long note, send `patch` (a unified diff of the changed lines with a little context) instead of `content`; it needs `expected_content_hash` too, and a patch that no longer fits is refused with the lines it expected. A patch keeps the body's line endings, and `\ No newline at end of file` may only end a hunk that reaches the end of the body. | | `mcp__tapper__node_move` | Call `node_read`, then relocate a node using the combined `hash` from `node_read` as `expected_hash`. A trashed node's id stays reserved until it is restored or purged. | | `mcp__tapper__node_delete` | Call `node_read`, then atomically remove 1–100 `nodes[]`, each carrying the combined `hash` from `node_read` as `expected_hash`. | @@ -285,7 +285,9 @@ listing resources. Text content is JSON rendered from the same public object as structuredContent. The message field preserves prose, diagnostics, and legacy formatted output. -Images remain in MCP image content blocks alongside JSON metadata. +Images remain in MCP image content blocks alongside JSON metadata. Every tool +declares the object's shape as its `outputSchema`; a tool whose result is only +text declares `{message}`. For node_list, node_grep, tag_list, node_links, and node_backlinks, follow next_offset until null, @@ -299,9 +301,10 @@ node_search pages with an opaque cursor instead: repeat the same query with next_cursor as cursor until next_cursor is absent (text output ends with a `next cursor:` line while more remain). An omitted or 0 limit lets the Hub choose the page size; -1 is not accepted. Hits are ordered by score, then KEG, then node id; -mode reports `keyword` when the Hub ranked without semantic matching. A KEG -outside the flight's cover is never searched, and a Hub without ranked search -fails with a message pointing at node_grep. +mode reports `keyword` when the Hub ranked without semantic matching. An empty +or omitted `kegs` searches every KEG the call's flight covers; naming a KEG +outside that cover fails with an error that names it, and a Hub without ranked +search fails with a message pointing at node_grep. The explore tools return truncated and have no cursor; refine the query. @@ -318,7 +321,7 @@ does not activate it or change the connection's pinned root. | `mcp__tapper__schema_validate` | Schema validation findings. | | `mcp__tapper__keg_info`, `mcp__tapper__node_stats` | KEG diagnostics and node statistics. | | `mcp__tapper__url` | The Hub web link for a KEG, note, flight, or namespace (`ref` as `42`, `@ns/keg/42`, `keg:~alias/42`, `+slug`, `@ns`), returned as `url`, `kind`, and canonical `ref`. Use it to hand the reader a clickable link; it never reads the note. | -| `mcp__tapper__lock_acquire`, `mcp__tapper__lock_status`, `mcp__tapper__lock_release` | Advisory locks; acquisition returns a private token for release. node_edit does not accept a lock token. | +| `mcp__tapper__lock_acquire`, `mcp__tapper__lock_status`, `mcp__tapper__lock_release` | Advisory locks; acquisition returns a private token for release. A held lock is waited for up to `wait_seconds` (default 10, max 60), then fails with `LOCK_TIMEOUT`. node_edit does not accept a lock token. | | `mcp__tapper__file_list`, `mcp__tapper__image_list` | Stored attachment names. | | `mcp__tapper__file_upload`, `mcp__tapper__image_upload` | Inline base64/data URI/embedded resource uploads; local stdio also accepts source_path. Link the returned stored filename. | | `mcp__tapper__image_download` | Image content block and metadata; local stdio optionally accepts dest_path. | diff --git a/internal/relay/client.go b/internal/relay/client.go index 6143b959..db7325ba 100644 --- a/internal/relay/client.go +++ b/internal/relay/client.go @@ -1,11 +1,13 @@ package relay import ( + "cmp" "context" "encoding/json" "errors" "fmt" "log/slog" + "math/rand/v2" "net/http" "net/url" "slices" @@ -31,6 +33,16 @@ var ErrIncompatible = errors.New("hub does not support this relay's protocol ver // Reconnecting would undo that, so it is permanent. var ErrDisconnected = errors.New("relay was disconnected from Hub by its owner; restart `tap relay` to reconnect") +// ErrRevoked means Hub revoked the credential the relay runs under (a +// sign-out, a revoked token). Reconnecting cannot fix it; the user has to +// sign in again. +var ErrRevoked = errors.New("hub revoked the relay's sign-in; run `tap auth login`, then restart `tap relay`") + +// errDialRejected is a dial Hub answered 401 or 403. It is ErrUnauthorized, +// but runHub gives it one retry with a forced refresh first: the local view +// of the token's expiry can be wrong (a skewed clock), and only Hub knows. +var errDialRejected = fmt.Errorf("%w (hub refused the token)", ErrUnauthorized) + const ( defaultCatalogInterval = time.Minute defaultReadTimeout = 90 * time.Second @@ -47,6 +59,12 @@ type Hub struct { // Token returns the current bearer token. It is called on every dial so a // refreshed token is picked up after a reconnect. Token func(ctx context.Context) (string, error) + // Refresh, when set, returns a freshly refreshed token whatever the + // cached one's recorded expiry says. The relay calls it to reauth a live + // session before Hub expires its token, to redial after + // credential_expired, and to retry a dial Hub refused. An error wrapping + // ErrUnauthorized means the sign-in is gone. Without it Token is used. + Refresh func(ctx context.Context) (string, error) } // Options configures a Client. @@ -210,26 +228,77 @@ func (c *Client) advertisedLimit() int { // supportedProtocols are the relay protocol versions this client speaks, // offered in register. Hub picks the highest it also speaks. -var supportedProtocols = []int{1, 2, 3} +var supportedProtocols = []int{1, 2, 3, 4} // runHub keeps one hub connected until ctx ends or that hub rejects the // relay for good. +// +// Each connection-level end code has its own arm: disconnected (the owner) +// and credential_revoked are final, credential_expired redials at once with +// a forced refresh, going_away redials after a short random delay, and any +// other code, one this relay does not know included, redials with backoff. func (c *Client) runHub(ctx context.Context, hub Hub) error { backoff := minBackoff + // force makes the next dial present a freshly refreshed token; + // rejectRetries is how many refused dials still get a forced retry. + force, rejectRetries := false, 1 for { - registered, err := c.runOnce(ctx, hub) + started := time.Now() + registered, err := c.runOnce(ctx, hub, force) + force = false if ctx.Err() != nil { return nil } + if registered { + backoff, rejectRetries = minBackoff, 1 + } + if errors.Is(err, errDialRejected) && rejectRetries > 0 && hub.Refresh != nil { + // Hub refused a token this relay thought was good. One dial + // with a forced refresh tells a skewed clock from a lost + // sign-in; a second refusal is final. + rejectRetries-- + force = true + c.logger.Info("hub refused the relay credential; retrying with a refreshed token", "hub", hub.URL) + continue + } if errors.Is(err, ErrUnauthorized) || errors.Is(err, ErrIncompatible) { return err } var hubErr *relaycontract.Error - if errors.As(err, &hubErr) && hubErr.Code == relaycontract.CodeDisconnected { - return ErrDisconnected - } - if registered { - backoff = minBackoff + if errors.As(err, &hubErr) { + switch hubErr.Code { + case relaycontract.CodeDisconnected: + return ErrDisconnected + case relaycontract.CodeCredentialRevoked: + return ErrRevoked + case relaycontract.CodeCredentialExpired: + // Whatever the local clock thinks, Hub says the token + // expired: the next dial refreshes it, and a refusal of that + // token still gets its one forced retry. + force, rejectRetries = true, 1 + if registered && time.Since(started) >= minBackoff { + // A session that ends this way faster than minBackoff + // takes the normal backoff, so a hub that keeps answering + // it cannot make the relay spin. + c.logger.Info("relay credential expired; reconnecting with a fresh token", "hub", hub.URL) + backoff = minBackoff + continue + } + case relaycontract.CodeGoingAway: + if registered { + // Hub is restarting. Come back at once, spread over a + // moment so every relay does not arrive together, and + // without counting it against the backoff. + wait := rand.N(minBackoff) + c.logger.Info("hub is restarting; reconnecting", "hub", hub.URL, "after", wait) + select { + case <-ctx.Done(): + return nil + case <-time.After(wait): + } + continue + } + } } c.logger.Warn("relay disconnected; reconnecting", "hub", hub.URL, "error", err, "backoff", backoff) select { @@ -258,12 +327,21 @@ func ConnectURL(hubURL string) (string, error) { return u.String(), nil } -func (c *Client) dial(ctx context.Context, hub Hub) (*websocket.Conn, error) { +// hubToken asks hub for its bearer token: a forced refresh when force is set +// and the hub can refresh, the current token otherwise. +func hubToken(ctx context.Context, hub Hub, force bool) (string, error) { + if force && hub.Refresh != nil { + return hub.Refresh(ctx) + } + return hub.Token(ctx) +} + +func (c *Client) dial(ctx context.Context, hub Hub, force bool) (*websocket.Conn, error) { target, err := ConnectURL(hub.URL) if err != nil { return nil, err } - token, err := hub.Token(ctx) + token, err := hubToken(ctx, hub, force) if err != nil { return nil, err } @@ -281,7 +359,7 @@ func (c *Client) dial(ctx context.Context, hub Hub) (*websocket.Conn, error) { if resp != nil { switch resp.StatusCode { case http.StatusUnauthorized, http.StatusForbidden: - return nil, ErrUnauthorized + return nil, errDialRejected case http.StatusBadRequest: if resp.Header.Get(apicontract.VersionHeader) == "" { return nil, fmt.Errorf("%w: %v", ErrIncompatible, err) @@ -351,6 +429,7 @@ func (c *Client) listModels(ctx context.Context) []relaycontract.Model { type session struct { c *Client conn *websocket.Conn + hub Hub // protocol is the version Hub chose in registered. Frames newer than it // must not be sent on this session. @@ -366,25 +445,40 @@ type session struct { offered map[string]struct{} // provider + "/" + model inflight map[string]context.CancelFunc + // reauthID is the id of the reauth awaiting Hub's answer, which the + // read loop hands to reauthReply. + reauthID string + reauthReply chan reauthResult + reauthSeq int + wg sync.WaitGroup } +// reauthResult is Hub's answer to a reauth: the new lifetime, or rejected. +type reauthResult struct { + expiresIn int + at time.Time + rejected string +} + func modelKey(provider, model string) string { return provider + "/" + model } -func (c *Client) runOnce(ctx context.Context, hub Hub) (registered bool, err error) { +func (c *Client) runOnce(ctx context.Context, hub Hub, force bool) (registered bool, err error) { c.mu.Lock() models := c.models c.mu.Unlock() - conn, err := c.dial(ctx, hub) + conn, err := c.dial(ctx, hub, force) if err != nil { return false, err } defer conn.CloseNow() s := &session{ - c: c, - conn: conn, - inflight: make(map[string]context.CancelFunc), + c: c, + conn: conn, + hub: hub, + inflight: make(map[string]context.CancelFunc), + reauthReply: make(chan reauthResult, 1), } s.setOffered(models) @@ -399,6 +493,9 @@ func (c *Client) runOnce(ctx context.Context, hub Hub) (registered bool, err err if err != nil { return false, err } + // The credential's lifetime counts from here, on the monotonic clock. + receivedAt := time.Now() + var expiresIn int switch env.Type { case relaycontract.TypeRegistered: var reg relaycontract.Registered @@ -409,6 +506,9 @@ func (c *Client) runOnce(ctx context.Context, hub Hub) (registered bool, err err return false, fmt.Errorf("%w: hub chose protocol %d", ErrIncompatible, reg.Protocol) } s.protocol = reg.Protocol + if reg.Protocol >= 4 { + expiresIn = reg.CredentialExpiresIn + } c.logger.Info("relay registered", "hub", hub.URL, "name", c.opts.Name, "protocol", reg.Protocol, "models", len(reg.Models)) if c.opts.OnRegistered != nil { c.opts.OnRegistered(hub.URL, reg) @@ -452,6 +552,13 @@ func (c *Client) runOnce(ctx context.Context, hub Hub) (registered bool, err err } else if len(c.toolServers) > 0 { c.logger.Warn("hub does not support relayed tools; serving models only", "hub", hub.URL, "protocol", s.protocol) } + if expiresIn > 0 { + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.keepCredential(sessCtx, receivedAt, expiresIn) + }() + } return true, s.readLoop(sessCtx) } @@ -521,9 +628,22 @@ func (s *session) readLoop(ctx context.Context) error { if cancel != nil { cancel() } + case relaycontract.TypeReauthed: + var r relaycontract.Reauthed + if env.Decode(&r) == nil { + s.deliverReauth(env.ID, reauthResult{expiresIn: r.CredentialExpiresIn, at: time.Now()}) + } case relaycontract.TypeError: var e relaycontract.Error - if env.Decode(&e) == nil && env.ID == "" { + if env.Decode(&e) != nil { + continue + } + if e.Code == relaycontract.CodeReauthRejected { + // Not an end: the old credential stands until it expires. + s.deliverReauth(env.ID, reauthResult{rejected: cmp.Or(e.Message, e.Code)}) + continue + } + if env.ID == "" { return &e } default: @@ -532,6 +652,115 @@ func (s *session) readLoop(ctx context.Context) error { } } +// refreshAt is the share of a credential's lifetime after which the relay +// reauths, leaving the rest for a refresh that has to be retried. +const refreshAt = 0.8 + +// reauthTimeout bounds the wait for Hub's answer to one reauth. +const reauthTimeout = 30 * time.Second + +// keepCredential reauths the session before Hub expires its credential: +// at refreshAt of each lifetime, timed on the monotonic clock from the frame +// that announced it, so a skewed wall clock on either side cannot move it. +// A failed or rejected reauth is retried halfway to the expiry while that is +// still a second or more away; after that the session ends with +// credential_expired and the relay redials with a fresh token. +func (s *session) keepCredential(ctx context.Context, from time.Time, expiresIn int) { + lifetime := time.Duration(expiresIn) * time.Second + deadline := from.Add(lifetime) + next := from.Add(time.Duration(float64(lifetime) * refreshAt)) + for { + timer := time.NewTimer(time.Until(next)) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + res, err := s.reauth(ctx) + if ctx.Err() != nil { + return + } + if err == nil && res.rejected == "" { + if res.expiresIn <= 0 { + // Hub no longer expires this session. + return + } + lifetime = time.Duration(res.expiresIn) * time.Second + deadline = res.at.Add(lifetime) + next = res.at.Add(time.Duration(float64(lifetime) * refreshAt)) + s.c.logger.Debug("relay credential renewed", "hub", s.hub.URL, "expiresIn", res.expiresIn) + continue + } + if err == nil { + err = errors.New(res.rejected) + } + wait := time.Until(deadline) / 2 + s.c.logger.Warn("relay could not renew its credential", "hub", s.hub.URL, "error", err, "retryIn", max(wait, 0)) + if wait < minBackoff { + return + } + next = time.Now().Add(wait) + } +} + +// reauth presents a freshly refreshed token and waits for Hub's answer. +func (s *session) reauth(ctx context.Context) (reauthResult, error) { + token, err := hubToken(ctx, s.hub, true) + if err != nil { + return reauthResult{}, err + } + if token == "" { + return reauthResult{}, ErrUnauthorized + } + // Drop an answer that arrived after an earlier reauth gave up on it. + select { + case <-s.reauthReply: + default: + } + s.mu.Lock() + s.reauthSeq++ + id := fmt.Sprintf("reauth-%d", s.reauthSeq) + s.reauthID = id + s.mu.Unlock() + defer func() { + s.mu.Lock() + s.reauthID = "" + s.mu.Unlock() + }() + if err := s.send(ctx, relaycontract.TypeReauth, id, relaycontract.Reauth{Token: token}); err != nil { + return reauthResult{}, err + } + timer := time.NewTimer(reauthTimeout) + defer timer.Stop() + select { + case <-ctx.Done(): + return reauthResult{}, ctx.Err() + case <-timer.C: + return reauthResult{}, errors.New("hub did not answer the reauth") + case res := <-s.reauthReply: + return res, nil + } +} + +// deliverReauth hands Hub's answer to the reauth awaiting it. An answer for +// no pending reauth is dropped. Hub echoes the reauth's id; an answer with +// no id is taken for the pending one. +func (s *session) deliverReauth(id string, res reauthResult) { + s.mu.Lock() + pending := s.reauthID + if pending == "" || (id != "" && id != pending) { + s.mu.Unlock() + return + } + s.reauthID = "" + s.mu.Unlock() + select { + case s.reauthReply <- res: + default: + } +} + func (s *session) startInfer(ctx context.Context, env relaycontract.Envelope) { var req relaycontract.Infer if err := env.Decode(&req); err != nil { @@ -755,7 +984,10 @@ func (c *Client) toolsChanged() { } // toolCatalog lists the servers that are up, in configuration order, within -// the protocol's limits on total tools and frame size. +// the protocol's limits on total tools and frame size. Over the frame limit, +// output schemas go first, largest first: they only describe results, which +// still pass through, so losing one costs less than losing a server. Only +// when every output schema is gone does it drop servers from the end. func (c *Client) toolCatalog() []relaycontract.ToolServer { out := []relaycontract.ToolServer{} total := 0 @@ -771,6 +1003,9 @@ func (c *Client) toolCatalog() []relaycontract.ToolServer { total += len(server.Tools) out = append(out, server) } + if raw, _ := json.Marshal(relaycontract.Tools{Servers: out}); len(raw) > relaycontract.MaxToolsFrameBytes { + c.stripOutputSchemas(out, len(raw)-relaycontract.MaxToolsFrameBytes) + } for { raw, _ := json.Marshal(relaycontract.Tools{Servers: out}) if len(raw) <= relaycontract.MaxToolsFrameBytes || len(out) == 0 { @@ -782,6 +1017,39 @@ func (c *Client) toolCatalog() []relaycontract.ToolServer { } } +// stripOutputSchemas removes output schemas from servers, largest first, +// until about excess bytes are saved. Tools slices come from +// ToolServer.catalog, which clones them, so this never touches a server's +// own listing. +func (c *Client) stripOutputSchemas(servers []relaycontract.ToolServer, excess int) { + type ref struct{ server, tool int } + var refs []ref + for i := range servers { + for j := range servers[i].Tools { + if len(servers[i].Tools[j].OutputSchema) > 0 { + refs = append(refs, ref{i, j}) + } + } + } + size := func(r ref) int { return len(servers[r.server].Tools[r.tool].OutputSchema) } + slices.SortStableFunc(refs, func(a, b ref) int { return size(b) - size(a) }) + // The field's key and separators go with the schema. + const fieldOverhead = len(`,"outputSchema":`) + stripped := 0 + for _, r := range refs { + if excess <= 0 { + break + } + tool := &servers[r.server].Tools[r.tool] + excess -= len(tool.OutputSchema) + fieldOverhead + tool.OutputSchema = nil + stripped++ + } + if stripped > 0 { + c.logger.Warn("relay mcp tools exceed the frame limit; leaving off output schemas", "tools", stripped) + } +} + func sameToolCatalog(a, b []relaycontract.ToolServer) bool { ra, _ := json.Marshal(a) rb, _ := json.Marshal(b) diff --git a/internal/relay/reauth_test.go b/internal/relay/reauth_test.go new file mode 100644 index 00000000..0d3eb0e9 --- /dev/null +++ b/internal/relay/reauth_test.go @@ -0,0 +1,445 @@ +package relay + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" + + "github.com/jlrickert/tapper/pkg/relaycontract" +) + +// handshakeV4 reads register and answers registered on protocol with the +// credential lifetime expiresIn (seconds). +func handshakeV4(t *testing.T, ctx context.Context, c *websocket.Conn, protocol, expiresIn int) { + t.Helper() + env := readEnv(t, ctx, c) + if env.Type != relaycontract.TypeRegister { + t.Fatalf("first frame = %q, want register", env.Type) + } + writeEnv(t, ctx, c, relaycontract.TypeRegistered, "", relaycontract.Registered{ + Protocol: protocol, Models: []relaycontract.CatalogBinding{}, CredentialExpiresIn: expiresIn, + }) +} + +// readFrame reads one frame, or reports false when none arrives within wait. +func readFrame(t *testing.T, c *websocket.Conn, wait time.Duration) (relaycontract.Envelope, bool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), wait) + defer cancel() + var env relaycontract.Envelope + if err := wsjson.Read(ctx, c, &env); err != nil { + if ctx.Err() != nil { + return env, false + } + t.Fatalf("read frame: %v", err) + } + return env, true +} + +// awaitReauth reads frames until a reauth, failing if none arrives before +// within: the hub would have ended the session by then. +func awaitReauth(t *testing.T, c *websocket.Conn, within time.Duration) (relaycontract.Envelope, relaycontract.Reauth) { + t.Helper() + deadline := time.Now().Add(within) + for { + env, ok := readFrame(t, c, time.Until(deadline)) + if !ok { + t.Fatalf("no reauth within %s; the hub would have expired the session", within) + } + if env.Type != relaycontract.TypeReauth { + continue + } + var r relaycontract.Reauth + if err := env.Decode(&r); err != nil { + t.Fatal(err) + } + return env, r + } +} + +// refreshingHub is a hub whose Refresh hands out fresh-1, fresh-2, … +func refreshingHub(hub *fakeHub, token string, refreshes *atomic.Int32) Hub { + h := hubFor(hub, token) + h.Refresh = func(context.Context) (string, error) { + return "fresh-" + string(rune('0'+refreshes.Add(1))), nil + } + return h +} + +// assertNoRedial fails if the relay dials the hub again within wait. +func assertNoRedial(t *testing.T, hub *fakeHub, wait time.Duration) { + t.Helper() + select { + case got := <-hub.auth: + t.Fatalf("relay redialed (%s); the session should have continued", got) + case <-time.After(wait): + } +} + +// Issue 295: a session whose token lives two seconds stays up across +// several lifetimes, in-flight work included, because the relay reauths +// before each expiry. +func TestReauthKeepsSessionAcrossCredentialExpiry(t *testing.T) { + fp, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var refreshes atomic.Int32 + startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{refreshingHub(hub, "tok", &refreshes)} + }) + ctx := context.Background() + conn := hub.accept(t) + <-hub.auth + const ttl = 2 + handshakeV4(t, ctx, conn, relaycontract.ProtocolVersion, ttl) + registeredAt := time.Now() + + // A long request in flight across the expiry. + writeEnv(t, ctx, conn, relaycontract.TypeInfer, "long", relaycontract.Infer{ + API: relaycontract.APIOpenAIChatCompletions, Provider: "ollama", Model: "qwen3:8b", + Body: json.RawMessage(`{"messages":[{"role":"user","content":"block"}]}`), + }) + <-fp.blocked + + for i := 1; i <= 2; i++ { + env, r := awaitReauth(t, conn, ttl*time.Second) + if env.ID == "" { + t.Fatal("reauth has no id") + } + if want := "fresh-" + string(rune('0'+i)); r.Token != want { + t.Fatalf("reauth %d token = %q, want the force-refreshed %q", i, r.Token, want) + } + if i == 1 && time.Since(registeredAt) >= ttl*time.Second { + t.Fatalf("first reauth came %s after registered, at or past the expiry", time.Since(registeredAt)) + } + writeEnv(t, ctx, conn, relaycontract.TypeReauthed, env.ID, relaycontract.Reauthed{CredentialExpiresIn: ttl}) + } + assertNoRedial(t, hub, 0) + + // The request that started under the first token is still live on the + // same connection. + writeEnv(t, ctx, conn, relaycontract.TypeCancel, "long", nil) + for { + env, ok := readFrame(t, conn, 5*time.Second) + if !ok { + t.Fatal("in-flight request did not answer its cancel") + } + if env.Type == relaycontract.TypeReauth { + writeEnv(t, ctx, conn, relaycontract.TypeReauthed, env.ID, relaycontract.Reauthed{CredentialExpiresIn: ttl}) + continue + } + var e relaycontract.Error + if env.ID != "long" || env.Type != relaycontract.TypeError || env.Decode(&e) != nil || e.Code != relaycontract.CodeCancelled { + t.Fatalf("frame = %s %s %s, want the in-flight request's cancellation", env.ID, env.Type, env.Payload) + } + break + } +} + +// Old hubs keep working: a protocol 3 session, or a protocol 4 session that +// advertises no expiry, never sees a reauth frame it would not understand. +func TestNoReauthWithoutAdvertisedExpiry(t *testing.T) { + for name, tc := range map[string]struct{ protocol, expiresIn int }{ + "protocol 3": {3, 1}, + "protocol 4 no expiry": {relaycontract.ProtocolVersion, 0}, + } { + t.Run(name, func(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var refreshes atomic.Int32 + startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{refreshingHub(hub, "tok", &refreshes)} + }) + ctx := context.Background() + conn := hub.accept(t) + handshakeV4(t, ctx, conn, tc.protocol, tc.expiresIn) + if env, ok := readFrame(t, conn, 1500*time.Millisecond); ok && env.Type == relaycontract.TypeReauth { + t.Fatalf("relay sent reauth on a %s session", name) + } + writeEnv(t, ctx, conn, relaycontract.TypePing, "", nil) + for { + env := readEnv(t, ctx, conn) + if env.Type == relaycontract.TypeReauth { + t.Fatalf("relay sent reauth on a %s session", name) + } + if env.Type == relaycontract.TypePong { + break + } + } + if n := refreshes.Load(); n != 0 { + t.Fatalf("relay refreshed %d times with no expiry to beat", n) + } + }) + } +} + +// A rejected reauth does not end the session: the old credential stands +// until Hub expires it. +func TestRejectedReauthKeepsSession(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var refreshes atomic.Int32 + startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{refreshingHub(hub, "tok", &refreshes)} + }) + ctx := context.Background() + conn := hub.accept(t) + <-hub.auth + handshakeV4(t, ctx, conn, relaycontract.ProtocolVersion, 2) + env, _ := awaitReauth(t, conn, 2*time.Second) + writeEnv(t, ctx, conn, relaycontract.TypeError, env.ID, relaycontract.Error{Code: relaycontract.CodeReauthRejected, Message: "token is for another account"}) + writeEnv(t, ctx, conn, relaycontract.TypePing, "", nil) + if got := readEnv(t, ctx, conn); got.Type != relaycontract.TypePong { + t.Fatalf("frame = %q, want pong on the same session", got.Type) + } + assertNoRedial(t, hub, 200*time.Millisecond) +} + +// recordLogs is a slog handler that keeps every message. +type recordLogs struct { + mu sync.Mutex + msgs []string +} + +func (r *recordLogs) Enabled(context.Context, slog.Level) bool { return true } +func (r *recordLogs) Handle(_ context.Context, rec slog.Record) error { + r.mu.Lock() + r.msgs = append(r.msgs, rec.Message) + r.mu.Unlock() + return nil +} +func (r *recordLogs) WithAttrs([]slog.Attr) slog.Handler { return r } +func (r *recordLogs) WithGroup(string) slog.Handler { return r } + +func (r *recordLogs) has(msg string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, m := range r.msgs { + if m == msg { + return true + } + } + return false +} + +// Issue 334: every end code Hub can send has its own outcome. The final +// ones stop the relay with their own error, each naming what the user can +// do; the retryable ones redial, each logging its own reason. An owner +// disconnect still says so. +func TestEndCodesHaveDistinctOutcomes(t *testing.T) { + final := map[string]struct { + want error + mention string + }{ + relaycontract.CodeDisconnected: {ErrDisconnected, "owner"}, + relaycontract.CodeCredentialRevoked: {ErrRevoked, "tap auth login"}, + } + messages := map[string]string{} + for code, tc := range final { + t.Run(code, func(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var refreshes atomic.Int32 + _, done := startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{refreshingHub(hub, "tok", &refreshes)} + }) + ctx := context.Background() + conn := hub.accept(t) + <-hub.auth + handshakeV4(t, ctx, conn, relaycontract.ProtocolVersion, 3600) + writeEnv(t, ctx, conn, relaycontract.TypeError, "", relaycontract.Error{Code: code, Message: "bye"}) + select { + case err := <-done: + if !errors.Is(err, tc.want) { + t.Fatalf("Run = %v, want %v", err, tc.want) + } + if !strings.Contains(err.Error(), tc.mention) { + t.Fatalf("message %q does not mention %q", err, tc.mention) + } + messages[code] = err.Error() + done <- err + case <-time.After(5 * time.Second): + t.Fatalf("relay kept running after %s", code) + } + assertNoRedial(t, hub, 200*time.Millisecond) + }) + } + if messages[relaycontract.CodeDisconnected] == messages[relaycontract.CodeCredentialRevoked] { + t.Fatal("owner disconnect and revocation read the same") + } + if strings.Contains(messages[relaycontract.CodeCredentialRevoked], "owner") { + t.Fatalf("revocation reported as an owner disconnect: %q", messages[relaycontract.CodeCredentialRevoked]) + } + + retry := map[string]struct { + log string + withinFirst time.Duration // a redial within this, with no backoff + wantToken string + }{ + relaycontract.CodeCredentialExpired: {"relay credential expired; reconnecting with a fresh token", minBackoff / 2, "Bearer fresh-1"}, + relaycontract.CodeGoingAway: {"hub is restarting; reconnecting", minBackoff + 200*time.Millisecond, "Bearer tok"}, + "some_future_code": {"relay disconnected; reconnecting", 5 * time.Second, "Bearer tok"}, + } + logs := map[string]bool{} + for code, tc := range retry { + t.Run(code, func(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + rec := &recordLogs{} + var refreshes atomic.Int32 + _, done := startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{refreshingHub(hub, "tok", &refreshes)} + o.Logger = slog.New(rec) + }) + ctx := context.Background() + conn := hub.accept(t) + <-hub.auth + handshakeV4(t, ctx, conn, relaycontract.ProtocolVersion, 3600) + time.Sleep(minBackoff + 100*time.Millisecond) + writeEnv(t, ctx, conn, relaycontract.TypeError, "", relaycontract.Error{Code: code, Message: "later"}) + _ = conn.Close(websocket.StatusTryAgainLater, code) + select { + case got := <-hub.auth: + if got != tc.wantToken { + t.Fatalf("redial auth = %q, want %q", got, tc.wantToken) + } + case err := <-done: + done <- err + t.Fatalf("relay stopped after %s: %v", code, err) + case <-time.After(tc.withinFirst): + t.Fatalf("relay did not redial within %s after %s", tc.withinFirst, code) + } + if !rec.has(tc.log) { + t.Fatalf("logs %v, want %q", rec.msgs, tc.log) + } + logs[tc.log] = true + handshake(t, ctx, hub.accept(t)) + }) + } + if len(logs) != len(retry) { + t.Fatalf("retryable end codes share log messages: %v", logs) + } +} + +// Issue 335: Hub refuses a token the relay's clock thinks is still good. +// One forced refresh decides: the fresh token connects. +func TestRefusedDialRetriesOnceWithForcedRefresh(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var refreshes atomic.Int32 + startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{refreshingHub(hub, "revoked", &refreshes)} + }) + select { + case got := <-hub.auth: + if got != "Bearer fresh-1" { + t.Fatalf("auth = %q, want the force-refreshed token", got) + } + case <-time.After(5 * time.Second): + t.Fatal("relay gave up instead of retrying with a refreshed token") + } + handshake(t, context.Background(), hub.accept(t)) +} + +// Revocation still stops the relay: a refreshed token Hub refuses too is +// final after exactly one retry, and a refresh Hub rejects outright is +// final at once. +func TestRevokedSignInStillStopsTheRelay(t *testing.T) { + cases := map[string]func(context.Context) (string, error){ + "refused again": func(context.Context) (string, error) { return "revoked", nil }, + "refresh rejected": func(context.Context) (string, error) { + return "", errors.Join(ErrUnauthorized, errors.New("invalid_grant")) + }, + } + for name, refresh := range cases { + t.Run(name, func(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var calls atomic.Int32 + _, done := startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + h := hubFor(hub, "revoked") + h.Refresh = func(ctx context.Context) (string, error) { + calls.Add(1) + return refresh(ctx) + } + o.Hubs = []Hub{h} + }) + select { + case err := <-done: + if !errors.Is(err, ErrUnauthorized) { + t.Fatalf("Run = %v, want ErrUnauthorized", err) + } + done <- err + case <-time.After(5 * time.Second): + t.Fatal("relay kept retrying a revoked sign-in") + } + if n := calls.Load(); n != 1 { + t.Fatalf("forced refreshes = %d, want exactly 1", n) + } + }) + } + + // And mid-session, on a session that reauths. + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var refreshes atomic.Int32 + _, done := startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{refreshingHub(hub, "tok", &refreshes)} + }) + ctx := context.Background() + conn := hub.accept(t) + <-hub.auth + handshakeV4(t, ctx, conn, relaycontract.ProtocolVersion, 2) + writeEnv(t, ctx, conn, relaycontract.TypeError, "", relaycontract.Error{Code: relaycontract.CodeCredentialRevoked, Message: "signed out"}) + select { + case err := <-done: + if !errors.Is(err, ErrRevoked) { + t.Fatalf("Run = %v, want ErrRevoked", err) + } + done <- err + case <-time.After(5 * time.Second): + t.Fatal("revocation did not stop the relay") + } +} + +// After credential_expired the redial presents a forced refresh, and if Hub +// refuses that token it gets one more forced refresh before giving up. +func TestCredentialExpiredRedialGetsOneRefusedRetry(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var calls atomic.Int32 + startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + h := hubFor(hub, "tok") + h.Refresh = func(context.Context) (string, error) { + if calls.Add(1) == 1 { + return "revoked", nil // not yet visible to this hub + } + return "fresh", nil + } + o.Hubs = []Hub{h} + }) + ctx := context.Background() + conn := hub.accept(t) + <-hub.auth + handshakeV4(t, ctx, conn, relaycontract.ProtocolVersion, 3600) + time.Sleep(minBackoff + 100*time.Millisecond) + writeEnv(t, ctx, conn, relaycontract.TypeError, "", relaycontract.Error{Code: relaycontract.CodeCredentialExpired}) + _ = conn.Close(websocket.StatusTryAgainLater, "credential expired") + select { + case got := <-hub.auth: + if got != "Bearer fresh" { + t.Fatalf("auth = %q, want the second forced refresh", got) + } + case <-time.After(5 * time.Second): + t.Fatal("relay stopped after a refused redial") + } + handshake(t, ctx, hub.accept(t)) +} diff --git a/internal/relay/relay_test.go b/internal/relay/relay_test.go index 08d70244..34af5c66 100644 --- a/internal/relay/relay_test.go +++ b/internal/relay/relay_test.go @@ -11,6 +11,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -495,8 +496,8 @@ func TestRegisterOffersEveryProtocolInVersion1Envelope(t *testing.T) { if err := env.Decode(®); err != nil { t.Fatal(err) } - if !slices.Equal(reg.Relay.Protocols, []int{1, 2, 3}) { - t.Fatalf("protocols = %v, want [1 2 3]", reg.Relay.Protocols) + if !slices.Equal(reg.Relay.Protocols, []int{1, 2, 3, 4}) { + t.Fatalf("protocols = %v, want [1 2 3 4]", reg.Relay.Protocols) } // An old hub answers with protocol 1 and the session works as before. writeEnv(t, ctx, conn, relaycontract.TypeRegistered, "", relaycontract.Registered{Protocol: 1, Models: []relaycontract.CatalogBinding{}}) @@ -513,7 +514,7 @@ func TestHubChoosingUnofferedProtocolIsPermanent(t *testing.T) { ctx := context.Background() conn := hub.accept(t) readEnv(t, ctx, conn) - writeEnv(t, ctx, conn, relaycontract.TypeRegistered, "", relaycontract.Registered{Protocol: 4, Models: []relaycontract.CatalogBinding{}}) + writeEnv(t, ctx, conn, relaycontract.TypeRegistered, "", relaycontract.Registered{Protocol: relaycontract.ProtocolVersion + 1, Models: []relaycontract.CatalogBinding{}}) select { case err := <-done: if !errors.Is(err, ErrIncompatible) { @@ -915,3 +916,72 @@ func TestProviderPriorityIsAdvertised(t *testing.T) { } } } + +// countingTokens hands out tok-1, tok-2, … one per dial, like a token source +// that refreshes an expired credential. +func countingTokens(n *atomic.Int32) func(context.Context) (string, error) { + return func(context.Context) (string, error) { + return fmt.Sprintf("tok-%d", n.Add(1)), nil + } +} + +func TestRunHubReconnectsAfterCredentialExpired(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var dials atomic.Int32 + _, done := startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{{URL: hub.srv.URL, Token: countingTokens(&dials)}} + }) + ctx := context.Background() + conn := hub.accept(t) + if got := <-hub.auth; got != "Bearer tok-1" { + t.Fatalf("first dial auth = %q", got) + } + handshake(t, ctx, conn) + // A session that lived a while, as one does until its token expires. + time.Sleep(minBackoff + 100*time.Millisecond) + writeEnv(t, ctx, conn, relaycontract.TypeError, "", relaycontract.Error{ + Code: relaycontract.CodeCredentialExpired, Message: "credential expired", + }) + _ = conn.Close(websocket.StatusTryAgainLater, "credential expired") + + select { + case got := <-hub.auth: + if got != "Bearer tok-2" { + t.Fatalf("reconnect auth = %q, want a fresh token", got) + } + case err := <-done: + done <- err + t.Fatalf("relay stopped after credential_expired: %v", err) + case <-time.After(minBackoff / 2): + t.Fatal("relay did not redial promptly after credential_expired") + } + handshake(t, ctx, hub.accept(t)) +} + +// A hub that answers every session with credential_expired straight away +// must not make the relay redial in a tight loop. +func TestRunHubBacksOffWhenCredentialExpiresAtOnce(t *testing.T) { + _, psrv := newFakeProvider(t, "qwen3:8b") + hub := newFakeHub(t) + var dials atomic.Int32 + startRelay(t, hub, []*Provider{ollama(t, psrv.URL)}, func(o *Options) { + o.Hubs = []Hub{{URL: hub.srv.URL, Token: countingTokens(&dials)}} + }) + ctx := context.Background() + conn := hub.accept(t) + <-hub.auth + handshake(t, ctx, conn) + writeEnv(t, ctx, conn, relaycontract.TypeError, "", relaycontract.Error{Code: relaycontract.CodeCredentialExpired}) + _ = conn.Close(websocket.StatusTryAgainLater, "credential expired") + select { + case <-hub.auth: + t.Fatal("relay redialed without backing off") + case <-time.After(minBackoff / 2): + } + select { + case <-hub.auth: + case <-time.After(5 * time.Second): + t.Fatal("relay never redialed") + } +} diff --git a/internal/relay/tools.go b/internal/relay/tools.go index 4d2037f0..786fe698 100644 --- a/internal/relay/tools.go +++ b/internal/relay/tools.go @@ -299,6 +299,13 @@ func contractTool(tool *sdkmcp.Tool) (relaycontract.Tool, error) { Description: tool.Description, InputSchema: schema, } + // An output schema Hub would refuse is left off rather than costing the + // tool: it only describes results, and the results still pass through. + if tool.OutputSchema != nil { + if out, err := json.Marshal(tool.OutputSchema); err == nil && len(out) <= relaycontract.MaxToolSchemaBytes && out[0] == '{' { + t.OutputSchema = out + } + } if a := tool.Annotations; a != nil { if t.Title == "" { t.Title = a.Title diff --git a/internal/relay/tools_test.go b/internal/relay/tools_test.go index dca9c80d..86486f3b 100644 --- a/internal/relay/tools_test.go +++ b/internal/relay/tools_test.go @@ -3,6 +3,7 @@ package relay import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -51,10 +52,11 @@ func textResult(text string) *sdkmcp.CallToolResult { func addEchoTool(server *sdkmcp.Server) { server.AddTool(&sdkmcp.Tool{ - Name: "echo", - Description: "Echoes its message.", - InputSchema: map[string]any{"type": "object", "properties": map[string]any{"message": map[string]any{"type": "string"}}}, - Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true}, + Name: "echo", + Description: "Echoes its message.", + InputSchema: map[string]any{"type": "object", "properties": map[string]any{"message": map[string]any{"type": "string"}}}, + OutputSchema: map[string]any{"type": "object", "properties": map[string]any{"echoed": map[string]any{"type": "string"}}}, + Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true}, }, func(_ context.Context, req *sdkmcp.CallToolRequest) (*sdkmcp.CallToolResult, error) { var args struct { Message string `json:"message"` @@ -307,10 +309,16 @@ func TestHTTPToolServerCatalogAndCall(t *testing.T) { if a := tool.Annotations; a == nil || a.ReadOnlyHint == nil || !*a.ReadOnlyHint { t.Fatalf("echo annotations = %+v, want readOnly", a) } + if !strings.Contains(string(tool.OutputSchema), `"echoed"`) { + t.Fatalf("echo outputSchema = %s, want the server's own", tool.OutputSchema) + } case "sleep": if a := tool.Annotations; a == nil || a.DestructiveHint == nil || !*a.DestructiveHint { t.Fatalf("sleep annotations = %+v, want destructive", a) } + if len(tool.OutputSchema) != 0 { + t.Fatalf("sleep declares no outputSchema, got %s", tool.OutputSchema) + } } } @@ -534,3 +542,101 @@ func TestToolEnv(t *testing.T) { t.Fatalf("inherit = %v, want %v", all, environ) } } + +// bigMCP serves n tools whose input and output schemas each carry a +// description of about inBytes and outBytes (0 leaves an output schema off). +func bigMCP(t *testing.T, n, inBytes, outBytes int) string { + t.Helper() + server := sdkmcp.NewServer(&sdkmcp.Implementation{Name: "big", Version: "1"}, nil) + for i := range n { + tool := &sdkmcp.Tool{ + Name: fmt.Sprintf("tool%03d", i), + InputSchema: map[string]any{"type": "object", "description": strings.Repeat("i", inBytes)}, + } + if outBytes > 0 { + tool.OutputSchema = map[string]any{"type": "object", "description": strings.Repeat("o", outBytes)} + } + server.AddTool(tool, func(context.Context, *sdkmcp.CallToolRequest) (*sdkmcp.CallToolResult, error) { + return textResult("ok"), nil + }) + } + srv := httptest.NewServer(sdkmcp.NewStreamableHTTPHandler(func(*http.Request) *sdkmcp.Server { return server }, nil)) + t.Cleanup(srv.Close) + return srv.URL +} + +func waitBothServers(t *testing.T, ctx context.Context, conn *websocket.Conn, a, b string) relaycontract.Tools { + t.Helper() + return waitTools(t, ctx, conn, func(x relaycontract.Tools) bool { return hasServer(a)(x) || hasServer(b)(x) }) +} + +// Over the frame limit, output schemas go before whole servers do. +func TestToolCatalogStripsOutputSchemasBeforeDroppingServers(t *testing.T) { + const tools, schema = 40, 60_000 // two servers × 40 × 60 KB ≈ 4.8 MB of output schemas + hub := newFakeHub(t) + startToolRelay(t, hub, []*ToolServer{ + newToolServer(t, ToolServerConfig{Name: "one", URL: bigMCP(t, tools, 10, schema)}), + newToolServer(t, ToolServerConfig{Name: "two", URL: bigMCP(t, tools, 10, schema)}), + }, nil) + ctx := context.Background() + conn := hub.accept(t) + conn.SetReadLimit(16 << 20) + handshakeProtocol(t, ctx, conn, 2) + got := waitTools(t, ctx, conn, func(x relaycontract.Tools) bool { return hasServer("one")(x) && hasServer("two")(x) }) + if raw, _ := json.Marshal(got); len(raw) > relaycontract.MaxToolsFrameBytes { + t.Fatalf("tools frame is %d bytes, over the %d limit", len(raw), relaycontract.MaxToolsFrameBytes) + } + kept := 0 + for _, s := range got.Servers { + if len(s.Tools) != tools { + t.Fatalf("server %s offers %d tools, want all %d", s.Name, len(s.Tools), tools) + } + for _, tool := range s.Tools { + if len(tool.InputSchema) == 0 { + t.Fatalf("%s/%s lost its input schema", s.Name, tool.Name) + } + if len(tool.OutputSchema) > 0 { + kept++ + } + } + } + if kept == 0 || kept == 2*tools { + t.Fatalf("kept %d of %d output schemas, want only as many stripped as needed", kept, 2*tools) + } +} + +// When input schemas alone overflow the frame, a server still has to go. +func TestToolCatalogDropsServerWhenInputSchemasOverflow(t *testing.T) { + const tools, schema = 40, 60_000 + hub := newFakeHub(t) + startToolRelay(t, hub, []*ToolServer{ + newToolServer(t, ToolServerConfig{Name: "one", URL: bigMCP(t, tools, schema, 0)}), + newToolServer(t, ToolServerConfig{Name: "two", URL: bigMCP(t, tools, schema, 0)}), + }, nil) + ctx := context.Background() + conn := hub.accept(t) + conn.SetReadLimit(16 << 20) + handshakeProtocol(t, ctx, conn, 2) + got := waitBothServers(t, ctx, conn, "one", "two") + // Let both servers come up and the catalog settle. + time.Sleep(500 * time.Millisecond) + writeEnv(t, ctx, conn, relaycontract.TypePing, "", nil) + for { + env := readEnv(t, ctx, conn) + if env.Type == relaycontract.TypePong { + break + } + if env.Type == relaycontract.TypeTools { + got = relaycontract.Tools{} + if err := env.Decode(&got); err != nil { + t.Fatal(err) + } + } + } + if raw, _ := json.Marshal(got); len(raw) > relaycontract.MaxToolsFrameBytes { + t.Fatalf("tools frame is %d bytes, over the limit", len(raw)) + } + if len(got.Servers) != 1 || got.Servers[0].Name != "one" { + t.Fatalf("servers = %v, want only the first", got.Servers) + } +} diff --git a/internal/testapi/context.go b/internal/testapi/context.go new file mode 100644 index 00000000..e6e025ee --- /dev/null +++ b/internal/testapi/context.go @@ -0,0 +1,77 @@ +package testapi + +// The Hub resolves GET /api/v1/info?flight= the way Tapper's orient does. +// A qualified @namespace/+slug names one flight. A bare +slug is matched +// against every flight the user can read (the list GET /api/v1/flights +// returns), not only the personal namespace: exactly one match resolves to +// it, none answers HubContextFlightNotFound, and several answer +// HubContextFlightAmbiguous. + +// HubContextWork is a GET /api/v1/info body in the shape tapper-hub serves +// (its handler.InfoResponse, documented in tapper-hub's +// docs/tapper-hub-api-v1.md), for user jared and flight @foldwise/+work. The +// flight's cover is the Hub-composed cover: @foldwise/dev is listed, and +// @foldwise/linked is reached through dev's links at distance 1. +// @foldwise/private is outside the cover. Unknown fields stand in for ones a +// newer Hub adds. +const HubContextWork = `{ + "user": { + "user_id": 1, + "username": "jared", + "display_name": "Jared", + "email": "jared@example.com", + "created_at": "2026-01-01T00:00:00Z", + "default_namespace": "foldwise", + "namespaces": ["foldwise", "jared"], + "personal_namespace": "jared" + }, + "flight": { + "name": "@foldwise/+work", + "title": "Work", + "cover": [ + {"ref": "@foldwise/dev", "role": "editor", "source": "@foldwise/dev", "distance": 0, "path": ["@foldwise/dev"]}, + {"ref": "@foldwise/linked", "role": "viewer", "source": "@foldwise/dev", "distance": 1, "path": ["@foldwise/dev", "@foldwise/linked"]} + ] + }, + "flight_error": null, + "kegs": [ + {"ref": "@foldwise/dev", "namespace": "foldwise", "alias": "dev", "title": "Development", + "description": "", "visibility": "private", "keg_version": "", "role": "admin", + "flight_cap": "editor", "effective_role": "editor", "added_later": true}, + {"ref": "@foldwise/linked", "namespace": "foldwise", "alias": "linked", "title": "Linked", + "description": "", "visibility": "private", "keg_version": "", "role": "admin", + "flight_cap": "viewer", "effective_role": "viewer"}, + {"ref": "@foldwise/private", "namespace": "foldwise", "alias": "private", "title": "Outside the flight", + "description": "", "visibility": "private", "keg_version": "", "role": "admin", + "flight_cap": "", "effective_role": "admin"} + ], + "unknown_top_level": "ignored" +}` + +// HubContextFlightNotFound is HubContextWork's answer when the requested +// flight is missing or unreadable: still 200, with flight null and the +// reason in flight_error. +const HubContextFlightNotFound = `{ + "user": {"user_id": 1, "username": "jared", "default_namespace": "foldwise"}, + "flight": null, + "flight_error": "flight not found", + "kegs": [ + {"ref": "@foldwise/dev", "namespace": "foldwise", "alias": "dev", "title": "Development", + "description": "", "visibility": "private", "keg_version": "", "role": "admin", + "flight_cap": "", "effective_role": "admin"} + ] +}` + +// HubContextFlightAmbiguous is the answer to a bare +work when the user can +// read more than one flight with that slug: still 200, with flight null and +// the same reason orient gives when it resolves the slug itself. +const HubContextFlightAmbiguous = `{ + "user": {"user_id": 1, "username": "jared", "default_namespace": "foldwise"}, + "flight": null, + "flight_error": "flight \"+work\" is ambiguous; use @namespace/+work", + "kegs": [ + {"ref": "@foldwise/dev", "namespace": "foldwise", "alias": "dev", "title": "Development", + "description": "", "visibility": "private", "keg_version": "", "role": "admin", + "flight_cap": "", "effective_role": "admin"} + ] +}` diff --git a/pkg/apicontract/contract.go b/pkg/apicontract/contract.go index b1057212..aba00026 100644 --- a/pkg/apicontract/contract.go +++ b/pkg/apicontract/contract.go @@ -15,7 +15,52 @@ import ( "time" ) -const Revision = "2026-10-12" +// Revision is the contract this build speaks: the revision a client sends +// on every REST request and the newest one a Hub serves. +const Revision = "2026-10-17" + +// previousRevisions are older revisions a Hub still accepts, so released +// clients keep working through an upgrade. 2026-10-12 differs from +// 2026-10-17 only in POST /api/v1/@{ns}/kegs/{alias}/archive/restore, which +// 2026-10-17 answers with 202 and a job instead of 200 {imported,removed}. +// A handler whose behavior changed between revisions reads the negotiated +// one with RequestRevision and must not serve the new behavior to a request +// made under an older revision. +var previousRevisions = []string{"2026-10-12"} + +// SupportedRevisions lists every revision a Hub accepts, newest first. The +// slice is a fresh copy on each call. +func SupportedRevisions() []string { + return append([]string{Revision}, previousRevisions...) +} + +func supported(rev string) bool { + if rev == Revision { + return true + } + for _, old := range previousRevisions { + if rev == old { + return true + } + } + return false +} + +type revisionKey struct{} + +// RequestRevision reports the contract revision Middleware accepted for the +// request carrying ctx, or "" when the request did not pass the REST gate. +func RequestRevision(ctx context.Context) string { + rev, _ := ctx.Value(revisionKey{}).(string) + return rev +} + +// WithRequestRevision returns ctx recording rev as the negotiated revision, +// as Middleware does for every accepted REST request. Tests of handlers that +// answer per revision use it to stand in for the gate. +func WithRequestRevision(ctx context.Context, rev string) context.Context { + return context.WithValue(ctx, revisionKey{}, rev) +} const VersionHeader = "Tapper-API-Version" const ClientHeader = "Tapper-Client-Version" const Required = "API_VERSION_REQUIRED" @@ -52,14 +97,15 @@ func failure(code, message, server string, requested, supported []string) *Compa return &CompatibilityError{Message: message, Code: code, Requested: requested, Supported: supported, ServerVersion: server} } -// Validate accepts exactly one unmodified contract revision header value. +// Validate accepts exactly one unmodified contract revision header value +// naming one of SupportedRevisions. func Validate(h http.Header, server string) *CompatibilityError { values := h.Values(VersionHeader) if len(values) == 0 { - return failure(Required, "send "+VersionHeader+": "+Revision, server, values, []string{Revision}) + return failure(Required, "send "+VersionHeader+": "+Revision, server, values, SupportedRevisions()) } - if len(values) != 1 || values[0] != Revision { - return failure(Unsupported, "unsupported REST contract; upgrade Tapper and Hub together", server, values, []string{Revision}) + if len(values) != 1 || !supported(values[0]) { + return failure(Unsupported, "unsupported REST contract; upgrade Tapper and Hub together", server, values, SupportedRevisions()) } return nil } @@ -78,13 +124,15 @@ func LogFailure(ctx context.Context, logger *slog.Logger, level slog.Level, clie // Middleware gates all /api/v1 paths, including unknown operations, before auth. // Discovery is build-only and remains usable when database services are down. +// An accepted request carries its revision in the response header and in its +// context (RequestRevision). func Middleware(server string, logger *slog.Logger) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/api/version" && r.Method == http.MethodGet { w.Header().Set("Cache-Control", "no-store") w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(Discovery{server, []string{Revision}}) + _ = json.NewEncoder(w).Encode(Discovery{server, SupportedRevisions()}) return } if r.URL.Path == "/api/v1" || strings.HasPrefix(r.URL.Path, "/api/v1/") { @@ -96,7 +144,9 @@ func Middleware(server string, logger *slog.Logger) func(http.Handler) http.Hand _ = json.NewEncoder(w).Encode(err) return } - w.Header().Set(VersionHeader, Revision) + rev := r.Header.Get(VersionHeader) + w.Header().Set(VersionHeader, rev) + r = r.WithContext(WithRequestRevision(r.Context(), rev)) } next.ServeHTTP(w, r) }) @@ -108,12 +158,75 @@ type check struct { err error } +// DefaultDiscoveryCacheTTL is how long a cached discovery success is trusted +// when Session.CacheTTL is unset. +const DefaultDiscoveryCacheTTL = time.Hour + +// DiscoveryCache persists successful discovery beyond one Session, so a new +// invocation can skip GET /api/version. Keys name a Hub base URL and this +// package's Revision. Only successes are stored; failures stay with the +// Session that saw them. Implementations must be safe for concurrent use and +// treat every error as a miss. +type DiscoveryCache interface { + // Get returns when key last passed discovery. + Get(key string) (time.Time, bool) + // Put records that key passed discovery at checked. + Put(key string, checked time.Time) + // Delete forgets key after the Hub rejected the cached contract. + Delete(key string) +} + // Session caches discovery per Hub for one invocation or MCP connection. +// +// Cache, when set, extends a success past the Session for CacheTTL +// (DefaultDiscoveryCacheTTL when zero), measured with Now (time.Now when +// nil). A stale entry is safe: the Hub still validates every request, and a +// typed rejection evicts the entry. type Session struct { mu sync.Mutex checks map[string]*check ClientVersion string Logger *slog.Logger + + Cache DiscoveryCache + CacheTTL time.Duration + Now func() time.Time +} + +// discoveryCacheKey names base under the current Revision, so a client built +// for another revision never trusts this one's result. +func discoveryCacheKey(base string) string { return Revision + " " + base } + +func (s *Session) now() time.Time { + if s.Now != nil { + return s.Now() + } + return time.Now() +} + +// cachedSuccess reports whether the persistent cache vouches for base. +func (s *Session) cachedSuccess(base string) bool { + if s.Cache == nil { + return false + } + checked, ok := s.Cache.Get(discoveryCacheKey(base)) + if !ok { + return false + } + ttl := s.CacheTTL + if ttl <= 0 { + ttl = DefaultDiscoveryCacheTTL + } + age := s.now().Sub(checked) + return age >= 0 && age < ttl +} + +// evict drops base from the persistent cache after the Hub rejected the +// contract the cache vouched for. +func (s *Session) evict(base string) { + if s.Cache != nil { + s.Cache.Delete(discoveryCacheKey(base)) + } } func NewSession(client string, logger *slog.Logger) *Session { @@ -146,7 +259,14 @@ func (s *Session) Check(ctx context.Context, client *http.Client, base string) e c := &check{done: make(chan struct{})} s.checks[base] = c s.mu.Unlock() + if s.cachedSuccess(base) { + close(c.done) + return nil + } c.err = discover(ctx, client, base) + if c.err == nil && s.Cache != nil { + s.Cache.Put(discoveryCacheKey(base), s.now()) + } LogFailure(ctx, s.Logger, slog.LevelError, s.ClientVersion, c.err, "") close(c.done) return c.err @@ -237,6 +357,7 @@ func Do(client *http.Client, req *http.Request) (*http.Response, error) { var e CompatibilityError if len(raw) <= 65536 && json.Unmarshal(raw, &e) == nil && (e.Code == Required || e.Code == Unsupported) && !e.OperationPerformed { resp.Body.Close() + s.evict(strings.TrimRight(base.String(), "/")) LogFailure(req.Context(), s.Logger, slog.LevelError, s.ClientVersion, &e, resp.Header.Get("X-Request-ID")) return nil, &e } diff --git a/pkg/apicontract/contract_test.go b/pkg/apicontract/contract_test.go index 7e7412f2..aac711fb 100644 --- a/pkg/apicontract/contract_test.go +++ b/pkg/apicontract/contract_test.go @@ -24,13 +24,20 @@ func TestGateBeforeApplication(t *testing.T) { code string }{ {"missing", nil, Required}, {"empty", []string{""}, Unsupported}, {"valid", []string{Revision}, ""}, + {"previous", []string{"2026-10-12"}, ""}, {"mixed", []string{Revision, "2026-10-12"}, Unsupported}, + {"retired", []string{"2026-09-28"}, Unsupported}, {"duplicate", []string{Revision, Revision}, Unsupported}, {"comma", []string{Revision + ", " + Revision}, Unsupported}, {"malformed", []string{" " + Revision}, Unsupported}, {"unsupported", []string{"2099-01-01"}, Unsupported}, } { t.Run(tc.name, func(t *testing.T) { called := false + seen := "" var logs bytes.Buffer - h := Middleware("0.24.0", slog.New(slog.NewJSONHandler(&logs, nil)))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { called = true; w.WriteHeader(204) })) + h := Middleware("0.24.0", slog.New(slog.NewJSONHandler(&logs, nil)))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + seen = RequestRevision(r.Context()) + w.WriteHeader(204) + })) req := httptest.NewRequest("POST", "/api/v1/@ns/kegs/k/nodes", strings.NewReader("private content")) req.Header[http.CanonicalHeaderKey(VersionHeader)] = tc.values req.Header.Set("Authorization", "Bearer secret") @@ -38,8 +45,8 @@ func TestGateBeforeApplication(t *testing.T) { rec := httptest.NewRecorder() h.ServeHTTP(rec, req) if tc.code == "" { - if !called || rec.Code != 204 || rec.Header().Get(VersionHeader) != Revision { - t.Fatal(rec, called) + if !called || rec.Code != 204 || rec.Header().Get(VersionHeader) != tc.values[0] || seen != tc.values[0] { + t.Fatal(rec, called, seen) } return } @@ -50,7 +57,7 @@ func TestGateBeforeApplication(t *testing.T) { if err := json.Unmarshal(rec.Body.Bytes(), &e); err != nil { t.Fatal(err) } - if e.Code != tc.code || e.OperationPerformed || len(e.Supported) != 1 { + if e.Code != tc.code || e.OperationPerformed || strings.Join(e.Supported, ",") != "2026-10-17,2026-10-12" { t.Fatalf("%+v", e) } if !strings.Contains(logs.String(), `"level":"WARN"`) || strings.Contains(logs.String(), "secret") || strings.Contains(logs.String(), "private content") { @@ -60,6 +67,22 @@ func TestGateBeforeApplication(t *testing.T) { } } +// A Hub keeps accepting the previous revision so released clients survive +// the upgrade, but a client always speaks the newest one. +func TestRevisionPolicy(t *testing.T) { + if Revision != "2026-10-17" { + t.Fatal(Revision) + } + got := SupportedRevisions() + got[0] = "mutated" + if strings.Join(SupportedRevisions(), ",") != "2026-10-17,2026-10-12" { + t.Fatal(SupportedRevisions()) + } + if RequestRevision(context.Background()) != "" { + t.Fatal("a request outside the gate has no revision") + } +} + func TestDiscoveryAndExemptions(t *testing.T) { h := Middleware("build-release", nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) })) for _, path := range []string{"/health", "/mcp", "/oauth/token", "/home", "/api/v10"} { @@ -75,7 +98,7 @@ func TestDiscoveryAndExemptions(t *testing.T) { if err := json.Unmarshal(r.Body.Bytes(), &d); err != nil { t.Fatal(err) } - if r.Code != 200 || r.Header().Get("Cache-Control") != "no-store" || d.ServerVersion != "build-release" || len(d.APIVersions) != 1 || d.APIVersions[0] != Revision { + if r.Code != 200 || r.Header().Get("Cache-Control") != "no-store" || d.ServerVersion != "build-release" || strings.Join(d.APIVersions, ",") != "2026-10-17,2026-10-12" { t.Fatal(r, d) } } @@ -85,8 +108,9 @@ func TestClientDiscoveryMatrix(t *testing.T) { name, body, code string status int }{ - {"matching", `{"server_version":"0.24.0","api_versions":["2026-10-12"]}`, "", 200}, - {"newer compatible", `{"server_version":"99.0.0","api_versions":["2026-10-12","2099-01-01"]}`, "", 200}, + {"matching", `{"server_version":"0.24.0","api_versions":["2026-10-17","2026-10-12"]}`, "", 200}, + {"newer compatible", `{"server_version":"99.0.0","api_versions":["2026-10-17","2099-01-01"]}`, "", 200}, + {"older hub", `{"server_version":"0.29.1","api_versions":["2026-10-12"]}`, Unsupported, 200}, {"newer incompatible", `{"server_version":"99.0.0","api_versions":["2099-01-01"]}`, Unsupported, 200}, {"legacy", "", DiscoveryUnavailable, 404}, {"redirect", "", DiscoveryUnavailable, 302}, {"malformed", "oops", DiscoveryInvalid, 200}, {"missing versions", `{"server_version":"v1"}`, DiscoveryInvalid, 200}, @@ -223,3 +247,153 @@ func TestDeploymentMismatchPreservesErrorAndDoesNotReplay(t *testing.T) { t.Fatal(logs.String()) } } + +// memoryDiscoveryCache is an in-memory DiscoveryCache for tests. +type memoryDiscoveryCache struct { + mu sync.Mutex + entries map[string]time.Time +} + +func (c *memoryDiscoveryCache) Get(key string) (time.Time, bool) { + c.mu.Lock() + defer c.mu.Unlock() + at, ok := c.entries[key] + return at, ok +} + +func (c *memoryDiscoveryCache) Put(key string, at time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + if c.entries == nil { + c.entries = map[string]time.Time{} + } + c.entries[key] = at +} + +func (c *memoryDiscoveryCache) Delete(key string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.entries, key) +} + +func (c *memoryDiscoveryCache) len() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} + +// discoveryCountingHub answers discovery with status and the operation with +// 204, or with a typed rejection when reject is set. +func discoveryCountingHub(t *testing.T, status int, reject *atomic.Bool) (*httptest.Server, *atomic.Int32) { + t.Helper() + var discoveries atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/version" { + discoveries.Add(1) + w.WriteHeader(status) + if status == http.StatusOK { + json.NewEncoder(w).Encode(Discovery{"v1", []string{Revision}}) + } + return + } + if reject != nil && reject.Load() { + w.WriteHeader(400) + json.NewEncoder(w).Encode(failure(Unsupported, "upgrade", "v2", []string{Revision}, []string{"2099-01-01"})) + return + } + w.WriteHeader(204) + })) + t.Cleanup(server.Close) + return server, &discoveries +} + +func cachedSession(cache DiscoveryCache, now *time.Time) *Session { + s := NewSession("test", nil) + s.Cache = cache + s.Now = func() time.Time { return *now } + return s +} + +func doOperation(t *testing.T, s *Session, server *httptest.Server) error { + t.Helper() + req, _ := http.NewRequestWithContext(WithSession(context.Background(), s), "GET", server.URL+"/api/v1/operation", nil) + resp, err := Do(server.Client(), req) + if err == nil { + resp.Body.Close() + } + return err +} + +func TestDiscoveryCacheHitSkipsDiscovery(t *testing.T) { + server, discoveries := discoveryCountingHub(t, http.StatusOK, nil) + cache := &memoryDiscoveryCache{} + now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) + + if err := doOperation(t, cachedSession(cache, &now), server); err != nil { + t.Fatal(err) + } + now = now.Add(30 * time.Minute) + if err := doOperation(t, cachedSession(cache, &now), server); err != nil { + t.Fatal(err) + } + if discoveries.Load() != 1 { + t.Fatalf("a fresh cached success must skip discovery; got %d discoveries", discoveries.Load()) + } +} + +func TestDiscoveryCacheExpiredRediscovers(t *testing.T) { + server, discoveries := discoveryCountingHub(t, http.StatusOK, nil) + cache := &memoryDiscoveryCache{} + now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) + + if err := doOperation(t, cachedSession(cache, &now), server); err != nil { + t.Fatal(err) + } + now = now.Add(DefaultDiscoveryCacheTTL + time.Second) + if err := doOperation(t, cachedSession(cache, &now), server); err != nil { + t.Fatal(err) + } + if discoveries.Load() != 2 { + t.Fatalf("an expired entry must rediscover; got %d discoveries", discoveries.Load()) + } +} + +func TestDiscoveryCacheFailureNotCached(t *testing.T) { + server, discoveries := discoveryCountingHub(t, http.StatusNotFound, nil) + cache := &memoryDiscoveryCache{} + now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) + + for i := 0; i < 2; i++ { + if err := doOperation(t, cachedSession(cache, &now), server); !IsCompatibility(err) { + t.Fatalf("want compatibility error, got %v", err) + } + } + if cache.len() != 0 || discoveries.Load() != 2 { + t.Fatalf("failures stay session-scoped; cache %d, discoveries %d", cache.len(), discoveries.Load()) + } +} + +func TestDiscoveryCacheEvictedOnCompatibilityRejection(t *testing.T) { + var reject atomic.Bool + server, discoveries := discoveryCountingHub(t, http.StatusOK, &reject) + cache := &memoryDiscoveryCache{} + now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) + + if err := doOperation(t, cachedSession(cache, &now), server); err != nil { + t.Fatal(err) + } + reject.Store(true) + if err := doOperation(t, cachedSession(cache, &now), server); !IsCompatibility(err) { + t.Fatalf("want compatibility error, got %v", err) + } + if cache.len() != 0 { + t.Fatal("a typed rejection must evict the cached success") + } + reject.Store(false) + if err := doOperation(t, cachedSession(cache, &now), server); err != nil { + t.Fatal(err) + } + if discoveries.Load() != 2 { + t.Fatalf("the next session must rediscover after eviction; got %d discoveries", discoveries.Load()) + } +} diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index a7aef98a..4b1d5367 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -8,7 +8,6 @@ import ( "slices" "strconv" "strings" - "time" "github.com/jlrickert/cli-toolkit/toolkit" "github.com/jlrickert/tapper/pkg/tapper" @@ -106,7 +105,10 @@ func RunWithProfile(ctx context.Context, rt *toolkit.Runtime, args []string, pro logCLIInvocation(deps, args, execErr) reportCLIInvocation(deps, execErr) if deps.InvocationReporter != nil { - flushCtx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + // Close waits only for a send already in flight and spools the rest, + // including this invocation's event, for the next process to send. + // The deadline is a ceiling on that wait plus the spool write. + flushCtx, cancel := context.WithTimeout(context.Background(), 2*tapper.InvocationTelemetryInFlightWait) deps.InvocationReporter.Close(flushCtx) cancel() } diff --git a/pkg/cli/cmd_agent.go b/pkg/cli/cmd_agent.go index 7e686eb4..31c63be0 100644 --- a/pkg/cli/cmd_agent.go +++ b/pkg/cli/cmd_agent.go @@ -28,7 +28,9 @@ Members read a namespace's agents; owners and admins change them. The agent is the unit of authority: its tools say what it may do, and its flight (--flight @ns/+slug) is the memory it works in. An agent with no flight -has no KEG access. Apps and tokens bind to an agent, not a flight. +has no KEG access. An agent holds no credentials of its own: it reaches Hub +only through 'tap mcp' (TAP_AGENT), which applies its tool allowlist and +passes its flight to Hub. Tools are tool group ids (keg:read, keg:write, keg:admin, flight:read, flight:admin, agent:read, agent:admin, discover) and single MCP tool names. No diff --git a/pkg/cli/cmd_archive.go b/pkg/cli/cmd_archive.go index 2dbe1509..330318c9 100644 --- a/pkg/cli/cmd_archive.go +++ b/pkg/cli/cmd_archive.go @@ -11,13 +11,12 @@ import ( func NewArchiveCmd(deps *Deps) *cobra.Command { cmd := &cobra.Command{ Use: "archive", - Short: "import and export keg archives", - Long: `Export nodes to a tar archive or import nodes from one.`, + Short: "export keg archives", + Long: `Export nodes to a tar archive.`, } cmd.AddCommand( NewArchiveExportCmd(deps), - NewArchiveImportCmd(deps), ) return cmd } @@ -53,37 +52,6 @@ func NewArchiveExportCmd(deps *Deps) *cobra.Command { return cmd } -func NewArchiveImportCmd(deps *Deps) *cobra.Command { - var opts tapper.ImportOptions - - cmd := &cobra.Command{ - Use: "import ARCHIVE", - Short: "import nodes from a keg archive", - Args: cobra.ExactArgs(1), - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if len(args) == 0 { - return nil, cobra.ShellCompDirectiveDefault - } - return nil, cobra.ShellCompDirectiveNoFileComp - }, - RunE: func(cmd *cobra.Command, args []string) error { - applyKegTargetProfile(deps, &opts.KegTargetOptions) - opts.Input = args[0] - imported, err := deps.Tap.Import(cmd.Context(), opts) - if err != nil { - return err - } - for _, id := range imported { - if _, err := fmt.Fprintln(cmd.OutOrStdout(), id.Path()); err != nil { - return err - } - } - return nil - }, - } - return cmd -} - func splitArchiveNodeList(raw string) []string { if strings.TrimSpace(raw) == "" { return nil diff --git a/pkg/cli/cmd_auth.go b/pkg/cli/cmd_auth.go index d554dfa8..a80304db 100644 --- a/pkg/cli/cmd_auth.go +++ b/pkg/cli/cmd_auth.go @@ -117,12 +117,10 @@ func runAuthLogin(ctx context.Context, deps *Deps, p authLoginParams) (*authLogi // PersistentPreRunE, so PathService is available for the store location — // no second resolution pass needed. storePath := deps.Tap.PathService.AuthStorePath() - store, err := tapper.LoadAuthStore(ctx, rt, storePath) - if err != nil { - return nil, err - } - store.Set(tapper.CanonicalHubURL(hubURL), *entry) - if err := store.Save(ctx, rt, storePath); err != nil { + // A login is a new identity: it gets a key of its own rather than + // inheriting the previous login's. + entry.AccountKey = tapper.NewAuthAccountKey() + if err := tapper.SaveAuthLogin(ctx, rt, storePath, tapper.CanonicalHubURL(hubURL), *entry); err != nil { return nil, err } diff --git a/pkg/cli/cmd_auth_test.go b/pkg/cli/cmd_auth_test.go index d419eecc..27298ec4 100644 --- a/pkg/cli/cmd_auth_test.go +++ b/pkg/cli/cmd_auth_test.go @@ -294,6 +294,63 @@ func TestAuthLoginCmd_PersistsStoreAndPrintsHub(t *testing.T) { require.Equal(t, "read write", entry.Scope) } +// Each login names itself with a fresh account key, so state kept for one +// login (the telemetry spool) is never carried into the next. +func TestAuthLoginCmd_MintsAccountKeyPerLogin(t *testing.T) { + t.Parallel() + sb := newTestSandbox(t) + hook := stubDeviceLoginHook(func(_ context.Context, _ *toolkit.Runtime, _ tapper.AuthLoginDeviceOptions) (*tapper.AuthEntry, error) { + return &tapper.AuthEntry{AccessToken: "stub-access-token", RefreshToken: "stub-refresh"}, nil + }) + accountKey := func() string { + var parsed struct { + Hubs map[string]struct { + AccountKey string `yaml:"account_key"` + } `yaml:"hubs"` + } + require.NoError(t, yaml.Unmarshal(sb.MustReadFile("~/.local/state/tapper/auth.yaml"), &parsed)) + return parsed.Hubs["https://hub.example.com"].AccountKey + } + + res := newAuthProcess(t, hook, "auth", "login", "--hub", "https://hub.example.com").Run(sb.Context(), sb.Runtime()) + require.NoError(t, res.Err) + first := accountKey() + require.NotEmpty(t, first) + + res = newAuthProcess(t, hook, "auth", "login", "--hub", "https://hub.example.com").Run(sb.Context(), sb.Runtime()) + require.NoError(t, res.Err) + require.NotEmpty(t, accountKey()) + require.NotEqual(t, first, accountKey()) +} + +// A login over an existing one replaces its tokens and keeps the other +// hubs' logins. +func TestAuthLoginCmd_ReplacesExistingLogin(t *testing.T) { + t.Parallel() + sb := newTestSandbox(t) + seedAuthStore(t, sb, map[string]tapper.AuthEntry{ + "https://hub.example.com": {AccessToken: "thub_oldaccess000", RefreshToken: "old-refresh"}, + "https://other.example.com": {AccessToken: "thub_otheraccess0"}, + }) + hook := stubDeviceLoginHook(func(context.Context, *toolkit.Runtime, tapper.AuthLoginDeviceOptions) (*tapper.AuthEntry, error) { + return &tapper.AuthEntry{AccessToken: "thub_newaccess000", RefreshToken: "new-refresh"}, nil + }) + + res := newAuthProcess(t, hook, "auth", "login", "--hub", "https://hub.example.com").Run(sb.Context(), sb.Runtime()) + require.NoError(t, res.Err) + + var parsed struct { + Hubs map[string]struct { + AccessToken string `yaml:"access_token"` + RefreshToken string `yaml:"refresh_token"` + } `yaml:"hubs"` + } + require.NoError(t, yaml.Unmarshal(sb.MustReadFile("~/.local/state/tapper/auth.yaml"), &parsed)) + require.Equal(t, "thub_newaccess000", parsed.Hubs["https://hub.example.com"].AccessToken) + require.Equal(t, "new-refresh", parsed.Hubs["https://hub.example.com"].RefreshToken) + require.Equal(t, "thub_otheraccess0", parsed.Hubs["https://other.example.com"].AccessToken) +} + func TestAuthDeviceLabelIncludesHostnameWhenRuntimeHasProcessInfo(t *testing.T) { t.Parallel() rt, err := toolkit.NewRuntime(toolkit.WithProcessInfo(toolkit.ProcessInfo{ diff --git a/pkg/cli/cmd_backlinks.go b/pkg/cli/cmd_backlinks.go index 4ce7b155..b2230de6 100644 --- a/pkg/cli/cmd_backlinks.go +++ b/pkg/cli/cmd_backlinks.go @@ -9,6 +9,7 @@ import ( func NewBacklinksCmd(deps *Deps) *cobra.Command { var opts tapper.BacklinksOptions + var idOnly bool cmd := &cobra.Command{ Use: "backlinks NODE_ID [NODE_ID...]", @@ -24,6 +25,9 @@ Default format: "%i\t%d\t%t".`, RunE: func(cmd *cobra.Command, args []string) error { opts.NodeIDs = args applyKegTargetProfile(deps, &opts.KegTargetOptions) + if idOnly { + opts.Format = tapper.IDOnlyFormat + } nodes, err := deps.Tap.Backlinks(cmd.Context(), opts) if err != nil { @@ -36,7 +40,7 @@ Default format: "%i\t%d\t%t".`, }, } - cmd.Flags().BoolVarP(&opts.IdOnly, "id-only", "", false, "show only ids") + cmd.Flags().BoolVar(&idOnly, "id-only", false, "show only ids (shorthand for --format %i)") cmd.Flags().BoolVar(&opts.Reverse, "reverse", false, "list nodes in reverse order") cmd.Flags().IntVarP(&opts.Limit, "limit", "n", 0, "maximum number of results (0 for no limit)") cmd.Flags().IntVar(&opts.Offset, "offset", 0, "skip the first N results") diff --git a/pkg/cli/cmd_flight.go b/pkg/cli/cmd_flight.go index 2408166c..db950cc2 100644 --- a/pkg/cli/cmd_flight.go +++ b/pkg/cli/cmd_flight.go @@ -72,6 +72,12 @@ func newFlightShowCmd(deps *Deps) *cobra.Command { } fmt.Fprintf(out, "source: %s\n", flight.Source) fmt.Fprintf(out, "visibility: %s\n", flight.Visibility) + if flight.ManageKegs { + fmt.Fprintln(out, "manageKegs: true (a request rooted on this flight administers its namespace's kegs)") + } + if flight.ManageFlights { + fmt.Fprintln(out, "manageFlights: true (a request rooted on this flight creates, edits, and deletes its namespace's flights)") + } if len(flight.Cover) > 0 { fmt.Fprintln(out, "cover:") for _, c := range flight.Cover { @@ -98,6 +104,7 @@ func newFlightShowCmd(deps *Deps) *cobra.Command { func newFlightCreateCmd(deps *Deps) *cobra.Command { var description, title, visibility, instructions, instructionsFile string var coverSpecs []string + var manage, manageKegs, manageFlights bool cmd := &cobra.Command{ Use: "create ", Short: "create a Hub-backed flight", @@ -114,9 +121,11 @@ func newFlightCreateCmd(deps *Deps) *cobra.Command { flight, err := deps.Tap.CreateFlight(cmd.Context(), tapper.CreateFlightOptions{ Ref: args[0], Title: title, Description: description, - Visibility: visibility, - Instructions: body, - Cover: cover, + Visibility: visibility, + Instructions: body, + Cover: cover, + ManageKegs: manageKegs || manage, + ManageFlights: manageFlights || manage, }) if err != nil { return err @@ -129,6 +138,10 @@ func newFlightCreateCmd(deps *Deps) *cobra.Command { _ = cmd.RegisterFlagCompletionFunc("description", cobra.NoFileCompletions) addFlightWriteFlags(cmd, &title, &instructions, &instructionsFile, &coverSpecs) cmd.Flags().StringVar(&visibility, "visibility", tapper.FlightVisibilityPrivate, "flight visibility: public or private") + cmd.Flags().BoolVar(&manageKegs, "manage-kegs", false, "let a request rooted on this flight administer every keg in its namespace (create, delete, settings, visibility, grants), within your own role") + cmd.Flags().BoolVar(&manageFlights, "manage-flights", false, "let a request rooted on this flight create, edit, and delete its namespace's flights, within your own role") + cmd.Flags().BoolVar(&manage, "manage", false, "deprecated: sets --manage-kegs and --manage-flights") + _ = cmd.Flags().MarkHidden("manage") return cmd } @@ -136,7 +149,7 @@ func newFlightEditCmd(deps *Deps) *cobra.Command { cmd := &cobra.Command{ Use: "edit ", Short: "edit a Hub-backed flight's manifest in the default editor", - Long: `Opens the flight manifest (title, visibility, subflights, cover, instructions) as YAML in the configured editor with a yaml-language-server schema modeline; every save is applied to the hub. Piped stdin applies a full manifest without opening an editor.`, + Long: `Opens the flight manifest (title, visibility, subflights, cover, manage, instructions) as YAML in the configured editor with a yaml-language-server schema modeline; every save is applied to the hub. Piped stdin applies a full manifest without opening an editor.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { current, err := deps.Tap.GetFlight(cmd.Context(), tapper.GetFlightOptions{Name: args[0]}) diff --git a/pkg/cli/cmd_grep.go b/pkg/cli/cmd_grep.go index 71ed6cdb..a7e93b7f 100644 --- a/pkg/cli/cmd_grep.go +++ b/pkg/cli/cmd_grep.go @@ -9,6 +9,7 @@ import ( func NewGrepCmd(deps *Deps) *cobra.Command { var opts tapper.GrepOptions + var idOnly bool cmd := &cobra.Command{ Use: "grep PATTERN", @@ -26,6 +27,9 @@ matching lines. RunE: func(cmd *cobra.Command, args []string) error { opts.Query = args[0] applyKegTargetProfile(deps, &opts.KegTargetOptions) + if idOnly { + opts.Format = tapper.IDOnlyFormat + } nodes, err := deps.Tap.Grep(cmd.Context(), opts) if err != nil { @@ -38,7 +42,7 @@ matching lines. }, } - cmd.Flags().BoolVarP(&opts.IdOnly, "id-only", "", false, "show only ids") + cmd.Flags().BoolVar(&idOnly, "id-only", false, "show only ids (shorthand for --format %i)") cmd.Flags().BoolVar(&opts.Reverse, "reverse", false, "list nodes in reverse order") cmd.Flags().IntVarP(&opts.Limit, "limit", "n", 0, "maximum number of results (0 for no limit)") cmd.Flags().IntVar(&opts.Offset, "offset", 0, "skip the first N results") diff --git a/pkg/cli/cmd_links.go b/pkg/cli/cmd_links.go index 3e1b7778..04e320a2 100644 --- a/pkg/cli/cmd_links.go +++ b/pkg/cli/cmd_links.go @@ -9,6 +9,7 @@ import ( func NewLinksCmd(deps *Deps) *cobra.Command { var opts tapper.LinksOptions + var idOnly bool cmd := &cobra.Command{ Use: "links NODE_ID [NODE_ID...]", @@ -24,6 +25,9 @@ Default format: "%i\t%d\t%t".`, RunE: func(cmd *cobra.Command, args []string) error { opts.NodeIDs = args applyKegTargetProfile(deps, &opts.KegTargetOptions) + if idOnly { + opts.Format = tapper.IDOnlyFormat + } nodes, err := deps.Tap.Links(cmd.Context(), opts) if err != nil { @@ -36,7 +40,7 @@ Default format: "%i\t%d\t%t".`, }, } - cmd.Flags().BoolVarP(&opts.IdOnly, "id-only", "", false, "show only ids") + cmd.Flags().BoolVar(&idOnly, "id-only", false, "show only ids (shorthand for --format %i)") cmd.Flags().BoolVar(&opts.Reverse, "reverse", false, "list nodes in reverse order") cmd.Flags().IntVarP(&opts.Limit, "limit", "n", 0, "maximum number of results (0 for no limit)") cmd.Flags().IntVar(&opts.Offset, "offset", 0, "skip the first N results") diff --git a/pkg/cli/cmd_list.go b/pkg/cli/cmd_list.go index 0d7c4f5a..b79360bf 100644 --- a/pkg/cli/cmd_list.go +++ b/pkg/cli/cmd_list.go @@ -9,6 +9,7 @@ import ( func NewListCmd(deps *Deps) *cobra.Command { opts := tapper.ListOptions{} + var idOnly bool cmd := &cobra.Command{ Use: "list", @@ -32,6 +33,9 @@ Use --offset to skip the first N results (for pagination). Use --sort to order by "id", "updated", "created", or "accessed".`, RunE: func(cmd *cobra.Command, args []string) error { applyKegTargetProfile(deps, &opts.KegTargetOptions) + if idOnly { + opts.Format = tapper.IDOnlyFormat + } nodes, err := deps.Tap.List(cmd.Context(), opts) if err != nil { return err @@ -47,7 +51,7 @@ Use --sort to order by "id", "updated", "created", or "accessed".`, }, } - cmd.Flags().BoolVarP(&opts.IdOnly, "id-only", "", false, "show only ids") + cmd.Flags().BoolVar(&idOnly, "id-only", false, "show only ids (shorthand for --format %i)") cmd.Flags().BoolVar(&opts.Reverse, "reverse", false, "list nodes in reverse order") cmd.Flags().IntVarP(&opts.Limit, "limit", "n", 0, "maximum number of results (0 for no limit)") cmd.Flags().IntVar(&opts.Offset, "offset", 0, "skip the first N results") diff --git a/pkg/cli/cmd_lock.go b/pkg/cli/cmd_lock.go index b01cbb10..0e3ac36a 100644 --- a/pkg/cli/cmd_lock.go +++ b/pkg/cli/cmd_lock.go @@ -43,9 +43,12 @@ func newLockAcquireCmd(deps *Deps) *cobra.Command { var opts tapper.LockOptions cmd := &cobra.Command{ - Use: "acquire NODE_ID", - Short: "acquire a cross-process lock on a node", - Long: "Acquire a cross-process lock and print the token to stdout.", + Use: "acquire NODE_ID", + Short: "acquire a cross-process lock on a node", + Long: `Acquire a cross-process lock and print the token to stdout. + +A lock another holder has is waited for up to --wait (default 10s, at most +60s); a lock still held then fails with a lock timeout.`, ValidArgsFunction: nodeIDCompletionFunc(deps, 1), Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -59,6 +62,7 @@ func newLockAcquireCmd(deps *Deps) *cobra.Command { return err }, } + cmd.Flags().DurationVar(&opts.Wait, "wait", 0, "how long to wait for a held lock (default 10s, max 60s)") return cmd } diff --git a/pkg/cli/cmd_lock_test.go b/pkg/cli/cmd_lock_test.go index 8d34d46e..2967fe56 100644 --- a/pkg/cli/cmd_lock_test.go +++ b/pkg/cli/cmd_lock_test.go @@ -3,6 +3,7 @@ package cli_test import ( "strings" "testing" + "time" testutils "github.com/jlrickert/cli-toolkit/sandbox" "github.com/stretchr/testify/require" @@ -213,3 +214,29 @@ func TestMeta_ReadIgnoresLockToken(t *testing.T) { Run(fx.Context(), fx.Runtime()) require.NoError(t, metaRes.Err, "meta read should succeed despite wrong lock token; stderr: %s", string(metaRes.Stderr)) } + +func TestLock_AcquireHeldLockTimesOutAtWait(t *testing.T) { + t.Parallel() + fx := NewSandbox(t, testutils.WithFixture("testuser", "~")) + first := NewProcess(t, false, "lock", "acquire", "0").Run(fx.Context(), fx.Runtime()) + require.NoError(t, first.Err) + + start := time.Now() + second := NewProcess(t, false, "lock", "acquire", "0", "--wait", "500ms").Run(fx.Context(), fx.Runtime()) + elapsed := time.Since(start) + require.Error(t, second.Err, "a held lock must not be acquired twice") + require.Contains(t, string(second.Stderr), "lock acquire timeout") + require.Less(t, elapsed, 5*time.Second) +} + +func TestLock_AcquireFreeLockWithWaitSucceeds(t *testing.T) { + t.Parallel() + fx := NewSandbox(t, testutils.WithFixture("testuser", "~")) + res := NewProcess(t, false, "lock", "acquire", "0", "--wait", "500ms").Run(fx.Context(), fx.Runtime()) + require.NoError(t, res.Err) + require.NotEmpty(t, strings.TrimSpace(string(res.Stdout))) + + negative := NewProcess(t, false, "lock", "acquire", "0", "--wait", "-1s").Run(fx.Context(), fx.Runtime()) + require.Error(t, negative.Err) + require.Contains(t, string(negative.Stderr), "must not be negative") +} diff --git a/pkg/cli/cmd_mcp.go b/pkg/cli/cmd_mcp.go index 8ad6b12e..ee8b51b0 100644 --- a/pkg/cli/cmd_mcp.go +++ b/pkg/cli/cmd_mcp.go @@ -55,7 +55,11 @@ per-command permission prompts.`, // Config-driven selection is re-resolved on initialize/orient. defaults.Flight = "" } - srv := mcp.NewServer(deps.Tap, Version, defaults, mcpServerOptions(rt.Logger(), deps.InvocationReporter)) + opts := mcpServerOptions(rt.Logger(), deps.InvocationReporter) + if deps.Tap != nil { + opts.DiscoveryCache = tapper.NewAPIDiscoveryCache(rt, deps.Tap.PathService) + } + srv := mcp.NewServer(deps.Tap, Version, defaults, opts) // A launched session runs a Hub agent (TAP_AGENT=@ns/name): serve // only its tools. If the agent cannot be loaded the session fails // closed to orient/guide rather than quietly offering everything. diff --git a/pkg/cli/cmd_relay.go b/pkg/cli/cmd_relay.go index da8034e5..2fc3d0b6 100644 --- a/pkg/cli/cmd_relay.go +++ b/pkg/cli/cmd_relay.go @@ -87,10 +87,14 @@ permission settings. Who may call them is decided on Hub, where each server is enabled and shared on its own. A relay.mcp entry with the same name replaces a built-in, or removes it with enabled: false. -The relay authenticates to each hub with that hub's token from 'tap auth login'. -It runs in the foreground until interrupted, reconnecting if a connection drops. -Disconnecting it from a hub's Account → Relay page stops it for that hub for -good; the other hubs keep being served. Run it again to reconnect. +The relay authenticates to each hub with that hub's token from 'tap auth login' +and renews it on the live connection before the hub would expire it, so +in-flight requests are not cut. It runs in the foreground until interrupted, +reconnecting if a connection drops or the hub restarts. Disconnecting it from a +hub's Account → Relay page stops it for that hub for good; the other hubs keep +being served. Run it again to reconnect. A revoked sign-in (a sign-out, a +revoked token) also stops it for that hub: run 'tap auth login', then start +the relay again. Experimental and unstable: expect this to change.`, Args: cobra.NoArgs, diff --git a/pkg/cli/cmd_root.go b/pkg/cli/cmd_root.go index b79e30c8..dc4eefed 100644 --- a/pkg/cli/cmd_root.go +++ b/pkg/cli/cmd_root.go @@ -242,7 +242,12 @@ func NewRootCmd(deps *Deps) *cobra.Command { return err } - ctx = apicontract.WithSession(ctx, apicontract.NewSession(Version, rt.Logger())) + // Discovery successes persist under CacheRoot, so an invocation + // within the TTL goes straight to its first REST request. + contract := apicontract.NewSession(Version, rt.Logger()) + contract.Cache = tapper.NewAPIDiscoveryCache(rt, tap.PathService) + contract.Now = rt.Clock().Now + ctx = apicontract.WithSession(ctx, contract) // Renew any expired hub credentials before the command runs so // one-off commands and the long-running MCP server both start diff --git a/pkg/cli/cmd_tags.go b/pkg/cli/cmd_tags.go index 74eadd30..5a6a288c 100644 --- a/pkg/cli/cmd_tags.go +++ b/pkg/cli/cmd_tags.go @@ -10,6 +10,7 @@ import ( func NewTagsCmd(deps *Deps) *cobra.Command { var opts tapper.TagsOptions + var idOnly bool cmd := &cobra.Command{ Use: "list [EXPR]", @@ -74,6 +75,9 @@ Examples: opts.Query = args[0] } applyKegTargetProfile(deps, &opts.KegTargetOptions) + if idOnly { + opts.Format = tapper.IDOnlyFormat + } items, err := deps.Tap.Tags(cmd.Context(), opts) if err != nil { @@ -86,7 +90,7 @@ Examples: }, } - cmd.Flags().BoolVarP(&opts.IdOnly, "id-only", "", false, "show only ids when TAG is provided") + cmd.Flags().BoolVar(&idOnly, "id-only", false, "show only ids when TAG is provided (shorthand for --format %i)") cmd.Flags().BoolVar(&opts.Reverse, "reverse", false, "list in reverse order") cmd.Flags().IntVarP(&opts.Limit, "limit", "n", 0, "maximum number of results (0 for no limit)") cmd.Flags().IntVar(&opts.Offset, "offset", 0, "skip the first N results") diff --git a/pkg/cli/keg_target_flags.go b/pkg/cli/keg_target_flags.go index e52773ef..e483c1fe 100644 --- a/pkg/cli/keg_target_flags.go +++ b/pkg/cli/keg_target_flags.go @@ -87,7 +87,7 @@ func nodeIDCompletionFunc(deps *Deps, maxArgs int) func(*cobra.Command, []string applyKegTargetProfile(deps, &kegOpts) ids, err := deps.Tap.List(cmd.Context(), tapper.ListOptions{ KegTargetOptions: kegOpts, - IdOnly: true, + Format: tapper.IDOnlyFormat, }) if err != nil { return nil, cobra.ShellCompDirectiveNoFileComp @@ -116,7 +116,7 @@ func nodeAndNameCompletionFunc( case 0: ids, err := deps.Tap.List(cmd.Context(), tapper.ListOptions{ KegTargetOptions: kegOpts, - IdOnly: true, + Format: tapper.IDOnlyFormat, }) if err != nil { return nil, cobra.ShellCompDirectiveNoFileComp @@ -152,7 +152,7 @@ func nodeIDWithLocalFileCompletionFunc(deps *Deps) func(*cobra.Command, []string applyKegTargetProfile(deps, &kegOpts) ids, err := deps.Tap.List(cmd.Context(), tapper.ListOptions{ KegTargetOptions: kegOpts, - IdOnly: true, + Format: tapper.IDOnlyFormat, }) if err != nil { return nil, cobra.ShellCompDirectiveNoFileComp diff --git a/pkg/keg/archive.go b/pkg/keg/archive.go index cee83042..87b51d12 100644 --- a/pkg/keg/archive.go +++ b/pkg/keg/archive.go @@ -1,7 +1,6 @@ package keg import ( - "archive/tar" "bytes" "compress/gzip" "context" @@ -55,7 +54,25 @@ type archiveManifestNode struct { // coherent read snapshot. LocalKeg materializes the archive before returning, // so the operation boundary is released while the caller reads the artifact // and a slow download does not block same-keg writers. -func (k *LocalKeg) ExportNodes(ctx context.Context, opts ExportNodesOptions) (io.ReadCloser, error) { +// +// When opts.Spool is set, ExportNodes owns it: on any error the spool has +// been discarded, so the caller must not reuse it. +func (k *LocalKeg) ExportNodes(ctx context.Context, opts ExportNodesOptions) (rc io.ReadCloser, err error) { + if opts.Spool != nil { + defer func() { + if err == nil { + return + } + if rc != nil { + // The read boundary failed after the spool's reader was + // opened. Closing it releases the spool; Discard below is + // idempotent per the ArchiveSpool contract. + err = errors.Join(err, rc.Close()) + rc = nil + } + err = errors.Join(err, opts.Spool.Discard()) + }() + } return withKegReadValue(ctx, k, func(ctx context.Context) (io.ReadCloser, error) { return k.exportNodes(ctx, opts) }) @@ -115,18 +132,26 @@ func (k *LocalKeg) exportNodes(ctx context.Context, opts ExportNodesOptions) (io } } - var artifact bytes.Buffer - if err := k.writeArchive(ctx, &artifact, ids, snapshotRepo, opts); err != nil { + if opts.Spool == nil { + var artifact bytes.Buffer + if err := k.writeArchive(ctx, &artifact, ids, snapshotRepo, opts); err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(artifact.Bytes())), nil + } + // ExportNodes discards the spool on every error return, this one included. + if err := k.writeArchive(ctx, opts.Spool, ids, snapshotRepo, opts); err != nil { return nil, err } - return io.NopCloser(bytes.NewReader(artifact.Bytes())), nil + return opts.Spool.Reader() } // writeArchive writes the complete keg-archive representation to w. The caller // decides whether w is a materialized buffer or a streaming destination. func (k *LocalKeg) writeArchive(ctx context.Context, w io.Writer, ids []NodeId, snapshotRepo RepositorySnapshots, opts ExportNodesOptions) error { - gz := gzip.NewWriter(w) - tw := tar.NewWriter(gz) + compressed := &countingWriter{w: w} + gz := gzip.NewWriter(compressed) + aw := newArchiveWriter(gz, opts.Limits) withSettings := len(opts.NodeIDs) == 0 && strings.TrimSpace(opts.Query) == "" && !opts.SkipZeroNode manifest := archiveManifest{ @@ -146,10 +171,10 @@ func (k *LocalKeg) writeArchive(ctx context.Context, w io.Writer, ids []NodeId, if err != nil { return fmt.Errorf("unable to encode keg settings for archive: %w", err) } - if err := writeTarFile(tw, "keg-archive/keg.yaml", rawSettings); err != nil { + if err := aw.writeFile("keg-archive/keg.yaml", rawSettings); err != nil { return err } - if err := k.writeArchiveSchemas(ctx, tw, &manifest); err != nil { + if err := k.writeArchiveSchemas(ctx, aw, &manifest); err != nil { return err } } @@ -176,51 +201,64 @@ func (k *LocalKeg) writeArchive(ctx context.Context, w io.Writer, ids []NodeId, } base := filepath.ToSlash(filepath.Join("keg-archive", "nodes", id.Path())) - if err := writeTarFile(tw, base+"/README.md", content); err != nil { + if err := aw.writeFile(base+"/README.md", content); err != nil { return err } - if err := writeTarFile(tw, base+"/meta.yaml", meta); err != nil { + if err := aw.writeFile(base+"/meta.yaml", meta); err != nil { return err } - if err := writeTarFile(tw, base+"/stats.json", statsBytes); err != nil { + if err := aw.writeFile(base+"/stats.json", statsBytes); err != nil { return err } if opts.WithAssets { - if err := k.writeArchiveAssets(ctx, tw, base, id); err != nil { + if err := k.writeArchiveAssets(ctx, aw, base, id); err != nil { return err } } entry := archiveManifestNode{SourceID: id.Path(), SourceHash: stats.Hash()} if opts.WithHistory { - count, err := k.writeArchiveHistory(ctx, tw, base, id, snapshotRepo) + count, err := k.writeArchiveHistory(ctx, aw, base, id, snapshotRepo) if err != nil { return err } entry.RevisionCount = count } manifest.Nodes = append(manifest.Nodes, entry) + if err := checkCompressedSize(compressed.n, opts.Limits); err != nil { + return err + } } rawManifest, err := json.MarshalIndent(manifest, "", " ") if err != nil { return fmt.Errorf("unable to encode archive manifest: %w", err) } - if err := writeTarFile(tw, "keg-archive/manifest.json", rawManifest); err != nil { + if err := aw.writeFile(archiveManifestName, rawManifest); err != nil { return err } - if err := tw.Close(); err != nil { - return fmt.Errorf("unable to finalize archive: %w", err) + if err := aw.close(); err != nil { + return err } if err := gz.Close(); err != nil { return fmt.Errorf("unable to finalize archive compression: %w", err) } + return checkCompressedSize(compressed.n, opts.Limits) +} + +// checkCompressedSize fails an export whose gzip stream has outgrown the +// compressed ceiling. Mid-export the count trails what gzip has buffered, so +// only the final check is exact. +func checkCompressedSize(n int64, lim ArchiveLimits) error { + if lim.MaxCompressedBytes > 0 && n > lim.MaxCompressedBytes { + return &ArchiveLimitError{Limit: ArchiveLimitCompressedBytes, Max: lim.MaxCompressedBytes, Actual: n} + } return nil } // writeArchiveSchemas adds keg-level schema definitions to a full archive. -func (k *LocalKeg) writeArchiveSchemas(ctx context.Context, tw *tar.Writer, manifest *archiveManifest) error { +func (k *LocalKeg) writeArchiveSchemas(ctx context.Context, aw *archiveWriter, manifest *archiveManifest) error { store, ok := repoSchemas(k.Repo) if !ok { return nil @@ -244,7 +282,7 @@ func (k *LocalKeg) writeArchiveSchemas(ctx context.Context, tw *tar.Writer, mani if _, err := validateSchemaDefinitionForType(typeName, rawSchema); err != nil { return fmt.Errorf("schema %q is invalid: %w", typeName, err) } - if err := writeTarFile(tw, "keg-archive/schemas/"+filename, rawSchema); err != nil { + if err := aw.writeFile("keg-archive/schemas/"+filename, rawSchema); err != nil { return err } manifest.Schemas = append(manifest.Schemas, typeName) @@ -254,7 +292,7 @@ func (k *LocalKeg) writeArchiveSchemas(ctx context.Context, tw *tar.Writer, mani } // writeArchiveAssets adds assets/ and images/ entries for one node. -func (k *LocalKeg) writeArchiveAssets(ctx context.Context, tw *tar.Writer, base string, id NodeId) error { +func (k *LocalKeg) writeArchiveAssets(ctx context.Context, aw *archiveWriter, base string, id NodeId) error { if files, ok := k.Repo.(RepositoryFiles); ok { names, err := files.ListFiles(ctx, id) if err != nil && !errors.Is(err, ErrNotExist) { @@ -265,7 +303,7 @@ func (k *LocalKeg) writeArchiveAssets(ctx context.Context, tw *tar.Writer, base if err != nil { return fmt.Errorf("unable to read file %s for node %s: %w", name, id.Path(), err) } - if err := writeTarFile(tw, base+"/assets/"+name, data); err != nil { + if err := aw.writeFile(base+"/assets/"+name, data); err != nil { return err } } @@ -280,7 +318,7 @@ func (k *LocalKeg) writeArchiveAssets(ctx context.Context, tw *tar.Writer, base if err != nil { return fmt.Errorf("unable to read image %s for node %s: %w", name, id.Path(), err) } - if err := writeTarFile(tw, base+"/images/"+name, data); err != nil { + if err := aw.writeFile(base+"/images/"+name, data); err != nil { return err } } @@ -295,7 +333,7 @@ func (k *LocalKeg) writeArchiveAssets(ctx context.Context, tw *tar.Writer, base if err != nil { return err } - if err := writeTarFile(tw, base+"/videos/"+name, data); err != nil { + if err := aw.writeFile(base+"/videos/"+name, data); err != nil { return err } } @@ -305,7 +343,7 @@ func (k *LocalKeg) writeArchiveAssets(ctx context.Context, tw *tar.Writer, base // writeArchiveHistory adds snapshot payloads and the per-node snapshot index // for one node, returning the revision count. -func (k *LocalKeg) writeArchiveHistory(ctx context.Context, tw *tar.Writer, base string, id NodeId, snapshotRepo RepositorySnapshots) (int, error) { +func (k *LocalKeg) writeArchiveHistory(ctx context.Context, aw *archiveWriter, base string, id NodeId, snapshotRepo RepositorySnapshots) (int, error) { history, err := snapshotRepo.ListSnapshots(ctx, id) if err != nil { return 0, fmt.Errorf("unable to list snapshots for node %s: %w", id.Path(), err) @@ -328,13 +366,13 @@ func (k *LocalKeg) writeArchiveHistory(ctx context.Context, tw *tar.Writer, base return 0, fmt.Errorf("unable to encode snapshot %d stats for node %s: %w", snap.ID, id.Path(), err) } snapBase := base + "/snapshots/" + fmt.Sprintf("%d", snap.ID) - if err := writeTarFile(tw, snapBase+".full", snapContent); err != nil { + if err := aw.writeFile(snapBase+".full", snapContent); err != nil { return 0, err } - if err := writeTarFile(tw, snapBase+".meta", snapMeta); err != nil { + if err := aw.writeFile(snapBase+".meta", snapMeta); err != nil { return 0, err } - if err := writeTarFile(tw, snapBase+".stats", statsBytes); err != nil { + if err := aw.writeFile(snapBase+".stats", statsBytes); err != nil { return 0, err } } @@ -342,7 +380,7 @@ func (k *LocalKeg) writeArchiveHistory(ctx context.Context, tw *tar.Writer, base if err != nil { return 0, fmt.Errorf("unable to encode snapshot index for node %s: %w", id.Path(), err) } - if err := writeTarFile(tw, base+"/snapshots/index.json", rawIndex); err != nil { + if err := aw.writeFile(base+"/snapshots/index.json", rawIndex); err != nil { return 0, err } return len(history), nil @@ -362,17 +400,24 @@ func (k *LocalKeg) importNodes(ctx context.Context, r io.Reader, opts ImportNode if err := k.checkKegExists(ctx); err != nil { return nil, fmt.Errorf("failed to import nodes: %w", err) } + if opts.ReplaceAll && opts.AssignNewIDs { + return nil, fmt.Errorf("import cannot both replace the keg and assign new ids: %w", ErrInvalid) + } + progress := func(phase string, done, total int) { + if opts.Progress != nil { + opts.Progress(ImportProgress{Phase: phase, Done: done, Total: total}) + } + } - data, err := io.ReadAll(r) + entries, err := openArchiveEntries(ctx, r, opts) if err != nil { - return nil, fmt.Errorf("unable to read archive stream: %w", err) + return nil, err } - entries, err := readArchiveEntries(data) + + rawManifest, ok, err := entries.Read("keg-archive/manifest.json") if err != nil { return nil, err } - - rawManifest, ok := entries["keg-archive/manifest.json"] if !ok { return nil, fmt.Errorf("archive manifest missing: %w", ErrInvalid) } @@ -392,6 +437,12 @@ func (k *LocalKeg) importNodes(ctx context.Context, r io.Reader, opts ImportNode if manifest.WithConfig { manifest.WithSettings = true } + // Only a full-keg export carries keg settings, so WithSettings is what + // marks an archive as holding the whole keg. Replacing a keg from a node + // selection would prune every node the selection left out. + if opts.ReplaceAll && !manifest.WithSettings { + return nil, fmt.Errorf("replacing a keg needs a full-keg archive; this one was exported with a node selection: %w", ErrInvalid) + } archivedSchemas, err := readArchiveSchemas(entries, manifest) if err != nil { @@ -457,7 +508,8 @@ func (k *LocalKeg) importNodes(ctx context.Context, r io.Reader, opts ImportNode return nil, err } - for _, sourceID := range ordered { + for i, sourceID := range ordered { + progress(ImportPhaseValidating, i, len(ordered)) newID := mapping[sourceID] base := filepath.ToSlash(filepath.Join("keg-archive", "nodes", sourceID)) content, err := readRequiredArchiveEntry(entries, base+"/README.md") @@ -565,7 +617,8 @@ func (k *LocalKeg) importNodes(ctx context.Context, r io.Reader, opts ImportNode } } - for _, sourceID := range ordered { + for i, sourceID := range ordered { + progress(ImportPhaseImporting, i, len(ordered)) newID := mapping[sourceID] nodeManifest := manifestNodes[sourceID] base := filepath.ToSlash(filepath.Join("keg-archive", "nodes", sourceID)) @@ -631,9 +684,22 @@ func (k *LocalKeg) importNodes(ctx context.Context, r io.Reader, opts ImportNode return nil, fmt.Errorf("unable to restore keg settings after import: %w", err) } } + var removed int64 + if opts.ReplaceAll { + progress(ImportPhasePruning, 0, 0) + keep := make([]NodeId, 0, len(ordered)+1) + for _, sourceID := range ordered { + keep = append(keep, mapping[sourceID]) + } + removed, err = k.pruneNodesExcept(ctx, keep) + if err != nil { + return nil, fmt.Errorf("unable to remove nodes the archive does not carry: %w", err) + } + } // A trash restore updates the dex itself, once for its whole batch, // rather than rebuilding it from every node for each restored one. if !opts.restoreFromTrash { + progress(ImportPhaseIndexing, 0, 0) if err := k.rebuildDexFromRepo(ctx); err != nil { return nil, err } @@ -648,9 +714,67 @@ func (k *LocalKeg) importNodes(ctx context.Context, r io.Reader, opts ImportNode for _, sourceID := range ordered { imported = append(imported, ImportedNode{SourceID: sourceID, SourceHash: manifestNodes[sourceID].SourceHash, ID: mapping[sourceID]}) } + if opts.Report != nil { + *opts.Report = ImportReport{Removed: removed} + } return imported, nil } +// pruneNodesExcept deletes every node not in keep, in bulk when the +// repository supports it. Nodes go through the repository rather than +// RemoveNodes so links in the surviving, just-imported content stay as the +// archive wrote them. +// +// Node 0 is always kept: it is the keg's landing node, and removing it makes +// the keg read as uninitialized. +func (k *LocalKeg) pruneNodesExcept(ctx context.Context, keep []NodeId) (int64, error) { + keep = append(keep[:len(keep):len(keep)], NodeId{ID: 0}) + if pruner, ok := k.Repo.(RepositoryNodePruner); ok { + return pruner.PruneNodesExcept(ctx, keep) + } + kept := make(map[string]struct{}, len(keep)) + for _, id := range keep { + kept[id.Path()] = struct{}{} + } + ids, err := k.Repo.ListNodes(ctx) + if err != nil { + return 0, err + } + var removed int64 + for _, id := range ids { + if _, ok := kept[id.Path()]; ok { + continue + } + if err := k.Repo.DeleteNode(ctx, id); err != nil { + return removed, err + } + removed++ + } + return removed, nil +} + +// openArchiveEntries indexes the archive an import reads. A caller-staged +// Source is indexed in place; a stream is first staged into memory as a plain +// tar, which costs the archive's uncompressed size but never a second copy of +// every payload. +func openArchiveEntries(ctx context.Context, r io.Reader, opts ImportNodesOptions) (archiveEntries, error) { + if opts.Source != nil { + if r != nil { + return nil, fmt.Errorf("import takes a reader or a staged source, not both: %w", ErrInvalid) + } + return indexArchive(ctx, opts.Source, opts.Limits) + } + if r == nil { + return nil, fmt.Errorf("import has no archive: %w", ErrInvalid) + } + var staged bytes.Buffer + if _, err := StageArchive(ctx, r, &staged, opts.Limits); err != nil { + return nil, err + } + // StageArchive has already enforced the limits on this tar. + return indexArchive(ctx, bytes.NewReader(staged.Bytes()), ArchiveLimits{}) +} + func (k *LocalKeg) cleanupUnusedImportReservations(ctx context.Context, ids []NodeId) { for _, id := range ids { content, err := k.Repo.ReadContent(ctx, id) @@ -663,7 +787,7 @@ func (k *LocalKeg) cleanupUnusedImportReservations(ctx context.Context, ids []No // importNodeHistory replays a node's archived snapshot revisions. func (k *LocalKeg) importNodeHistory( ctx context.Context, - entries map[string][]byte, + entries archiveEntries, base, sourceID string, newID NodeId, nodeManifest archiveManifestNode, @@ -671,7 +795,10 @@ func (k *LocalKeg) importNodeHistory( snapshotRepo RepositorySnapshots, opts ImportNodesOptions, ) error { - rawIndex, ok := entries[base+"/snapshots/index.json"] + rawIndex, ok, err := entries.Read(base + "/snapshots/index.json") + if err != nil { + return err + } if !ok { if nodeManifest.RevisionCount > 0 { return fmt.Errorf("archive node %s missing snapshots/index.json: %w", sourceID, ErrInvalid) @@ -719,6 +846,7 @@ func (k *LocalKeg) importNodeHistory( Kind: SnapshotContentKindFull, Base: expectedParent, Data: content, + Hash: hashSnapshotBytes(k.Runtime, content), }, }) if err != nil { @@ -818,70 +946,6 @@ func (k *LocalKeg) loadNodeDataForDex(ctx context.Context, id NodeId) (*NodeData // -- archive entry helpers -func writeTarFile(tw *tar.Writer, name string, data []byte) error { - header := &tar.Header{ - Name: filepath.ToSlash(name), - Mode: 0o644, - Size: int64(len(data)), - Typeflag: tar.TypeReg, - } - if err := tw.WriteHeader(header); err != nil { - return fmt.Errorf("unable to write archive header for %s: %w", name, err) - } - if _, err := tw.Write(data); err != nil { - return fmt.Errorf("unable to write archive payload for %s: %w", name, err) - } - return nil -} - -func readArchiveEntries(data []byte) (map[string][]byte, error) { - gz, err := gzip.NewReader(bytes.NewReader(data)) - if err == nil { - defer gz.Close() - return readTarEntries(tar.NewReader(gz)) - } - - entries, tarErr := readTarEntries(tar.NewReader(bytes.NewReader(data))) - if tarErr == nil { - return entries, nil - } - - return nil, fmt.Errorf("unable to open archive stream: gzip=%v; tar=%v", err, tarErr) -} - -func readTarEntries(tr *tar.Reader) (map[string][]byte, error) { - entries := map[string][]byte{} - for { - header, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, fmt.Errorf("unable to read archive entry: %w", err) - } - if header.FileInfo().IsDir() { - continue - } - if !safeArchiveEntryName(header.Name) { - return nil, fmt.Errorf("unsafe archive entry %q: %w", header.Name, ErrInvalid) - } - payload, err := io.ReadAll(tr) - if err != nil { - return nil, fmt.Errorf("unable to read archive payload %s: %w", header.Name, err) - } - entries[filepath.ToSlash(header.Name)] = payload - } - return entries, nil -} - -func readRequiredArchiveEntry(entries map[string][]byte, path string) ([]byte, error) { - value, ok := entries[path] - if !ok { - return nil, ErrInvalid - } - return value, nil -} - const ( archiveSchemasDir = "keg-archive/schemas" archiveSchemasPrefix = archiveSchemasDir + "/" @@ -891,11 +955,11 @@ type archiveSchemaStore struct { schemas map[string][]byte } -func readArchiveSchemas(entries map[string][]byte, manifest archiveManifest) (*archiveSchemaStore, error) { +func readArchiveSchemas(entries archiveEntries, manifest archiveManifest) (*archiveSchemaStore, error) { manifestSchemas := make(map[string]struct{}, len(manifest.Schemas)) for _, typeName := range manifest.Schemas { typeName = strings.TrimSpace(typeName) - if err := ValidSchemaTypeName(typeName); err != nil { + if err := ValidSchemaStorageName(typeName); err != nil { return nil, fmt.Errorf("invalid archive manifest schema %q: %w", typeName, err) } if _, exists := manifestSchemas[typeName]; exists { @@ -905,7 +969,7 @@ func readArchiveSchemas(entries map[string][]byte, manifest archiveManifest) (*a } store := &archiveSchemaStore{schemas: map[string][]byte{}} - for name, data := range entries { + for _, name := range entries.WithPrefix(archiveSchemasDir) { if name == archiveSchemasDir { return nil, fmt.Errorf("invalid archive schema entry %q: %w", name, ErrInvalid) } @@ -913,6 +977,10 @@ func readArchiveSchemas(entries map[string][]byte, manifest archiveManifest) (*a continue } + data, err := readRequiredArchiveEntry(entries, name) + if err != nil { + return nil, err + } typeName, err := archiveSchemaTypeFromEntry(name) if err != nil { return nil, err @@ -979,7 +1047,7 @@ func (s *archiveSchemaStore) ListSchemas(ctx context.Context) ([]string, error) func (s *archiveSchemaStore) ReadSchema(ctx context.Context, typeName string) ([]byte, error) { _ = ctx - if err := ValidSchemaTypeName(typeName); err != nil { + if err := ValidSchemaStorageName(typeName); err != nil { return nil, err } data, ok := s.schemas[strings.TrimSpace(typeName)] @@ -1013,7 +1081,7 @@ func (s *archiveSchemaStore) CreateSchema(ctx context.Context, typeName string, func (s *archiveSchemaStore) DeleteSchema(ctx context.Context, typeName string) error { _ = ctx typeName = strings.TrimSpace(typeName) - if err := ValidSchemaTypeName(typeName); err != nil { + if err := ValidSchemaStorageName(typeName); err != nil { return err } if _, ok := s.schemas[typeName]; !ok { @@ -1231,71 +1299,105 @@ func restoreImportedNodeAssets(ctx context.Context, repo Repository, id NodeId, return nil } -// writeArchiveEntriesAssets writes a v3 archive's assets/ and images/ payloads -// for one node onto the repository. -func writeArchiveEntriesAssets(ctx context.Context, repo Repository, entries map[string][]byte, base string, id NodeId) error { +// writeArchiveEntriesAssets writes a v3 archive's assets/, images/, and +// videos/ payloads for one node onto the repository. +func writeArchiveEntriesAssets(ctx context.Context, repo Repository, entries archiveEntries, base string, id NodeId) error { filesRepo, hasFiles := repo.(RepositoryFiles) imagesRepo, hasImages := repo.(RepositoryImages) + videoPrefix := base + "/videos/" + for _, name := range entries.WithPrefix(videoPrefix) { + videos, ok := repo.(RepositoryVideos) + if !ok { + return ErrNotSupported + } + assetName := strings.TrimPrefix(name, videoPrefix) + if err := validAssetName(assetName); err != nil { + return err + } + data, err := readRequiredArchiveEntry(entries, name) + if err != nil { + return err + } + if err := ValidateVideo(data); err != nil { + return err + } + if err := videos.WriteVideo(ctx, id, assetName, data); err != nil { + return err + } + } + filePrefix := base + "/assets/" + for _, name := range entries.WithPrefix(filePrefix) { + if name == filePrefix { + continue + } + if !hasFiles { + return ErrNotSupported + } + assetName := name[len(filePrefix):] + if err := validAssetName(assetName); err != nil { + return err + } + data, err := readRequiredArchiveEntry(entries, name) + if err != nil { + return err + } + if err := filesRepo.WriteFile(ctx, id, assetName, data); err != nil { + return err + } + } + imagePrefix := base + "/images/" - videoPrefix := base + "/videos/" - for name, data := range entries { - switch { - case strings.HasPrefix(name, videoPrefix): - repo, ok := repo.(RepositoryVideos) - if !ok { - return ErrNotSupported - } - assetName := strings.TrimPrefix(name, videoPrefix) - if err := validAssetName(assetName); err != nil { - return err - } - if err := ValidateVideo(data); err != nil { - return err - } - if err := repo.WriteVideo(ctx, id, assetName, data); err != nil { - return err - } - case len(name) > len(filePrefix) && name[:len(filePrefix)] == filePrefix: - if !hasFiles { - return ErrNotSupported - } - assetName := name[len(filePrefix):] - if err := validAssetName(assetName); err != nil { - return err - } - if err := filesRepo.WriteFile(ctx, id, assetName, data); err != nil { - return err - } - case len(name) > len(imagePrefix) && name[:len(imagePrefix)] == imagePrefix: - if !hasImages { - return ErrNotSupported - } - assetName := name[len(imagePrefix):] - if err := validAssetName(assetName); err != nil { - return err - } - if err := imagesRepo.WriteImage(ctx, id, assetName, data); err != nil { - return err - } + for _, name := range entries.WithPrefix(imagePrefix) { + if name == imagePrefix { + continue + } + if !hasImages { + return ErrNotSupported + } + assetName := name[len(imagePrefix):] + if err := validAssetName(assetName); err != nil { + return err + } + data, err := readRequiredArchiveEntry(entries, name) + if err != nil { + return err + } + if err := imagesRepo.WriteImage(ctx, id, assetName, data); err != nil { + return err } } return nil } -func validateArchiveAssetEntries(entries map[string][]byte) error { - for name := range entries { - parts := strings.Split(name, "/") - if len(parts) < 4 || parts[0] != "keg-archive" || parts[1] != "nodes" { - continue +func validateArchiveAssetEntries(entries archiveEntries) error { + for _, name := range entries.WithPrefix("keg-archive/nodes/") { + if err := checkArchiveAssetEntry(name); err != nil { + return err } - switch parts[3] { - case "assets", "images", "videos": - assetName := strings.Join(parts[4:], "/") - if err := validAssetName(assetName); err != nil { - return fmt.Errorf("archive asset entry %q: %w", name, err) - } + } + return nil +} + +// checkArchiveAssetEntry rejects a node attachment entry (under a node's +// assets/, images/, or videos/) whose name import would refuse to write, so +// staging and import agree on which archives are valid. The error wraps both +// ErrInvalidAssetName and ErrInvalid: the archive, not the keg, is at fault. +// Other entries pass. +func checkArchiveAssetEntry(name string) error { + rest, ok := strings.CutPrefix(name, "keg-archive/nodes/") + if !ok { + return nil + } + parts := strings.Split(rest, "/") + if len(parts) < 2 { + return nil + } + switch parts[1] { + case "assets", "images", "videos": + if err := validAssetName(strings.Join(parts[2:], "/")); err != nil { + return fmt.Errorf("archive asset entry %q: %w: %w", name, err, ErrInvalid) } } return nil diff --git a/pkg/keg/archive_complete_test.go b/pkg/keg/archive_complete_test.go new file mode 100644 index 00000000..573fedcc --- /dev/null +++ b/pkg/keg/archive_complete_test.go @@ -0,0 +1,204 @@ +package keg_test + +import ( + "bytes" + "encoding/json" + "errors" + "testing" + + "github.com/jlrickert/tapper/pkg/keg" + "github.com/stretchr/testify/require" +) + +// A replacing import prunes every node the archive does not carry, so it must +// refuse an archive that was exported with a node selection: such an archive +// would otherwise wipe the rest of the keg, node 0 included. +func TestImportNodesReplaceAllRefusesPartialArchive(t *testing.T) { + t.Parallel() + src, ctx := limitsTestKeg(t, 3) + partials := map[string]keg.ExportNodesOptions{ + "node ids": {NodeIDs: []keg.NodeId{{ID: 1}}}, + "query": {Query: "golang"}, + "skip zero": {SkipZeroNode: true}, + } + for name, opts := range partials { + t.Run(name, func(t *testing.T) { + archive := mustExportArchive(t, src, opts) + + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + for range 4 { + _, err := dst.Create(ctx, &keg.CreateOptions{Body: []byte("# Extra\n")}) + require.NoError(t, err) + } + before, err := dst.Repo.ListNodes(ctx) + require.NoError(t, err) + + _, err = dst.ImportNodes(ctx, bytes.NewReader(archive), keg.ImportNodesOptions{ReplaceAll: true}) + require.ErrorIs(t, err, keg.ErrInvalid) + + after, err := dst.Repo.ListNodes(ctx) + require.NoError(t, err) + require.Equal(t, before, after) + require.Contains(t, after, keg.NodeId{ID: 0}) + view, err := dst.ReadNode(ctx, keg.NodeId{ID: 1}) + require.NoError(t, err) + require.Contains(t, string(view.Content), "# Extra") + }) + } + + t.Run("full archive", func(t *testing.T) { + archive := mustExportArchive(t, src, keg.ExportNodesOptions{}) + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + for range 5 { + _, err := dst.Create(ctx, &keg.CreateOptions{Body: []byte("# Extra\n")}) + require.NoError(t, err) + } + _, err := dst.ImportNodes(ctx, bytes.NewReader(archive), keg.ImportNodesOptions{ReplaceAll: true}) + require.NoError(t, err) + ids, err := dst.Repo.ListNodes(ctx) + require.NoError(t, err) + require.Equal(t, []keg.NodeId{{ID: 0}, {ID: 1}, {ID: 2}, {ID: 3}}, ids) + view, err := dst.ReadNode(ctx, keg.NodeId{ID: 1}) + require.NoError(t, err) + require.Contains(t, string(view.Content), "# Node 0") + }) +} + +// A full archive that somehow lacks node 0 still never removes the landing +// node when it replaces a keg. +func TestImportNodesReplaceAllKeepsNodeZero(t *testing.T) { + t.Parallel() + src, ctx := limitsTestKeg(t, 1) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{}) + entries := readArchiveEntriesForTest(t, archive) + var manifest map[string]any + require.NoError(t, json.Unmarshal(entries["keg-archive/manifest.json"], &manifest)) + var nodes []any + for _, n := range manifest["nodes"].([]any) { + if n.(map[string]any)["source_id"] != "0" { + nodes = append(nodes, n) + } + } + require.Len(t, nodes, 1) + manifest["nodes"] = nodes + rawManifest, err := json.Marshal(manifest) + require.NoError(t, err) + archive = replaceArchiveEntry(t, archive, "keg-archive/manifest.json", rawManifest) + + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + for range 3 { + _, err := dst.Create(ctx, &keg.CreateOptions{Body: []byte("# Extra\n")}) + require.NoError(t, err) + } + _, err = dst.ImportNodes(ctx, bytes.NewReader(archive), keg.ImportNodesOptions{ReplaceAll: true}) + require.NoError(t, err) + ids, err := dst.Repo.ListNodes(ctx) + require.NoError(t, err) + require.Equal(t, []keg.NodeId{{ID: 0}, {ID: 1}}, ids) +} + +func TestStageArchiveReportsCompleteness(t *testing.T) { + t.Parallel() + src, _ := limitsTestKeg(t, 2) + full := mustExportArchive(t, src, keg.ExportNodesOptions{}) + + stats, err := stageStats(t, full, keg.ArchiveLimits{}) + require.NoError(t, err) + require.True(t, stats.Complete, "full export") + + legacy := replaceArchiveEntry(t, full, "keg-archive/manifest.json", + legacyConfigManifest(t, readArchiveEntriesForTest(t, full)["keg-archive/manifest.json"])) + stats, err = stageStats(t, legacy, keg.ArchiveLimits{}) + require.NoError(t, err) + require.True(t, stats.Complete, "legacy with_config manifest") + + partials := map[string]keg.ExportNodesOptions{ + "node ids": {NodeIDs: []keg.NodeId{{ID: 1}}}, + "query": {Query: "golang"}, + "skip zero": {SkipZeroNode: true}, + } + for name, opts := range partials { + stats, err := stageStats(t, mustExportArchive(t, src, opts), keg.ArchiveLimits{}) + require.NoError(t, err, name) + require.False(t, stats.Complete, name) + } +} + +// ExportNodes owns a caller's spool: every failed export releases it exactly +// once, whether the failure came before, during, or after the write. +func TestExportNodesDiscardsSpoolOnEveryFailure(t *testing.T) { + t.Parallel() + k, ctx := limitsTestKeg(t, 1) + + t.Run("missing keg", func(t *testing.T) { + fx := NewSandbox(t) + empty := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + spool := &recordingSpool{} + _, err := empty.ExportNodes(ctx, keg.ExportNodesOptions{Spool: spool}) + require.Error(t, err) + require.Equal(t, 1, spool.discards) + }) + t.Run("bad query", func(t *testing.T) { + spool := &recordingSpool{} + _, err := k.ExportNodes(ctx, keg.ExportNodesOptions{Spool: spool, Query: ".created>"}) + require.Error(t, err) + require.Equal(t, 1, spool.discards) + }) + t.Run("limit", func(t *testing.T) { + spool := &recordingSpool{} + _, err := k.ExportNodes(ctx, keg.ExportNodesOptions{Spool: spool, Limits: keg.ArchiveLimits{MaxEntries: 1}}) + require.ErrorIs(t, err, keg.ErrArchiveTooLarge) + require.Equal(t, 1, spool.discards) + }) + t.Run("reader", func(t *testing.T) { + sentinel := errors.New("spool reader broke") + spool := &recordingSpool{readerErr: sentinel} + _, err := k.ExportNodes(ctx, keg.ExportNodesOptions{Spool: spool}) + require.ErrorIs(t, err, sentinel) + require.Equal(t, 1, spool.discards) + }) + t.Run("success", func(t *testing.T) { + spool := &recordingSpool{} + rc, err := k.ExportNodes(ctx, keg.ExportNodesOptions{Spool: spool}) + require.NoError(t, err) + require.NoError(t, rc.Close()) + require.Zero(t, spool.discards) + }) +} + +// failingWriter accepts limit bytes, then fails every write with err. +type failingWriter struct { + limit int + n int + err error +} + +func (w *failingWriter) Write(p []byte) (int, error) { + if w.n+len(p) > w.limit { + k := max(w.limit-w.n, 0) + w.n += k + return k, w.err + } + w.n += len(p) + return len(p), nil +} + +// A failure writing the staged copy is the destination's fault, not the +// archive's, so StageArchive must not report it as an invalid archive. +func TestStageArchiveReturnsSinkErrorsAsIs(t *testing.T) { + t.Parallel() + src, _ := limitsTestKeg(t, 2) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{WithAssets: true, WithHistory: true}) + sentinel := errors.New("disk full") + for _, limit := range []int{0, 600, 4096, 20000} { + _, err := keg.StageArchive(t.Context(), bytes.NewReader(archive), &failingWriter{limit: limit, err: sentinel}, keg.ArchiveLimits{}) + require.ErrorIs(t, err, sentinel, "limit %d", limit) + require.NotErrorIs(t, err, keg.ErrInvalid, "limit %d", limit) + } +} diff --git a/pkg/keg/archive_entries.go b/pkg/keg/archive_entries.go new file mode 100644 index 00000000..cddeba3c --- /dev/null +++ b/pkg/keg/archive_entries.go @@ -0,0 +1,42 @@ +package keg + +import ( + "sort" + "strings" +) + +// archiveEntries is read access to the file entries of one keg-archive. Names +// are slash-separated and unique; when the tar carried a name more than once, +// the last entry wins. +type archiveEntries interface { + // Read returns the payload of name. ok is false when the archive has no + // such entry. + Read(name string) (data []byte, ok bool, err error) + // Names returns every entry name in ascending order. + Names() []string + // WithPrefix returns the entry names that start with prefix, in ascending + // order. + WithPrefix(prefix string) []string +} + +// sortedNamesWithPrefix returns the run of names, which must be sorted, that +// start with prefix. +func sortedNamesWithPrefix(names []string, prefix string) []string { + start := sort.SearchStrings(names, prefix) + end := start + for end < len(names) && strings.HasPrefix(names[end], prefix) { + end++ + } + return names[start:end:end] +} + +func readRequiredArchiveEntry(entries archiveEntries, path string) ([]byte, error) { + value, ok, err := entries.Read(path) + if err != nil { + return nil, err + } + if !ok { + return nil, ErrInvalid + } + return value, nil +} diff --git a/pkg/keg/archive_history_internal_test.go b/pkg/keg/archive_history_internal_test.go new file mode 100644 index 00000000..cce8213f --- /dev/null +++ b/pkg/keg/archive_history_internal_test.go @@ -0,0 +1,54 @@ +package keg + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/jlrickert/cli-toolkit/sandbox" + "github.com/stretchr/testify/require" +) + +// snapshotWriteRecorder records the snapshot writes a repository receives. +type snapshotWriteRecorder struct { + *testMemoryRepository + writes []SnapshotWrite +} + +func (r *snapshotWriteRecorder) AppendSnapshot(ctx context.Context, id NodeId, in SnapshotWrite) (Snapshot, error) { + r.writes = append(r.writes, in) + return r.testMemoryRepository.AppendSnapshot(ctx, id, in) +} + +// A repository stores the content hash it is handed, so import must hand it +// one for every archived revision, exactly as taking a snapshot does. +func TestImportNodesHandsSnapshotContentHashToRepository(t *testing.T) { + fx := sandbox.NewSandbox(t, nil) + ctx := fx.Context() + src := NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + require.NoError(t, src.Init(ctx)) + require.NoError(t, src.UpdateSettings(ctx, func(cfg *Settings) { + if cfg.SchemaPolicy == nil { + cfg.SchemaPolicy = &SchemaPolicy{} + } + cfg.SchemaPolicy.Strict = false + })) + node, err := src.Create(ctx, &CreateOptions{Body: []byte("# Node\n")}) + require.NoError(t, err) + want, err := src.AppendSnapshot(ctx, node.ID, "first") + require.NoError(t, err) + require.NotEmpty(t, want.ContentHash) + rc, err := src.ExportNodes(ctx, ExportNodesOptions{WithHistory: true}) + require.NoError(t, err) + archive, err := io.ReadAll(rc) + require.NoError(t, err) + + repo := &snapshotWriteRecorder{testMemoryRepository: newTestMemoryRepo(fx.Runtime())} + dst := NewLocalKeg(repo, fx.Runtime()) + require.NoError(t, dst.Init(ctx)) + _, err = dst.ImportNodes(ctx, bytes.NewReader(archive), ImportNodesOptions{}) + require.NoError(t, err) + require.Len(t, repo.writes, 1) + require.Equal(t, want.ContentHash, repo.writes[0].Content.Hash) +} diff --git a/pkg/keg/archive_importer_test.go b/pkg/keg/archive_importer_test.go new file mode 100644 index 00000000..6c1f941d --- /dev/null +++ b/pkg/keg/archive_importer_test.go @@ -0,0 +1,65 @@ +package keg_test + +import ( + "bytes" + "testing" + + "github.com/jlrickert/tapper/pkg/keg" + "github.com/stretchr/testify/require" +) + +// Importing is a capability of a storage engine, not of every keg: a keg +// reached over a hub's API cannot import, while an engine keg restores a +// whole-keg export through NodeImporter and still restores from its trash. +func TestNodeImporterRestoresAWholeKegExport(t *testing.T) { + t.Parallel() + var remote keg.Keg = keg.NewRemoteKeg("http://hub.invalid", "token", NewSandbox(t).Runtime()) + _, ok := remote.(keg.NodeImporter) + require.False(t, ok, "a remote keg has no node import") + + src, ctx := limitsTestKeg(t, 3) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{WithHistory: true, WithAssets: true}) + + dst, _, _ := newTrashingKeg(t) + // The destination already holds more nodes than the archive, so the + // last one sits on an id the archive does not carry. + var stray keg.CreateResult + for range 4 { + var err error + stray, err = dst.Create(ctx, &keg.CreateOptions{Body: []byte("# Stray\n\nnot in the archive\n")}) + require.NoError(t, err) + } + require.Equal(t, 4, stray.ID.ID) + + var engine keg.Keg = dst + importer, ok := engine.(keg.NodeImporter) + require.True(t, ok, "an engine keg imports archives") + imported, err := importer.ImportNodes(ctx, bytes.NewReader(archive), keg.ImportNodesOptions{ReplaceAll: true}) + require.NoError(t, err) + require.Len(t, imported, 4, "node 0 and the three nodes") + for _, node := range imported { + want, err := src.ReadNode(ctx, node.ID) + require.NoError(t, err) + got, err := dst.ReadNode(ctx, node.ID) + require.NoError(t, err) + require.Equal(t, want.Content, got.Content) + files, err := dst.ListFiles(ctx, node.ID) + require.NoError(t, err) + if node.ID.ID > 0 { + require.Contains(t, files, "notes.txt") + } + } + exists, err := dst.NodeExists(ctx, stray.ID) + require.NoError(t, err) + require.False(t, exists, "a whole-keg restore removes nodes the archive does not carry") + + // Trash restore imports through the same engine path. + view, err := dst.ReadNode(ctx, imported[1].ID) + require.NoError(t, err) + _, err = dst.Remove(ctx, keg.NodeRemoveOptions{ID: view.ID, ExpectedHash: view.Hash()}) + require.NoError(t, err) + require.NoError(t, dst.RestoreTrashed(ctx, []keg.NodeId{view.ID})) + back, err := dst.ReadNode(ctx, view.ID) + require.NoError(t, err) + require.Equal(t, view.Content, back.Content) +} diff --git a/pkg/keg/archive_index.go b/pkg/keg/archive_index.go new file mode 100644 index 00000000..e2c2abab --- /dev/null +++ b/pkg/keg/archive_index.go @@ -0,0 +1,127 @@ +package keg + +import ( + "archive/tar" + "context" + "fmt" + "io" + "path/filepath" + "slices" + "strings" +) + +// ArchiveSource is random access to a staged, uncompressed keg-archive tar, +// such as the output of StageArchive. A *bytes.Reader satisfies it. +type ArchiveSource interface { + io.ReaderAt + // Size returns the length of the staged tar in bytes. + Size() int64 +} + +// archiveIndex serves archive entries from an ArchiveSource by offset, so an +// import holds only the entry it is reading. +type archiveIndex struct { + src io.ReaderAt + offsets map[string]archiveIndexEntry + names []string +} + +type archiveIndexEntry struct { + off int64 + size int64 +} + +// indexArchive scans the tar in src once and records where each file entry's +// payload lives. It applies the same rules as StageArchive: directory entries +// are skipped, unsafe names (overlong ones included) are rejected, names are +// slash-normalized, the last of duplicate names wins, and lim is enforced, +// MaxIndexBytes as the index grows, so its memory stays within that bound. +func indexArchive(ctx context.Context, src ArchiveSource, lim ArchiveLimits) (*archiveIndex, error) { + if lim.MaxTarBytes > 0 && src.Size() > lim.MaxTarBytes { + return nil, &ArchiveLimitError{Limit: ArchiveLimitTarBytes, Max: lim.MaxTarBytes, Actual: src.Size()} + } + sr := io.NewSectionReader(src, 0, src.Size()) + tr := tar.NewReader(sr) + idx := &archiveIndex{src: src, offsets: map[string]archiveIndexEntry{}} + entries := 0 + var index int64 + for { + if err := ctx.Err(); err != nil { + return nil, err + } + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, archiveStreamError(err) + } + if header.FileInfo().IsDir() { + continue + } + if !safeArchiveEntryName(header.Name) { + return nil, unsafeArchiveEntryError(header.Name) + } + // A sparse entry's payload is not stored contiguously, so it has no + // single offset. StageArchive rewrites sparse entries as regular ones. + if header.Typeflag == tar.TypeGNUSparse || isPAXSparse(header) { + return nil, fmt.Errorf("sparse archive entry %q is not supported: %w", header.Name, ErrInvalid) + } + entries++ + if lim.MaxEntries > 0 && entries > lim.MaxEntries { + return nil, &ArchiveLimitError{Limit: ArchiveLimitEntries, Max: int64(lim.MaxEntries), Actual: int64(entries), Entry: header.Name} + } + if lim.MaxEntryBytes > 0 && header.Size > lim.MaxEntryBytes { + return nil, &ArchiveLimitError{Limit: ArchiveLimitEntryBytes, Max: lim.MaxEntryBytes, Actual: header.Size, Entry: header.Name} + } + name := filepath.ToSlash(header.Name) + index += archiveIndexCost(name) + if lim.MaxIndexBytes > 0 && index > lim.MaxIndexBytes { + return nil, &ArchiveLimitError{Limit: ArchiveLimitIndexBytes, Max: lim.MaxIndexBytes, Actual: index, Entry: name} + } + // After Next the section reader sits at the start of the payload. + off, err := sr.Seek(0, io.SeekCurrent) + if err != nil { + return nil, archiveStreamError(err) + } + if off+header.Size > src.Size() { + return nil, archiveStreamError(io.ErrUnexpectedEOF) + } + if _, seen := idx.offsets[name]; !seen { + idx.names = append(idx.names, name) + } + idx.offsets[name] = archiveIndexEntry{off: off, size: header.Size} + } + slices.Sort(idx.names) + return idx, nil +} + +func isPAXSparse(header *tar.Header) bool { + for key := range header.PAXRecords { + if strings.HasPrefix(key, "GNU.sparse.") { + return true + } + } + return false +} + +func (x *archiveIndex) Read(name string) ([]byte, bool, error) { + entry, ok := x.offsets[name] + if !ok { + return nil, false, nil + } + data := make([]byte, entry.size) + if n, err := x.src.ReadAt(data, entry.off); n < len(data) { + if err == nil || err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, false, fmt.Errorf("unable to read archive payload %s: %w", name, err) + } + return data, true, nil +} + +func (x *archiveIndex) Names() []string { return x.names } + +func (x *archiveIndex) WithPrefix(prefix string) []string { + return sortedNamesWithPrefix(x.names, prefix) +} diff --git a/pkg/keg/archive_limits.go b/pkg/keg/archive_limits.go new file mode 100644 index 00000000..89fdad39 --- /dev/null +++ b/pkg/keg/archive_limits.go @@ -0,0 +1,88 @@ +package keg + +import ( + "errors" + "fmt" +) + +// ErrArchiveTooLarge reports a keg-archive that exceeds an ArchiveLimits +// ceiling. *ArchiveLimitError unwraps to it. +var ErrArchiveTooLarge = errors.New("archive exceeds limit") + +// Archive limit names carried by ArchiveLimitError.Limit. +const ( + ArchiveLimitEntries = "entries" + ArchiveLimitEntryBytes = "entry_bytes" + ArchiveLimitTarBytes = "tar_bytes" + ArchiveLimitCompressedBytes = "compressed_bytes" + ArchiveLimitIndexBytes = "index_bytes" +) + +// ArchiveLimits bounds a keg-archive. Export and import measure the same +// quantities the same way, so an archive that export produces under a set of +// limits is accepted by an import under the same limits. A zero field is +// unlimited. +type ArchiveLimits struct { + // MaxEntries bounds the number of file entries. Directory entries are not + // counted; export never writes them. + MaxEntries int + // MaxEntryBytes bounds the payload size of any one entry. + MaxEntryBytes int64 + // MaxTarBytes bounds the uncompressed tar stream, headers, padding, and + // end-of-archive marker included. + MaxTarBytes int64 + // MaxCompressedBytes bounds the gzip stream. + MaxCompressedBytes int64 + // MaxIndexBytes bounds the memory an import's entry index takes: the + // sum, over every file entry, of its name's length plus + // ArchiveIndexEntryOverhead. Entry names are caller-supplied, and the + // index holds every one of them for the whole import. + MaxIndexBytes int64 +} + +// ArchiveIndexEntryOverhead is what MaxIndexBytes charges each entry on top +// of its name: the bookkeeping an index keeps per entry. +const ArchiveIndexEntryOverhead = 64 + +// archiveIndexCost is what one entry adds to the measure MaxIndexBytes +// bounds. +func archiveIndexCost(name string) int64 { + return int64(len(name)) + ArchiveIndexEntryOverhead +} + +// ArchiveLimitError reports which ArchiveLimits ceiling an archive crossed. +// Limits are checked while the archive streams, so for the entry count and +// the byte totals Actual is the value at the point the ceiling was crossed, +// not the archive's final total. +type ArchiveLimitError struct { + // Limit is one of the ArchiveLimit* names. + Limit string + // Max is the configured ceiling. + Max int64 + // Actual is the measured value that exceeded Max. + Actual int64 + // Entry names the entry being processed, when there is one. + Entry string +} + +func (e *ArchiveLimitError) Error() string { + switch e.Limit { + case ArchiveLimitEntries: + return fmt.Sprintf("archive has more than %d entries", e.Max) + case ArchiveLimitEntryBytes: + return fmt.Sprintf("archive entry %q is %d bytes, over the %d byte limit", e.Entry, e.Actual, e.Max) + case ArchiveLimitTarBytes: + return fmt.Sprintf("archive expands beyond the %d byte limit", e.Max) + case ArchiveLimitCompressedBytes: + return fmt.Sprintf("archive is larger than the %d byte limit", e.Max) + case ArchiveLimitIndexBytes: + return fmt.Sprintf("archive entry names need more than the %d byte index limit", e.Max) + default: + return fmt.Sprintf("archive exceeds %s limit of %d", e.Limit, e.Max) + } +} + +func (e *ArchiveLimitError) Unwrap() error { return ErrArchiveTooLarge } + +// tarBlock is the tar record size; headers and payloads are padded to it. +const tarBlock = 512 diff --git a/pkg/keg/archive_limits_test.go b/pkg/keg/archive_limits_test.go new file mode 100644 index 00000000..4630c037 --- /dev/null +++ b/pkg/keg/archive_limits_test.go @@ -0,0 +1,478 @@ +package keg_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "testing" + + "github.com/jlrickert/tapper/pkg/keg" + "github.com/stretchr/testify/require" +) + +// limitsTestKeg returns a keg with history and assets, so its archive holds +// every kind of entry export writes. +func limitsTestKeg(t *testing.T, nodes int) (*keg.LocalKeg, context.Context) { + t.Helper() + fx := NewSandbox(t) + ctx := fx.Context() + k := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, k, ctx) + for i := range nodes { + node, err := k.Create(ctx, &keg.CreateOptions{Body: fmt.Appendf(nil, "# Node %d\n\nsee ../0\n", i)}) + require.NoError(t, err) + require.NoError(t, k.WriteFile(ctx, node.ID, "notes.txt", bytes.Repeat([]byte("n"), 100*(i+1)))) + _, err = k.AppendSnapshot(ctx, node.ID, "rev") + require.NoError(t, err) + } + return k, ctx +} + +func stageStats(t *testing.T, archive []byte, lim keg.ArchiveLimits) (keg.ArchiveStageStats, error) { + t.Helper() + return keg.StageArchive(t.Context(), bytes.NewReader(archive), io.Discard, lim) +} + +func largestEntry(t *testing.T, archive []byte) int64 { + t.Helper() + var largest int64 + for _, data := range readArchiveEntriesForTest(t, archive) { + largest = max(largest, int64(len(data))) + } + return largest +} + +func requireLimitError(t *testing.T, err error, limit string) { + t.Helper() + var limitErr *keg.ArchiveLimitError + require.ErrorAs(t, err, &limitErr) + require.Equal(t, limit, limitErr.Limit) + require.ErrorIs(t, err, keg.ErrArchiveTooLarge) +} + +// Every export must be restorable: for each ceiling, export and import accept +// an archive at exactly the measured value and refuse it one below, naming the +// same limit. +func TestArchiveLimitsAgreeBetweenExportAndImport(t *testing.T) { + t.Parallel() + k, ctx := limitsTestKeg(t, 6) + opts := keg.ExportNodesOptions{WithHistory: true, WithAssets: true} + archive := mustExportArchive(t, k, opts) + stats, err := stageStats(t, archive, keg.ArchiveLimits{}) + require.NoError(t, err) + require.Equal(t, len(readArchiveEntriesForTest(t, archive)), stats.Entries) + require.Equal(t, int64(len(archive)), stats.CompressedBytes) + + measured := map[string]keg.ArchiveLimits{ + keg.ArchiveLimitEntries: {MaxEntries: stats.Entries}, + keg.ArchiveLimitEntryBytes: {MaxEntryBytes: largestEntry(t, archive)}, + keg.ArchiveLimitTarBytes: {MaxTarBytes: stats.TarBytes}, + keg.ArchiveLimitCompressedBytes: {MaxCompressedBytes: int64(len(archive))}, + keg.ArchiveLimitIndexBytes: {MaxIndexBytes: stats.IndexBytes}, + } + below := func(lim keg.ArchiveLimits) keg.ArchiveLimits { + switch { + case lim.MaxEntries > 0: + lim.MaxEntries-- + case lim.MaxEntryBytes > 0: + lim.MaxEntryBytes-- + case lim.MaxTarBytes > 0: + lim.MaxTarBytes-- + case lim.MaxIndexBytes > 0: + lim.MaxIndexBytes-- + default: + lim.MaxCompressedBytes-- + } + return lim + } + for limit, lim := range measured { + t.Run(limit, func(t *testing.T) { + atLimit := opts + atLimit.Limits = lim + again := mustExportArchive(t, k, atLimit) + _, err := stageStats(t, again, lim) + require.NoError(t, err) + + over := opts + over.Limits = below(lim) + _, err = k.ExportNodes(ctx, over) + requireLimitError(t, err, limit) + _, err = stageStats(t, archive, below(lim)) + requireLimitError(t, err, limit) + }) + } +} + +func TestImportNodesEnforcesLimits(t *testing.T) { + t.Parallel() + src, ctx := limitsTestKeg(t, 3) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{WithAssets: true}) + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + + _, err := dst.ImportNodes(ctx, bytes.NewReader(archive), keg.ImportNodesOptions{Limits: keg.ArchiveLimits{MaxEntries: 2}}) + requireLimitError(t, err, keg.ArchiveLimitEntries) + + var staged bytes.Buffer + _, err = keg.StageArchive(ctx, bytes.NewReader(archive), &staged, keg.ArchiveLimits{}) + require.NoError(t, err) + _, err = dst.ImportNodes(ctx, nil, keg.ImportNodesOptions{ + Source: bytes.NewReader(staged.Bytes()), + Limits: keg.ArchiveLimits{MaxEntryBytes: 10}, + }) + requireLimitError(t, err, keg.ArchiveLimitEntryBytes) +} + +func TestImportNodesFromStagedSource(t *testing.T) { + t.Parallel() + src, ctx := limitsTestKeg(t, 2) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{WithHistory: true, WithAssets: true}) + var staged bytes.Buffer + stats, err := keg.StageArchive(ctx, bytes.NewReader(archive), &staged, keg.ArchiveLimits{}) + require.NoError(t, err) + require.Equal(t, int64(staged.Len()), stats.TarBytes) + require.Equal(t, 3, stats.Nodes) + + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + imported, err := dst.ImportNodes(ctx, nil, keg.ImportNodesOptions{Source: bytes.NewReader(staged.Bytes())}) + require.NoError(t, err) + require.Len(t, imported, 3) + data, err := dst.ReadFile(ctx, keg.NodeId{ID: 2}, "notes.txt") + require.NoError(t, err) + require.Len(t, data, 200) + + _, err = dst.ImportNodes(ctx, bytes.NewReader(archive), keg.ImportNodesOptions{Source: bytes.NewReader(staged.Bytes())}) + require.ErrorIs(t, err, keg.ErrInvalid) +} + +func TestStageArchiveRejectsUnreadableArchives(t *testing.T) { + t.Parallel() + src, ctx := limitsTestKeg(t, 1) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{}) + cases := map[string][]byte{ + "junk": []byte("not an archive"), + "truncated": archive[:len(archive)/2], + "empty": nil, + "unsafe": tarWith(t, map[string]string{"keg-archive/manifest.json": `{"format":"keg-archive/v3"}`, "keg-archive/nodes/1/assets/../../../x": "x"}), + "v2": tarWith(t, map[string]string{"keg-archive/manifest.json": `{"format":"keg-archive/v2"}`}), + "missing": tarWith(t, map[string]string{"keg-archive/nodes/1/README.md": "# x"}), + } + for name, data := range cases { + t.Run(name, func(t *testing.T) { + _, err := keg.StageArchive(ctx, bytes.NewReader(data), io.Discard, keg.ArchiveLimits{}) + require.ErrorIs(t, err, keg.ErrInvalid) + }) + } +} + +func tarWith(t *testing.T, entries map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for name, body := range entries { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + return buf.Bytes() +} + +// A tar may carry a name twice; the last entry wins, as it always has. +func TestImportNodesUsesLastDuplicateEntry(t *testing.T) { + t.Parallel() + src, ctx := limitsTestKeg(t, 1) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{NodeIDs: []keg.NodeId{{ID: 1}}}) + gz, err := gzip.NewReader(bytes.NewReader(archive)) + require.NoError(t, err) + raw, err := io.ReadAll(gz) + require.NoError(t, err) + // Drop the end-of-archive marker and append a second README. + var buf bytes.Buffer + buf.Write(raw[:len(raw)-1024]) + tw := tar.NewWriter(&buf) + body := []byte("# Replaced\n") + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "keg-archive/nodes/1/README.md", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})) + _, err = tw.Write(body) + require.NoError(t, err) + require.NoError(t, tw.Close()) + + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + _, err = dst.ImportNodes(ctx, bytes.NewReader(buf.Bytes()), keg.ImportNodesOptions{}) + require.NoError(t, err) + view, err := dst.ReadNode(ctx, keg.NodeId{ID: 1}) + require.NoError(t, err) + require.Contains(t, string(view.Content), "# Replaced") +} + +func TestImportNodesReplaceAllRemovesNodesNotInArchive(t *testing.T) { + t.Parallel() + src, ctx := limitsTestKeg(t, 2) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{WithAssets: true}) + + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + for range 4 { + _, err := dst.Create(ctx, &keg.CreateOptions{Body: []byte("# Extra\n")}) + require.NoError(t, err) + } + + var report keg.ImportReport + var phases []string + _, err := dst.ImportNodes(ctx, bytes.NewReader(archive), keg.ImportNodesOptions{ + ReplaceAll: true, + Report: &report, + Progress: func(p keg.ImportProgress) { + if len(phases) == 0 || phases[len(phases)-1] != p.Phase { + phases = append(phases, p.Phase) + } + }, + }) + require.NoError(t, err) + require.Equal(t, int64(2), report.Removed) + require.Equal(t, []string{keg.ImportPhaseValidating, keg.ImportPhaseImporting, keg.ImportPhasePruning, keg.ImportPhaseIndexing}, phases) + ids, err := dst.Repo.ListNodes(ctx) + require.NoError(t, err) + require.Equal(t, []keg.NodeId{{ID: 0}, {ID: 1}, {ID: 2}}, ids) + + _, err = dst.ImportNodes(ctx, bytes.NewReader(archive), keg.ImportNodesOptions{ReplaceAll: true, AssignNewIDs: true}) + require.ErrorIs(t, err, keg.ErrInvalid) +} + +type recordingSpool struct { + bytes.Buffer + read, discarded bool + discards int + readerErr error +} + +func (s *recordingSpool) Reader() (io.ReadCloser, error) { + s.read = true + if s.readerErr != nil { + return nil, s.readerErr + } + return io.NopCloser(bytes.NewReader(s.Bytes())), nil +} + +func (s *recordingSpool) Discard() error { + s.discarded = true + s.discards++ + return nil +} + +func TestExportNodesWritesToSpool(t *testing.T) { + t.Parallel() + k, ctx := limitsTestKeg(t, 2) + spool := &recordingSpool{} + rc, err := k.ExportNodes(ctx, keg.ExportNodesOptions{Spool: spool}) + require.NoError(t, err) + data, err := io.ReadAll(rc) + require.NoError(t, err) + require.True(t, spool.read) + require.Equal(t, spool.Bytes(), data) + + failed := &recordingSpool{} + _, err = k.ExportNodes(ctx, keg.ExportNodesOptions{Spool: failed, Limits: keg.ArchiveLimits{MaxEntries: 1}}) + require.True(t, errors.Is(err, keg.ErrArchiveTooLarge)) + require.True(t, failed.discarded) + require.False(t, failed.read) +} + +// The archive that started this work held 10,990 entries. A larger keg with +// history and assets must survive export, a replacing import into a keg that +// already has other nodes, and a second export unchanged. +func TestArchiveRoundTripOfLargeKeg(t *testing.T) { + if testing.Short() { + t.Skip("large archive round trip") + } + t.Parallel() + fx := NewSandbox(t) + ctx := fx.Context() + src := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, src, ctx) + png := tinyPNG(t) + ids := make([]keg.NodeId, 0, 540) + for i := range 540 { + node, err := src.Create(ctx, &keg.CreateOptions{Body: fmt.Appendf(nil, "# Node %d\n\nlinks ../%d\n", i, i/2)}) + require.NoError(t, err) + for file := range 14 { + require.NoError(t, src.WriteFile(ctx, node.ID, fmt.Sprintf("notes-%d.txt", file), fmt.Appendf(nil, "notes %d/%d", i, file))) + } + require.NoError(t, src.WriteImage(ctx, node.ID, "diagram.png", png)) + ids = append(ids, node.ID) + } + // Each new snapshot replays the keg's whole timeline, so the fixture gets + // its entry count mostly from assets and takes one batched snapshot round. + for rev := range 1 { + for start := 0; start < len(ids); start += keg.MaxMutationBatchSize { + batch := ids[start:min(start+keg.MaxMutationBatchSize, len(ids))] + requests := make([]keg.NodeSnapshotRequest, 0, len(batch)) + for _, id := range batch { + require.NoError(t, src.SetContent(ctx, id, fmt.Appendf(nil, "# Node %d\n\nrevision %d\n", id.ID, rev))) + requests = append(requests, keg.NodeSnapshotRequest{ID: id, Message: "rev"}) + } + _, err := src.AppendSnapshots(ctx, requests) + require.NoError(t, err) + } + } + opts := keg.ExportNodesOptions{WithHistory: true, WithAssets: true} + first := readArchiveEntriesForTest(t, mustExportArchive(t, src, opts)) + require.Greater(t, len(first), 10_990) + + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + for range 600 { + _, err := dst.Create(ctx, &keg.CreateOptions{Body: []byte("# Extra\n")}) + require.NoError(t, err) + } + var report keg.ImportReport + _, err := dst.ImportNodes(ctx, bytes.NewReader(mustExportArchive(t, src, opts)), keg.ImportNodesOptions{ReplaceAll: true, Report: &report}) + require.NoError(t, err) + require.Equal(t, int64(60), report.Removed) + + second := readArchiveEntriesForTest(t, mustExportArchive(t, dst, opts)) + require.Equal(t, len(first), len(second)) + for name, data := range first { + other, ok := second[name] + require.True(t, ok, "missing %s after round trip", name) + if name == "keg-archive/manifest.json" { + require.JSONEq(t, withoutExportedAt(t, data), withoutExportedAt(t, other)) + continue + } + require.Equal(t, data, other, "entry %s changed in round trip", name) + } +} + +func withoutExportedAt(t *testing.T, raw []byte) string { + t.Helper() + var manifest map[string]any + require.NoError(t, json.Unmarshal(raw, &manifest)) + delete(manifest, "exported_at") + out, err := json.Marshal(manifest) + require.NoError(t, err) + return string(out) +} + +// Directory entries are not part of an archive's contents, and a name too +// long for a plain tar header survives staging intact. +func TestStageArchiveSkipsDirectoriesAndKeepsLongNames(t *testing.T) { + t.Parallel() + long := "keg-archive/nodes/1/assets/" + strings.Repeat("n", 180) + ".txt" + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "keg-archive/", Mode: 0o755, Typeflag: tar.TypeDir})) + for name, body := range map[string]string{"keg-archive/manifest.json": `{"format":"keg-archive/v3","nodes":[]}`, long: "pax"} { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + + var staged bytes.Buffer + stats, err := keg.StageArchive(t.Context(), bytes.NewReader(buf.Bytes()), &staged, keg.ArchiveLimits{}) + require.NoError(t, err) + require.Equal(t, 2, stats.Entries) + tr := tar.NewReader(&staged) + var names []string + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + names = append(names, hdr.Name) + } + require.ElementsMatch(t, []string{"keg-archive/manifest.json", long}, names) +} + +// paxTar builds a tar whose entries carry the given names, as PAX headers do +// for names longer than the ustar fields. +func paxTar(t *testing.T, names ...string) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + manifest := `{"format":"keg-archive/v3","nodes":[]}` + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "keg-archive/manifest.json", Mode: 0o644, Size: int64(len(manifest)), Typeflag: tar.TypeReg})) + _, err := tw.Write([]byte(manifest)) + require.NoError(t, err) + for _, name := range names { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Typeflag: tar.TypeReg, Format: tar.FormatPAX})) + } + require.NoError(t, tw.Close()) + return buf.Bytes() +} + +func importStaged(t *testing.T, tarball []byte, lim keg.ArchiveLimits) error { + t.Helper() + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, fx.Context()) + _, err := dst.ImportNodes(fx.Context(), nil, keg.ImportNodesOptions{Source: bytes.NewReader(tarball), Limits: lim}) + return err +} + +// An entry name no export writes is refused, and the refusal does not echo +// the whole name back. +func TestArchiveRefusesOverlongEntryNames(t *testing.T) { + t.Parallel() + long := "keg-archive/nodes/1/assets/" + strings.Repeat("n", 1<<19) + tarball := paxTar(t, long) + + _, err := stageStats(t, tarball, keg.ArchiveLimits{}) + require.ErrorIs(t, err, keg.ErrInvalid) + require.Less(t, len(err.Error()), 1024, "the error does not carry the name") + + err = importStaged(t, tarball, keg.ArchiveLimits{}) + require.ErrorIs(t, err, keg.ErrInvalid) + require.Less(t, len(err.Error()), 1024) +} + +// A long name that is still a plausible path is kept. +func TestArchiveKeepsEntryNamesUpToTheCap(t *testing.T) { + t.Parallel() + prefix := "keg-archive/nodes/1/assets/" + name := prefix + strings.Repeat("n", 4096-len(prefix)) + stats, err := stageStats(t, paxTar(t, name), keg.ArchiveLimits{}) + require.NoError(t, err) + require.Equal(t, 2, stats.Entries) + require.NoError(t, importStaged(t, paxTar(t, name), keg.ArchiveLimits{})) +} + +// The memory an import's entry index takes is bounded, by staging and by an +// import from an already staged source alike. +func TestArchiveIndexBytesAreBounded(t *testing.T) { + t.Parallel() + names := make([]string, 200) + for i := range names { + names[i] = fmt.Sprintf("keg-archive/nodes/1/assets/%04d-%s", i, strings.Repeat("n", 2000)) + } + tarball := paxTar(t, names...) + stats, err := stageStats(t, tarball, keg.ArchiveLimits{}) + require.NoError(t, err) + require.Greater(t, stats.IndexBytes, int64(200*2000)) + + lim := keg.ArchiveLimits{MaxIndexBytes: 64 << 10} + _, err = stageStats(t, tarball, lim) + requireLimitError(t, err, keg.ArchiveLimitIndexBytes) + requireLimitError(t, importStaged(t, tarball, lim), keg.ArchiveLimitIndexBytes) + + fx := NewSandbox(t) + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, fx.Context()) + _, err = dst.ImportNodes(fx.Context(), bytes.NewReader(tarball), keg.ImportNodesOptions{Limits: lim}) + requireLimitError(t, err, keg.ArchiveLimitIndexBytes) + + require.NoError(t, importStaged(t, tarball, keg.ArchiveLimits{MaxIndexBytes: stats.IndexBytes}), "an archive at the limit is accepted") +} diff --git a/pkg/keg/archive_stage.go b/pkg/keg/archive_stage.go new file mode 100644 index 00000000..e929a25f --- /dev/null +++ b/pkg/keg/archive_stage.go @@ -0,0 +1,417 @@ +package keg + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "path/filepath" +) + +// archiveManifestName is the archive entry that describes the archive. +const archiveManifestName = "keg-archive/manifest.json" + +// countingWriter counts the bytes written through it. +type countingWriter struct { + w io.Writer + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} + +// tarBudgetWriter refuses any write that would take the tar past +// MaxTarBytes, holding back room for the end-of-archive marker until close +// writes it. Nothing past the ceiling ever reaches the destination. +type tarBudgetWriter struct { + w *countingWriter + a *archiveWriter +} + +func (b tarBudgetWriter) Write(p []byte) (int, error) { + if max := b.a.lim.MaxTarBytes; max > 0 { + total := b.w.n + int64(len(p)) + if !b.a.closing { + total += 2 * tarBlock + } + if total > max { + return 0, &ArchiveLimitError{Limit: ArchiveLimitTarBytes, Max: max, Actual: total, Entry: b.a.entry} + } + } + return b.w.Write(p) +} + +// archiveWriter writes keg-archive tar entries and enforces ArchiveLimits. +// Export and staging share it, so both measure an archive the same way. +type archiveWriter struct { + tw *tar.Writer + tar *countingWriter + lim ArchiveLimits + entries int + // index is the archive's measure under MaxIndexBytes so far. + index int64 + // entry names the entry being written, for limit errors. + entry string + // closing is set while close writes the end-of-archive marker. + closing bool +} + +func newArchiveWriter(w io.Writer, lim ArchiveLimits) *archiveWriter { + cw := &countingWriter{w: w} + a := &archiveWriter{tar: cw, lim: lim} + a.tw = tar.NewWriter(tarBudgetWriter{w: cw, a: a}) + return a +} + +// admit counts one entry of the given size, failing before anything is +// written when it would cross the entry-count, entry-size, or index ceiling, +// or when its header and padded payload alone would take the tar past +// MaxTarBytes. +func (a *archiveWriter) admit(name string, size int64) error { + a.entry = name + a.entries++ + if a.lim.MaxEntries > 0 && a.entries > a.lim.MaxEntries { + return &ArchiveLimitError{Limit: ArchiveLimitEntries, Max: int64(a.lim.MaxEntries), Actual: int64(a.entries), Entry: name} + } + if a.lim.MaxEntryBytes > 0 && size > a.lim.MaxEntryBytes { + return &ArchiveLimitError{Limit: ArchiveLimitEntryBytes, Max: a.lim.MaxEntryBytes, Actual: size, Entry: name} + } + a.index += archiveIndexCost(filepath.ToSlash(name)) + if a.lim.MaxIndexBytes > 0 && a.index > a.lim.MaxIndexBytes { + return &ArchiveLimitError{Limit: ArchiveLimitIndexBytes, Max: a.lim.MaxIndexBytes, Actual: a.index, Entry: name} + } + if a.lim.MaxTarBytes > 0 && size >= 0 { + // The least this entry can add: one header block and its payload + // padded to a block, plus the end-of-archive marker. + if size > a.lim.MaxTarBytes { + return &ArchiveLimitError{Limit: ArchiveLimitTarBytes, Max: a.lim.MaxTarBytes, Actual: size, Entry: name} + } + padded := (size + tarBlock - 1) / tarBlock * tarBlock + if need := a.tar.n + tarBlock + padded + 2*tarBlock; need > a.lim.MaxTarBytes { + return &ArchiveLimitError{Limit: ArchiveLimitTarBytes, Max: a.lim.MaxTarBytes, Actual: need, Entry: name} + } + } + return nil +} + +// settle flushes the current entry's padding. The tar total is enforced on +// every write, so an entry that fits has already been checked. +func (a *archiveWriter) settle(name string) error { + if err := a.tw.Flush(); err != nil { + return fmt.Errorf("unable to write archive payload for %s: %w", name, err) + } + return nil +} + +func archiveFileHeader(name string, size int64) *tar.Header { + return &tar.Header{ + Name: filepath.ToSlash(name), + Mode: 0o644, + Size: size, + Typeflag: tar.TypeReg, + } +} + +// writeFile adds one entry from memory. +func (a *archiveWriter) writeFile(name string, data []byte) error { + if err := a.admit(name, int64(len(data))); err != nil { + return err + } + return a.writeAdmitted(name, data) +} + +// writeAdmitted adds an entry that admit has already counted. +func (a *archiveWriter) writeAdmitted(name string, data []byte) error { + if err := a.tw.WriteHeader(archiveFileHeader(name, int64(len(data)))); err != nil { + return fmt.Errorf("unable to write archive header for %s: %w", name, err) + } + if _, err := a.tw.Write(data); err != nil { + return fmt.Errorf("unable to write archive payload for %s: %w", name, err) + } + return a.settle(name) +} + +// copyFile adds one entry of exactly size bytes read from r. +func (a *archiveWriter) copyFile(name string, size int64, r io.Reader) error { + if err := a.admit(name, size); err != nil { + return err + } + if err := a.tw.WriteHeader(archiveFileHeader(name, size)); err != nil { + return fmt.Errorf("unable to write archive header for %s: %w", name, err) + } + if _, err := io.CopyN(a.tw, r, size); err != nil { + return fmt.Errorf("unable to read archive payload %s: %w", name, err) + } + return a.settle(name) +} + +// close writes the end-of-archive marker. +func (a *archiveWriter) close() error { + a.closing = true + if err := a.tw.Close(); err != nil { + return fmt.Errorf("unable to finalize archive: %w", err) + } + return nil +} + +// ArchiveStageStats describes an archive that StageArchive accepted. +type ArchiveStageStats struct { + // Entries is the number of file entries. + Entries int + // TarBytes is the size of the staged, uncompressed tar. + TarBytes int64 + // CompressedBytes is the number of input bytes read. It is zero when the + // input was a plain tar. + CompressedBytes int64 + // Nodes is the number of nodes the manifest lists. + Nodes int + // IndexBytes is the archive's measure under ArchiveLimits.MaxIndexBytes. + IndexBytes int64 + // Complete reports that the manifest says the archive carries the whole + // keg (with_settings, or the legacy with_config spelling). An archive + // exported with a node selection, a query, or without node 0 is not + // complete and cannot replace a keg. + Complete bool +} + +// countingReader counts bytes read and fails, naming limit, once more than +// max are read. It reads at most one byte past max. +type countingReader struct { + r io.Reader + n int64 + max int64 + limit string +} + +func (c *countingReader) Read(p []byte) (int, error) { + if c.max > 0 && c.n > c.max { + return 0, &ArchiveLimitError{Limit: c.limit, Max: c.max, Actual: c.n} + } + if c.max > 0 && int64(len(p)) > c.max-c.n+1 { + p = p[:c.max-c.n+1] + } + n, err := c.r.Read(p) + c.n += int64(n) + if c.max > 0 && c.n > c.max { + return n, &ArchiveLimitError{Limit: c.limit, Max: c.max, Actual: c.n} + } + return n, err +} + +// maxArchiveHeaderBytes bounds what the tar reader may read to reach one +// entry: the previous entry's padding, its header block, and any PAX or GNU +// long-name and long-link records before it. A record can otherwise be a +// megabyte, held in memory while it is parsed, and an archive can carry one +// per entry. It leaves ample room for a name of maxArchiveNameBytes. +const maxArchiveHeaderBytes = 64 << 10 + +// tarHeaderReader sits under the tar reader and, while it is between +// entries, refuses to read past a bound. +type tarHeaderReader struct { + r *countingReader + // end is the stream offset reading must not pass, or -1 inside an + // entry's payload. + end int64 +} + +func (h *tarHeaderReader) Read(p []byte) (int, error) { + if h.end >= 0 { + if h.r.n >= h.end { + return 0, fmt.Errorf("archive entry header is over the %d byte limit: %w", maxArchiveHeaderBytes, ErrInvalid) + } + if int64(len(p)) > h.end-h.r.n { + p = p[:h.end-h.r.n] + } + } + return h.r.Read(p) +} + +// next advances tr to the next entry, reading at most maxArchiveHeaderBytes +// to get there. +func (h *tarHeaderReader) next(tr *tar.Reader) (*tar.Header, error) { + h.end = h.r.n + maxArchiveHeaderBytes + defer func() { h.end = -1 }() + return tr.Next() +} + +// archiveSinkError marks a failure writing to StageArchive's destination, so +// it is not mistaken for an unreadable archive. +type archiveSinkError struct{ err error } + +func (e *archiveSinkError) Error() string { return e.err.Error() } +func (e *archiveSinkError) Unwrap() error { return e.err } + +// sinkWriter tags every write error from w as an *archiveSinkError. +type sinkWriter struct{ w io.Writer } + +func (s sinkWriter) Write(p []byte) (int, error) { + n, err := s.w.Write(p) + if err != nil { + err = &archiveSinkError{err: err} + } + return n, err +} + +// sinkFailure returns the destination's own error when err came from writing +// to it. +func sinkFailure(err error) (error, bool) { + var sinkErr *archiveSinkError + if errors.As(err, &sinkErr) { + return sinkErr.Unwrap(), true + } + return nil, false +} + +// StageArchive reads a keg-archive from r, gzip or plain tar, and writes it to +// w as a normalized plain tar that import can index by offset. Directory +// entries are dropped and names are slash-normalized; every other entry, +// duplicates included, is copied in order, so import resolves duplicates the +// same way it would from the original. +// +// StageArchive rejects unsafe entry names, attachment entries whose name is +// not a valid asset name (wrapping ErrInvalidAssetName), and oversized entry +// headers, and it requires a keg-archive/v3 manifest. It checks lim while +// streaming: MaxTarBytes bounds both every byte read from the uncompressed +// stream, extended headers included, and the staged output, which is checked +// before each write, so w never receives more than MaxTarBytes. Memory use is +// bounded by the manifest, not by the archive. A rejected archive returns an +// error wrapping ErrInvalid or ErrArchiveTooLarge; w may hold a partial tar. +// A failure writing to w is returned as is. +func StageArchive(ctx context.Context, r io.Reader, w io.Writer, lim ArchiveLimits) (ArchiveStageStats, error) { + var stats ArchiveStageStats + compressed := &countingReader{r: r, limit: ArchiveLimitCompressedBytes} + br := bufio.NewReader(compressed) + + var tarStream io.Reader = br + magic, _ := br.Peek(2) + gzipped := len(magic) == 2 && magic[0] == 0x1f && magic[1] == 0x8b + if gzipped { + compressed.max = lim.MaxCompressedBytes + gz, err := gzip.NewReader(br) + if err != nil { + return stats, archiveStreamError(err) + } + defer gz.Close() + tarStream = gz + } + + aw := newArchiveWriter(sinkWriter{w: w}, lim) + // Every byte of the uncompressed stream counts toward MaxTarBytes, not + // just what staging writes back out: extended headers, padding, and + // skipped entries are all decompressed and parsed. + raw := &tarHeaderReader{r: &countingReader{r: tarStream, max: lim.MaxTarBytes, limit: ArchiveLimitTarBytes}, end: -1} + tr := tar.NewReader(raw) + var manifest []byte + for { + if err := ctx.Err(); err != nil { + return stats, err + } + header, err := raw.next(tr) + if err == io.EOF { + break + } + if err != nil { + return stats, archiveStreamError(err) + } + if header.FileInfo().IsDir() { + // Read past any payload here, so the next header's bound + // covers only headers. + if _, err := io.Copy(io.Discard, tr); err != nil { + return stats, archiveStreamError(err) + } + continue + } + if !safeArchiveEntryName(header.Name) { + return stats, unsafeArchiveEntryError(header.Name) + } + name := filepath.ToSlash(header.Name) + if err := checkArchiveAssetEntry(name); err != nil { + return stats, err + } + if name == archiveManifestName { + // The manifest is the one entry held in memory, so it is admitted + // against the entry-size ceiling before it is buffered. + if err := aw.admit(name, header.Size); err != nil { + return stats, err + } + var buf bytes.Buffer + if _, err := io.CopyN(&buf, tr, header.Size); err != nil { + return stats, archiveStreamError(err) + } + manifest = buf.Bytes() + if err := aw.writeAdmitted(name, manifest); err != nil { + if sinkErr, ok := sinkFailure(err); ok { + return stats, sinkErr + } + return stats, err + } + continue + } + if err := aw.copyFile(name, header.Size, tr); err != nil { + if sinkErr, ok := sinkFailure(err); ok { + return stats, sinkErr + } + var limitErr *ArchiveLimitError + if errors.As(err, &limitErr) { + return stats, err + } + return stats, archiveStreamError(err) + } + } + if err := aw.close(); err != nil { + if sinkErr, ok := sinkFailure(err); ok { + return stats, sinkErr + } + return stats, err + } + // The tar can end, and the loop with it, before a limit error from the + // compressed stream surfaces through gzip, so check the count directly. + if err := checkCompressedSize(compressed.n, ArchiveLimits{MaxCompressedBytes: compressed.max}); err != nil { + return stats, err + } + + if manifest == nil { + return stats, fmt.Errorf("archive manifest missing: %w", ErrInvalid) + } + var head struct { + Format string `json:"format"` + WithSettings bool `json:"with_settings"` + WithConfig bool `json:"with_config"` + Nodes []json.RawMessage `json:"nodes"` + } + if err := json.Unmarshal(manifest, &head); err != nil { + return stats, fmt.Errorf("unable to parse archive manifest: %v: %w", err, ErrInvalid) + } + if head.Format != kegArchiveFormatV3 { + return stats, fmt.Errorf("unsupported archive format %q: %w", head.Format, ErrInvalid) + } + + stats.Entries = aw.entries + stats.IndexBytes = aw.index + stats.TarBytes = aw.tar.n + if gzipped { + stats.CompressedBytes = compressed.n + } + stats.Nodes = len(head.Nodes) + stats.Complete = head.WithSettings || head.WithConfig + return stats, nil +} + +// archiveStreamError classifies a failure to read the archive stream itself. +// Limit errors pass through; anything else means the input is not a readable +// keg-archive. +func archiveStreamError(err error) error { + var limitErr *ArchiveLimitError + if errors.As(err, &limitErr) { + return err + } + return fmt.Errorf("unable to read archive stream: %v: %w", err, ErrInvalid) +} diff --git a/pkg/keg/archive_stage_test.go b/pkg/keg/archive_stage_test.go new file mode 100644 index 00000000..0abb4db1 --- /dev/null +++ b/pkg/keg/archive_stage_test.go @@ -0,0 +1,259 @@ +package keg_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "fmt" + "io" + "math/rand/v2" + "strings" + "testing" + + "github.com/jlrickert/tapper/pkg/keg" + "github.com/stretchr/testify/require" +) + +const stageTestManifest = `{"format":"keg-archive/v3","nodes":[]}` + +type stageTestEntry struct { + name string + body []byte + pax map[string]string + fmt tar.Format +} + +// stageTestTar builds a tar holding a manifest and then entries. +func stageTestTar(t *testing.T, entries ...stageTestEntry) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + all := append([]stageTestEntry{{name: "keg-archive/manifest.json", body: []byte(stageTestManifest)}}, entries...) + for _, e := range all { + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: e.name, Mode: 0o644, Size: int64(len(e.body)), Typeflag: tar.TypeReg, + PAXRecords: e.pax, Format: e.fmt, + })) + _, err := tw.Write(e.body) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + return buf.Bytes() +} + +func gzipBytes(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, err := gz.Write(data) + require.NoError(t, err) + require.NoError(t, gz.Close()) + return buf.Bytes() +} + +// readCounter counts the bytes a reader hands out. +type readCounter struct { + r io.Reader + n int64 +} + +func (c *readCounter) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +// writeCounter counts the bytes written to it. +type writeCounter struct{ n int64 } + +func (c *writeCounter) Write(p []byte) (int, error) { + c.n += int64(len(p)) + return len(p), nil +} + +// One extended header record far larger than any name is refused while it is +// being read, for both PAX and GNU long-name records. +func TestStageArchiveRefusesOversizedHeaderRecords(t *testing.T) { + t.Parallel() + huge := strings.Repeat("x", 768<<10) + cases := map[string]stageTestEntry{ + "pax record": {name: "keg-archive/nodes/1/README.md", body: []byte("# x\n"), pax: map[string]string{"comment": huge}, fmt: tar.FormatPAX}, + "gnu long name": {name: "keg-archive/nodes/1/assets/" + huge, body: []byte("x"), fmt: tar.FormatGNU}, + } + for name, entry := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + tarball := stageTestTar(t, entry) + in := &readCounter{r: bytes.NewReader(tarball)} + _, err := keg.StageArchive(t.Context(), in, io.Discard, keg.ArchiveLimits{}) + require.ErrorIs(t, err, keg.ErrInvalid) + require.Less(t, len(err.Error()), 1024, "the error does not carry the record") + require.Less(t, in.n, int64(128<<10), "the record is refused before it is read in full") + }) + } +} + +// Extended headers count toward MaxTarBytes even though staging drops them: +// a small gzip whose PAX records expand past the limit is refused. +func TestStageArchiveCountsExtendedHeadersTowardTarBytes(t *testing.T) { + t.Parallel() + var entries []stageTestEntry + for i := range 64 { + entries = append(entries, stageTestEntry{ + name: fmt.Sprintf("keg-archive/nodes/1/assets/%02d.txt", i), + pax: map[string]string{"comment": strings.Repeat("c", 32<<10)}, + fmt: tar.FormatPAX, + }) + } + archive := gzipBytes(t, stageTestTar(t, entries...)) + require.Less(t, len(archive), 64<<10, "the upload itself is small") + + lim := keg.ArchiveLimits{MaxTarBytes: 1 << 20} + var staged writeCounter + in := &readCounter{r: bytes.NewReader(archive)} + _, err := keg.StageArchive(t.Context(), in, &staged, lim) + requireLimitError(t, err, keg.ArchiveLimitTarBytes) + require.Less(t, staged.n, int64(64<<10), "the staged output never got near the limit") + + stats, err := stageStats(t, archive, keg.ArchiveLimits{}) + require.NoError(t, err, "without a limit the same archive stages") + require.Equal(t, 65, stats.Entries) +} + +// Names long enough to need PAX or GNU long-name records stage under a limit +// set to the archive's exact size. +func TestStageArchiveAcceptsLongNamesAtTheLimit(t *testing.T) { + t.Parallel() + prefix := "keg-archive/nodes/1/assets/" + var entries []stageTestEntry + for i, n := range []int{150, 1000, 4096 - len(prefix) - 8} { + format := tar.FormatPAX + if i%2 == 1 { + format = tar.FormatGNU + } + entries = append(entries, stageTestEntry{ + name: fmt.Sprintf("%s%04d-%s", prefix, i, strings.Repeat("n", n-5)), + body: bytes.Repeat([]byte("b"), 700*(i+1)), + fmt: format, + }) + } + tarball := stageTestTar(t, entries...) + lim := keg.ArchiveLimits{MaxTarBytes: int64(len(tarball))} + for _, input := range [][]byte{tarball, gzipBytes(t, tarball)} { + var staged bytes.Buffer + stats, err := keg.StageArchive(t.Context(), bytes.NewReader(input), &staged, lim) + require.NoError(t, err) + require.Equal(t, 4, stats.Entries) + require.LessOrEqual(t, stats.TarBytes, lim.MaxTarBytes) + + tr := tar.NewReader(&staged) + var names []string + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + names = append(names, hdr.Name) + } + require.Len(t, names, 4) + for _, e := range entries { + require.Contains(t, names, e.name) + } + } +} + +// An archive bomb is refused before staging writes past MaxTarBytes, whether +// it is one huge entry or many that each fit. +func TestStageArchiveRefusesABombBeforeWritingPastTheLimit(t *testing.T) { + t.Parallel() + const limit = 1 << 20 + one := stageTestTar(t, stageTestEntry{name: "keg-archive/nodes/1/assets/bomb.bin", body: make([]byte, 16<<20)}) + var many []stageTestEntry + for i := range 64 { + many = append(many, stageTestEntry{name: fmt.Sprintf("keg-archive/nodes/1/assets/%02d.bin", i), body: make([]byte, 60<<10)}) + } + cases := map[string][]byte{"one entry": one, "many entries": stageTestTar(t, many...)} + for name, tarball := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + var staged writeCounter + _, err := keg.StageArchive(t.Context(), bytes.NewReader(gzipBytes(t, tarball)), &staged, keg.ArchiveLimits{MaxTarBytes: limit}) + requireLimitError(t, err, keg.ArchiveLimitTarBytes) + require.LessOrEqual(t, staged.n, int64(limit), "nothing past the limit reaches staging") + }) + } +} + +// A large archive under the limit stages in full. +func TestStageArchiveStagesALargeArchiveUnderTheLimit(t *testing.T) { + t.Parallel() + rng := rand.New(rand.NewPCG(1, 2)) + var entries []stageTestEntry + for i := range 48 { + body := make([]byte, 96<<10) + for j := range body { + body[j] = byte(rng.UintN(256)) + } + entries = append(entries, stageTestEntry{name: fmt.Sprintf("keg-archive/nodes/%d/assets/blob.bin", i), body: body}) + } + tarball := stageTestTar(t, entries...) + require.Greater(t, len(tarball), 4<<20) + + var staged bytes.Buffer + stats, err := keg.StageArchive(t.Context(), bytes.NewReader(gzipBytes(t, tarball)), &staged, keg.ArchiveLimits{MaxTarBytes: int64(len(tarball))}) + require.NoError(t, err) + require.Equal(t, 49, stats.Entries) + require.Equal(t, int64(staged.Len()), stats.TarBytes) + require.Equal(t, tarball, staged.Bytes(), "a normalized archive stages byte for byte") +} + +// An attachment entry whose name import would refuse to write is refused by +// staging, as an invalid asset name. +func TestStageArchiveRefusesHostileAssetNames(t *testing.T) { + t.Parallel() + for _, name := range []string{ + `keg-archive/nodes/1/assets/a\b.txt`, + "keg-archive/nodes/1/assets/./doc.txt", + "keg-archive/nodes/1/assets/sub/doc.txt", + "keg-archive/nodes/1/images/nested/pic.png", + "keg-archive/nodes/1/videos/./clip.mp4", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + tarball := stageTestTar(t, stageTestEntry{name: name, body: []byte("x")}) + var staged writeCounter + _, err := keg.StageArchive(t.Context(), bytes.NewReader(tarball), &staged, keg.ArchiveLimits{}) + require.ErrorIs(t, err, keg.ErrInvalidAssetName) + require.ErrorIs(t, err, keg.ErrInvalid) + }) + } +} + +// Ordinary asset names, dots and dashes included, stage and import. +func TestStageArchiveKeepsOrdinaryAssetNames(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + ctx := fx.Context() + src := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, src, ctx) + node, err := src.Create(ctx, &keg.CreateOptions{Body: []byte("# Assets\n")}) + require.NoError(t, err) + names := []string{"diagram.v2-final.txt", ".hidden-notes", "a..b.txt", "x-y_z.tar.gz"} + for _, name := range names { + require.NoError(t, src.WriteFile(ctx, node.ID, name, []byte(name))) + } + archive := mustExportArchive(t, src, keg.ExportNodesOptions{WithAssets: true}) + + var staged bytes.Buffer + _, err = keg.StageArchive(ctx, bytes.NewReader(archive), &staged, keg.ArchiveLimits{}) + require.NoError(t, err) + + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + _, err = dst.ImportNodes(ctx, nil, keg.ImportNodesOptions{Source: bytes.NewReader(staged.Bytes())}) + require.NoError(t, err) + files, err := dst.ListFiles(ctx, node.ID) + require.NoError(t, err) + require.ElementsMatch(t, names, files) +} diff --git a/pkg/keg/archive_test.go b/pkg/keg/archive_test.go index d53cc9a7..030f936a 100644 --- a/pkg/keg/archive_test.go +++ b/pkg/keg/archive_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "io" + "strings" "sync/atomic" "testing" @@ -656,3 +657,92 @@ func TestArchiveImportRejectsNestedAssetNameBeforeWritingNodes(t *testing.T) { require.NoError(t, err) require.False(t, exists) } + +// rewriteArchive replaces old with new in every entry's name and content, +// standing in for an archive another version wrote. +func rewriteArchive(t *testing.T, archive []byte, old, new string) []byte { + t.Helper() + gzr, err := gzip.NewReader(bytes.NewReader(archive)) + require.NoError(t, err) + defer gzr.Close() + + var raw bytes.Buffer + tr := tar.NewReader(gzr) + gzw := gzip.NewWriter(&raw) + tw := tar.NewWriter(gzw) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + data, err := io.ReadAll(tr) + require.NoError(t, err) + data = bytes.ReplaceAll(data, []byte(old), []byte(new)) + copyHeader := *header + copyHeader.Name = strings.ReplaceAll(header.Name, old, new) + copyHeader.Size = int64(len(data)) + require.NoError(t, tw.WriteHeader(©Header)) + _, err = tw.Write(data) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + return raw.Bytes() +} + +// A schema name earlier versions accepted, such as one with a space, still +// imports from an archive, and its nodes still validate against it. Only +// names that would escape the schemas directory are refused. +func TestArchiveImportKeepsLegacySchemaNames(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + ctx := fx.Context() + + const placeholder = "legacyplaceholdertype" + src := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, src, ctx) + require.NoError(t, src.CreateSchema(ctx, placeholder, bytes.ReplaceAll(archiveTaskSchema, []byte("task"), []byte(placeholder)))) + _, err := src.Create(ctx, &keg.CreateOptions{Body: []byte("# Standup\n"), Meta: []byte("type: " + placeholder + "\n")}) + require.NoError(t, err) + archive := mustExportArchive(t, src, keg.ExportNodesOptions{}) + + for _, name := range []string{"meeting notes", strings.Repeat("long", 20)} { + t.Run(name, func(t *testing.T) { + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + imported, err := dst.ImportNodes(ctx, bytes.NewReader(rewriteArchive(t, archive, placeholder, name)), keg.ImportNodesOptions{}) + require.NoError(t, err) + var standup keg.NodeId + for _, node := range imported { + if node.SourceID == "1" { + standup = node.ID + } + } + require.NotZero(t, standup.ID, "imported: %v", imported) + + raw, err := dst.ReadSchema(ctx, name) + require.NoError(t, err) + require.Contains(t, string(raw), "type: "+name) + result, err := dst.ValidateNode(ctx, standup) + require.NoError(t, err) + require.True(t, result.Valid, "issues: %v", result.Issues) + require.Equal(t, name, result.Type) + + // An imported legacy schema can still be edited, but no new + // schema can take such a name. + require.NoError(t, dst.WriteSchema(ctx, name, raw, keg.SchemaWriteOptions{ExpectedHash: keg.DocumentHash(raw)})) + other := name + " two" + require.ErrorIs(t, dst.CreateSchema(ctx, other, bytes.ReplaceAll(raw, []byte(name), []byte(other))), keg.ErrInvalid) + }) + } + + for _, name := range []string{"a..b", `a\b`} { + t.Run("refuses "+name, func(t *testing.T) { + dst := keg.NewLocalKeg(newTestMemoryRepo(fx.Runtime()), fx.Runtime()) + initNonStrictTestKeg(t, dst, ctx) + _, err := dst.ImportNodes(ctx, bytes.NewReader(rewriteArchive(t, archive, placeholder, name)), keg.ImportNodesOptions{}) + require.ErrorIs(t, err, keg.ErrInvalid) + }) + } +} diff --git a/pkg/keg/asset_name.go b/pkg/keg/asset_name.go index 402153b3..464a5543 100644 --- a/pkg/keg/asset_name.go +++ b/pkg/keg/asset_name.go @@ -34,7 +34,14 @@ func ValidateAssetName(name string) error { // archives plant ".." in an asset entry (e.g. keg-archive/nodes/1/assets/../../x) // to escape the keg root when imported (zip-slip, CWE-22). Import rejects an // archive containing any such entry before writing anything to the repository. +// +// A name longer than maxArchiveNameBytes is refused too. No export writes one, +// and a tar header can carry a name of up to a megabyte, which an import +// would otherwise keep in memory for every entry. func safeArchiveEntryName(name string) bool { + if len(name) > maxArchiveNameBytes { + return false + } name = filepath.ToSlash(name) if name == "" || filepath.IsAbs(name) || strings.HasPrefix(name, "/") { return false @@ -46,3 +53,16 @@ func safeArchiveEntryName(name string) bool { } return true } + +// maxArchiveNameBytes is the longest keg-archive entry name accepted, the +// usual PATH_MAX. +const maxArchiveNameBytes = 4096 + +// unsafeArchiveEntryError reports an entry safeArchiveEntryName refused, +// quoting at most the start of its name so a huge one is not echoed back. +func unsafeArchiveEntryError(name string) error { + if len(name) > maxArchiveNameBytes { + return fmt.Errorf("archive entry name of %d bytes starting %q is over the %d byte limit: %w", len(name), name[:64], maxArchiveNameBytes, ErrInvalid) + } + return fmt.Errorf("unsafe archive entry %q: %w", name, ErrInvalid) +} diff --git a/pkg/keg/dex.go b/pkg/keg/dex.go index 5e96ae85..ca2809a8 100644 --- a/pkg/keg/dex.go +++ b/pkg/keg/dex.go @@ -90,7 +90,11 @@ func WithSettings(cfg *Settings) DexOption { } idx, err := NewQueryFilteredIndexWithSort(entry.File, query, resolver, sortOrder) if err != nil { - return fmt.Errorf("dex: settings index %q: %w", entry.File, err) + // The query is stored keg configuration, not the current + // caller's input, so the parser's ErrInvalid is dropped: a + // read that trips over it must not be told to fix its own + // arguments. + return fmt.Errorf("dex: settings index %q: %v", entry.File, err) } d.custom = append(d.custom, idx) } diff --git a/pkg/keg/errors.go b/pkg/keg/errors.go index 379cd5d5..dd9936c4 100644 --- a/pkg/keg/errors.go +++ b/pkg/keg/errors.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "strings" "time" ) @@ -50,6 +51,11 @@ var ( // failures. ErrLock = errors.New("cannot acquire lock") + // ErrKegRestoring reports a write refused because the keg is being + // restored from an archive. The restore replaces the keg's contents, so a + // write would be lost; retry once the restore finishes. + ErrKegRestoring = errors.New("keg is being restored from an archive") + // ErrKegLockUpgrade reports an attempted read-to-write operation-boundary // upgrade. Callers must leave the read boundary before starting a write. ErrKegLockUpgrade = errors.New("cannot upgrade keg read boundary to write") @@ -78,6 +84,43 @@ func (e *PreconditionConflictError) Error() string { func (e *PreconditionConflictError) Unwrap() error { return ErrConflict } +// MissingPartHash names one node part written without its per-part +// precondition token: the part supplied ("content", "patch", or "meta"), the +// field that carries its token (expected_content_hash), and the node_read +// field that token comes from (content_hash). +type MissingPartHash struct { + Part string + Field string + TokenField string +} + +// PartPreconditionRequiredError reports a node edit that supplies parts +// without their per-part tokens. Missing lists every absent token, so one +// node_read fixes them all, and Keg and NodeID let a surface name that read. +// It unwraps to ErrPreconditionRequired so wire-code mapping and errors.Is +// checks keep working. +type PartPreconditionRequiredError struct { + // Item is the edit's position in its batch. + Item int + // Keg is the keg alias the caller targeted; empty for the default keg. + Keg string + NodeID string + Missing []MissingPartHash +} + +func (e *PartPreconditionRequiredError) Error() string { + if e == nil || len(e.Missing) == 0 { + return ErrPreconditionRequired.Error() + } + parts := make([]string, len(e.Missing)) + for i, m := range e.Missing { + parts[i] = fmt.Sprintf("%s requires %s (the %s from node_read)", m.Part, m.Field, m.TokenField) + } + return fmt.Sprintf("edit %d node %q: %s: %s", e.Item, e.NodeID, strings.Join(parts, "; "), ErrPreconditionRequired) +} + +func (e *PartPreconditionRequiredError) Unwrap() error { return ErrPreconditionRequired } + // AliasNotFoundError is a typed error that carries the missing alias for callers // that need richer diagnostic information. type AliasNotFoundError struct { diff --git a/pkg/keg/format_fields.go b/pkg/keg/format_fields.go index c7d3d062..52f34508 100644 --- a/pkg/keg/format_fields.go +++ b/pkg/keg/format_fields.go @@ -89,7 +89,7 @@ var LegacyFormatVerbs = map[byte]string{ // FormatVocabularyDescription is the one-line summary of the listing field // vocabulary. It is duplicated as a literal in the MCP tool schemas, which // require a constant struct tag; a test holds the two in agreement. -const FormatVocabularyDescription = "output format template. Legacy verbs %i (id), %t (title), %d (updated), %c (created), %a (accessed); %% is a literal percent. Named selectors use %{...}: a bare word names a metadata key such as %{type} or %{status}, a leading dot names a statistics field such as %{.accessCount} or %{.omega}, and %{tags} is the node's tag list. Backslash escapes are interpreted: \\t tab, \\n newline, \\r return, \\\\ backslash. Selectors other than id, title, and the three dates are resolved by the Hub for the returned page in one batch." +const FormatVocabularyDescription = "Output template per node. Verbs: %i id, %t title, %d updated, %c created, %a accessed, %% a literal percent. Selectors: %{key} a metadata key such as %{type}, %{.field} a stats field such as %{.accessCount}, %{tags} the tag list. Escapes: \\t, \\n, \\r, \\\\." func isIndexTimeField(name string) bool { return slices.Contains(IndexTimeFieldNames, name) diff --git a/pkg/keg/keg.go b/pkg/keg/keg.go index 40de8e3a..ea931774 100644 --- a/pkg/keg/keg.go +++ b/pkg/keg/keg.go @@ -255,5 +255,8 @@ func (k *LocalKeg) SetTarget(target *Target) { k.target = target } -// LocalKeg implements the full Keg interface. -var _ Keg = (*LocalKeg)(nil) +// LocalKeg implements the full Keg interface and imports archives. +var ( + _ Keg = (*LocalKeg)(nil) + _ NodeImporter = (*LocalKeg)(nil) +) diff --git a/pkg/keg/keg_aggregate.go b/pkg/keg/keg_aggregate.go index 50bdef6d..dd1ee6ed 100644 --- a/pkg/keg/keg_aggregate.go +++ b/pkg/keg/keg_aggregate.go @@ -545,12 +545,20 @@ func (k *LocalKeg) listEntries(ctx context.Context, opts ListEntriesOptions) (*L return &ListEntriesResult{Query: strings.TrimSpace(opts.Query), Entries: entries, Tags: tags, IndexedCount: indexed, NodeCount: len(ids)}, nil } +// ReadNodes reads the selected nodes. A touched read waits a bounded time for +// the keg write lock; when the lock stays busy it serves the nodes untouched, +// since the content matters to the caller and the access record does not. func (k *LocalKeg) ReadNodes(ctx context.Context, opts ReadNodesOptions) ([]NodeView, error) { - fn := func(ctx context.Context) ([]NodeView, error) { return k.readNodes(ctx, opts) } if opts.Touch { - return withKegWriteValue(ctx, k, fn) + views, err := withKegWriteValue(touchLockContext(ctx), k, func(ctx context.Context) ([]NodeView, error) { + return k.readNodes(ctx, opts) + }) + if !touchLockBusy(ctx, err) { + return views, err + } + opts.Touch = false } - return withKegReadValue(ctx, k, fn) + return withKegReadValue(ctx, k, func(ctx context.Context) ([]NodeView, error) { return k.readNodes(ctx, opts) }) } func (k *LocalKeg) readNodes(ctx context.Context, opts ReadNodesOptions) ([]NodeView, error) { @@ -573,28 +581,8 @@ func (k *LocalKeg) readNodes(ctx context.Context, opts ReadNodesOptions) ([]Node } } } - var touchBackups []*aggregateBackup if opts.Touch { - touchBackups = make([]*aggregateBackup, 0, len(ids)) - for _, id := range ids { - view, err := k.ReadNode(ctx, id) - if err != nil { - if errors.Is(err, ErrNotExist) { - return nil, fmt.Errorf("node %s not found: %w", id.Path(), err) - } - return nil, err - } - backup, err := k.captureAggregateBackup(ctx, view) - if err != nil { - return nil, err - } - touchBackups = append(touchBackups, backup) - } - for _, id := range ids { - if err := k.Touch(ctx, id); err != nil { - return nil, errors.Join(err, k.restoreTouchBackups(ctx, touchBackups)) - } - } + return k.readTouchedNodes(ctx, ids) } read := k.ReadNode @@ -608,9 +596,6 @@ func (k *LocalKeg) readNodes(ctx context.Context, opts ReadNodesOptions) ([]Node if errors.Is(err, ErrNotExist) { err = fmt.Errorf("node %s not found: %w", id.Path(), err) } - if opts.Touch { - err = errors.Join(err, k.restoreTouchBackups(ctx, touchBackups)) - } return nil, err } views = append(views, *view) @@ -618,6 +603,102 @@ func (k *LocalKeg) readNodes(ctx context.Context, opts ReadNodesOptions) ([]Node return views, nil } +// readTouchedNodes reads and touches every id in order, refreshing the dirty +// index at most once at the end. Any failure restores every touch already +// written, so either all ids are touched or none are. The caller holds the +// keg write operation. +func (k *LocalKeg) readTouchedNodes(ctx context.Context, ids []NodeId) ([]NodeView, error) { + if err := k.checkKegExists(ctx); err != nil { + return nil, fmt.Errorf("failed to read node: %w", err) + } + views := make([]NodeView, 0, len(ids)) + backups := make([]*aggregateBackup, 0, len(ids)) + refresh := false + fail := func(err error) ([]NodeView, error) { + return nil, errors.Join(err, k.restoreTouchBackups(ctx, backups)) + } + for _, id := range ids { + err := k.withNodeLock(ctx, id, func(lockCtx context.Context) error { + view, backup, changed, err := k.readTouchedNode(lockCtx, id) + if backup != nil { + backups = append(backups, backup) + } + if err != nil { + return err + } + refresh = refresh || changed + views = append(views, *view) + return nil + }) + if err != nil { + if errors.Is(err, ErrNotExist) { + err = fmt.Errorf("node %s not found: %w", id.Path(), err) + } + return fail(err) + } + } + if refresh { + if err := k.refreshDirtyIndex(ctx); err != nil { + return fail(err) + } + } + // A repeated id is touched once per occurrence; every view of it shows + // the stats after the last one. + last := make(map[NodeId]*NodeStats, len(views)) + for _, view := range views { + last[view.ID] = view.Stats + } + for i := range views { + if final := last[views[i].ID]; views[i].Stats != final { + stats, err := cloneNodeStats(ctx, final) + if err != nil { + return fail(err) + } + views[i].Stats = stats + } + } + return views, nil +} + +// readTouchedNode reads id, records one access, and returns the view with the +// touched stats. It also returns the backup that undoes the touch, which is +// non-nil once the touch may have written anything, and whether the dirty +// index needs a refresh. The caller holds the keg write operation and the +// node lock. +func (k *LocalKeg) readTouchedNode(ctx context.Context, id NodeId) (*NodeView, *aggregateBackup, bool, error) { + view, err := k.readNode(ctx, id) + if err != nil { + return nil, nil, false, err + } + backup, err := k.captureAggregateBackup(ctx, view) + if err != nil { + return nil, nil, false, err + } + stats, err := cloneNodeStats(ctx, view.Stats) + if err != nil { + return nil, nil, false, err + } + refresh, err := k.touchNoRefresh(ctx, id, view.Meta, stats) + if err != nil { + return nil, backup, false, err + } + if refresh { + // The touch wrote metadata, which feeds the view's hashes. + view, err = k.readNode(ctx, id) + return view, backup, true, err + } + // Reread stats rather than reuse the touched copy: a repository may + // normalize what it stores, and the view shows what a reread would. + stored, err := k.Repo.ReadStats(ctx, id) + if err != nil { + return nil, backup, false, err + } + if stored != nil { + view.Stats = stored + } + return view, backup, false, nil +} + func (k *LocalKeg) RelatedNodes(ctx context.Context, opts RelatedNodesOptions) (*RelatedNodesResult, error) { return withKegReadValue(ctx, k, func(ctx context.Context) (*RelatedNodesResult, error) { return k.relatedNodes(ctx, opts) }) } @@ -821,7 +902,7 @@ func (k *LocalKeg) removeNodes(ctx context.Context, opts RemoveNodesOptions) (Re result.Failure = newBatchFailure(item.ID, err) return result, nil } - if err := checkExpectedHash("node "+item.ID.Path(), item.ExpectedHash, view.Hash(), nodeRecoveryContent(view)); err != nil { + if err := checkExpectedHash("node "+item.ID.Path(), nodeHashSource, item.ExpectedHash, view.Hash(), nodeRecoveryContent(view)); err != nil { result.Failure = newBatchFailure(item.ID, err) return result, nil } @@ -870,6 +951,11 @@ func (k *LocalKeg) createSchema(ctx context.Context, typeName string, data []byt if !ok { return ErrNotSupported } + // A new schema takes the strict name; one restored from an archive is + // written as it was named. + if err := ValidSchemaTypeName(typeName); err != nil { + return err + } if _, err := validateSchemaDefinitionForType(typeName, data); err != nil { return err } @@ -882,12 +968,19 @@ func (k *LocalKeg) createSchema(ctx context.Context, typeName string, data []byt return nil } +// OpenNode opens one node. Like ReadNodes, a touched open whose keg lock stays +// busy serves the node untouched. func (k *LocalKeg) OpenNode(ctx context.Context, opts NodeOpenOptions) (*NodeView, error) { - fn := func(ctx context.Context) (*NodeView, error) { return k.openNode(ctx, opts) } if opts.Touch { - return withKegWriteValue(ctx, k, fn) + view, err := withKegWriteValue(touchLockContext(ctx), k, func(ctx context.Context) (*NodeView, error) { + return k.openNode(ctx, opts) + }) + if !touchLockBusy(ctx, err) { + return view, err + } + opts.Touch = false } - return withKegReadValue(ctx, k, fn) + return withKegReadValue(ctx, k, func(ctx context.Context) (*NodeView, error) { return k.openNode(ctx, opts) }) } func (k *LocalKeg) openNode(ctx context.Context, opts NodeOpenOptions) (*NodeView, error) { @@ -903,21 +996,16 @@ func (k *LocalKeg) openNode(ctx context.Context, opts NodeOpenOptions) (*NodeVie if err := k.validateAggregateLock(lockCtx, opts.ID, opts.LockToken); err != nil { return err } + if !opts.Touch { + view, err = k.readNode(lockCtx, opts.ID) + return err + } var backup *aggregateBackup - if opts.Touch { - before, err := k.ReadNode(lockCtx, opts.ID) - if err != nil { - return err - } - backup, err = k.captureAggregateBackup(lockCtx, before) - if err != nil { - return err - } - if err := k.Touch(lockCtx, opts.ID); err != nil { - return err - } + var refresh bool + view, backup, refresh, err = k.readTouchedNode(lockCtx, opts.ID) + if err == nil && refresh { + err = k.refreshDirtyIndex(lockCtx) } - view, err = k.ReadNode(lockCtx, opts.ID) if err != nil && backup != nil { err = errors.Join(err, k.restoreTouchBackups(lockCtx, []*aggregateBackup{backup})) } @@ -1051,9 +1139,11 @@ func (k *LocalKeg) restoreAggregateBackup(ctx context.Context, backup *aggregate return errors.Join(errs...) } +// restoreTouchBackups undoes touches newest first, so a node touched twice +// ends at the state its first backup captured. func (k *LocalKeg) restoreTouchBackups(ctx context.Context, backups []*aggregateBackup) error { var errs []error - for _, backup := range backups { + for _, backup := range slices.Backward(backups) { if backup == nil { continue } diff --git a/pkg/keg/keg_batch.go b/pkg/keg/keg_batch.go index c1da9c29..1aab447e 100644 --- a/pkg/keg/keg_batch.go +++ b/pkg/keg/keg_batch.go @@ -10,6 +10,17 @@ import ( var nodePlaceholderPattern = regexp.MustCompile(`\{\{node:([^{}]+)\}\}`) +// A placeholder expands to a bare id, which is not a link: a node link is +// ../ID. These match a placeholder that is the whole target of an inline link +// or a reference definition, the one position where a bare id can only be a +// mistake, so expansion can supply the ../ the caller left out. An inline +// link may carry a title after the target. A footnote definition ([^label]:) +// is not a reference definition: its text is prose, so it is left alone. +var ( + inlineLinkPlaceholderPattern = regexp.MustCompile(`\]\([ \t]*(\{\{node:[^{}]+\}\})((?:[ \t]+(?:"[^"\n]*"|'[^'\n]*'|\([^)\n]*\)))?)[ \t]*\)`) + refDefPlaceholderPattern = regexp.MustCompile(`(?m)^( {0,3}\[[^\]^\n][^\]\n]*\]:[ \t]*)(\{\{node:[^{}]+\}\})([ \t]|$)`) +) + type BatchMutationError struct { Index int Key string @@ -129,8 +140,10 @@ func (k *LocalKeg) createNodes(ctx context.Context, nodes []NodeCreate) ([]Creat } func expandNodePlaceholders(body []byte, ids map[string]NodeId) ([]byte, error) { + text := inlineLinkPlaceholderPattern.ReplaceAllString(string(body), "](../${1}${2})") + text = refDefPlaceholderPattern.ReplaceAllString(text, "${1}../${2}${3}") var unknown string - out := nodePlaceholderPattern.ReplaceAllStringFunc(string(body), func(match string) string { + out := nodePlaceholderPattern.ReplaceAllStringFunc(text, func(match string) string { parts := nodePlaceholderPattern.FindStringSubmatch(match) key := strings.TrimSpace(parts[1]) id, ok := ids[key] @@ -229,9 +242,10 @@ func (k *LocalKeg) updateNodes(ctx context.Context, updates []NodeUpdateOptions) } results := make([]NodeUpdateResult, len(prepared)) createdSnapshots := false + history := &snapshotTimeline{} for i, item := range prepared { if item.opts.SnapshotBefore { - if _, err := k.appendSnapshotNoRefresh(ctx, item.opts.ID, "before batch update"); err != nil { + if _, err := k.appendSnapshotNoRefresh(ctx, history, item.opts.ID, "before batch update"); err != nil { return nil, &BatchMutationError{Index: i, NodeID: item.opts.ID, Err: err} } createdSnapshots = true @@ -270,7 +284,7 @@ func (k *LocalKeg) updateNodes(ctx context.Context, updates []NodeUpdateOptions) return nil, err } if createdSnapshots { - if err := k.refreshSnapshotGeneratedIndexes(ctx); err != nil { + if err := k.refreshSnapshotGeneratedIndexesFrom(ctx, history); err != nil { return nil, err } } @@ -366,14 +380,15 @@ func (k *LocalKeg) AppendSnapshots(ctx context.Context, nodes []NodeSnapshotRequ } } out := make([]Snapshot, len(nodes)) + history := &snapshotTimeline{} for i, item := range nodes { - snap, err := k.appendSnapshotNoRefresh(ctx, item.ID, item.Message) + snap, err := k.appendSnapshotNoRefresh(ctx, history, item.ID, item.Message) if err != nil { return nil, &BatchMutationError{Index: i, NodeID: item.ID, Err: err} } out[i] = snap } - if err := k.refreshSnapshotGeneratedIndexes(ctx); err != nil { + if err := k.refreshSnapshotGeneratedIndexesFrom(ctx, history); err != nil { return nil, err } return out, nil diff --git a/pkg/keg/keg_batch_test.go b/pkg/keg/keg_batch_test.go index 3b4253be..9dc26639 100644 --- a/pkg/keg/keg_batch_test.go +++ b/pkg/keg/keg_batch_test.go @@ -62,6 +62,72 @@ func TestCreateNodesResolvesPlaceholdersAndPreservesOrder(t *testing.T) { require.Contains(t, string(two), "[First](../1)") } +// A placeholder written as a bare link target gains the ../ a node link needs, +// so the link registers; everywhere else it stays the bare id, so ../ and +// keg: references written in full are not prefixed twice. +func TestCreateNodesLinksBarePlaceholderTargets(t *testing.T) { + k, ctx := newStrictBatchKeg(t) + body := "# Source\n\n" + + "[Inline]({{node:target}})\n" + + "[Spaced]( {{node:target}} )\n" + + "[Relative](../{{node:target}})\n" + + "Cross: keg:@ns/k/{{node:target}}\n" + + "See node {{node:target}}.\n\n" + + "[ref]: {{node:target}}\n" + results, err := k.CreateNodes(ctx, []keg.NodeCreate{ + {Key: "source", Schema: "task", Body: []byte(body), Meta: []byte("type: task\n")}, + {Key: "target", Schema: "task", Body: []byte("# Target\n"), Meta: []byte("type: task\n")}, + }) + require.NoError(t, err) + raw, err := k.GetContent(ctx, results[0].ID) + require.NoError(t, err) + got := string(raw) + require.Contains(t, got, "[Inline](../2)") + require.Contains(t, got, "[Spaced](../2)") + require.Contains(t, got, "[Relative](../2)") + require.NotContains(t, got, "../../2") + require.Contains(t, got, "Cross: keg:@ns/k/2\n") + require.Contains(t, got, "See node 2.") + require.Contains(t, got, "[ref]: ../2\n") + + content, err := keg.ParseContent(k.Runtime, raw, keg.MarkdownContentFilename) + require.NoError(t, err) + require.Contains(t, content.Links, results[1].ID) +} + +// A titled inline link still gains its ../, and a footnote definition is +// prose, not a link target, so its placeholder stays the bare id. +func TestCreateNodesPlaceholderTitlesAndFootnotes(t *testing.T) { + k, ctx := newStrictBatchKeg(t) + body := "# Source\n\n" + + "[Titled]({{node:target}} \"T\")\n" + + "[Single]({{node:target}} 'T')\n" + + "[Paren]( {{node:target}} (T) )\n" + + "See[^1].\n\n" + + "[^1]: {{node:target}}\n" + + "[ref]: {{node:target}}\n" + results, err := k.CreateNodes(ctx, []keg.NodeCreate{ + {Key: "source", Schema: "task", Body: []byte(body), Meta: []byte("type: task\n")}, + {Key: "target", Schema: "task", Body: []byte("# Target\n"), Meta: []byte("type: task\n")}, + }) + require.NoError(t, err) + raw, err := k.GetContent(ctx, results[0].ID) + require.NoError(t, err) + got := string(raw) + require.Contains(t, got, "[Titled](../2 \"T\")\n") + require.Contains(t, got, "[Single](../2 'T')\n") + require.Contains(t, got, "[Paren](../2 (T))\n") + require.Contains(t, got, "[^1]: 2\n") + require.Contains(t, got, "[ref]: ../2\n") +} + +func TestCreateNodesRejectsUnknownPlaceholderLinkTarget(t *testing.T) { + k, ctx := newStrictBatchKeg(t) + _, err := k.CreateNodes(ctx, []keg.NodeCreate{{Key: "one", Schema: "task", Body: []byte("# One\n\n[Missing]({{node:nope}})\n"), Meta: []byte("type: task\n")}}) + require.ErrorIs(t, err, keg.ErrInvalid) + require.ErrorContains(t, err, `unknown node placeholder "nope"`) +} + func TestCreateNodesPreflightFailureLeavesKegUnchanged(t *testing.T) { k, ctx := newStrictBatchKeg(t) _, err := k.CreateNodes(ctx, []keg.NodeCreate{{Key: "one", Schema: "task", Body: []byte("# One\n"), Meta: []byte("type: task\n")}, {Key: "two", Schema: "task", Body: []byte("# Two\n\n[Missing](../{{node:nope}})\n"), Meta: []byte("type: task\n")}}) diff --git a/pkg/keg/keg_iface.go b/pkg/keg/keg_iface.go index 0dfb05ce..74878a88 100644 --- a/pkg/keg/keg_iface.go +++ b/pkg/keg/keg_iface.go @@ -149,13 +149,17 @@ type Keg interface { // OpenNode validates any held advisory lock against opts.LockToken, // optionally records an access touch, and returns one coherent node view. - // If the operation fails after touching, the touch is rolled back. + // If the operation fails after touching, the touch is rolled back. A touch + // that cannot get the keg write lock within a bounded wait is skipped and + // the node is read untouched. OpenNode(ctx context.Context, opts NodeOpenOptions) (*NodeView, error) // ReadNodes reads either opts.NodeIDs in caller order or the nodes selected // by opts.Query in dex order; the selectors are mutually exclusive. All // views come from one coherent keg snapshot. When Touch is set, either every - // selected node is touched or all touch side effects are rolled back. + // selected node is touched or all touch side effects are rolled back; a + // touch that cannot get the keg write lock within a bounded wait is + // skipped and the nodes are read untouched. ReadNodes(ctx context.Context, opts ReadNodesOptions) ([]NodeView, error) // UpdateNode validates advisory-lock ownership and an optional expected @@ -309,10 +313,6 @@ type Keg interface { // read snapshot, and the caller must Close the returned reader. ExportNodes(ctx context.Context, opts ExportNodesOptions) (io.ReadCloser, error) - // ImportNodes loads a keg-archive stream into the keg, replacing - // existing nodes with matching ids, and rebuilds derived state. - ImportNodes(ctx context.Context, r io.Reader, opts ImportNodesOptions) ([]ImportedNode, error) - // Cross-process locks // Lock acquires a cross-process advisory lock on a node. @@ -444,9 +444,38 @@ type ExportNodesOptions struct { WithAssets bool // Source labels the archive manifest with the origin keg reference. Source string + // Limits bounds the archive. Export fails with *ArchiveLimitError rather + // than produce an archive an import under the same limits would refuse. + // Zero fields are unlimited. + Limits ArchiveLimits + // Spool, when set, receives the archive instead of an in-memory buffer, + // and the returned reader reads it back. ExportNodes owns the spool: when + // it returns an error the spool has been discarded. + Spool ArchiveSpool +} + +// ArchiveSpool holds an exported archive between the write, which happens +// inside the keg's read boundary, and the caller's read, which does not. +type ArchiveSpool interface { + io.Writer + // Reader returns the spooled archive. Closing it releases the spool. + Reader() (io.ReadCloser, error) + // Discard releases a spool whose export failed. It must be idempotent and + // safe to call after the reader returned by Reader has been closed. + Discard() error } -// ImportNodesOptions configures Keg.ImportNodes. +// NodeImporter loads keg-archives into a keg. It is not part of Keg: only a +// storage engine that owns the keg's data implements it (LocalKeg, and a +// hub's own engine for its archive restore). A keg reached over a hub's API +// has no import; the hub loads an archive through its restore instead. +type NodeImporter interface { + // ImportNodes loads a keg-archive stream into the keg, replacing + // existing nodes with matching ids, and rebuilds derived state. + ImportNodes(ctx context.Context, r io.Reader, opts ImportNodesOptions) ([]ImportedNode, error) +} + +// ImportNodesOptions configures NodeImporter.ImportNodes. type ImportNodesOptions struct { // AssignNewIDs allocates fresh sequential node ids for the archive's // nodes instead of landing them on their archive ids. Links between @@ -468,6 +497,56 @@ type ImportNodesOptions struct { // rather than its own, its links to itself following. Restoring from // the trash always sets it to the entry's current id. restoreTo *NodeId + // Source, when set, is a staged plain-tar archive (see StageArchive) read + // by offset in place of the reader passed to ImportNodes, which must then + // be nil. Import memory is bounded by the largest entry rather than the + // archive. + Source ArchiveSource + // Limits bounds the archive. Zero fields are unlimited. + Limits ArchiveLimits + // ReplaceAll removes every node the archive does not carry, so the keg + // ends up holding exactly the archive's nodes. Removal deletes nodes + // through the repository without rewriting links in the surviving nodes, + // and node 0 is never removed. It needs a full-keg archive (see + // ArchiveStageStats.Complete) and fails with ErrInvalid on an archive + // exported with a node selection. It cannot be combined with AssignNewIDs. + ReplaceAll bool + // Progress, when set, is called as the import advances. + Progress func(ImportProgress) + // Report, when set, is filled in with what the import changed beyond the + // imported nodes. + Report *ImportReport +} + +// Import progress phases reported through ImportNodesOptions.Progress. +const ( + ImportPhaseValidating = "validating" + ImportPhaseImporting = "importing" + ImportPhasePruning = "pruning" + ImportPhaseIndexing = "indexing" +) + +// ImportProgress reports how far an import has advanced through one phase. +// Done and Total count nodes; Total is zero when a phase has no count. +type ImportProgress struct { + Phase string + Done int + Total int +} + +// ImportReport describes side effects of an import. +type ImportReport struct { + // Removed counts the nodes ReplaceAll deleted. + Removed int64 +} + +// RepositoryNodePruner is an optional Repository capability that deletes every +// node not in keep in one operation, returning how many went. ReplaceAll uses +// it when present and falls back to DeleteNode per node otherwise. +type RepositoryNodePruner interface { + // PruneNodesExcept deletes every node whose id is not in keep and returns + // the number deleted. + PruneNodesExcept(ctx context.Context, keep []NodeId) (int64, error) } // ImportedNode maps an archive source id to the node id it landed on. diff --git a/pkg/keg/keg_local_content.go b/pkg/keg/keg_local_content.go index cb45856c..55768ae5 100644 --- a/pkg/keg/keg_local_content.go +++ b/pkg/keg/keg_local_content.go @@ -371,8 +371,7 @@ func (k *LocalKeg) touch(ctx context.Context, id NodeId) error { return fmt.Errorf("failed to touch node: %w", err) } - now := k.Runtime.Clock().Now() - + var refresh bool if err := k.withNodeLock(ctx, id, func(lockCtx context.Context) error { // Verify the node truly exists (has content) under the lock. exists, exErr := k.nodeExistsWithContent(lockCtx, id) @@ -383,30 +382,58 @@ func (k *LocalKeg) touch(ctx context.Context, id NodeId) error { return fmt.Errorf("node %s not found: %w", id.Path(), ErrNotExist) } - meta, stats, err := k.getMetaAndStats(lockCtx, id) - if errors.Is(err, ErrNotExist) { - meta = NewMeta(lockCtx, now) - stats = NewStats(now) - } else if err != nil { + meta, err := k.Repo.ReadMeta(lockCtx, id) + if err != nil && !errors.Is(err, ErrNotExist) { return fmt.Errorf("failed to read node metadata: %w", err) } - if stats == nil { - stats = &NodeStats{} - } - - stats.SetAccessed(now) - stats.IncrementAccessCount() - stats.EnsureTimes(now) - if err := k.Repo.WriteMeta(lockCtx, id, []byte(meta.ToYAML())); err != nil { - return err + stats, err := k.getStats(lockCtx, id) + if err != nil { + return fmt.Errorf("failed to read node stats: %w", err) } - return k.Repo.WriteStats(lockCtx, id, stats) + refresh, err = k.touchNoRefresh(lockCtx, id, meta, stats) + return err }); err != nil { return err } + if !refresh { + return nil + } return k.refreshDirtyIndex(ctx) } +// touchNoRefresh records one access in stats, which the caller read under the +// node lock along with rawMeta, the node's stored metadata. It leaves existing +// metadata alone and writes it only for a node that has none. +// +// It does not refresh the dirty index, so a batch pays for that once. It +// reports whether the index needs it, which is only when the metadata it +// wrote hashes differently from what was stored. Stats never change a dirty +// row: the row falls back to the stats title only when content has none, and +// a touch never changes the title, so a stats-only write leaves the index as +// a rebuild would make it. +func (k *LocalKeg) touchNoRefresh(ctx context.Context, id NodeId, rawMeta []byte, stats *NodeStats) (bool, error) { + now := k.Runtime.Clock().Now() + if stats == nil { + stats = &NodeStats{} + } + stats.SetAccessed(now) + stats.IncrementAccessCount() + stats.EnsureTimes(now) + + refresh := false + if len(bytes.TrimSpace(rawMeta)) == 0 { + created := []byte(NewMeta(ctx, now).ToYAML()) + if err := k.Repo.WriteMeta(ctx, id, created); err != nil { + return false, err + } + refresh = hashSnapshotBytes(k.Runtime, created) != hashSnapshotBytes(k.Runtime, rawMeta) + } + if err := k.Repo.WriteStats(ctx, id, stats); err != nil { + return false, err + } + return refresh, nil +} + // Node returns a Node handle bound to this keg's repository and runtime for the // given id. It performs no I/O; content, metadata, and stats are loaded lazily // by the Node's own methods. diff --git a/pkg/keg/keg_local_lock.go b/pkg/keg/keg_local_lock.go index 3d6aca14..61ff341c 100644 --- a/pkg/keg/keg_local_lock.go +++ b/pkg/keg/keg_local_lock.go @@ -2,6 +2,7 @@ package keg import ( "context" + "errors" "fmt" ) @@ -22,7 +23,7 @@ func (k *LocalKeg) Lock(ctx context.Context, id NodeId) (LockInfo, error) { if !ok { return LockInfo{}, ErrNotSupported } - token, err := locker.AcquireLock(ctx, id) + token, err := acquireNodeLock(ctx, locker, id) if err != nil { return LockInfo{}, err } @@ -71,3 +72,23 @@ func (k *LocalKeg) ForceUnlock(ctx context.Context, id NodeId) error { } return locker.ForceReleaseLock(ctx, id) } + +// acquireNodeLock acquires id's lock within the bound WithNodeLockWait set on +// ctx, if any. A lock still held when the bound passes fails with +// ErrLockTimeout; a canceled caller fails with its own error. +func acquireNodeLock(ctx context.Context, locker RepositoryLock, id NodeId) (LockToken, error) { + wait := NodeLockWaitFrom(ctx) + if wait <= 0 { + return locker.AcquireLock(ctx, id) + } + bounded, cancel := context.WithTimeout(ctx, wait) + defer cancel() + token, err := locker.AcquireLock(bounded, id) + if err == nil { + return token, nil + } + if ctx.Err() == nil && errors.Is(bounded.Err(), context.DeadlineExceeded) && !errors.Is(err, ErrLockTimeout) { + return "", fmt.Errorf("%w: node %s still locked after %s: %w", ErrLockTimeout, id.Path(), wait, err) + } + return "", err +} diff --git a/pkg/keg/keg_local_move.go b/pkg/keg/keg_local_move.go index 59703bcd..3e6967ba 100644 --- a/pkg/keg/keg_local_move.go +++ b/pkg/keg/keg_local_move.go @@ -33,7 +33,7 @@ func (k *LocalKeg) move(ctx context.Context, opts NodeMoveOptions) ([]NodeId, er if err != nil { return nil, err } - if err := checkExpectedHash("node "+src.Path(), opts.ExpectedHash, current.Hash(), nodeRecoveryContent(current)); err != nil { + if err := checkExpectedHash("node "+src.Path(), nodeHashSource, opts.ExpectedHash, current.Hash(), nodeRecoveryContent(current)); err != nil { return nil, err } if src.Equals(dst) { @@ -144,7 +144,7 @@ func (k *LocalKeg) remove(ctx context.Context, opts NodeRemoveOptions) ([]NodeId if err != nil { return nil, err } - if err := checkExpectedHash("node "+id.Path(), opts.ExpectedHash, current.Hash(), nodeRecoveryContent(current)); err != nil { + if err := checkExpectedHash("node "+id.Path(), nodeHashSource, opts.ExpectedHash, current.Hash(), nodeRecoveryContent(current)); err != nil { return nil, err } return k.removeUnchecked(ctx, id, manualRemoval) diff --git a/pkg/keg/keg_local_settings.go b/pkg/keg/keg_local_settings.go index 9c9593da..3ac12c28 100644 --- a/pkg/keg/keg_local_settings.go +++ b/pkg/keg/keg_local_settings.go @@ -91,7 +91,7 @@ func (k *LocalKeg) setSettings(ctx context.Context, data []byte, opts SettingsWr if err != nil { return fmt.Errorf("failed to read current settings: %w", err) } - if err := checkExpectedHash("settings", opts.ExpectedHash, current.Hash(), current.Raw()); err != nil { + if err := checkExpectedHash("settings", settingsHashSource, opts.ExpectedHash, current.Hash(), current.Raw()); err != nil { return err } return k.replaceSettings(ctx, data) diff --git a/pkg/keg/keg_omega.go b/pkg/keg/keg_omega.go index 22e07e07..3a3b709d 100644 --- a/pkg/keg/keg_omega.go +++ b/pkg/keg/keg_omega.go @@ -40,12 +40,7 @@ type snapshotOmegaReplayer struct { latestLinks map[string][]NodeId } -func (k *LocalKeg) replaySnapshotTimeline(ctx context.Context, records []timelineSnapshotState) (snapshotTimelineReplay, error) { - schemas, err := k.loadOmegaSchemas(ctx) - if err != nil { - return snapshotTimelineReplay{}, err - } - +func replaySnapshotTimelineWith(schemas map[string]*SchemaDefinition, records []timelineSnapshotState) snapshotTimelineReplay { replayer := snapshotOmegaReplayer{ schemas: schemas, latest: make(map[string]snapshotNodeState, len(records)), @@ -62,7 +57,7 @@ func (k *LocalKeg) replaySnapshotTimeline(ctx context.Context, records []timelin } out.Steps = append(out.Steps, step) } - return out, nil + return out } func (k *LocalKeg) loadOmegaSchemas(ctx context.Context) (map[string]*SchemaDefinition, error) { @@ -263,18 +258,21 @@ func omegaAttributeScore(enum map[string]float64, rawValue string) (float64, boo return clampOmegaScore(score), true } -func (k *LocalKeg) computePendingSnapshotOmega(ctx context.Context, pending timelineSnapshotState) (*float64, error) { - records, err := k.loadTimelineSnapshotStates(ctx) - if err != nil { +// computePendingSnapshotOmega replays history with pending added and returns +// pending's omega. A nil history loads a fresh one. +func (k *LocalKeg) computePendingSnapshotOmega(ctx context.Context, history *snapshotTimeline, pending timelineSnapshotState) (*float64, error) { + if history == nil { + history = &snapshotTimeline{} + } + if err := history.load(ctx, k); err != nil { return nil, err } + records := make([]timelineSnapshotState, 0, len(history.records)+1) + records = append(records, history.records...) records = append(records, pending) sortTimelineSnapshotStates(records) - replay, err := k.replaySnapshotTimeline(ctx, records) - if err != nil { - return nil, err - } + replay := replaySnapshotTimelineWith(history.schemas, records) for _, step := range replay.Steps { if !step.Record.snapshot.Node.Equals(pending.snapshot.Node) || step.Record.snapshot.ID != pending.snapshot.ID { continue diff --git a/pkg/keg/keg_operation.go b/pkg/keg/keg_operation.go index 5629f43f..0722c2bf 100644 --- a/pkg/keg/keg_operation.go +++ b/pkg/keg/keg_operation.go @@ -1,9 +1,64 @@ package keg -import "context" +import ( + "context" + "errors" + "time" +) type localKegOperationContextKey struct{} +type lockWaitContextKey struct{} + +// touchLockWait bounds how long a read that records an access waits for the +// keg write lock. The access is bookkeeping, so a read behind a long writer +// gives it up and serves the content rather than queue for the lock. +const touchLockWait = 2 * time.Second + +// WithLockWait returns ctx carrying a bound on how long a repository waits to +// acquire the keg write lock. A repository that honors it fails with +// ErrLockTimeout once d passes without the lock; the bound covers acquiring +// the lock, not the work done under it. A d of zero or less leaves the wait +// unbounded, which is the default. +// +// The bound is advisory. It lives only in ctx and is not part of the Hub API, +// so it never crosses the wire: it applies only where the Repository runs in +// the same process as the caller and reads it with LockWaitFrom. A Repository +// that does not, the in-memory test repository included, waits as it always +// has, and a touched read there waits for the lock instead of skipping the +// touch. Callers must not rely on the bound for correctness, only to keep a +// read from queueing behind a long writer where it is honored. +func WithLockWait(ctx context.Context, d time.Duration) context.Context { + return context.WithValue(ctx, lockWaitContextKey{}, d) +} + +// LockWaitFrom returns the keg lock wait bound carried by ctx, or zero when +// the wait is unbounded. Repository implementations call it when they acquire +// the keg write lock; see WithLockWait for which ones do. +func LockWaitFrom(ctx context.Context) time.Duration { + d, _ := ctx.Value(lockWaitContextKey{}).(time.Duration) + if d < 0 { + return 0 + } + return d +} + +// touchLockContext bounds the keg lock wait of a touched read, keeping any +// bound the caller already set. +func touchLockContext(ctx context.Context) context.Context { + if LockWaitFrom(ctx) > 0 { + return ctx + } + return WithLockWait(ctx, touchLockWait) +} + +// touchLockBusy reports whether a touched read failed only because the keg +// lock stayed busy, so the read can be served without the touch. A canceled +// or expired ctx also surfaces as ErrLockTimeout; that read fails as it is. +func touchLockBusy(ctx context.Context, err error) bool { + return errors.Is(err, ErrLockTimeout) && ctx.Err() == nil +} + func localKegOperationActive(ctx context.Context, k *LocalKeg) bool { held, _ := ctx.Value(localKegOperationContextKey{}).(*LocalKeg) return held == k diff --git a/pkg/keg/keg_remote.go b/pkg/keg/keg_remote.go index d23cfd7e..4aeb67d6 100644 --- a/pkg/keg/keg_remote.go +++ b/pkg/keg/keg_remote.go @@ -1049,57 +1049,6 @@ func (k *RemoteKeg) ExportNodes(ctx context.Context, opts ExportNodesOptions) (i return resp.Body, nil } -// ImportNodes implements Keg via POST /archive, streaming r as the request -// body. -func (k *RemoteKeg) ImportNodes(ctx context.Context, r io.Reader, opts ImportNodesOptions) ([]ImportedNode, error) { - q := url.Values{} - if opts.AssignNewIDs { - q.Set("assign_new_ids", "1") - } - if opts.HistoryIfSupported { - q.Set("history_if_supported", "1") - } - if opts.SourceAlias != "" { - q.Set("source_alias", opts.SourceAlias) - } - if opts.TargetAlias != "" { - q.Set("target_alias", opts.TargetAlias) - } - path := "/archive" - if enc := q.Encode(); enc != "" { - path += "?" + enc - } - resp, err := k.do(ctx, http.MethodPost, path, r, "application/gzip", nil) - if err != nil { - return nil, err - } - body, err := k.readBody(resp, "ImportNodes", http.StatusOK, http.StatusCreated) - if err != nil { - return nil, err - } - var result struct { - Imported []struct { - Source string `json:"source"` - SourceHash string `json:"source_hash"` - ID string `json:"id"` - } `json:"imported"` - } - if err := json.Unmarshal(body, &result); err != nil { - return nil, NewBackendError("remote", "ImportNodes", 0, - fmt.Errorf("invalid response: %w", err), false) - } - out := make([]ImportedNode, 0, len(result.Imported)) - for _, item := range result.Imported { - id, err := ParseNode(item.ID) - if err != nil { - return nil, NewBackendError("remote", "ImportNodes", 0, - fmt.Errorf("invalid imported node id %q: %w", item.ID, err), false) - } - out = append(out, ImportedNode{SourceID: item.Source, SourceHash: item.SourceHash, ID: *id}) - } - return out, nil -} - // --- Cross-process locks --- // remoteLockInfo is the hub's lock-state wire shape, shared by LockNode and @@ -1125,11 +1074,17 @@ func (li remoteLockInfo) toLockInfo() LockInfo { return info } -// Lock implements Keg via POST /nodes/{id}/lock. The hub owns the lease; no -// client-side renewal goroutine runs. +// Lock implements Keg via POST /nodes/{id}/lock, with ?wait= carrying the +// WithNodeLockWait bound. The hub owns the lease; no client-side renewal +// goroutine runs. func (k *RemoteKeg) Lock(ctx context.Context, id NodeId) (LockInfo, error) { var info remoteLockInfo path := fmt.Sprintf("/nodes/%d/lock", id.ID) + if wait := NodeLockWaitFrom(ctx); wait > 0 { + // The Hub takes whole seconds; round up so a sub-second bound still + // waits rather than asking for no wait at all. + path += "?wait=" + strconv.Itoa(int((wait+time.Second-1)/time.Second)) + } if err := k.postJSON(ctx, path, "Lock", struct{}{}, &info, http.StatusOK, http.StatusCreated); err != nil { return LockInfo{}, err } diff --git a/pkg/keg/keg_remote_test.go b/pkg/keg/keg_remote_test.go index bee395de..fb675f8a 100644 --- a/pkg/keg/keg_remote_test.go +++ b/pkg/keg/keg_remote_test.go @@ -502,24 +502,6 @@ func newMockOpsHub(t *testing.T, f *sandbox.Sandbox, token string) *mockOpsHub { w.Header().Set("Content-Type", "application/gzip") _, _ = io.Copy(w, rc) }) - mux.HandleFunc("POST /archive", func(w http.ResponseWriter, r *http.Request) { - imported, err := backing.ImportNodes(r.Context(), r.Body, kegpkg.ImportNodesOptions{ - AssignNewIDs: r.URL.Query().Get("assign_new_ids") == "1", - }) - if err != nil { - h.kegError(w, err) - return - } - type importedWire struct { - Source string `json:"source"` - ID string `json:"id"` - } - out := make([]importedWire, len(imported)) - for i, n := range imported { - out[i] = importedWire{Source: n.SourceID, ID: n.ID.Path()} - } - h.writeJSON(w, http.StatusOK, map[string]any{"imported": out}) - }) // Locks mux.HandleFunc("POST /nodes/{id}/lock", func(w http.ResponseWriter, r *http.Request) { @@ -686,6 +668,22 @@ func TestRemoteKegRoundTripBasics(t *testing.T) { require.Len(t, nodeIDs, 4) } +func TestRemoteCreateNodesLinksBarePlaceholderTargets(t *testing.T) { + t.Parallel() + fx, hub, remote := newRemoteKegFixture(t) + ctx := fx.Context() + require.NoError(t, hub.backing.CreateSchema(ctx, "note", []byte("type: note\n"))) + + created, err := remote.CreateNodes(ctx, []kegpkg.NodeCreate{ + {Key: "source", Schema: "note", Body: []byte("# Source\n\n[Target]({{node:target}})\n")}, + {Key: "target", Schema: "note", Body: []byte("# Target\n")}, + }) + require.NoError(t, err) + source, err := remote.ReadNode(ctx, created[0].ID) + require.NoError(t, err) + require.Contains(t, string(source.Content), fmt.Sprintf("[Target](../%d)", created[1].ID.ID)) +} + func TestRemoteMutationBatchesPreserveOrderAndAtomicity(t *testing.T) { t.Parallel() fx, hub, remote := newRemoteKegFixture(t) @@ -802,31 +800,6 @@ func TestRemoteKegExportImport(t *testing.T) { require.Contains(t, string(content), "Alpha body") } -func TestRemoteKegImportRoundTrip(t *testing.T) { - t.Parallel() - f, _, rk := newRemoteKegFixture(t) - ctx := f.Context() - - // Export a node from a local keg and import it into the remote keg - // with fresh ids. - src := kegpkg.NewLocalKeg(newTestMemoryRepo(f.Runtime()), f.Runtime()) - initNonStrictTestKeg(t, src, ctx) - srcID, err := src.Create(ctx, &kegpkg.CreateOptions{Body: []byte("# Imported node\n\ntravels by archive")}) - require.NoError(t, err) - - rc, err := src.ExportNodes(ctx, kegpkg.ExportNodesOptions{NodeIDs: []kegpkg.NodeId{srcID.ID}}) - require.NoError(t, err) - defer rc.Close() - - imported, err := rk.ImportNodes(ctx, rc, kegpkg.ImportNodesOptions{AssignNewIDs: true}) - require.NoError(t, err) - require.Len(t, imported, 1) - - content, err := rk.GetContent(ctx, imported[0].ID) - require.NoError(t, err) - require.Contains(t, string(content), "travels by archive") -} - // TestRemoteKegSingleRoundTrip is the headline guarantee of the RemoteKeg // refactor: each Keg operation is exactly one HTTP request. func TestRemoteKegSingleRoundTrip(t *testing.T) { diff --git a/pkg/keg/keg_settings.go b/pkg/keg/keg_settings.go index 22d59471..57964d27 100644 --- a/pkg/keg/keg_settings.go +++ b/pkg/keg/keg_settings.go @@ -277,11 +277,17 @@ func ParseKegSettings(data []byte) (*Settings, error) { // ParseKegSettingsStrict parses raw user-supplied settings data for persistence. // It rejects user-defined index entries that collide with required system // indexes or duplicate another user index. +// +// Every failure wraps ErrInvalid: the bytes are the caller's, so a missing +// kegv, bad YAML, or a bad index query is theirs to fix. func ParseKegSettingsStrict(data []byte) (*Settings, error) { cfg, err := parseKegSettings(data, true) if cfg != nil { cfg.raw = append([]byte(nil), data...) } + if err != nil && !errors.Is(err, ErrInvalid) { + err = fmt.Errorf("%w: %w", err, ErrInvalid) + } return cfg, err } @@ -439,6 +445,13 @@ func userIndexEntries(entries []IndexEntry, strict bool) ([]IndexEntry, error) { if _, ok := seen[name]; ok { return nil, fmt.Errorf("duplicate user index %q", name) } + if strict && strings.TrimSpace(entry.Query) != "" { + // Refuse at the edit what would otherwise fail every later read + // of this keg when the index is built. + if _, err := ParseQueryExpression(entry.Query); err != nil { + return nil, fmt.Errorf("index %q query: %w", name, err) + } + } seen[name] = struct{}{} out = append(out, entry) } diff --git a/pkg/keg/keg_snapshots.go b/pkg/keg/keg_snapshots.go index 57ec8971..36e79daf 100644 --- a/pkg/keg/keg_snapshots.go +++ b/pkg/keg/keg_snapshots.go @@ -16,7 +16,7 @@ func (k *LocalKeg) AppendSnapshot(ctx context.Context, id NodeId, msg string) (S } func (k *LocalKeg) appendSnapshot(ctx context.Context, id NodeId, msg string, refreshIndexes bool) (Snapshot, error) { - out, err := k.appendSnapshotNoRefresh(ctx, id, msg) + out, err := k.appendSnapshotNoRefresh(ctx, nil, id, msg) if err != nil { return out, err } @@ -28,7 +28,10 @@ func (k *LocalKeg) appendSnapshot(ctx context.Context, id NodeId, msg string, re return out, nil } -func (k *LocalKeg) appendSnapshotLocked(ctx context.Context, id NodeId, msg string) (Snapshot, error) { +// appendSnapshotLocked appends a snapshot of node id's current state. history +// is the batch's snapshot timeline, which the append reads and extends; nil +// loads a fresh one for this append alone. The caller holds the node lock. +func (k *LocalKeg) appendSnapshotLocked(ctx context.Context, history *snapshotTimeline, id NodeId, msg string) (Snapshot, error) { snapshots, ok := repoSnapshots(k.Repo) if !ok { return Snapshot{}, ErrNotSupported @@ -73,7 +76,7 @@ func (k *LocalKeg) appendSnapshotLocked(ctx context.Context, id NodeId, msg stri if err != nil { return Snapshot{}, fmt.Errorf("failed to prepare snapshot timeline event: %w", err) } - omega, err := k.computePendingSnapshotOmega(ctx, pending) + omega, err := k.computePendingSnapshotOmega(ctx, history, pending) if err != nil { return Snapshot{}, fmt.Errorf("failed to compute snapshot omega: %w", err) } @@ -99,10 +102,15 @@ func (k *LocalKeg) appendSnapshotLocked(ctx context.Context, id NodeId, msg stri if err != nil { return Snapshot{}, err } + if history != nil { + if err := history.add(ctx, k, id, out); err != nil { + return Snapshot{}, err + } + } return out, nil } -func (k *LocalKeg) appendSnapshotNoRefresh(ctx context.Context, id NodeId, msg string) (Snapshot, error) { +func (k *LocalKeg) appendSnapshotNoRefresh(ctx context.Context, history *snapshotTimeline, id NodeId, msg string) (Snapshot, error) { if err := k.checkKegExists(ctx); err != nil { return Snapshot{}, fmt.Errorf("failed to snapshot node: %w", err) } @@ -111,7 +119,7 @@ func (k *LocalKeg) appendSnapshotNoRefresh(ctx context.Context, id NodeId, msg s } var out Snapshot err := k.withNodeLock(ctx, id, func(lockCtx context.Context) error { - snap, err := k.appendSnapshotLocked(lockCtx, id, msg) + snap, err := k.appendSnapshotLocked(lockCtx, history, id, msg) if err != nil { return err } @@ -124,11 +132,11 @@ func (k *LocalKeg) appendSnapshotNoRefresh(ctx context.Context, id NodeId, msg s return out, nil } -func (k *LocalKeg) refreshSnapshotIndexesAfterPolicy(ctx context.Context, created int) error { +func (k *LocalKeg) refreshSnapshotIndexesAfterPolicy(ctx context.Context, history *snapshotTimeline, created int) error { if created == 0 { return nil } - if err := k.refreshSnapshotGeneratedIndexes(ctx); err != nil { + if err := k.refreshSnapshotGeneratedIndexesFrom(ctx, history); err != nil { return fmt.Errorf("failed to refresh snapshot indexes: %w", err) } return nil diff --git a/pkg/keg/keg_touch_internal_test.go b/pkg/keg/keg_touch_internal_test.go new file mode 100644 index 00000000..ed36f19a --- /dev/null +++ b/pkg/keg/keg_touch_internal_test.go @@ -0,0 +1,254 @@ +package keg + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/jlrickert/cli-toolkit/sandbox" + "github.com/stretchr/testify/require" +) + +// touchCountingRepo counts the writes a touch should keep to a minimum, and +// can fail the stats write of one node. +type touchCountingRepo struct { + *testMemoryRepository + dirtyWrites atomic.Int32 + metaWrites atomic.Int32 + failStats atomic.Pointer[NodeId] +} + +func (r *touchCountingRepo) WriteIndex(ctx context.Context, name string, data []byte) error { + if name == DirtyIndexName { + r.dirtyWrites.Add(1) + } + return r.testMemoryRepository.WriteIndex(ctx, name, data) +} + +func (r *touchCountingRepo) WriteMeta(ctx context.Context, id NodeId, data []byte) error { + r.metaWrites.Add(1) + return r.testMemoryRepository.WriteMeta(ctx, id, data) +} + +func (r *touchCountingRepo) WriteStats(ctx context.Context, id NodeId, stats *NodeStats) error { + if fail := r.failStats.Load(); fail != nil && *fail == id { + return errors.New("injected stats write failure") + } + return r.testMemoryRepository.WriteStats(ctx, id, stats) +} + +func (r *touchCountingRepo) reset() { + r.dirtyWrites.Store(0) + r.metaWrites.Store(0) +} + +func newTouchTestKeg(t *testing.T) (*LocalKeg, *touchCountingRepo, context.Context) { + t.Helper() + fx := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + repo := &touchCountingRepo{testMemoryRepository: newTestMemoryRepo(fx.Runtime())} + k := NewLocalKeg(repo, fx.Runtime()) + ctx := t.Context() + require.NoError(t, k.Init(ctx)) + require.NoError(t, k.UpdateSettings(ctx, func(cfg *Settings) { cfg.SchemaPolicy.Strict = false })) + return k, repo, ctx +} + +func createTouchTestNodes(t *testing.T, ctx context.Context, k *LocalKeg, n int) []NodeId { + t.Helper() + ids := make([]NodeId, 0, n) + for range n { + created, err := k.Create(ctx, &CreateOptions{Body: []byte("# Node\n\nbody\n"), Meta: []byte("tags:\n - kept\n")}) + require.NoError(t, err) + ids = append(ids, created.ID) + } + return ids +} + +func TestReadNodesTouchRefreshesDirtyIndexAtMostOnce(t *testing.T) { + k, repo, ctx := newTouchTestKeg(t) + ids := createTouchTestNodes(t, ctx, k, 8) + metaBefore := make(map[NodeId][]byte, len(ids)) + for _, id := range ids { + raw, err := repo.ReadMeta(ctx, id) + require.NoError(t, err) + metaBefore[id] = raw + } + + repo.reset() + views, err := k.ReadNodes(ctx, ReadNodesOptions{NodeIDs: ids, Touch: true}) + require.NoError(t, err) + require.Len(t, views, len(ids)) + require.LessOrEqual(t, repo.dirtyWrites.Load(), int32(1)) + require.Zero(t, repo.metaWrites.Load(), "a touch must not rewrite existing metadata") + for i, id := range ids { + require.Equal(t, 1, views[i].Stats.AccessCount()) + raw, err := repo.ReadMeta(ctx, id) + require.NoError(t, err) + require.Equal(t, metaBefore[id], raw) + stored, err := repo.ReadStats(ctx, id) + require.NoError(t, err) + require.Equal(t, 1, stored.AccessCount()) + } +} + +func TestReadNodesTouchCreatesMissingMeta(t *testing.T) { + k, repo, ctx := newTouchTestKeg(t) + ids := createTouchTestNodes(t, ctx, k, 4) + require.NoError(t, repo.updateNode(ctx, ids[1], func(n *memoryNode) { n.meta = nil })) + require.NoError(t, k.refreshDirtyIndex(ctx)) + + repo.reset() + views, err := k.ReadNodes(ctx, ReadNodesOptions{NodeIDs: ids, Touch: true}) + require.NoError(t, err) + require.Len(t, views, len(ids)) + require.LessOrEqual(t, repo.dirtyWrites.Load(), int32(1)) + require.Equal(t, int32(1), repo.metaWrites.Load(), "only the node without metadata gets it written") + + stored, err := repo.GetIndex(ctx, DirtyIndexName) + require.NoError(t, err) + rebuilt, err := k.buildDirtyIndexData(ctx) + require.NoError(t, err) + require.Equal(t, string(rebuilt), string(stored)) +} + +func TestTouchKeepsDirtyIndexTitleFromStats(t *testing.T) { + k, repo, ctx := newTouchTestKeg(t) + created, err := k.Create(ctx, &CreateOptions{Body: []byte("# Placeholder\n")}) + require.NoError(t, err) + // Blank content has no title, so the dirty row takes the stats title. + require.NoError(t, repo.updateNode(ctx, created.ID, func(n *memoryNode) { n.content = []byte("\n") })) + stats, err := repo.ReadStats(ctx, created.ID) + require.NoError(t, err) + stats.SetTitle("Title From Stats") + require.NoError(t, repo.testMemoryRepository.WriteStats(ctx, created.ID, stats)) + require.NoError(t, k.refreshDirtyIndex(ctx)) + before, err := repo.GetIndex(ctx, DirtyIndexName) + require.NoError(t, err) + require.Contains(t, string(before), "Title From Stats") + + _, err = k.OpenNode(ctx, NodeOpenOptions{ID: created.ID, Touch: true}) + require.NoError(t, err) + _, err = k.ReadNodes(ctx, ReadNodesOptions{NodeIDs: []NodeId{created.ID}, Touch: true}) + require.NoError(t, err) + require.NoError(t, k.Touch(ctx, created.ID)) + + after, err := repo.GetIndex(ctx, DirtyIndexName) + require.NoError(t, err) + require.Equal(t, string(before), string(after)) + rebuilt, err := k.buildDirtyIndexData(ctx) + require.NoError(t, err) + require.Equal(t, string(rebuilt), string(after)) + stored, err := repo.ReadStats(ctx, created.ID) + require.NoError(t, err) + require.Equal(t, "Title From Stats", stored.Title()) + require.Equal(t, 3, stored.AccessCount()) +} + +func TestReadNodesTouchRollsBackEveryTouchOnFailure(t *testing.T) { + k, repo, ctx := newTouchTestKeg(t) + ids := createTouchTestNodes(t, ctx, k, 3) + // The first id appears twice so the rollback must undo both touches. + request := []NodeId{ids[0], ids[1], ids[0], ids[2]} + repo.failStats.Store(&ids[2]) + + _, err := k.ReadNodes(ctx, ReadNodesOptions{NodeIDs: request, Touch: true}) + require.ErrorContains(t, err, "injected stats write failure") + for _, id := range ids { + stored, err := repo.ReadStats(ctx, id) + require.NoError(t, err) + require.Zero(t, stored.AccessCount(), "node %s keeps no touch", id.Path()) + } +} + +func TestReadNodesTouchRepeatedIDShowsFinalStats(t *testing.T) { + k, _, ctx := newTouchTestKeg(t) + ids := createTouchTestNodes(t, ctx, k, 2) + + views, err := k.ReadNodes(ctx, ReadNodesOptions{NodeIDs: []NodeId{ids[0], ids[1], ids[0]}, Touch: true}) + require.NoError(t, err) + require.Len(t, views, 3) + require.Equal(t, 2, views[0].Stats.AccessCount()) + require.Equal(t, 1, views[1].Stats.AccessCount()) + require.Equal(t, 2, views[2].Stats.AccessCount()) +} + +// busyKegRepo reports the keg write lock as busy to any write that bounds its +// wait, as a repository does when another writer holds the lock past it. +type busyKegRepo struct { + *testMemoryRepository + busy atomic.Bool + lastWait atomic.Int64 +} + +func (r *busyKegRepo) WithKegWrite(ctx context.Context, fn func(context.Context) error) error { + r.lastWait.Store(int64(LockWaitFrom(ctx))) + if r.busy.Load() && LockWaitFrom(ctx) > 0 { + return fmt.Errorf("keg row lock: %w", ErrLockTimeout) + } + return r.testMemoryRepository.WithKegWrite(ctx, fn) +} + +func newBusyTestKeg(t *testing.T) (*LocalKeg, *busyKegRepo, context.Context) { + t.Helper() + fx := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + repo := &busyKegRepo{testMemoryRepository: newTestMemoryRepo(fx.Runtime())} + k := NewLocalKeg(repo, fx.Runtime()) + ctx := t.Context() + require.NoError(t, k.Init(ctx)) + require.NoError(t, k.UpdateSettings(ctx, func(cfg *Settings) { cfg.SchemaPolicy.Strict = false })) + return k, repo, ctx +} + +func TestTouchedReadsBoundTheKegLockWait(t *testing.T) { + k, repo, ctx := newBusyTestKeg(t) + created, err := k.Create(ctx, &CreateOptions{Body: []byte("# Bounded\n")}) + require.NoError(t, err) + + _, err = k.ReadNodes(ctx, ReadNodesOptions{NodeIDs: []NodeId{created.ID}, Touch: true}) + require.NoError(t, err) + require.Equal(t, int64(touchLockWait), repo.lastWait.Load()) + + _, err = k.OpenNode(WithLockWait(ctx, time.Minute), NodeOpenOptions{ID: created.ID, Touch: true}) + require.NoError(t, err) + require.Equal(t, int64(time.Minute), repo.lastWait.Load(), "a caller's own bound wins") + + require.NoError(t, k.Touch(ctx, created.ID)) + require.Zero(t, repo.lastWait.Load(), "an explicit Touch waits as long as it takes") +} + +func TestTouchedReadsServeContentWhenTheKegLockIsBusy(t *testing.T) { + k, repo, ctx := newBusyTestKeg(t) + created, err := k.Create(ctx, &CreateOptions{Body: []byte("# Busy\n\nbody\n")}) + require.NoError(t, err) + repo.busy.Store(true) + + views, err := k.ReadNodes(ctx, ReadNodesOptions{NodeIDs: []NodeId{created.ID}, Touch: true}) + require.NoError(t, err) + require.Len(t, views, 1) + require.Contains(t, string(views[0].Content), "# Busy") + require.Zero(t, views[0].Stats.AccessCount()) + + view, err := k.OpenNode(ctx, NodeOpenOptions{ID: created.ID, Touch: true}) + require.NoError(t, err) + require.Contains(t, string(view.Content), "# Busy") + require.Zero(t, view.Stats.AccessCount()) + + stats, err := repo.ReadStats(ctx, created.ID) + require.NoError(t, err) + require.Zero(t, stats.AccessCount()) +} + +func TestTouchedReadFailsWhenItsContextIsDone(t *testing.T) { + k, repo, ctx := newBusyTestKeg(t) + created, err := k.Create(ctx, &CreateOptions{Body: []byte("# Done\n")}) + require.NoError(t, err) + repo.busy.Store(true) + + canceled, cancel := context.WithCancel(ctx) + cancel() + _, err = k.ReadNodes(canceled, ReadNodesOptions{NodeIDs: []NodeId{created.ID}, Touch: true}) + require.ErrorIs(t, err, ErrLockTimeout) +} diff --git a/pkg/keg/merge.go b/pkg/keg/merge.go index 5eb656a6..df8baf87 100644 --- a/pkg/keg/merge.go +++ b/pkg/keg/merge.go @@ -49,7 +49,7 @@ func (k *LocalKeg) mergeNodes(ctx context.Context, opts MergeNodesOptions) (Merg if err != nil { return MergeNodesResult{}, fmt.Errorf("merge source %s: %w", source.Path(), err) } - if err := checkExpectedHash("node "+source.Path(), opts.ExpectedSourceHash, current.Hash(), nodeRecoveryContent(current)); err != nil { + if err := checkExpectedHash("node "+source.Path(), mergeSourceHashSource, opts.ExpectedSourceHash, current.Hash(), nodeRecoveryContent(current)); err != nil { return MergeNodesResult{}, err } results, err := k.updateNodes(ctx, []NodeUpdateOptions{opts.Target}) diff --git a/pkg/keg/orientation_test.go b/pkg/keg/orientation_test.go index 9049f10f..4a925633 100644 --- a/pkg/keg/orientation_test.go +++ b/pkg/keg/orientation_test.go @@ -2,6 +2,7 @@ package keg import ( "context" + "github.com/jlrickert/tapper/pkg/apicontract" "io" "net/http" "reflect" @@ -13,7 +14,7 @@ type orientationRoundTripFunc func(*http.Request) (*http.Response, error) func (f orientationRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { if req.URL.Path == "/api/version" { - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"server_version":"test","api_versions":["2026-10-12"]}`))}, nil + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"server_version":"test","api_versions":["` + apicontract.Revision + `"]}`))}, nil } return f(req) } diff --git a/pkg/keg/precondition.go b/pkg/keg/precondition.go index 91086270..645d77f2 100644 --- a/pkg/keg/precondition.go +++ b/pkg/keg/precondition.go @@ -28,15 +28,32 @@ func DocumentHash(data []byte) string { return fmt.Sprintf("%x", sha256.Sum256(data)) } -func requireExpectedHash(resource, expected string) error { +// hashSource names where a guarded write's token comes from: the request +// field that carries it and the read tool whose hash fills it. A missing token +// names both, so the caller's next step is in the message itself. +type hashSource struct { + Field string + Read string +} + +var ( + nodeHashSource = hashSource{Field: "expected_hash", Read: "node_read"} + mergeSourceHashSource = hashSource{Field: "expected_source_hash", Read: "node_read"} + schemaHashSource = hashSource{Field: "expected_hash", Read: "schema_read"} + settingsHashSource = hashSource{Field: "expected_hash", Read: "keg_settings_read"} + nodeContentHashSource = hashSource{Field: "expected_content_hash", Read: "node_read"} + nodeMetaHashSource = hashSource{Field: "expected_meta_hash", Read: "node_read"} +) + +func requireExpectedHash(resource string, source hashSource, expected string) error { if expected != "" { return nil } - return fmt.Errorf("%s: %w", resource, ErrPreconditionRequired) + return fmt.Errorf("%s: %s is required: call %s and pass its hash: %w", resource, source.Field, source.Read, ErrPreconditionRequired) } -func checkExpectedHash(resource, expected, current string, content []byte) error { - if err := requireExpectedHash(resource, expected); err != nil { +func checkExpectedHash(resource string, source hashSource, expected, current string, content []byte) error { + if err := requireExpectedHash(resource, source, expected); err != nil { return err } if expected == current { @@ -95,7 +112,7 @@ func checkNodeUpdatePrecondition(ctx context.Context, rt *toolkit.Runtime, opts } } if opts.ExpectedContentHash == "" && opts.ExpectedMetaHash == "" { - err := checkExpectedHash(resource, opts.ExpectedHash, existing.Hash(), nodeRecoveryContent(existing)) + err := checkExpectedHash(resource, nodeHashSource, opts.ExpectedHash, existing.Hash(), nodeRecoveryContent(existing)) var pc *PreconditionConflictError if errors.As(err, &pc) { pc.CurrentContentHash = currentContentHash @@ -104,16 +121,16 @@ func checkNodeUpdatePrecondition(ctx context.Context, rt *toolkit.Runtime, opts return err } if opts.HasContent { - if opts.ExpectedContentHash == "" { - return fmt.Errorf("%s content (expected content hash): %w", resource, ErrPreconditionRequired) + if err := requireExpectedHash(resource+" content", nodeContentHashSource, opts.ExpectedContentHash); err != nil { + return err } if opts.ExpectedContentHash != currentContentHash { return conflict() } } if opts.HasMeta { - if opts.ExpectedMetaHash == "" { - return fmt.Errorf("%s metadata (expected meta hash): %w", resource, ErrPreconditionRequired) + if err := requireExpectedHash(resource+" metadata", nodeMetaHashSource, opts.ExpectedMetaHash); err != nil { + return err } if opts.ExpectedMetaHash != currentMetaHash { return conflict() diff --git a/pkg/keg/query_expr.go b/pkg/keg/query_expr.go index 5e4f04ca..03128b5f 100644 --- a/pkg/keg/query_expr.go +++ b/pkg/keg/query_expr.go @@ -27,12 +27,14 @@ type QueryExpr struct { } // ParseQueryExpression compiles raw into a QueryExpr that can be evaluated with -// EvaluateQueryExpression. Returns an error if raw is empty or syntactically -// invalid. +// EvaluateQueryExpression. Returns an error wrapping ErrInvalid if raw is empty +// or syntactically invalid: the expression is the caller's input, so a parse +// failure is theirs to fix. Callers parsing a stored expression rather than a +// caller's own must reclassify (see WithSettings). func ParseQueryExpression(raw string) (QueryExpr, error) { node, err := parseQueryExpression(raw) if err != nil { - return QueryExpr{}, err + return QueryExpr{}, fmt.Errorf("%w: %w", err, ErrInvalid) } return QueryExpr{root: node}, nil } diff --git a/pkg/keg/remote_errors.go b/pkg/keg/remote_errors.go index c9654c84..bd58dae9 100644 --- a/pkg/keg/remote_errors.go +++ b/pkg/keg/remote_errors.go @@ -32,7 +32,12 @@ const ( RemoteCodeOrientationDenied = "ORIENTATION_DENIED" RemoteCodeOrientationUnavailable = "ORIENTATION_UNAVAILABLE" RemoteCodeOrientationRootUnavailable = "ORIENTATION_ROOT_UNAVAILABLE" + RemoteCodeKegRestoring = "KEG_RESTORING" + RemoteCodeArchiveTooLarge = "ARCHIVE_TOO_LARGE" RemoteCodeInternal = "INTERNAL" + // RemoteCodeRateLimited is reported for a *RateLimitError. It has no + // sentinel row: an HTTP 429 is decoded into RateLimitError directly. + RemoteCodeRateLimited = "RATE_LIMITED" ) // remoteCodeTable pairs each sentinel with its wire code and HTTP status. @@ -42,6 +47,8 @@ var remoteCodeTable = []struct { status int }{ {ErrNotExist, RemoteCodeNotFound, http.StatusNotFound}, + {ErrKegRestoring, RemoteCodeKegRestoring, http.StatusConflict}, + {ErrArchiveTooLarge, RemoteCodeArchiveTooLarge, http.StatusRequestEntityTooLarge}, {ErrDestinationExists, RemoteCodeDestExists, http.StatusConflict}, {ErrExist, RemoteCodeExist, http.StatusConflict}, {ErrPreconditionRequired, RemoteCodePreconditionRequired, http.StatusPreconditionRequired}, diff --git a/pkg/keg/remote_precondition_test.go b/pkg/keg/remote_precondition_test.go index 0459297b..9b0f9af5 100644 --- a/pkg/keg/remote_precondition_test.go +++ b/pkg/keg/remote_precondition_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "testing" @@ -66,6 +67,37 @@ func TestRemoteKegDecodesPreconditionErrorsWithRecoveryFields(t *testing.T) { }) } +// TestPartPreconditionRequiredRoundTripsThroughTheHub pins that the typed +// missing-hash error keeps its wire code: a hub serving it answers +// PRECONDITION_REQUIRED with its message, and the client decodes that back to +// ErrPreconditionRequired with the message intact. +func TestPartPreconditionRequiredRoundTripsThroughTheHub(t *testing.T) { + t.Parallel() + typed := &keg.PartPreconditionRequiredError{NodeID: "3", Missing: []keg.MissingPartHash{ + {Part: "content", Field: "expected_content_hash", TokenField: "content_hash"}, + {Part: "meta", Field: "expected_meta_hash", TokenField: "meta_hash"}, + }} + require.ErrorIs(t, typed, keg.ErrPreconditionRequired) + code, status := keg.RemoteErrorCode(fmt.Errorf("edit: %w", typed)) + require.Equal(t, keg.RemoteCodePreconditionRequired, code) + require.Equal(t, http.StatusPreconditionRequired, status) + + srv := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + body, _ := json.Marshal(map[string]any{"error": typed.Error(), "code": code, "operationPerformed": false}) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + rk := keg.NewRemoteKeg(srv.URL, "", nil) + _, err := rk.UpdateNodes(context.Background(), []keg.NodeUpdateOptions{{ + ID: keg.NodeId{ID: 3}, Content: []byte("# New\n"), HasContent: true, + }}) + require.ErrorIs(t, err, keg.ErrPreconditionRequired) + require.Contains(t, err.Error(), "content requires expected_content_hash") + require.Contains(t, err.Error(), "meta requires expected_meta_hash") +} + func TestRemoteKegUpdateNodesSendsPartTokensAndDecodesPartConflicts(t *testing.T) { t.Parallel() var body map[string]any diff --git a/pkg/keg/repo_lock.go b/pkg/keg/repo_lock.go index d30a7ba3..6ae82db1 100644 --- a/pkg/keg/repo_lock.go +++ b/pkg/keg/repo_lock.go @@ -11,6 +11,48 @@ import ( // DefaultLockTTL is the default time-to-live for a cross-process lock. const DefaultLockTTL = 5 * time.Minute +// DefaultNodeLockWait is how long a node lock acquisition waits for a held +// lock when the caller names no bound. MaxNodeLockWait caps any requested +// bound, so a caller never hangs for the lock's whole TTL. +const ( + DefaultNodeLockWait = 10 * time.Second + MaxNodeLockWait = 60 * time.Second +) + +// ClampNodeLockWait returns the node lock wait bound for a requested d: the +// default when d is zero or less, and at most MaxNodeLockWait. +func ClampNodeLockWait(d time.Duration) time.Duration { + switch { + case d <= 0: + return DefaultNodeLockWait + case d > MaxNodeLockWait: + return MaxNodeLockWait + default: + return d + } +} + +type nodeLockWaitContextKey struct{} + +// WithNodeLockWait returns ctx carrying a bound on how long Keg.Lock waits for +// a node lock another holder has. RemoteKeg sends it to the Hub as +// ?wait=; LocalKeg gives up acquiring after d. Either way a lock still +// held when the bound passes fails with ErrLockTimeout. A d of zero or less +// leaves the bound to the backend. +func WithNodeLockWait(ctx context.Context, d time.Duration) context.Context { + return context.WithValue(ctx, nodeLockWaitContextKey{}, d) +} + +// NodeLockWaitFrom returns the node lock wait bound carried by ctx, or zero +// when the caller set none. +func NodeLockWaitFrom(ctx context.Context) time.Duration { + d, _ := ctx.Value(nodeLockWaitContextKey{}).(time.Duration) + if d < 0 { + return 0 + } + return d +} + // LockToken is an opaque string identifying lock ownership. type LockToken string diff --git a/pkg/keg/schema.go b/pkg/keg/schema.go index 33ce4abe..360b1357 100644 --- a/pkg/keg/schema.go +++ b/pkg/keg/schema.go @@ -8,8 +8,10 @@ import ( "fmt" "math" "path/filepath" + "regexp" "sort" "strings" + "unicode/utf8" "github.com/google/jsonschema-go/jsonschema" "github.com/jlrickert/tapper/pkg/schemas" @@ -321,14 +323,21 @@ func (k *LocalKeg) schemaSelectionRequiredMessage(ctx context.Context, op schema if op == schemaWriteUpdate { msg += " (an edit that leaves metadata unchanged keeps the node's type and needs none)" } - if names, err := k.listSchemas(ctx); err == nil { - if len(names) == 0 { - msg += "; this keg defines no schemas, so an admin must add one or relax schemaPolicy" - } else { - msg += "; this keg defines: " + strings.Join(names, ", ") - } + return k.withKegSchemaList(ctx, msg) +} + +// withKegSchemaList appends the schemas this keg defines to msg, so a caller +// who picked a wrong or missing schema sees the valid choices without another +// call. Listing is best effort: msg is returned unchanged when it fails. +func (k *LocalKeg) withKegSchemaList(ctx context.Context, msg string) string { + names, err := k.listSchemas(ctx) + if err != nil { + return msg } - return msg + if len(names) == 0 { + return msg + "; this keg defines no schemas, so an admin must add one or relax schemaPolicy" + } + return msg + "; this keg defines: " + strings.Join(names, ", ") } // projectNodeWrite resolves one live create/update's explicit schema @@ -338,6 +347,13 @@ func (k *LocalKeg) schemaSelectionRequiredMessage(ctx context.Context, op schema // resolved by precedence. func (k *LocalKeg) projectNodeWrite(ctx context.Context, op schemaWriteOperation, id NodeId, node *NodeData, schema string, candidates ...schemaTypeCandidate) (*SchemaValidationResult, error) { selected := strings.TrimSpace(schema) + if selected != "" { + // Every backend refuses a malformed selection the same way, before + // it is compared with declarations or looked up as a file. + if err := ValidSchemaTypeName(selected); err != nil { + return schemaSelectionError(id, selected, k.withKegSchemaList(ctx, strings.TrimSuffix(err.Error(), ": "+ErrInvalid.Error()))) + } + } if id.ID != 0 { var declared schemaTypeCandidate inherited := "" @@ -386,7 +402,7 @@ func (k *LocalKeg) projectNodeWrite(ctx context.Context, op schemaWriteOperation if result.Issues[i].Field == "meta.type" { result.Issues[i].Field = "schema" if strings.HasPrefix(result.Issues[i].Message, "unknown type ") { - result.Issues[i].Message = "unknown schema " + strings.TrimPrefix(result.Issues[i].Message, "unknown type ") + result.Issues[i].Message = k.withKegSchemaList(ctx, "unknown schema "+strings.TrimPrefix(result.Issues[i].Message, "unknown type ")) } } } @@ -415,7 +431,7 @@ func ParseSchemaDefinition(data []byte) (*SchemaDefinition, error) { func validateSchemaDefinitionForType(typeName string, data []byte) (*SchemaDefinition, error) { typeName = strings.TrimSpace(typeName) - if err := ValidSchemaTypeName(typeName); err != nil { + if err := ValidSchemaStorageName(typeName); err != nil { return nil, err } parsed, err := ParseSchemaDefinition(data) @@ -759,7 +775,41 @@ func normalizeRelationDirection(raw string) (string, bool) { } } +// SchemaTypeNameMaxLength bounds a schema type name, in characters. +const SchemaTypeNameMaxLength = 64 + +// SchemaTypeNamePattern is ValidSchemaTypeName's character rule as a regular +// expression that both Go and JSON Schema accept, so a tool's input schema can +// refuse a bad name before any backend sees it. A type name is one word: no +// whitespace, control characters, or path separators, and no "..". +const SchemaTypeNamePattern = `^(?:[^\s/\\.\x00-\x1f]|\.[^\s/\\.\x00-\x1f])+\.?$` + +var schemaTypeNameRE = regexp.MustCompile(SchemaTypeNamePattern) + +// ValidSchemaTypeName reports whether typeName can name a new schema. The +// name becomes a file name and is what callers pass as a write's schema, so a +// sentence or a path is refused rather than looked up. A schema that already +// exists answers to ValidSchemaStorageName instead. func ValidSchemaTypeName(typeName string) error { + typeName = strings.TrimSpace(typeName) + if typeName == "" || typeName == "." || typeName == ".." { + return fmt.Errorf("schema type is required: %w", ErrInvalid) + } + if utf8.RuneCountInString(typeName) > SchemaTypeNameMaxLength { + return fmt.Errorf("invalid schema type: longer than %d characters; a schema type is a short name such as \"task\": %w", SchemaTypeNameMaxLength, ErrInvalid) + } + if filepath.IsAbs(typeName) || !schemaTypeNameRE.MatchString(typeName) { + return fmt.Errorf("invalid schema type %q: a schema type is one word with no spaces or path separators, such as \"task\": %w", typeName, ErrInvalid) + } + return nil +} + +// ValidSchemaStorageName reports whether typeName can name a schema that +// already exists: one stored in a keg or carried by an archive. Earlier +// versions accepted any name that stays inside the schemas directory, so +// only that is required here; ValidSchemaTypeName's stricter rule applies +// when a schema is created. +func ValidSchemaStorageName(typeName string) error { typeName = strings.TrimSpace(typeName) if typeName == "" || typeName == "." || typeName == ".." { return fmt.Errorf("schema type is required: %w", ErrInvalid) @@ -772,7 +822,7 @@ func ValidSchemaTypeName(typeName string) error { func SchemaFilename(typeName string) (string, error) { typeName = strings.TrimSpace(typeName) - if err := ValidSchemaTypeName(typeName); err != nil { + if err := ValidSchemaStorageName(typeName); err != nil { return "", err } return typeName + SchemaFileSuffix, nil @@ -823,7 +873,7 @@ func (k *LocalKeg) writeSchema(ctx context.Context, typeName string, data []byte if err != nil { return err } - if err := checkExpectedHash("schema "+typeName, opts.ExpectedHash, DocumentHash(current), current); err != nil { + if err := checkExpectedHash("schema "+typeName, schemaHashSource, opts.ExpectedHash, DocumentHash(current), current); err != nil { return err } return store.WriteSchema(ctx, typeName, data) @@ -842,7 +892,7 @@ func (k *LocalKeg) deleteSchema(ctx context.Context, typeName string, opts Schem if err != nil { return err } - if err := checkExpectedHash("schema "+typeName, opts.ExpectedHash, DocumentHash(current), current); err != nil { + if err := checkExpectedHash("schema "+typeName, schemaHashSource, opts.ExpectedHash, DocumentHash(current), current); err != nil { return err } return store.DeleteSchema(ctx, typeName) diff --git a/pkg/keg/schema_selection_test.go b/pkg/keg/schema_selection_test.go index 55df7e0b..dcf5e9b6 100644 --- a/pkg/keg/schema_selection_test.go +++ b/pkg/keg/schema_selection_test.go @@ -3,6 +3,7 @@ package keg_test import ( "context" "errors" + "regexp" "strings" "testing" @@ -123,6 +124,14 @@ func TestExplicitSchemaSelectionPersistsReplacesAndRejectsConflicts(t *testing.T _, err = k.Create(ctx, &keg.CreateOptions{Schema: "missing", Body: []byte("# Unknown\n")}) require.ErrorIs(t, err, keg.ErrSchemaInvalid) require.Contains(t, err.Error(), "schema: unknown schema \"missing\"") + require.Contains(t, err.Error(), "this keg defines:", "an unknown schema names the keg's types") + + // A selection that cannot be a type name is refused by the keg itself, + // so a backend without an input schema in front behaves the same. + _, err = k.Create(ctx, &keg.CreateOptions{Schema: "a note about planning", Body: []byte("# Prose\n")}) + require.ErrorIs(t, err, keg.ErrSchemaInvalid) + require.Contains(t, err.Error(), "one word with no spaces") + require.Contains(t, err.Error(), "this keg defines:") } func TestSchemaSelectionBatchFailureIsAtomic(t *testing.T) { @@ -394,3 +403,23 @@ func TestResolveValidationModeParity(t *testing.T) { require.Equal(t, keg.ValidationModeWarn, keg.ResolveValidationMode(keg.WithValidationActor(ctx, keg.ValidationActorHuman), nil)) require.Equal(t, keg.ValidationModeOff, keg.ResolveValidationMode(keg.WithValidationMode(keg.WithValidationActor(ctx, keg.ValidationActorHuman), keg.ValidationModeOff), policy)) } + +func TestValidSchemaTypeNameMatchesItsPattern(t *testing.T) { + t.Parallel() + pattern := regexp.MustCompile(keg.SchemaTypeNamePattern) + for name, valid := range map[string]bool{ + "task": true, "meeting-note": true, "v2.task": true, "Task_2": true, + "": false, ".": false, "..": false, "a..b": false, "a/b": false, `a\b`: false, + "a note": false, "tab\tname": false, strings.Repeat("x", keg.SchemaTypeNameMaxLength+1): false, + } { + err := keg.ValidSchemaTypeName(name) + if valid { + require.NoError(t, err, name) + } else { + require.ErrorIs(t, err, keg.ErrInvalid, name) + } + if len(name) <= keg.SchemaTypeNameMaxLength && name != "" { + require.Equal(t, valid, pattern.MatchString(name), "pattern and rule disagree on %q", name) + } + } +} diff --git a/pkg/keg/snapshot_batch_internal_test.go b/pkg/keg/snapshot_batch_internal_test.go new file mode 100644 index 00000000..5c12294a --- /dev/null +++ b/pkg/keg/snapshot_batch_internal_test.go @@ -0,0 +1,199 @@ +package keg + +import ( + "context" + "testing" + "time" + + "github.com/jlrickert/cli-toolkit/toolkit" + "github.com/stretchr/testify/require" +) + +var snapshotBatchStart = time.Date(2026, 3, 2, 9, 0, 0, 0, time.UTC) + +// newSnapshotBatchKeg builds a keg whose omega values depend on each other: +// a note scored by the evidence it links to and by the evidence linking back +// to it, with one snapshot of history already present. It returns the node +// ids in snapshot order. +func newSnapshotBatchKeg(t *testing.T) (*LocalKeg, *toolkit.Runtime, []NodeId) { + t.Helper() + k, rt := newSnapshotIndexTestKeg(t) + ctx := t.Context() + setSnapshotIndexClock(t, rt, snapshotBatchStart) + require.NoError(t, k.CreateSchema(ctx, "evidence", []byte("type: evidence\n"))) + require.NoError(t, k.CreateSchema(ctx, "note", []byte(`type: note +relations: + - name: support + type: evidence + maturity: + - direction: links + attribute: status + weight: 2 + enum: + draft: 0.25 + ready: 1 + - name: review + type: evidence + maturity: + - direction: backlinks + attribute: certainty + weight: 1 +`))) + evidence, err := k.Create(ctx, &CreateOptions{Body: []byte("# Evidence\n"), Meta: []byte("type: evidence\nstatus: draft\n")}) + require.NoError(t, err) + _, err = k.AppendSnapshot(ctx, evidence.ID, "first evidence") + require.NoError(t, err) + require.NoError(t, k.SetMeta(ctx, evidence.ID, mustParseMeta(t, "type: evidence\nstatus: ready\n"))) + source, err := k.Create(ctx, &CreateOptions{ + Body: []byte("# Source\n\n[Evidence](../" + evidence.ID.Path() + ")\n"), + Meta: []byte("type: note\n"), + }) + require.NoError(t, err) + review, err := k.Create(ctx, &CreateOptions{ + Body: []byte("# Review\n\n[Source](../" + source.ID.Path() + ")\n"), + Meta: []byte("type: evidence\ncertainty: 0.5\n"), + }) + require.NoError(t, err) + return k, rt, []NodeId{evidence.ID, source.ID, review.ID} +} + +func mustParseMeta(t *testing.T, raw string) *NodeMeta { + t.Helper() + meta, err := ParseMeta(t.Context(), []byte(raw)) + require.NoError(t, err) + return meta +} + +type snapshotOutcome struct { + snapshotOmegas map[string][]*float64 + liveOmegas map[string]*float64 + timeline string + dirty string +} + +func snapshotOutcomeOf(t *testing.T, ctx context.Context, k *LocalKeg, ids []NodeId) snapshotOutcome { + t.Helper() + out := snapshotOutcome{snapshotOmegas: map[string][]*float64{}, liveOmegas: map[string]*float64{}} + for _, id := range ids { + snaps, err := k.ListSnapshots(ctx, id) + require.NoError(t, err) + for _, snap := range snaps { + _, _, _, stats, err := k.GetSnapshot(ctx, id, snap.ID, SnapshotReadOptions{}) + require.NoError(t, err) + out.snapshotOmegas[id.Path()] = append(out.snapshotOmegas[id.Path()], statsOmega(stats)) + } + stats, err := k.GetStats(ctx, id) + require.NoError(t, err) + out.liveOmegas[id.Path()] = statsOmega(stats) + } + timeline, err := k.ReadIndex(ctx, TimelineIndexName) + require.NoError(t, err) + dirty, err := k.ReadIndex(ctx, DirtyIndexName) + require.NoError(t, err) + out.timeline, out.dirty = string(timeline), string(dirty) + return out +} + +func statsOmega(stats *NodeStats) *float64 { + if omega, ok := stats.Omega(); ok { + return &omega + } + return nil +} + +func TestAppendSnapshotsMatchesSequentialAppends(t *testing.T) { + batched, batchedRT, ids := newSnapshotBatchKeg(t) + sequential, sequentialRT, _ := newSnapshotBatchKeg(t) + ctx := t.Context() + at := snapshotBatchStart.Add(time.Hour) + + setSnapshotIndexClock(t, batchedRT, at) + requests := make([]NodeSnapshotRequest, 0, len(ids)) + for _, id := range ids { + requests = append(requests, NodeSnapshotRequest{ID: id, Message: "checkpoint"}) + } + _, err := batched.AppendSnapshots(ctx, requests) + require.NoError(t, err) + + setSnapshotIndexClock(t, sequentialRT, at) + for _, id := range ids { + _, err := sequential.AppendSnapshot(ctx, id, "checkpoint") + require.NoError(t, err) + } + + want := snapshotOutcomeOf(t, ctx, sequential, ids) + got := snapshotOutcomeOf(t, ctx, batched, ids) + require.Equal(t, want, got) + require.NotNil(t, got.snapshotOmegas[ids[1].Path()][0], "the source scores against evidence snapshotted earlier in the batch") +} + +func TestRunSnapshotPolicyMatchesSequentialAppends(t *testing.T) { + policy, policyRT, ids := newSnapshotBatchKeg(t) + sequential, sequentialRT, _ := newSnapshotBatchKeg(t) + ctx := t.Context() + idle := 10 * time.Minute + for _, k := range []*LocalKeg{policy, sequential} { + require.NoError(t, k.UpdateSettings(ctx, func(cfg *Settings) { + cfg.Snapshots = &SnapshotSettings{Mode: SnapshotModeAuto, IdleAfter: idle.String()} + })) + } + at := snapshotBatchStart.Add(time.Hour) + + setSnapshotIndexClock(t, policyRT, at) + result, err := policy.RunSnapshotPolicy(ctx) + require.NoError(t, err) + created := make([]NodeId, 0, len(result.Created)) + for _, snap := range result.Created { + created = append(created, snap.Node) + } + require.Contains(t, created, ids[1]) + + setSnapshotIndexClock(t, sequentialRT, at) + for _, id := range created { + _, err := sequential.AppendSnapshot(ctx, id, AutoSnapshotMessage(idle)) + require.NoError(t, err) + } + + all, err := policy.ListNodes(ctx) + require.NoError(t, err) + require.Equal(t, snapshotOutcomeOf(t, ctx, sequential, all), snapshotOutcomeOf(t, ctx, policy, all)) +} + +// perNodeRepo hides the batch-read capability, so reads go node by node. +type perNodeRepo struct { + Repository + RepositorySnapshots +} + +func TestDirtyIndexIsTheSameWithBatchReads(t *testing.T) { + k, rt, ids := newSnapshotBatchKeg(t) + ctx := t.Context() + repo := k.Repo.(*testMemoryRepository) + setSnapshotIndexClock(t, rt, snapshotBatchStart.Add(time.Hour)) + _, err := k.AppendSnapshot(ctx, ids[1], "clean") + require.NoError(t, err) + untitled, err := k.Create(ctx, &CreateOptions{Body: []byte("# Placeholder\n")}) + require.NoError(t, err) + require.NoError(t, repo.updateNode(ctx, untitled.ID, func(n *memoryNode) { + n.content = []byte("\n") + n.stats.SetTitle("Title From Stats") + })) + bare, err := k.Create(ctx, &CreateOptions{Body: []byte("# Bare\n")}) + require.NoError(t, err) + require.NoError(t, repo.updateNode(ctx, bare.ID, func(n *memoryNode) { n.meta = nil })) + + batched, err := k.buildDirtyIndexData(ctx) + require.NoError(t, err) + perNode, err := NewLocalKeg(perNodeRepo{Repository: repo, RepositorySnapshots: repo}, rt).buildDirtyIndexData(ctx) + require.NoError(t, err) + require.Equal(t, string(perNode), string(batched)) + + rows := decodeJSONLines[dirtyIndexRow](t, batched) + byNode := map[string]dirtyIndexRow{} + for _, row := range rows { + byNode[row.Node] = row + } + require.NotContains(t, byNode, ids[1].Path(), "a node matching its latest snapshot is clean") + require.Equal(t, "Title From Stats", byNode[untitled.ID.Path()].Title) + require.Empty(t, byNode[bare.ID.Path()].CurrentMetaHash) +} diff --git a/pkg/keg/snapshot_indexes.go b/pkg/keg/snapshot_indexes.go index 42c6b588..0147ed40 100644 --- a/pkg/keg/snapshot_indexes.go +++ b/pkg/keg/snapshot_indexes.go @@ -57,8 +57,59 @@ type timelineSnapshotState struct { stats *NodeStats } +// snapshotTimeline is the keg's snapshot history and omega schemas, loaded +// once for a batch of appends instead of once per append. It loads on first +// use, and each append adds the record of the snapshot it wrote, so it stays +// what a fresh load would return. It is valid only within one keg write +// operation, where nothing else appends snapshots or changes schemas. +type snapshotTimeline struct { + loaded bool + records []timelineSnapshotState + schemas map[string]*SchemaDefinition +} + +func (t *snapshotTimeline) load(ctx context.Context, k *LocalKeg) error { + if t.loaded { + return nil + } + records, err := k.loadTimelineSnapshotStates(ctx) + if err != nil { + return err + } + schemas, err := k.loadOmegaSchemas(ctx) + if err != nil { + return err + } + t.records, t.schemas, t.loaded = records, schemas, true + return nil +} + +// add records snapshot snap of node id, which the caller just appended. +func (t *snapshotTimeline) add(ctx context.Context, k *LocalKeg, id NodeId, snap Snapshot) error { + if !t.loaded { + return nil + } + snapshots, ok := repoSnapshots(k.Repo) + if !ok { + return nil + } + state, err := k.loadTimelineSnapshotState(ctx, snapshots, id, snap.ID) + if err != nil { + return err + } + t.records = append(t.records, state) + sortTimelineSnapshotStates(t.records) + return nil +} + func (k *LocalKeg) refreshSnapshotGeneratedIndexes(ctx context.Context) error { - timeline, replay, err := k.buildTimelineIndexDataWithReplay(ctx) + return k.refreshSnapshotGeneratedIndexesFrom(ctx, nil) +} + +// refreshSnapshotGeneratedIndexesFrom is refreshSnapshotGeneratedIndexes over +// a batch's timeline; a nil timeline loads a fresh one. +func (k *LocalKeg) refreshSnapshotGeneratedIndexesFrom(ctx context.Context, history *snapshotTimeline) error { + timeline, replay, err := k.buildTimelineIndexDataFrom(ctx, history) if err != nil { return err } @@ -88,14 +139,18 @@ func (k *LocalKeg) buildTimelineIndexData(ctx context.Context) ([]byte, error) { } func (k *LocalKeg) buildTimelineIndexDataWithReplay(ctx context.Context) ([]byte, snapshotTimelineReplay, error) { - records, err := k.loadTimelineSnapshotStates(ctx) - if err != nil { - return nil, snapshotTimelineReplay{}, err + return k.buildTimelineIndexDataFrom(ctx, nil) +} + +func (k *LocalKeg) buildTimelineIndexDataFrom(ctx context.Context, history *snapshotTimeline) ([]byte, snapshotTimelineReplay, error) { + if history == nil { + history = &snapshotTimeline{} } - replay, err := k.replaySnapshotTimeline(ctx, records) - if err != nil { + if err := history.load(ctx, k); err != nil { return nil, snapshotTimelineReplay{}, err } + records := history.records + replay := replaySnapshotTimelineWith(history.schemas, records) rows := make([]timelineIndexRow, 0, len(replay.Steps)) latestLinks := make(map[string][]NodeId, len(records)) @@ -149,24 +204,9 @@ func (k *LocalKeg) loadTimelineSnapshotStates(ctx context.Context) ([]timelineSn return nil, fmt.Errorf("list snapshots for node %s: %w", id.Path(), err) } for _, snap := range nodeSnapshots { - loaded, content, meta, stats, err := snapshots.GetSnapshot(ctx, id, snap.ID, SnapshotReadOptions{ResolveContent: true}) - if err != nil && errors.Is(err, ErrConflict) { - if unchecked, ok := k.Repo.(interface { - readContentAtUnchecked(context.Context, NodeId, RevisionID) ([]byte, error) - }); ok { - loaded, _, meta, stats, err = snapshots.GetSnapshot(ctx, id, snap.ID, SnapshotReadOptions{}) - if err == nil { - content, err = unchecked.readContentAtUnchecked(ctx, id, snap.ID) - } - } - } - if err != nil { - return nil, fmt.Errorf("read snapshot node %s rev %d: %w", id.Path(), snap.ID, err) - } - loaded.Node = id - state, err := timelineSnapshotStateFromPayload(ctx, k.Runtime, loaded, content, meta, stats) + state, err := k.loadTimelineSnapshotState(ctx, snapshots, id, snap.ID) if err != nil { - return nil, fmt.Errorf("parse snapshot node %s rev %d: %w", id.Path(), snap.ID, err) + return nil, err } records = append(records, state) } @@ -176,6 +216,29 @@ func (k *LocalKeg) loadTimelineSnapshotStates(ctx context.Context) ([]timelineSn return records, nil } +func (k *LocalKeg) loadTimelineSnapshotState(ctx context.Context, snapshots RepositorySnapshots, id NodeId, rev RevisionID) (timelineSnapshotState, error) { + loaded, content, meta, stats, err := snapshots.GetSnapshot(ctx, id, rev, SnapshotReadOptions{ResolveContent: true}) + if err != nil && errors.Is(err, ErrConflict) { + if unchecked, ok := k.Repo.(interface { + readContentAtUnchecked(context.Context, NodeId, RevisionID) ([]byte, error) + }); ok { + loaded, _, meta, stats, err = snapshots.GetSnapshot(ctx, id, rev, SnapshotReadOptions{}) + if err == nil { + content, err = unchecked.readContentAtUnchecked(ctx, id, rev) + } + } + } + if err != nil { + return timelineSnapshotState{}, fmt.Errorf("read snapshot node %s rev %d: %w", id.Path(), rev, err) + } + loaded.Node = id + state, err := timelineSnapshotStateFromPayload(ctx, k.Runtime, loaded, content, meta, stats) + if err != nil { + return timelineSnapshotState{}, fmt.Errorf("parse snapshot node %s rev %d: %w", id.Path(), rev, err) + } + return state, nil +} + func timelineSnapshotStateFromPayload(ctx context.Context, rt *toolkit.Runtime, snap Snapshot, contentBytes, metaBytes []byte, stats *NodeStats) (timelineSnapshotState, error) { content, err := ParseContent(rt, contentBytes, FormatMarkdown) if err != nil { @@ -228,35 +291,29 @@ func (k *LocalKeg) buildDirtyIndexData(ctx context.Context) ([]byte, error) { } slices.SortFunc(ids, func(a, b NodeId) int { return a.Compare(b) }) + // Read each node's content once; a node without content has no row. + type dirtyCandidate struct { + id NodeId + hash string + title string + latest Snapshot + hasRev bool + } snapshots, supportsSnapshots := repoSnapshots(k.Repo) - rows := make([]dirtyIndexRow, 0) + candidates := make([]dirtyCandidate, 0, len(ids)) for _, id := range ids { - exists, err := k.nodeExistsWithContent(ctx, id) - if err != nil { - return nil, fmt.Errorf("check node %s for %s index: %w", id.Path(), DirtyIndexName, err) - } - if !exists { - continue - } - contentBytes, err := k.Repo.ReadContent(ctx, id) if err != nil { - return nil, fmt.Errorf("read node %s content for %s index: %w", id.Path(), DirtyIndexName, err) - } - metaBytes, err := k.Repo.ReadMeta(ctx, id) - if err != nil { - if !errors.Is(err, ErrNotExist) { - return nil, fmt.Errorf("read node %s meta for %s index: %w", id.Path(), DirtyIndexName, err) + if errors.Is(err, ErrNotExist) { + continue } - metaBytes = nil + return nil, fmt.Errorf("read node %s content for %s index: %w", id.Path(), DirtyIndexName, err) } content, err := ParseContent(k.Runtime, contentBytes, FormatMarkdown) if err != nil { return nil, fmt.Errorf("parse node %s content for %s index: %w", id.Path(), DirtyIndexName, err) } - - var latest Snapshot - hasSnapshot := false + candidate := dirtyCandidate{id: id, hash: content.Hash, title: content.Title} if supportsSnapshots { nodeSnapshots, err := snapshots.ListSnapshots(ctx, id) if err != nil { @@ -265,39 +322,103 @@ func (k *LocalKeg) buildDirtyIndexData(ctx context.Context) ([]byte, error) { } } if len(nodeSnapshots) > 0 { - latest = nodeSnapshots[len(nodeSnapshots)-1] - hasSnapshot = true + candidate.latest = nodeSnapshots[len(nodeSnapshots)-1] + candidate.hasRev = true } } + candidates = append(candidates, candidate) + } + + present := make([]NodeId, 0, len(candidates)) + for _, candidate := range candidates { + present = append(present, candidate.id) + } + metas, err := k.readDirtyIndexMeta(ctx, present) + if err != nil { + return nil, err + } - currentHash := content.Hash - currentMetaHash := hashSnapshotBytes(k.Runtime, metaBytes) - if hasSnapshot && latest.ContentHash == currentHash && latest.MetaHash == currentMetaHash { + rows := make([]dirtyIndexRow, 0) + var untitled []NodeId + for _, candidate := range candidates { + currentMetaHash := hashSnapshotBytes(k.Runtime, metas[candidate.id.Path()]) + if candidate.hasRev && candidate.latest.ContentHash == candidate.hash && candidate.latest.MetaHash == currentMetaHash { continue } - - title := content.Title - if title == "" { - if stats, err := k.Repo.ReadStats(ctx, id); err == nil && stats != nil { - title = stats.Title() - } + if candidate.title == "" { + untitled = append(untitled, candidate.id) } - rows = append(rows, dirtyIndexRow{ V: 1, - Node: id.Path(), - CurrentHash: currentHash, + Node: candidate.id.Path(), + CurrentHash: candidate.hash, CurrentMetaHash: currentMetaHash, - SnapshotRevision: int64(latest.ID), - SnapshotHash: latest.ContentHash, - SnapshotMetaHash: latest.MetaHash, - Title: title, + SnapshotRevision: int64(candidate.latest.ID), + SnapshotHash: candidate.latest.ContentHash, + SnapshotMetaHash: candidate.latest.MetaHash, + Title: candidate.title, }) } + // A row whose content has no title falls back to the stats title. + if len(untitled) > 0 { + titles := k.readDirtyIndexStatsTitles(ctx, untitled) + for i := range rows { + if rows[i].Title == "" { + rows[i].Title = titles[rows[i].Node] + } + } + } + return jsonLines(rows) } +// readDirtyIndexMeta returns raw metadata keyed by node id path, leaving out +// nodes that have none. It reads the whole set at once when the repository +// supports batch reads. +func (k *LocalKeg) readDirtyIndexMeta(ctx context.Context, ids []NodeId) (map[string][]byte, error) { + if batch, ok := repositoryBatchRead(k.Repo); ok { + metas, err := batch.ReadMetaBatch(ctx, ids) + if err != nil { + return nil, fmt.Errorf("read node meta for %s index: %w", DirtyIndexName, err) + } + return metas, nil + } + metas := make(map[string][]byte, len(ids)) + for _, id := range ids { + raw, err := k.Repo.ReadMeta(ctx, id) + if err != nil { + if errors.Is(err, ErrNotExist) { + continue + } + return nil, fmt.Errorf("read node %s meta for %s index: %w", id.Path(), DirtyIndexName, err) + } + metas[id.Path()] = raw + } + return metas, nil +} + +// readDirtyIndexStatsTitles returns the stats title keyed by node id path. A +// node whose stats cannot be read has no title, as it would on a single read. +func (k *LocalKeg) readDirtyIndexStatsTitles(ctx context.Context, ids []NodeId) map[string]string { + titles := make(map[string]string, len(ids)) + if batch, ok := repositoryBatchRead(k.Repo); ok { + stats, err := batch.ReadStatsBatch(ctx, ids) + if err == nil { + for key, s := range stats { + titles[key] = s.Title() + } + return titles + } + } + for _, id := range ids { + if stats, err := k.Repo.ReadStats(ctx, id); err == nil && stats != nil { + titles[id.Path()] = stats.Title() + } + } + return titles +} + func sortTimelineSnapshotStates(records []timelineSnapshotState) { slices.SortFunc(records, compareTimelineSnapshotState) } diff --git a/pkg/keg/snapshot_policy.go b/pkg/keg/snapshot_policy.go index 441b762e..2bab8bf9 100644 --- a/pkg/keg/snapshot_policy.go +++ b/pkg/keg/snapshot_policy.go @@ -82,8 +82,9 @@ func (k *LocalKeg) runSnapshotPolicy(ctx context.Context) (SnapshotPolicyResult, result.Scanned = len(ids) msg := AutoSnapshotMessage(idleAfter) + history := &snapshotTimeline{} for _, id := range ids { - snap, created, err := k.appendPolicySnapshotIfEligible(ctx, id, idleAfter, msg) + snap, created, err := k.appendPolicySnapshotIfEligible(ctx, history, id, idleAfter, msg) if err != nil { if errors.Is(err, ErrNotExist) { continue @@ -94,13 +95,13 @@ func (k *LocalKeg) runSnapshotPolicy(ctx context.Context) (SnapshotPolicyResult, result.Created = append(result.Created, snap) } } - if err := k.refreshSnapshotIndexesAfterPolicy(ctx, len(result.Created)); err != nil { + if err := k.refreshSnapshotIndexesAfterPolicy(ctx, history, len(result.Created)); err != nil { return result, err } return result, nil } -func (k *LocalKeg) appendPolicySnapshotIfEligible(ctx context.Context, id NodeId, idleAfter time.Duration, msg string) (Snapshot, bool, error) { +func (k *LocalKeg) appendPolicySnapshotIfEligible(ctx context.Context, history *snapshotTimeline, id NodeId, idleAfter time.Duration, msg string) (Snapshot, bool, error) { var out Snapshot created := false err := k.withNodeLock(ctx, id, func(lockCtx context.Context) error { @@ -111,7 +112,7 @@ func (k *LocalKeg) appendPolicySnapshotIfEligible(ctx context.Context, id NodeId if !eligible { return nil } - snap, err := k.appendSnapshotLocked(lockCtx, id, msg) + snap, err := k.appendSnapshotLocked(lockCtx, history, id, msg) if err != nil { return err } diff --git a/pkg/keg/surface_batch.go b/pkg/keg/surface_batch.go index 0a927f96..fe1605a1 100644 --- a/pkg/keg/surface_batch.go +++ b/pkg/keg/surface_batch.go @@ -50,7 +50,7 @@ func (k *LocalKeg) preflightMutation(ctx context.Context, ids []int, hashes []st if err != nil { return err } - if err := checkExpectedHash(fmt.Sprintf("node %d", id), hashes[i], node.Hash(), nodeRecoveryContent(node)); err != nil { + if err := checkExpectedHash(fmt.Sprintf("node %d", id), nodeHashSource, hashes[i], node.Hash(), nodeRecoveryContent(node)); err != nil { return err } } diff --git a/pkg/keg/trash.go b/pkg/keg/trash.go index 31f50a6a..40ea26f3 100644 --- a/pkg/keg/trash.go +++ b/pkg/keg/trash.go @@ -471,16 +471,24 @@ func addDexLink(dex *Dex, src, dst NodeId) { // good -- purged since the node was trashed, neither live nor in the trash -- // at node 0, as purging did to the links live nodes held. Links to a live // node or to another trashed node are left alone. -func (k *LocalKeg) mapPurgedLinks(ctx context.Context, entries map[string][]byte, ordered []string, mapping map[string]NodeId) error { +func (k *LocalKeg) mapPurgedLinks(ctx context.Context, entries archiveEntries, ordered []string, mapping map[string]NodeId) error { for _, sourceID := range ordered { base := "keg-archive/nodes/" + sourceID var targets []NodeId - if raw, ok := entries[base+"/README.md"]; ok { + raw, ok, err := entries.Read(base + "/README.md") + if err != nil { + return err + } + if ok { if content, err := ParseContent(k.Runtime, raw, MarkdownContentFilename); err == nil { targets = append(targets, content.Links...) } } - if raw, ok := entries[base+"/stats.json"]; ok { + raw, ok, err = entries.Read(base + "/stats.json") + if err != nil { + return err + } + if ok { if stats, err := ParseStats(ctx, raw); err == nil { targets = append(targets, stats.Links()...) } diff --git a/pkg/mcp/api_contract_test.go b/pkg/mcp/api_contract_test.go index 930516b2..eeab515f 100644 --- a/pkg/mcp/api_contract_test.go +++ b/pkg/mcp/api_contract_test.go @@ -70,10 +70,7 @@ func TestMCPRESTCompatibilityInitializationAndDeployment(t *testing.T) { result, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "orient", Arguments: map[string]any{}}) require.NoError(t, err) require.True(t, result.IsError) - raw, err := json.Marshal(result.StructuredContent) - require.NoError(t, err) - var payload map[string]any - require.NoError(t, json.Unmarshal(raw, &payload)) + payload := errorPayload(t, result) require.Equal(t, apicontract.Unsupported, payload["code"]) require.Equal(t, false, payload["operationPerformed"]) require.Equal(t, []any{"2099-01-01"}, payload["supported_api_versions"]) diff --git a/pkg/mcp/error_classification_test.go b/pkg/mcp/error_classification_test.go new file mode 100644 index 00000000..f8e0355e --- /dev/null +++ b/pkg/mcp/error_classification_test.go @@ -0,0 +1,222 @@ +package mcp_test + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" + + "github.com/jlrickert/tapper/pkg/keg" + "github.com/jlrickert/tapper/pkg/mcp" + "github.com/jlrickert/tapper/pkg/tapper" +) + +// TestMCP_ClientFixableErrorsAreRefusals drives each bad input through its +// tool and asserts the failure names the caller's mistake: a client code and +// operationPerformed false, never INTERNAL with an unknown outcome. +func TestMCP_ClientFixableErrorsAreRefusals(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + nodeID := createNodeForTest(t, session, ctx, "# Target\n", "") + settingsHash := readSettingsHash(t, session, ctx, "") + + cases := []struct { + name string + tool string + args map[string]any + code string + }{ + {"unbalanced query", "node_list", map[string]any{"query": "(golang"}, keg.RemoteCodeInvalid}, + {"truncated query", "node_list", map[string]any{"query": "golang and"}, keg.RemoteCodeInvalid}, + {"query on node_read", "node_read", map[string]any{"query": "or"}, keg.RemoteCodeInvalid}, + {"non-numeric node id", "node_read", map[string]any{"node_ids": []string{"abc"}}, keg.RemoteCodeInvalid}, + {"leading-zero node id", "node_stats", map[string]any{"node_id": "007"}, keg.RemoteCodeInvalid}, + {"missing node", "node_read", map[string]any{"node_ids": []string{"9999"}}, keg.RemoteCodeNotFound}, + {"stats of a missing node", "node_stats", map[string]any{"node_id": "9999"}, keg.RemoteCodeNotFound}, + {"settings keg and kegs", "keg_settings_read", map[string]any{"keg": "a", "kegs": []string{"@a/b"}}, keg.RemoteCodeInvalid}, + {"settings full read of two kegs", "keg_settings_read", map[string]any{"kegs": []string{"@a/b", "@a/c"}, "minimal": false}, keg.RemoteCodeInvalid}, + {"removed keg_list all", "keg_list", map[string]any{"all": true}, "INVALID_ARGUMENT"}, + {"settings without kegv", "keg_settings_edit", map[string]any{"data": "title: x\n", "expected_hash": settingsHash}, keg.RemoteCodeInvalid}, + {"settings bad yaml", "keg_settings_edit", map[string]any{"data": "kegv: [\n", "expected_hash": settingsHash}, keg.RemoteCodeInvalid}, + {"settings bad index query", "keg_settings_edit", map[string]any{ + "data": "kegv: 2025-07\nindexes:\n - file: broken.md\n query: \"(a\"\n", "expected_hash": settingsHash, + }, keg.RemoteCodeInvalid}, + {"bad base64", "file_upload", map[string]any{"node_id": nodeID, "filename": "a.txt", "data_base64": "!!!"}, keg.RemoteCodeInvalid}, + {"bad data uri", "file_upload", map[string]any{"node_id": nodeID, "filename": "a.txt", "source_uri": "data:text/plain"}, keg.RemoteCodeInvalid}, + {"no upload source", "file_upload", map[string]any{"node_id": nodeID, "filename": "a.txt"}, keg.RemoteCodeInvalid}, + {"unknown guide topic", "guide", map[string]any{"topic": "nope"}, keg.RemoteCodeInvalid}, + } + for _, tc := range cases { + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: tc.tool, Arguments: tc.args}) + require.NoError(t, err, tc.name) + payload := errorPayload(t, res) + require.Equal(t, tc.code, payload["code"], "%s: %v", tc.name, payload["message"]) + require.Equal(t, false, payload["operationPerformed"], tc.name) + require.NotEmpty(t, payload["action"], tc.name) + } + + // None of the refusals wrote anything. + require.Equal(t, settingsHash, readSettingsHash(t, session, ctx, "")) + + // Overshoot: an existing node's stats still read. + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "node_stats", Arguments: map[string]any{"node_id": nodeID}}) + require.NoError(t, err) + require.False(t, res.IsError, "stats of an existing node") +} + +// TestMCP_StoredBadIndexQueryIsNotTheCallersFault is the overshoot side: a +// query stored in keg settings (written before edits were validated) fails +// every listing, and that failure must not tell a caller whose own query is +// fine to fix its arguments. +func TestMCP_StoredBadIndexQueryIsNotTheCallersFault(t *testing.T) { + t.Parallel() + ctx := context.Background() + rt := newTestSandbox(t).Runtime() + tap := newMemoryTap(t, ctx, rt) + resolved, err := tap.KegResolver(ctx, tapper.KegTargetOptions{}, tapper.FlightRoleAdmin) + require.NoError(t, err) + local, ok := resolved.(*keg.LocalKeg) + require.True(t, ok) + cfg, err := local.Settings(ctx) + require.NoError(t, err) + cfg.Indexes = append(cfg.Indexes, keg.IndexEntry{File: "broken.md", Query: "(a"}) + require.NoError(t, local.Repo.WriteSettings(ctx, cfg), "seed the stored query below the strict edit check") + + session := connectFlightSession(t, ctx, mcp.NewServer(tap, "test", mcp.KegDefaults{}), nil) + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "node_list", Arguments: map[string]any{"query": "hello"}}) + require.NoError(t, err) + payload := errorPayload(t, res) + require.Equal(t, keg.RemoteCodeInternal, payload["code"], "%v", payload["message"]) + require.Nil(t, payload["operationPerformed"]) + require.Contains(t, payload["message"], "unable to list keg: dex: settings index \"broken.md\"", + "the caller's valid query must not be blamed for the stored one") +} + +// providerFailures is a flight and agent provider whose creates fail with a +// fixed error, standing in for a Hub's answer on the wire. +type providerFailures struct { + fakeAgents + createErr error +} + +func (p *providerFailures) CreateAgent(context.Context, tapper.CreateAgentOptions) (*tapper.HubAgent, error) { + return nil, p.createErr +} +func (p *providerFailures) ListFlights(context.Context) ([]string, error) { return nil, nil } +func (p *providerFailures) GetFlight(context.Context, string) (*tapper.Flight, error) { + return nil, keg.ErrNotExist +} +func (p *providerFailures) CreateFlight(context.Context, tapper.CreateFlightOptions) (*tapper.Flight, error) { + return nil, p.createErr +} +func (p *providerFailures) UpdateFlight(context.Context, tapper.UpdateFlightOptions) (*tapper.Flight, error) { + return nil, p.createErr +} +func (p *providerFailures) DeleteFlight(context.Context, tapper.DeleteFlightOptions) error { + return p.createErr +} + +// TestMCP_CreateFailuresByKind is the error kind × tool table for the create +// tools: a Hub refusal is reported as a refusal with its own guidance, and only +// an unclassified failure keeps the unknown outcome. +func TestMCP_CreateFailuresByKind(t *testing.T) { + t.Parallel() + kinds := []struct { + name string + err error + code string + performed any + action string + retry any + }{ + {"duplicate name", keg.RemoteErrorFromCode(keg.RemoteCodeConflict, http.StatusConflict, "flight or agent already exists"), + keg.RemoteCodeConflict, false, "conflicts with current state", nil}, + {"rate limited", keg.NewRateLimitError(1500*time.Millisecond, "slow down", nil), + keg.RemoteCodeRateLimited, false, "retryAfterSeconds", float64(2)}, + {"keg restoring", keg.RemoteErrorFromCode(keg.RemoteCodeKegRestoring, http.StatusConflict, "keg is being restored"), + keg.RemoteCodeKegRestoring, false, "being restored", nil}, + {"archive too large", keg.RemoteErrorFromCode(keg.RemoteCodeArchiveTooLarge, http.StatusRequestEntityTooLarge, "archive exceeds limit"), + keg.RemoteCodeArchiveTooLarge, false, "size or count limit", nil}, + {"internal failure", errors.New("database connection reset"), + keg.RemoteCodeInternal, nil, "outcome is unknown", nil}, + } + tools := []struct { + tool string + args map[string]any + }{ + {"flight_create", map[string]any{"ref": "@me/+dup", "title": "Dup"}}, + {"agent_create", map[string]any{"agent": "@me/dup", "title": "Dup", "model": "@me/qwen3"}}, + } + for _, kind := range kinds { + provider := &providerFailures{fakeAgents: fakeAgents{byRef: map[string]tapper.HubAgent{}}, createErr: kind.err} + session, ctx := newTestSessionWithOpts(t, mcp.ServerOptions{FlightProvider: provider, AgentProvider: provider}) + for _, tool := range tools { + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: tool.tool, Arguments: tool.args}) + require.NoError(t, err) + payload := errorPayload(t, res) + label := kind.name + "/" + tool.tool + require.Equal(t, kind.code, payload["code"], "%s: %v", label, payload["message"]) + require.Equal(t, kind.performed, payload["operationPerformed"], label) + require.Contains(t, payload["action"], kind.action, label) + require.Equal(t, kind.retry, payload["retryAfterSeconds"], label) + require.Contains(t, payload["message"], kind.err.Error(), label) + } + } +} + +// TestMCP_LockedSessionNamesTheRepair pins errMCPFlightRequired's +// classification: nothing was performed, and the action sends the agent to +// repair the selection rather than to try another flight. A real per-call +// denial on an active session keeps the ordinary denial advice. +func TestMCP_LockedSessionNamesTheRepair(t *testing.T) { + backend := newRefreshFlightBackend("selection") + session, ctx := newRefreshFlightSession(t, backend, nil) + for _, args := range []map[string]any{{}, {"flight": "+other"}} { + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "flight_create", Arguments: args}) + require.NoError(t, err) + payload := errorPayload(t, res) + require.Equal(t, keg.RemoteCodeOrientationDenied, payload["code"]) + require.Equal(t, false, payload["operationPerformed"]) + require.Contains(t, payload["action"], "session_refresh") + require.Contains(t, payload["action"], "Passing a different flight will not unlock them") + } + + active, activeCtx := newPerCallFlightSession(t, newPerCallFlightBackend()) + denied, err := active.CallTool(activeCtx, &sdkmcp.CallToolParams{Name: "keg_list", Arguments: map[string]any{"flight": "+missing"}}) + require.NoError(t, err) + payload := errorPayload(t, denied) + require.Equal(t, keg.RemoteCodeOrientationDenied, payload["code"]) + require.Contains(t, payload["action"], "Pass a flight that covers this keg") + require.NotContains(t, payload["action"], "session_refresh") +} + +// TestMCP_RepeatFailureCountsTheFlight pins that the loop breaker digests the +// call as sent, flight included: following ORIENTATION_DENIED's advice to try +// another flight is not a repeat, while the same call on one flight still is. +func TestMCP_RepeatFailureCountsTheFlight(t *testing.T) { + session, ctx := newPerCallFlightSession(t, newPerCallFlightBackend()) + call := func(flight string) map[string]any { + t.Helper() + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "keg_list", Arguments: map[string]any{"flight": flight}}) + require.NoError(t, err) + payload := errorPayload(t, res) + require.Equal(t, keg.RemoteCodeOrientationDenied, payload["code"]) + return payload + } + + for _, flight := range []string{"+missing-a", "+missing-b", "+missing-c", "+missing-d"} { + payload := call(flight) + require.NotContains(t, payload, "repeat_count", "a call on another flight is a different call (%s)", flight) + } + + call("+missing-e") + call("+missing-e") + third := call("+missing-e") + require.Equal(t, float64(3), third["repeat_count"]) + require.Contains(t, third["action"], "stop retrying") + require.Equal(t, keg.RemoteCodeOrientationDenied, third["code"], "the repeat keeps its original code") +} diff --git a/pkg/mcp/error_contract_internal_test.go b/pkg/mcp/error_contract_internal_test.go index d3600742..54a288bb 100644 --- a/pkg/mcp/error_contract_internal_test.go +++ b/pkg/mcp/error_contract_internal_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/jlrickert/tapper/pkg/keg" + "github.com/jlrickert/tapper/pkg/tapper" ) // TestErrorResultsAlwaysCarryTheRecoveryContract is the guard that keeps the @@ -40,6 +41,8 @@ func TestErrorResultsAlwaysCarryTheRecoveryContract(t *testing.T) { {"lock timeout", keg.ErrLockTimeout}, {"not supported", keg.ErrNotSupported}, {"precondition required", keg.ErrPreconditionRequired}, + {"part precondition required", &keg.PartPreconditionRequiredError{NodeID: "3", Missing: []keg.MissingPartHash{{Part: "content", Field: "expected_content_hash", TokenField: "content_hash"}}}}, + {"empty edit", &tapper.EmptyEditError{NodeID: "3"}}, {"precondition conflict", &keg.PreconditionConflictError{Resource: "node 1", CurrentHash: "abc"}}, {"unauthorized", keg.ErrUnauthorized}, {"forbidden", keg.ErrForbidden}, diff --git a/pkg/mcp/flight_manage_test.go b/pkg/mcp/flight_manage_test.go new file mode 100644 index 00000000..793e0907 --- /dev/null +++ b/pkg/mcp/flight_manage_test.go @@ -0,0 +1,85 @@ +package mcp_test + +import ( + "testing" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" +) + +// flight_create sets each switch when asked, and flight_read reports them. +func TestMCP_FlightCreateSetsSwitches(t *testing.T) { + backend := newPerCallFlightBackend() + session, ctx := newPerCallFlightSession(t, backend) + + created, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "flight_create", Arguments: map[string]any{ + "ref": "@team/+desk", "manage_kegs": true, + }}) + require.NoError(t, err) + require.False(t, created.IsError, "%v", created.Content) + out := created.StructuredContent.(map[string]any) + require.Equal(t, true, out["manage_kegs"]) + require.Equal(t, false, out["manage_flights"], "only the switch asked for is set") + + read, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "flight_read", Arguments: map[string]any{"name": "@team/+desk"}}) + require.NoError(t, err) + require.Equal(t, true, read.StructuredContent.(map[string]any)["manage_kegs"]) + + plain, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "flight_read", Arguments: map[string]any{"name": "@team/+child"}}) + require.NoError(t, err) + require.Equal(t, false, plain.StructuredContent.(map[string]any)["manage_kegs"], "a flight without switches reads false") + require.Equal(t, false, plain.StructuredContent.(map[string]any)["manage_flights"]) +} + +// The deprecated manage argument sets both switches. +func TestMCP_FlightCreateManageAliasSetsBoth(t *testing.T) { + backend := newPerCallFlightBackend() + session, ctx := newPerCallFlightSession(t, backend) + + created, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "flight_create", Arguments: map[string]any{ + "ref": "@team/+desk", "manage": true, + }}) + require.NoError(t, err) + require.False(t, created.IsError, "%v", created.Content) + out := created.StructuredContent.(map[string]any) + require.Equal(t, true, out["manage_kegs"]) + require.Equal(t, true, out["manage_flights"]) +} + +// flight_edit changes a switch only when the call names it, or names the +// deprecated manage alias and leaves the switch out; an edit of anything +// else leaves both to the Hub to keep. +func TestMCP_FlightEditChangesSwitchesOnlyWhenNamed(t *testing.T) { + backend := newPerCallFlightBackend() + session, ctx := newPerCallFlightSession(t, backend) + + for _, args := range []map[string]any{ + {"ref": "@team/+child", "manage_kegs": true, "expected_hash": "h"}, + {"ref": "@team/+child", "manage_flights": false, "expected_hash": "h"}, + {"ref": "@team/+child", "manage": true, "manage_kegs": false, "expected_hash": "h"}, + {"ref": "@team/+child", "title": "Retitled", "expected_hash": "h"}, + } { + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "flight_edit", Arguments: args}) + require.NoError(t, err) + require.False(t, res.IsError, "%v", res.Content) + } + backend.mu.Lock() + defer backend.mu.Unlock() + require.Len(t, backend.updates, 4) + + require.NotNil(t, backend.updates[0].ManageKegs) + require.True(t, *backend.updates[0].ManageKegs) + require.Nil(t, backend.updates[0].ManageFlights, "a switch not named is kept") + + require.Nil(t, backend.updates[1].ManageKegs) + require.NotNil(t, backend.updates[1].ManageFlights) + require.False(t, *backend.updates[1].ManageFlights) + + require.NotNil(t, backend.updates[2].ManageKegs) + require.False(t, *backend.updates[2].ManageKegs, "an explicit switch wins over the alias") + require.NotNil(t, backend.updates[2].ManageFlights) + require.True(t, *backend.updates[2].ManageFlights, "the alias fills the switch left out") + + require.Nil(t, backend.updates[3].ManageKegs, "an edit that names no switch keeps them") + require.Nil(t, backend.updates[3].ManageFlights) +} diff --git a/pkg/mcp/flight_read_fresh_test.go b/pkg/mcp/flight_read_fresh_test.go new file mode 100644 index 00000000..50dab93c --- /dev/null +++ b/pkg/mcp/flight_read_fresh_test.go @@ -0,0 +1,38 @@ +package mcp_test + +import ( + "context" + "testing" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" +) + +func readFlightForTest(t *testing.T, ctx context.Context, session *sdkmcp.ClientSession, name string) map[string]any { + t.Helper() + result, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "flight_read", Arguments: map[string]any{"name": name}}) + require.NoError(t, err) + require.False(t, result.IsError, extractText(t, result)) + out, ok := result.StructuredContent.(map[string]any) + require.True(t, ok, "flight_read returns structured content") + return out +} + +// A long-running MCP server reads a flight as the Hub holds it now: an edit +// made elsewhere (another client, the web) shows on the next flight_read, +// with the hash a guarded edit needs, for the session's own flight and for +// any other. +func TestMCP_FlightReadSeesOutOfProcessEdits(t *testing.T) { + ctx, srv, rt := newOrientationServer(t, "+alpha") + session := connectFlightSession(t, ctx, srv, nil) + callOrient(t, ctx, session) + callCat(t, ctx, session) + + for _, slug := range []string{"beta", "alpha"} { + before := readFlightForTest(t, ctx, session, "+"+slug) + writeFlightCover(t, rt, slug, slug+" edited elsewhere", "personal") + after := readFlightForTest(t, ctx, session, "+"+slug) + require.Equal(t, slug+" edited elsewhere", after["instructions"], "+%s is read fresh", slug) + require.NotEqual(t, before["hash"], after["hash"], "+%s carries the current hash", slug) + } +} diff --git a/pkg/mcp/legacy_arguments.go b/pkg/mcp/legacy_arguments.go new file mode 100644 index 00000000..2393818c --- /dev/null +++ b/pkg/mcp/legacy_arguments.go @@ -0,0 +1,93 @@ +package mcp + +import ( + "context" + "encoding/json" + "strings" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/jlrickert/tapper/pkg/tapper" +) + +// Some arguments were dropped from the tool schemas because another argument +// already says the same thing. The schemas forbid properties they do not +// list, so a client written against an older schema would fail validation the +// moment it sent one. legacyArgumentsMiddleware rewrites them into their +// replacement before the SDK validates the call, so those clients keep +// working without the old names reappearing in any schema an agent reads. +// +// - id_only on node_list, node_grep, tag_list, node_backlinks, and +// node_links is format "%i", which prints the same lines. A true id_only +// wins over any format sent with it, as it always did. +// - keg on node_search named the one KEG to search. kegs now names any +// number of them, so a legacy keg is searched as if it were the only +// entry in kegs (see legacySearchKeg). + +// idOnlyTools are the listing tools that accepted id_only. +var idOnlyTools = map[string]bool{ + "node_list": true, "node_grep": true, "tag_list": true, "node_backlinks": true, "node_links": true, +} + +type legacySearchKegKey struct{} + +// legacySearchKeg returns the keg a pre-kegs node_search call named, or "". +// It travels outside the arguments because a bare name ("notes", meaning the +// caller's default namespace) is not a valid kegs entry; node_search resolves +// it the way the keg argument always was. +func legacySearchKeg(ctx context.Context) string { + keg, _ := ctx.Value(legacySearchKegKey{}).(string) + return keg +} + +func legacyArgumentsMiddleware(next sdkmcp.MethodHandler) sdkmcp.MethodHandler { + return func(ctx context.Context, method string, req sdkmcp.Request) (sdkmcp.Result, error) { + if method != "tools/call" { + return next(ctx, method, req) + } + params, _ := req.GetParams().(*sdkmcp.CallToolParamsRaw) + if params == nil || len(params.Arguments) == 0 || IsRelayedToolName(params.Name) { + return next(ctx, method, req) + } + switch { + case idOnlyTools[params.Name]: + rewriteArguments(params, func(args map[string]json.RawMessage) { + raw, ok := args["id_only"] + if !ok { + return + } + delete(args, "id_only") + var idOnly bool + if json.Unmarshal(raw, &idOnly) == nil && idOnly { + args["format"], _ = json.Marshal(tapper.IDOnlyFormat) + } + }) + case params.Name == "node_search": + rewriteArguments(params, func(args map[string]json.RawMessage) { + raw, ok := args["keg"] + if !ok { + return + } + delete(args, "keg") + var keg string + if json.Unmarshal(raw, &keg) == nil && strings.TrimSpace(keg) != "" { + ctx = context.WithValue(ctx, legacySearchKegKey{}, strings.TrimSpace(keg)) + } + }) + } + return next(ctx, method, req) + } +} + +// rewriteArguments applies edit to the call's arguments when they decode as +// an object, and leaves anything else for the SDK to refuse. +func rewriteArguments(params *sdkmcp.CallToolParamsRaw, edit func(map[string]json.RawMessage)) { + var args map[string]json.RawMessage + if json.Unmarshal(params.Arguments, &args) != nil || args == nil { + return + } + edit(args) + if raw, err := json.Marshal(args); err == nil { + params.Arguments = raw + } +} diff --git a/pkg/mcp/legacy_arguments_test.go b/pkg/mcp/legacy_arguments_test.go new file mode 100644 index 00000000..d9b0af73 --- /dev/null +++ b/pkg/mcp/legacy_arguments_test.go @@ -0,0 +1,73 @@ +package mcp_test + +import ( + "encoding/json" + "testing" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" +) + +// pageLines returns a listing call's structured lines. +func pageLines(t *testing.T, res *sdkmcp.CallToolResult) []string { + t.Helper() + require.Falsef(t, res.IsError, "call failed: %s", extractText(t, res)) + raw, err := json.Marshal(res.StructuredContent) + require.NoError(t, err) + var page struct { + Lines []string `json:"lines"` + } + require.NoError(t, json.Unmarshal(raw, &page)) + return page.Lines +} + +// id_only left the listing schemas because format "%i" prints the same +// lines. A client written against the old schema still sends it, and must get +// those lines rather than a validation failure. +func TestMCP_IDOnlyIsFormatForOldClients(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + + listed, err := session.ListTools(ctx, nil) + require.NoError(t, err) + for _, tool := range listed.Tools { + _, advertised := schemaProperties(t, tool.InputSchema)["id_only"] + require.Falsef(t, advertised, "%s still advertises id_only", tool.Name) + } + + cases := []struct { + tool string + args map[string]any + }{ + {"node_list", map[string]any{}}, + {"node_list", map[string]any{"reverse": true}}, + {"node_grep", map[string]any{"query": "Hel+o"}}, + {"tag_list", map[string]any{"query": "test"}}, + {"node_backlinks", map[string]any{"node_ids": []string{"0"}}}, + {"node_links", map[string]any{"node_ids": []string{"1"}}}, + } + call := func(tool string, base map[string]any, extra map[string]any) []string { + args := map[string]any{} + for k, v := range base { + args[k] = v + } + for k, v := range extra { + args[k] = v + } + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: tool, Arguments: args}) + require.NoError(t, err) + return pageLines(t, res) + } + for _, tc := range cases { + want := call(tc.tool, tc.args, map[string]any{"format": "%i"}) + require.NotEmptyf(t, want, "%s %v listed nothing; the case proves nothing", tc.tool, tc.args) + require.Equalf(t, want, call(tc.tool, tc.args, map[string]any{"id_only": true}), + "%s: id_only must print what format %%i prints", tc.tool) + // As before, id_only wins over a format sent with it. + require.Equalf(t, want, call(tc.tool, tc.args, map[string]any{"id_only": true, "format": "%t"}), + "%s: id_only must override format", tc.tool) + // id_only false asks for nothing, so the default rendering applies. + require.Equalf(t, call(tc.tool, tc.args, nil), call(tc.tool, tc.args, map[string]any{"id_only": false}), + "%s: id_only false must change nothing", tc.tool) + } +} diff --git a/pkg/mcp/mcptest/validate.go b/pkg/mcp/mcptest/validate.go new file mode 100644 index 00000000..e2b613a9 --- /dev/null +++ b/pkg/mcp/mcptest/validate.go @@ -0,0 +1,165 @@ +// Package mcptest holds test helpers for MCP servers built on package mcp. +package mcptest + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + + "github.com/google/jsonschema-go/jsonschema" + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// ValidatingClient returns an MCP client that checks every tool result the +// way a strict client does, and fails t when one would be rejected. The +// TypeScript SDK, for one, validates a result's structuredContent against the +// tool's declared outputSchema whether or not isError is set, and refuses a +// successful result that declares an outputSchema but carries no structured +// content; either one reaches the caller as a JSON-RPC error rather than the +// tool's answer. +// +// Schemas come from the session's own tool listing, fetched on the first call +// to a tool the client has not seen listed, so the client checks whatever +// server it connects to. +func ValidatingClient(t testing.TB, impl *sdkmcp.Implementation, opts *sdkmcp.ClientOptions) *sdkmcp.Client { + t.Helper() + client := sdkmcp.NewClient(impl, opts) + v := &validator{t: t, schemas: map[*sdkmcp.ClientSession]map[string]*jsonschema.Resolved{}} + client.AddSendingMiddleware(v.middleware) + return client +} + +type validator struct { + t testing.TB + mu sync.Mutex + schemas map[*sdkmcp.ClientSession]map[string]*jsonschema.Resolved +} + +func (v *validator) middleware(next sdkmcp.MethodHandler) sdkmcp.MethodHandler { + return func(ctx context.Context, method string, req sdkmcp.Request) (sdkmcp.Result, error) { + result, err := next(ctx, method, req) + if err != nil { + return result, err + } + session, _ := req.GetSession().(*sdkmcp.ClientSession) + switch method { + case "tools/list": + if listed, ok := result.(*sdkmcp.ListToolsResult); ok && listed != nil { + v.learn(session, listed.Tools) + } + case "tools/call": + params, _ := req.GetParams().(*sdkmcp.CallToolParams) + call, ok := result.(*sdkmcp.CallToolResult) + if ok && call != nil && params != nil { + v.check(ctx, session, params.Name, call) + } + } + return result, err + } +} + +// check fails t when a strict client would reject call. +func (v *validator) check(ctx context.Context, session *sdkmcp.ClientSession, tool string, call *sdkmcp.CallToolResult) { + schema := v.schema(ctx, session, tool) + if schema == nil { + return + } + if call.StructuredContent == nil { + if !call.IsError { + v.t.Errorf("%s declares an outputSchema but returned no structured content", tool) + } + return + } + if err := validate(schema, call.StructuredContent); err != nil { + v.t.Errorf("%s result (isError=%t) does not match its outputSchema: %v\nstructuredContent: %v", tool, call.IsError, err, call.StructuredContent) + } +} + +// schema returns tool's resolved outputSchema, listing the session's tools +// when it has not seen tool yet. A tool without an outputSchema, or one the +// listing does not offer, has none. +func (v *validator) schema(ctx context.Context, session *sdkmcp.ClientSession, tool string) *jsonschema.Resolved { + if session == nil { + return nil + } + v.mu.Lock() + known, seen := v.schemas[session][tool] + v.mu.Unlock() + if seen { + return known + } + var tools []*sdkmcp.Tool + for listed, err := range session.Tools(ctx, nil) { + if err != nil { + v.t.Errorf("list tools to validate %s: %v", tool, err) + return nil + } + tools = append(tools, listed) + } + v.learn(session, tools) + v.mu.Lock() + defer v.mu.Unlock() + return v.schemas[session][tool] +} + +// learn records the outputSchemas of listed tools for session. +func (v *validator) learn(session *sdkmcp.ClientSession, tools []*sdkmcp.Tool) { + if session == nil { + return + } + resolved := make(map[string]*jsonschema.Resolved, len(tools)) + for _, tool := range tools { + if tool == nil { + continue + } + schema, err := resolve(tool.OutputSchema) + if err != nil { + v.t.Errorf("%s outputSchema: %v", tool.Name, err) + continue + } + resolved[tool.Name] = schema + } + v.mu.Lock() + defer v.mu.Unlock() + if v.schemas[session] == nil { + v.schemas[session] = map[string]*jsonschema.Resolved{} + } + for name, schema := range resolved { + v.schemas[session][name] = schema + } +} + +// resolve parses an outputSchema as a client receives it; nil means the tool +// declares none. +func resolve(outputSchema any) (*jsonschema.Resolved, error) { + if outputSchema == nil { + return nil, nil + } + raw, err := json.Marshal(outputSchema) + if err != nil { + return nil, err + } + if string(raw) == "null" { + return nil, nil + } + var schema jsonschema.Schema + if err := json.Unmarshal(raw, &schema); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + return schema.Resolve(nil) +} + +// validate checks structured content as a client decodes it off the wire. +func validate(schema *jsonschema.Resolved, structured any) error { + raw, err := json.Marshal(structured) + if err != nil { + return err + } + var instance any + if err := json.Unmarshal(raw, &instance); err != nil { + return err + } + return schema.Validate(instance) +} diff --git a/pkg/mcp/node_edit_errors_test.go b/pkg/mcp/node_edit_errors_test.go new file mode 100644 index 00000000..8ffee44b --- /dev/null +++ b/pkg/mcp/node_edit_errors_test.go @@ -0,0 +1,132 @@ +package mcp_test + +import ( + "strings" + "testing" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" +) + +// TestMCP_EditRefusesProseAsSchema pins that a sentence passed as schema is +// refused by input validation, before the handler runs, with the code an +// agent reads as "fix the arguments". +func TestMCP_EditRefusesProseAsSchema(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + + nodeID := createNodeForTest(t, session, ctx, "# Typed\n", "") + _, metaHash := readNodePartHashes(t, session, ctx, nodeID) + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: "node_edit", + Arguments: batchEditArgs(map[string]any{ + "node_id": nodeID, + "meta": "tags:\n - typed\n", + "expected_meta_hash": metaHash, + "schema": "a note about the quarterly planning meeting", + }), + }) + require.NoError(t, err) + require.True(t, res.IsError) + structured := structuredMap(t, res) + require.Equal(t, "INVALID_ARGUMENT", structured["code"]) + require.Equal(t, false, structured["operationPerformed"]) + require.Contains(t, extractText(t, res), "schema") + + _, unchanged := readNodePartHashes(t, session, ctx, nodeID) + require.Equal(t, metaHash, unchanged, "a refused edit wrote metadata") +} + +// TestMCP_UnknownSchemaListsKegTypes pins that a well-formed but undefined +// schema names the types the keg does define, so the retry needs no +// schema_list round trip. +func TestMCP_UnknownSchemaListsKegTypes(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + + for _, name := range []string{"decision", "task"} { + created, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: "schema_create", + Arguments: map[string]any{"data": "type: " + name + "\n"}, + }) + require.NoError(t, err) + require.False(t, created.IsError, "schema create failed: %s", extractText(t, created)) + } + + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: "node_create", + Arguments: map[string]any{"nodes": []any{map[string]any{ + "key": "n", "schema": "meeting", "content": "# Meeting\n", + }}}, + }) + require.NoError(t, err) + require.True(t, res.IsError) + require.Equal(t, "SCHEMA_INVALID", structuredMap(t, res)["code"]) + text := extractText(t, res) + require.Contains(t, text, `unknown schema "meeting"`) + require.True(t, strings.Contains(text, "this keg defines: decision, task"), text) +} + +// TestMCP_RepeatedIdenticalFailureSaysStop pins the loop breaker: the third +// identical failing call keeps its code and gains repeat_count and a stop +// action; changed arguments or a success start the count again. +func TestMCP_RepeatedIdenticalFailureSaysStop(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + + nodeID := createNodeForTest(t, session, ctx, "# Looping\n\nBody.\n", "") + bad := batchEditArgs(map[string]any{"node_id": nodeID, "content": "# Looping\n\nNew body.\n"}) + call := func(args map[string]any) map[string]any { + t.Helper() + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "node_edit", Arguments: args}) + require.NoError(t, err) + if !res.IsError { + return nil + } + return structuredMap(t, res) + } + + for attempt := 1; attempt <= 2; attempt++ { + got := call(bad) + require.Equal(t, "PRECONDITION_REQUIRED", got["code"]) + require.NotContains(t, got, "repeat_count", "attempt %d is an ordinary retry", attempt) + } + third := call(bad) + require.Equal(t, "PRECONDITION_REQUIRED", third["code"], "the original code is kept") + require.EqualValues(t, 3, third["repeat_count"]) + action, _ := third["action"].(string) + require.Contains(t, action, "failed 3 times; stop retrying it") + require.Contains(t, action, "node_read(node_id=", "the original guidance is kept after the stop instruction") + require.EqualValues(t, 4, call(bad)["repeat_count"]) + + // Different arguments are a different call: the count starts over. + changed := batchEditArgs(map[string]any{"node_id": nodeID, "content": "# Looping\n\nOther body.\n"}) + require.NotContains(t, call(changed), "repeat_count") + + // Following the advice works, and the success clears the tool's count. + contentHash, _ := readNodePartHashes(t, session, ctx, nodeID) + require.Nil(t, call(batchEditArgs(map[string]any{ + "node_id": nodeID, "content": "# Looping\n\nNew body.\n", "expected_content_hash": contentHash, + })), "the corrected call failed") + require.NotContains(t, call(bad), "repeat_count") +} + +// TestMCP_RepeatedInvalidArgumentsSaysStop covers failures the SDK raises +// before any handler runs: they are counted too, and keep INVALID_ARGUMENT. +func TestMCP_RepeatedInvalidArgumentsSaysStop(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + + args := batchEditArgs(map[string]any{"node_id": "1", "meta": "tags: []\n", "schema": "a note about planning"}) + var last map[string]any + for range repeatThreshold { + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "node_edit", Arguments: args}) + require.NoError(t, err) + require.True(t, res.IsError) + last = structuredMap(t, res) + } + require.Equal(t, "INVALID_ARGUMENT", last["code"]) + require.EqualValues(t, repeatThreshold, last["repeat_count"]) +} + +const repeatThreshold = 3 diff --git a/pkg/mcp/output_schema.go b/pkg/mcp/output_schema.go new file mode 100644 index 00000000..f1ec12ad --- /dev/null +++ b/pkg/mcp/output_schema.go @@ -0,0 +1,81 @@ +package mcp + +import ( + "fmt" + "reflect" + + "github.com/google/jsonschema-go/jsonschema" +) + +// Every tool declares an outputSchema for its structuredContent, so a client +// (and the Hub's tool inspector) can see a result's shape before calling. +// +// publicResponseMiddleware shapes every result the same way: the tool's own +// structured fields, plus message, the result's text. A tool whose result is +// only text therefore still returns an object, {"message": "..."}, and +// declares exactly that (textOutputSchema) rather than nothing. +// +// The schemas leave additionalProperties open. They document the fields a +// client may rely on; a newer server adding a field must not make an older +// client that validates results reject them. + +// outputMessageDescription describes the message field every result carries. +const outputMessageDescription = "The result as text, for clients that read text. For a tool with no other fields, it is the whole result." + +// textOutput is the structured content of a tool whose result is only text. +type textOutput struct { + Message string `json:"message" jsonschema:"The result as text."` +} + +// textOutputSchema is the outputSchema of a tool whose result is only text. +func textOutputSchema() *jsonschema.Schema { + return outputSchema[textOutput]() +} + +// outputSchema derives a tool's outputSchema from T, the type its handler +// puts in StructuredContent, and adds the message field the public response +// adds to every result. +func outputSchema[T any]() *jsonschema.Schema { + schema, err := jsonschema.For[T](nil) + if err != nil { + panic(fmt.Sprintf("derive MCP output schema for %T: %v", *new(T), err)) + } + openSchema(schema) + if schema.Properties == nil { + schema.Properties = map[string]*jsonschema.Schema{} + } + if _, ok := schema.Properties["message"]; !ok { + schema.Properties["message"] = &jsonschema.Schema{Type: "string", Description: outputMessageDescription} + } + return schema +} + +// openSchema removes the additionalProperties: false that schema inference +// puts on every struct, throughout schema. +func openSchema(schema *jsonschema.Schema) { + if schema == nil { + return + } + if isFalseSchema(schema.AdditionalProperties) { + schema.AdditionalProperties = nil + } else { + openSchema(schema.AdditionalProperties) + } + for _, property := range schema.Properties { + openSchema(property) + } + openSchema(schema.Items) + for _, def := range schema.Defs { + openSchema(def) + } + for _, alt := range schema.AnyOf { + openSchema(alt) + } + for _, alt := range schema.OneOf { + openSchema(alt) + } +} + +func isFalseSchema(s *jsonschema.Schema) bool { + return s != nil && reflect.DeepEqual(s, &jsonschema.Schema{Not: &jsonschema.Schema{}}) +} diff --git a/pkg/mcp/output_schema_test.go b/pkg/mcp/output_schema_test.go new file mode 100644 index 00000000..81144f9f --- /dev/null +++ b/pkg/mcp/output_schema_test.go @@ -0,0 +1,286 @@ +package mcp_test + +import ( + "encoding/json" + "sync" + "testing" + + "github.com/google/jsonschema-go/jsonschema" + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" + + "github.com/jlrickert/tapper/pkg/mcp" + "github.com/jlrickert/tapper/pkg/mcp/mcptest" + "github.com/jlrickert/tapper/pkg/tapapi" +) + +// Every tool declares the shape of its structured result, and every result a +// test receives is checked against it: a client that validates results (as +// the MCP spec asks clients to) must never reject one of ours. + +var ( + declaredOutputsOnce sync.Once + declaredOutputs map[string]*jsonschema.Resolved + declaredOutputsErr error +) + +// fullToolListing lists every tool a server offers with all providers set, +// on a shared filesystem (where the local attachment variants register) or +// not. +func fullToolListing(t *testing.T, sharedFS bool) []*sdkmcp.Tool { + t.Helper() + session, ctx := newPlainTestSession(t, mcp.ServerOptions{ + AgentProvider: &fakeAgents{}, + NamespaceProvider: &fakeNamespaces{}, + ExploreProvider: &fakeExplore{}, + SharedFilesystem: sharedFS, + }) + listed, err := session.ListTools(ctx, nil) + require.NoError(t, err) + require.NotEmpty(t, listed.Tools) + return listed.Tools +} + +// outputSchemas resolves each tool's declared outputSchema, by tool name. +// The shared-filesystem variants of a tool declare the same shape as the +// others, so one schema per name serves every server. +func outputSchemas(t *testing.T) map[string]*jsonschema.Resolved { + t.Helper() + declaredOutputsOnce.Do(func() { + declaredOutputs = map[string]*jsonschema.Resolved{} + for _, shared := range []bool{false, true} { + for _, tool := range fullToolListing(t, shared) { + if tool.OutputSchema == nil { + continue + } + raw, err := json.Marshal(tool.OutputSchema) + if err != nil { + declaredOutputsErr = err + return + } + var schema jsonschema.Schema + if err := json.Unmarshal(raw, &schema); err != nil { + declaredOutputsErr = err + return + } + resolved, err := schema.Resolve(nil) + if err != nil { + declaredOutputsErr = err + return + } + declaredOutputs[tool.Name] = resolved + } + } + }) + require.NoError(t, declaredOutputsErr) + return declaredOutputs +} + +// validateOutput validates structured content as a client decodes it. +func validateOutput(schema *jsonschema.Resolved, structured any) error { + raw, err := json.Marshal(structured) + if err != nil { + return err + } + var instance any + if err := json.Unmarshal(raw, &instance); err != nil { + return err + } + return schema.Validate(instance) +} + +// newTestClient is the client every test session connects with: it checks +// each result, failed or not, as a strict client would. +func newTestClient(t *testing.T, name string, opts *sdkmcp.ClientOptions) *sdkmcp.Client { + t.Helper() + return mcptest.ValidatingClient(t, &sdkmcp.Implementation{Name: name, Version: "0.1"}, opts) +} + +func TestMCP_EveryToolDeclaresAnOutputSchema(t *testing.T) { + t.Parallel() + for _, shared := range []bool{false, true} { + for _, tool := range fullToolListing(t, shared) { + require.NotNilf(t, tool.OutputSchema, "%s declares no outputSchema", tool.Name) + raw, err := json.Marshal(tool.OutputSchema) + require.NoError(t, err) + var schema struct { + Type string `json:"type"` + Properties map[string]json.RawMessage `json:"properties"` + Additional json.RawMessage `json:"additionalProperties"` + } + require.NoError(t, json.Unmarshal(raw, &schema)) + require.Equalf(t, "object", schema.Type, "%s: structured content is always an object", tool.Name) + require.Containsf(t, schema.Properties, "message", "%s: every result carries its text as message", tool.Name) + require.Emptyf(t, schema.Additional, "%s: a newer server's extra field must not fail an older client", tool.Name) + } + } +} + +// outputFields lists an outputSchema's top-level properties. +func outputFields(t *testing.T, tool *sdkmcp.Tool) map[string]json.RawMessage { + t.Helper() + raw, err := json.Marshal(tool.OutputSchema) + require.NoError(t, err) + var schema struct { + Properties map[string]json.RawMessage `json:"properties"` + } + require.NoError(t, json.Unmarshal(raw, &schema)) + return schema.Properties +} + +// The read tools agents page through most declare the fields they page by, +// and what they actually return matches. +func TestMCP_ReadToolOutputsMatchTheirSchemas(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + listed, err := session.ListTools(ctx, nil) + require.NoError(t, err) + byName := map[string]*sdkmcp.Tool{} + for _, tool := range listed.Tools { + byName[tool.Name] = tool + } + wantFields := map[string][]string{ + "node_list": {"lines", "offset", "limit", "next_offset", "has_more", "pagination"}, + "node_grep": {"lines", "next_offset", "has_more"}, + "node_read": {"nodes"}, + } + for name, fields := range wantFields { + props := outputFields(t, byName[name]) + for _, field := range fields { + require.Containsf(t, props, field, "%s outputSchema", name) + } + } + schemas := outputSchemas(t) + for name, args := range map[string]map[string]any{ + "node_list": {"limit": 1}, + "node_grep": {"query": "Hello"}, + "node_read": {"node_ids": []string{"0", "1"}}, + } { + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: name, Arguments: args}) + require.NoError(t, err) + require.Falsef(t, res.IsError, "%s: %s", name, extractText(t, res)) + require.NoErrorf(t, validateOutput(schemas[name], res.StructuredContent), "%s result", name) + } + + // The schema really does constrain: a result missing a required field + // or carrying the wrong type fails it. + require.Error(t, validateOutput(schemas["node_list"], map[string]any{"lines": "not a list"})) + require.Error(t, validateOutput(schemas["node_read"], map[string]any{"nodes": []any{map[string]any{"node_id": "1"}}})) +} + +func TestMCP_NodeSearchOutputMatchesItsSchema(t *testing.T) { + t.Parallel() + provider := &recordingNodeSearch{} + session, ctx := newTestSessionWithOpts(t, mcp.ServerOptions{NodeSearchProvider: provider}) + listed, err := session.ListTools(ctx, nil) + require.NoError(t, err) + for _, tool := range listed.Tools { + if tool.Name == "node_search" { + props := outputFields(t, tool) + for _, field := range []string{"hits", "next_cursor", "mode", "warnings", "pagination"} { + require.Contains(t, props, field) + } + } + } + + schema := outputSchemas(t)["node_search"] + for _, resp := range []string{ + `{"hits":[{"ref":"keg:@acme/notes/12","keg":"@acme/notes","node_id":"12","title":"Festival","score":4.25,"fields":["title"],"sources":["keyword"],"snippet":"festival","highlights":[{"start":0,"end":8}]}],"next_cursor":"c2","terms":[{"term":"festival","lexemes":["festiv"],"typo":false}],"mode":"hybrid","warnings":["partial"]}`, + `{"hits":[]}`, + } { + provider.resp = decodeSearchResponse(t, resp) + res := callNodeSearch(t, session, ctx, map[string]any{"query": "festival"}) + require.False(t, res.IsError, extractText(t, res)) + require.NoError(t, validateOutput(schema, res.StructuredContent)) + } +} + +func decodeSearchResponse(t *testing.T, raw string) *tapapi.SearchResponse { + t.Helper() + var resp tapapi.SearchResponse + require.NoError(t, json.Unmarshal([]byte(raw), &resp)) + return &resp +} + +// errorPayload decodes a failed result's public payload from its leading +// text block, the only place a failure carries it. +func errorPayload(t *testing.T, res *sdkmcp.CallToolResult) map[string]any { + t.Helper() + require.True(t, res.IsError, "want a failed result") + require.NotEmpty(t, res.Content) + text, ok := res.Content[0].(*sdkmcp.TextContent) + require.Truef(t, ok, "first content block is %T, want text", res.Content[0]) + var payload map[string]any + require.NoErrorf(t, json.Unmarshal([]byte(text.Text), &payload), "leading text is not a JSON object: %q", text.Text) + return payload +} + +// A tool's outputSchema describes its success. A failure that carried its +// error payload as structured content would fail that schema in a client +// that validates results, which then reports a JSON-RPC error instead of +// the tool's answer; so failures carry the payload as JSON text alone. +func TestMCP_ErrorResultsCarryNoStructuredContent(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + call := func(name string, args map[string]any) *sdkmcp.CallToolResult { + t.Helper() + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: name, Arguments: args}) + require.NoError(t, err) + return res + } + require.False(t, call("orient", nil).IsError) + + read := call("node_read", map[string]any{"node_ids": []string{"1"}}) + require.False(t, read.IsError, extractText(t, read)) + node := read.StructuredContent.(map[string]any)["nodes"].([]any)[0].(map[string]any) + + for _, tc := range []struct { + name string + tool string + args map[string]any + code string + }{ + {"missing node", "node_read", map[string]any{"node_ids": []string{"999999"}}, "NOT_FOUND"}, + {"edit without hashes", "node_edit", map[string]any{"nodes": []any{map[string]any{"node_id": "1", "content": "# Changed"}}}, "PRECONDITION_REQUIRED"}, + {"stale hash", "node_edit", map[string]any{"nodes": []any{map[string]any{"node_id": "1", "expected_content_hash": "stale", "content": "# Changed"}}}, "CONFLICT"}, + {"invalid arguments", "node_list", map[string]any{"limit": "many"}, "INVALID_ARGUMENT"}, + } { + t.Run(tc.name, func(t *testing.T) { + res := call(tc.tool, tc.args) + require.True(t, res.IsError, "%s: want a failure", tc.tool) + require.Nil(t, res.StructuredContent, "%s: a failure carries no structured content", tc.tool) + payload := errorPayload(t, res) + require.Equal(t, tc.code, payload["code"]) + require.NotEmpty(t, payload["message"]) + require.NotEmpty(t, payload["action"]) + require.Contains(t, payload, "operationPerformed") + }) + } + + // Still current: the refused edits changed nothing. + again := call("node_read", map[string]any{"node_ids": []string{"1"}}) + require.Equal(t, node["content_hash"], again.StructuredContent.(map[string]any)["nodes"].([]any)[0].(map[string]any)["content_hash"]) +} + +// The other side of the line: a success keeps its structured content, and its +// text is the same payload. +func TestMCP_SuccessResultsKeepStructuredContent(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + for _, tc := range []struct { + tool string + args map[string]any + }{ + {"orient", nil}, + {"node_read", map[string]any{"node_ids": []string{"1"}}}, + {"node_list", map[string]any{"limit": 1}}, + } { + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: tc.tool, Arguments: tc.args}) + require.NoError(t, err) + require.False(t, res.IsError, "%s: %s", tc.tool, extractText(t, res)) + require.NotNil(t, res.StructuredContent, tc.tool) + structured, err := json.Marshal(res.StructuredContent) + require.NoError(t, err) + require.JSONEq(t, string(structured), res.Content[0].(*sdkmcp.TextContent).Text, tc.tool) + } +} diff --git a/pkg/mcp/precondition_read_test.go b/pkg/mcp/precondition_read_test.go index d1cbdda9..68e5da74 100644 --- a/pkg/mcp/precondition_read_test.go +++ b/pkg/mcp/precondition_read_test.go @@ -3,6 +3,7 @@ package mcp_test import ( "context" "encoding/json" + "fmt" "strings" "testing" @@ -289,8 +290,13 @@ func TestPrecondition_RemoveBatchUsesDistinctTokensAtomically(t *testing.T) { } } +// structuredMap returns a result's public payload: a success's structured +// content, or a failure's JSON text, the only place a failure carries it. func structuredMap(t *testing.T, result *sdkmcp.CallToolResult) map[string]any { t.Helper() + if result.IsError { + return errorPayload(t, result) + } require.NotNil(t, result.StructuredContent) raw, err := json.Marshal(result.StructuredContent) require.NoError(t, err) @@ -555,6 +561,79 @@ func TestPrecondition_NodeEditRequiresMatchingPartHashes(t *testing.T) { require.Contains(t, conflict["currentContent"], "first") } +// TestPrecondition_NodeEditMissingHashNamesTheRead pins the recovery for an +// edit sent without its tokens: the action is a copyable node_read for this +// keg and node, and every missing token is reported in one error so a single +// read fixes the call. +func TestPrecondition_NodeEditMissingHashNamesTheRead(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + + nodeID := createNodeForTest(t, session, ctx, "# Guarded\n\nBody.\n", "tags:\n - guarded\n") + + patchOnly, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: "node_edit", + Arguments: map[string]any{ + "keg": "personal", + "nodes": []any{map[string]any{"node_id": nodeID, "patch": "@@ -1 +1 @@\n-# Guarded\n+# Patched\n"}}, + }, + }) + require.NoError(t, err) + require.True(t, patchOnly.IsError) + structured := structuredMap(t, patchOnly) + require.Equal(t, "PRECONDITION_REQUIRED", structured["code"]) + require.Equal(t, false, structured["operationPerformed"]) + require.Equal(t, nodeID, structured["node_id"]) + require.Equal(t, []any{"expected_content_hash"}, structured["missing"]) + action, _ := structured["action"].(string) + require.Contains(t, action, fmt.Sprintf(`node_read(keg="personal", node_id=%q)`, nodeID)) + require.Contains(t, action, "content_hash as expected_content_hash") + require.Contains(t, extractText(t, patchOnly), action, "the action is appended to the message") + + both, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: "node_edit", + Arguments: batchEditArgs(map[string]any{ + "node_id": nodeID, + "content": "# Guarded\n\nNew body.\n", + "meta": "tags:\n - changed\n", + }), + }) + require.NoError(t, err) + require.True(t, both.IsError) + structured = structuredMap(t, both) + require.Equal(t, []any{"expected_content_hash", "expected_meta_hash"}, structured["missing"]) + action, _ = structured["action"].(string) + require.Contains(t, action, fmt.Sprintf("node_read(node_id=%q)", nodeID)) + require.Contains(t, action, "content_hash as expected_content_hash and meta_hash as expected_meta_hash") + text := extractText(t, both) + require.Contains(t, text, "content requires expected_content_hash") + require.Contains(t, text, "meta requires expected_meta_hash") +} + +// TestPrecondition_WholeNodeMissingHashNamesFieldAndRead covers the tools +// that take the combined hash: their refusals name the field and node_read. +func TestPrecondition_WholeNodeMissingHashNamesFieldAndRead(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + + nodeID := createNodeForTest(t, session, ctx, "# Doomed\n", "") + for _, tc := range []struct { + tool string + args map[string]any + }{ + {"node_delete", map[string]any{"nodes": []any{map[string]any{"node_id": nodeID, "expected_hash": ""}}}}, + {"node_move", map[string]any{"source_id": nodeID, "dest_id": "900", "expected_hash": ""}}, + } { + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: tc.tool, Arguments: tc.args}) + require.NoError(t, err) + require.True(t, res.IsError, tc.tool) + require.Equal(t, "PRECONDITION_REQUIRED", structuredMap(t, res)["code"], tc.tool) + text := extractText(t, res) + require.Contains(t, text, "expected_hash is required", tc.tool) + require.Contains(t, text, "node_read", tc.tool) + } +} + // TestCat_TextIsSummaryWhenRowsCarryDocuments pins that node_read sends each // document once: in nodes[], not again in the text message. func TestCat_TextIsSummaryWhenRowsCarryDocuments(t *testing.T) { diff --git a/pkg/mcp/precondition_schema_test.go b/pkg/mcp/precondition_schema_test.go index 620d8138..1e75b731 100644 --- a/pkg/mcp/precondition_schema_test.go +++ b/pkg/mcp/precondition_schema_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/jlrickert/tapper/pkg/keg" ) func listedToolSchemas(t *testing.T) map[string]map[string]any { @@ -134,12 +136,13 @@ func TestMCP_MutationDescriptionsTeachReadMergeRetryProtocol(t *testing.T) { } } -// TestMCP_AuthorityBearingToolsDescribeFlight pins that the injected flight -// property is also documented. schemaWithFlight adds `flight` to every -// authority-bearing tool's schema, but an agent reading descriptions rather -// than raw schemas saw no mention of it and reported the parameter as missing -// from keg_create. A property nothing describes reads as absent. -func TestMCP_AuthorityBearingToolsDescribeFlight(t *testing.T) { +// TestMCP_FlightIsExplainedOnceOnTheParameter pins where flight selection is +// documented. schemaWithFlight adds `flight` to every authority-bearing tool; +// an injected property nothing describes reads as absent (an agent once +// reported it missing from keg_create), so the property carries its own +// description. Tool descriptions do not repeat it: the same sentence on every +// tool buried each tool's own text, and the operating rules explain it once. +func TestMCP_FlightIsExplainedOnceOnTheParameter(t *testing.T) { t.Parallel() session, ctx := newTestSession(t) result, err := session.ListTools(ctx, nil) @@ -147,20 +150,31 @@ func TestMCP_AuthorityBearingToolsDescribeFlight(t *testing.T) { checked := 0 for _, tool := range result.Tools { + require.NotEmptyf(t, strings.TrimSpace(tool.Description), "tool %q has no description", tool.Name) + require.NotContainsf(t, tool.Description, "Accepts an optional flight", + "tool %q repeats the flight explanation its parameter carries", tool.Name) schema, err := json.Marshal(tool.InputSchema) require.NoError(t, err) var object struct { - Properties map[string]json.RawMessage `json:"properties"` + Properties map[string]struct { + Description string `json:"description"` + } `json:"properties"` } require.NoError(t, json.Unmarshal(schema, &object)) - if _, ok := object.Properties["flight"]; !ok { + flight, ok := object.Properties["flight"] + if !ok { continue } checked++ - require.Containsf(t, strings.ToLower(tool.Description), "flight", - "tool %q accepts a flight property but never mentions it in its description", tool.Name) + require.Containsf(t, flight.Description, "Omit it", + "tool %q: the flight property must say what omitting it does", tool.Name) + require.NotContainsf(t, tool.Description, flight.Description, + "tool %q repeats its flight property's description", tool.Name) } require.Greater(t, checked, 0, "no tool exposed a flight property; the injection may have broken") + + instructions := session.InitializeResult().Instructions + require.Contains(t, instructions, "`flight`", "the operating rules must explain the flight argument once") } // TestMCP_WriteToolsStateTheContentContract keeps the two rules an agent @@ -189,3 +203,27 @@ func TestMCP_WriteToolsStateTheContentContract(t *testing.T) { require.Contains(t, byName["node_read"], "content_hash") require.Contains(t, byName["node_read"], "meta_hash") } + +// TestMCP_SchemaSelectionFieldsAreConstrained pins the schema field's rule in +// the advertised input schemas: the same pattern and length bound +// keg.ValidSchemaTypeName enforces, so a client or the SDK refuses a sentence +// before any backend sees it. The guard tokens stay optional. +func TestMCP_SchemaSelectionFieldsAreConstrained(t *testing.T) { + t.Parallel() + schemas := listedToolSchemas(t) + + fields := map[string]map[string]any{ + "node_edit": schemaProperty(t, schemas["node_edit"], schemaArrayItem(t, schemas["node_edit"], "nodes"), "schema"), + "node_create": schemaProperty(t, schemas["node_create"], schemaArrayItem(t, schemas["node_create"], "nodes"), "schema"), + "node_merge": schemaProperty(t, schemas["node_merge"], schemas["node_merge"], "schema"), + } + for tool, field := range fields { + require.Equal(t, keg.SchemaTypeNamePattern, field["pattern"], tool) + require.EqualValues(t, keg.SchemaTypeNameMaxLength, field["maxLength"], tool) + } + + editItem := schemaArrayItem(t, schemas["node_edit"], "nodes") + requireSchemaField(t, editItem, "schema", false) + requireSchemaField(t, editItem, "expected_content_hash", false) + requireSchemaField(t, editItem, "expected_meta_hash", false) +} diff --git a/pkg/mcp/providers.go b/pkg/mcp/providers.go index 66402840..cbeba474 100644 --- a/pkg/mcp/providers.go +++ b/pkg/mcp/providers.go @@ -9,6 +9,7 @@ import ( "net/http" "sort" "strings" + "sync" "github.com/jlrickert/tapper/pkg/apicontract" "github.com/jlrickert/tapper/pkg/keg" @@ -235,11 +236,13 @@ func (p *localOrientationProvider) Load(ctx context.Context) (*Orientation, erro if strings.TrimSpace(ref) == "" { return p.resolveUnpinned(ctx, "") } + pre := p.prefetchOrientation(ctx, true) + defer pre.wait() flight, err := p.tap.FlightService.GetFlightFresh(ctx, ref) if err != nil { return nil, err } - return p.resolve(ctx, flight, "") + return p.resolve(ctx, flight, "", pre) } func (p *localOrientationProvider) Resolve(ctx context.Context, rootRef, selected string) (*Orientation, error) { @@ -263,6 +266,8 @@ func (p *localOrientationProvider) Resolve(ctx context.Context, rootRef, selecte if strings.TrimSpace(rootRef) == "" { return p.resolveUnpinned(ctx, selected) } + pre := p.prefetchOrientation(ctx, true) + defer pre.wait() root, err := p.tap.FlightService.GetFlightFresh(ctx, rootRef) if err != nil { if errors.Is(err, keg.ErrNotExist) || errors.Is(err, keg.ErrForbidden) || errors.Is(err, keg.ErrUnauthorized) { @@ -270,10 +275,28 @@ func (p *localOrientationProvider) Resolve(ctx context.Context, rootRef, selecte } return nil, fmt.Errorf("%w: refresh launch root %q: %w", ErrOrientationUnavailable, rootRef, err) } - return p.resolve(ctx, root, selected) + return p.resolve(ctx, root, selected, pre) } func (p *localOrientationProvider) resolveUnpinned(ctx context.Context, selected string) (*Orientation, error) { + // The flight catalog, the KEG catalog, the selected flight, and the Hub + // identity are independent, so they load together. + pinned := strings.TrimSpace(selected) != "" + pre := p.prefetchOrientation(ctx, pinned) + defer pre.wait() + var ( + active *tapper.Flight + loadErr error + loaded sync.WaitGroup + ) + if pinned { + loaded.Add(1) + go func() { + defer loaded.Done() + active, loadErr = p.tap.FlightService.GetFlightFresh(ctx, selected) + }() + } + defer loaded.Wait() var warnings []string catalog, err := p.tap.FlightService.FlightCatalog(ctx, "", &warnings) available := make([]string, 0, len(catalog)) @@ -283,7 +306,7 @@ func (p *localOrientationProvider) resolveUnpinned(ctx context.Context, selected if err != nil { return nil, fmt.Errorf("%w: list identity-accessible flights: %w", ErrOrientationUnavailable, err) } - authorized, kegWarnings, err := p.tap.IdentityKegCatalog(ctx) + authorized, kegWarnings, err := pre.catalog() if err != nil { return nil, err } @@ -297,7 +320,7 @@ func (p *localOrientationProvider) resolveUnpinned(ctx context.Context, selected if err := FinalizeOrientation(orientation); err != nil { return nil, err } - authority := &tapper.OrientationAuthority{FullAccess: true, AvailableFlights: available, Revision: orientation.Revision} + authority := &tapper.OrientationAuthority{FullAccess: true, AvailableFlights: available, Revision: orientation.Revision, Agent: p.tap.ActiveAgent()} payload, err := tapper.BuildOrientationPayload(nil, localUnpinnedInstructions(warnings), p.tap.ActiveLaunch(), authorized, warnings, authority) if err != nil { return nil, err @@ -305,7 +328,7 @@ func (p *localOrientationProvider) resolveUnpinned(ctx context.Context, selected orientation.Payload = payload return orientation, nil } - active, loadErr := p.tap.FlightService.GetFlightFresh(ctx, selected) + loaded.Wait() if loadErr != nil { return nil, fmt.Errorf("%w: selected flight %q is unavailable: %w", ErrOrientationDenied, selected, loadErr) } @@ -326,7 +349,7 @@ func (p *localOrientationProvider) resolveUnpinned(ctx context.Context, selected Kegs: kegs, AggregateKegs: append([]tapper.OrientationKeg(nil), kegs...), Warnings: warnings, } if orientation.Root != nil { - if err := p.bindRootAuthority(ctx, orientation); err != nil { + if err := p.bindRootAuthority(ctx, orientation, pre); err != nil { return nil, err } } @@ -337,6 +360,7 @@ func (p *localOrientationProvider) resolveUnpinned(ctx context.Context, selected Active: active, Path: orientation.Path, AvailableFlights: available, Revision: orientation.Revision, Children: tapper.ImmediateFlightChildren(active, catalog), FullAccess: true, + Agent: p.tap.ActiveAgent(), } payload, err := tapper.BuildOrientationPayload(active, "", p.tap.ActiveLaunch(), kegs, warnings, authority) if err != nil { @@ -372,7 +396,10 @@ func (p *localOrientationProvider) orientationIdentityForSource(ctx context.Cont return "", fmt.Errorf("%w: authenticated identity for selected Hub %q is unavailable", ErrOrientationUnavailable, source) } -func (p *localOrientationProvider) resolve(ctx context.Context, root *tapper.Flight, selected string) (*Orientation, error) { +// resolve loads the flight graph rooted at root and selects from it. pre +// carries the KEG catalog and Hub identity, already loading alongside the +// graph. +func (p *localOrientationProvider) resolve(ctx context.Context, root *tapper.Flight, selected string, pre *orientationPrefetch) (*Orientation, error) { graph, err := p.tap.FlightService.ResolveFlightGraph(ctx, root) if err != nil { return nil, fmt.Errorf("%w: load flight graph rooted at %s: %w", ErrOrientationUnavailable, root.Name, err) @@ -387,7 +414,7 @@ func (p *localOrientationProvider) resolve(ctx context.Context, root *tapper.Fli return nil, fmt.Errorf("%w: load active composed cover: %w", ErrOrientationUnavailable, err) } active = tapper.ComposedFlight(active) - authorized, warnings, err := p.tap.IdentityKegCatalog(ctx) + authorized, warnings, err := pre.catalog() if err != nil { return nil, err } @@ -407,7 +434,7 @@ func (p *localOrientationProvider) resolve(ctx context.Context, root *tapper.Fli Kegs: kegs, AggregateKegs: aggregate, Warnings: warnings, } if orientation.Root != nil { - if err := p.bindRootAuthority(ctx, orientation); err != nil { + if err := p.bindRootAuthority(ctx, orientation, pre); err != nil { return nil, err } } @@ -418,6 +445,7 @@ func (p *localOrientationProvider) resolve(ctx context.Context, root *tapper.Fli Root: root, Active: active, Path: path, AvailableFlights: orientation.AvailableFlights, Children: tapper.ImmediateFlightChildren(active, graphFlights(graph)), Revision: orientation.Revision, + Agent: p.tap.ActiveAgent(), } discovery := kegs if strings.TrimSpace(selected) == "" { @@ -432,14 +460,16 @@ func (p *localOrientationProvider) resolve(ctx context.Context, root *tapper.Fli } func (p *localOrientationProvider) Render(ctx context.Context, flight *tapper.Flight) (*Orientation, error) { - authorized, warnings, err := p.tap.IdentityKegCatalog(ctx) + pre := p.prefetchOrientation(ctx, flight != nil) + defer pre.wait() + authorized, warnings, err := pre.catalog() if err != nil { return nil, err } kegs := tapper.ProjectOrientationKegs(flight, authorized) orientation := &Orientation{Root: flight, Flight: flight, Path: []string{flight.Name}, Kegs: kegs, AggregateKegs: append([]tapper.OrientationKeg(nil), kegs...), Warnings: warnings} if orientation.Root != nil { - if err := p.bindRootAuthority(ctx, orientation); err != nil { + if err := p.bindRootAuthority(ctx, orientation, pre); err != nil { return nil, err } } @@ -448,6 +478,7 @@ func (p *localOrientationProvider) Render(ctx context.Context, flight *tapper.Fl } authority := &tapper.OrientationAuthority{ Root: flight, Active: flight, Path: orientation.Path, Revision: orientation.Revision, + Agent: p.tap.ActiveAgent(), } payload, err := tapper.BuildOrientationPayload(flight, "", p.tap.ActiveLaunch(), kegs, warnings, authority) if err != nil { @@ -690,14 +721,16 @@ func (p localIdentityProvider) Identities(ctx context.Context) ([]AuthIdentity, return out, nil } -// bindRootAuthority binds the live proof to this connection's selected Hub. -func (p *localOrientationProvider) bindRootAuthority(ctx context.Context, orientation *Orientation) error { - _, entry, err := p.tap.ConfigService.SelectedHub("") - if err != nil { - return err +// bindRootAuthority binds the live proof to this connection's selected Hub, +// using the identity pre already loaded when it has one. +func (p *localOrientationProvider) bindRootAuthority(ctx context.Context, orientation *Orientation, pre *orientationPrefetch) error { + var err error + if pre != nil && pre.withIdentity { + pre.wait() + orientation.RootHub, orientation.Identity, err = pre.rootHub, pre.identity, pre.identityErr + } else { + orientation.RootHub, orientation.Identity, err = p.rootIdentity(ctx) } - orientation.RootHub = tapper.CanonicalConfiguredHubURL(entry.URL) - orientation.Identity, err = p.orientationIdentityForSource(ctx, orientation.RootHub) if err != nil { return err } @@ -708,12 +741,79 @@ func (p *localOrientationProvider) bindRootAuthority(ctx context.Context, orient return nil } +// rootIdentity resolves the selected Hub and the canonical identity +// authenticated to it. +func (p *localOrientationProvider) rootIdentity(ctx context.Context) (rootHub, identity string, err error) { + _, entry, err := p.tap.ConfigService.SelectedHub("") + if err != nil { + return "", "", err + } + rootHub = tapper.CanonicalConfiguredHubURL(entry.URL) + identity, err = p.orientationIdentityForSource(ctx, rootHub) + return rootHub, identity, err +} + +// orientationPrefetch loads the parts of an orientation that do not depend on +// the flight graph, the identity KEG catalog and the selected Hub identity, +// while the caller loads the graph. Callers read each result where the +// sequential code made the request, so errors surface in the same order. +type orientationPrefetch struct { + wg sync.WaitGroup + withIdentity bool + + authorized []tapper.OrientationKeg + warnings []string + catalogErr error + + rootHub string + identity string + identityErr error +} + +// prefetchOrientation starts the KEG catalog and, when withIdentity, the Hub +// identity. Callers defer wait so no request outlives the orientation. +func (p *localOrientationProvider) prefetchOrientation(ctx context.Context, withIdentity bool) *orientationPrefetch { + pre := &orientationPrefetch{withIdentity: withIdentity} + pre.wg.Add(1) + go func() { + defer pre.wg.Done() + pre.authorized, pre.warnings, pre.catalogErr = p.tap.IdentityKegCatalog(ctx) + }() + if withIdentity { + pre.wg.Add(1) + go func() { + defer pre.wg.Done() + pre.rootHub, pre.identity, pre.identityErr = p.rootIdentity(ctx) + }() + } + return pre +} + +func (pre *orientationPrefetch) wait() { pre.wg.Wait() } + +// catalog returns the identity KEG catalog once it has loaded. +func (pre *orientationPrefetch) catalog() ([]tapper.OrientationKeg, []string, error) { + pre.wait() + return pre.authorized, pre.warnings, pre.catalogErr +} + // KegDeletionProvider permanently removes catalog KEGs. type KegDeletionProvider interface { // DeleteKeg removes the explicit canonical KEG and all its data, including snapshots. DeleteKeg(context.Context, string) error } +// KegDeletionAuthorizer is a KegDeletionProvider that authorizes a delete +// with the session's full authority itself, as a Hub does: that authority may +// reach past the session flight's cover (a flight managing the namespace's +// KEGs), so keg_delete leaves the decision to it instead of pre-checking the +// cover. +type KegDeletionAuthorizer interface { + // AuthorizesKegDeletion reports whether DeleteKeg authorizes the delete + // itself, so keg_delete skips its cover pre-check. + AuthorizesKegDeletion() bool +} + func (p localKegDiscoveryProvider) DeleteKeg(ctx context.Context, ref string) error { _, err := p.tap.KegDelete(ctx, tapper.KegDeleteOptions{Keg: ref}) return err diff --git a/pkg/mcp/public_response.go b/pkg/mcp/public_response.go index 74b0f0b0..b42efb2c 100644 --- a/pkg/mcp/public_response.go +++ b/pkg/mcp/public_response.go @@ -86,16 +86,71 @@ func publicToolResponse(call *sdkmcp.CallToolResult) (*sdkmcp.CallToolResult, er // A nonempty bounded page supplies a safe next request; an empty page ends it. func pagedLinesResult(lines []string, offset, limit int) *sdkmcp.CallToolResult { result := linesResult(lines) - var next any - var more any = false - if limit > 0 && len(lines) > 0 { - next = offset + limit - more = nil + out := pagedLinesOutput{ + Lines: append([]string{}, lines...), Offset: offset, Limit: limit, + HasMore: new(bool), + Pagination: "Repeat the same query with next_offset as offset until next_offset is null. A final nonempty page may be followed by an empty page. Pages are live, not a stable snapshot; reverse reverses each page only.", } - result.StructuredContent = map[string]any{ - "lines": append([]string{}, lines...), "offset": offset, "limit": limit, - "next_offset": next, "has_more": more, - "pagination": "Repeat the same query with next_offset as offset until next_offset is null. A final nonempty page may be followed by an empty page. Pages are live, not a stable snapshot; reverse reverses each page only.", + if limit > 0 && len(lines) > 0 { + next := offset + limit + out.NextOffset, out.HasMore = &next, nil } + result.StructuredContent = out return result } + +// pagedLinesOutput is the structured content of the offset-paged listing +// tools: node_list, node_grep, tag_list, node_backlinks, and node_links. +type pagedLinesOutput struct { + Lines []string `json:"lines" jsonschema:"The page's lines, one per node rendered by format. tag_list without a query lists tags; node_grep without format follows each node with its matched lines."` + Offset int `json:"offset" jsonschema:"The offset this page starts at."` + Limit int `json:"limit" jsonschema:"The page size applied; 0 means unlimited."` + NextOffset *int `json:"next_offset" jsonschema:"The offset of the next page, or null when no page follows."` + HasMore *bool `json:"has_more" jsonschema:"False when no page follows; null when one may."` + Pagination string `json:"pagination" jsonschema:"How to page this listing."` +} + +// errorResultMiddleware sends a failed tool result without structured +// content. A tool's outputSchema describes its success, and clients that +// validate results (the TypeScript SDK among them) check structuredContent +// against it even when isError is set, so an error payload there turns a +// readable tool error into a JSON-RPC -32602. The payload travels as the +// result's leading JSON text block instead, which publicToolResponse +// already renders; a relayed failure that carries structured content but no +// JSON text gets one rendered from it. +func errorResultMiddleware(next sdkmcp.MethodHandler) sdkmcp.MethodHandler { + return func(ctx context.Context, method string, req sdkmcp.Request) (sdkmcp.Result, error) { + result, err := next(ctx, method, req) + if err != nil || method != "tools/call" { + return result, err + } + call, ok := result.(*sdkmcp.CallToolResult) + if !ok || call == nil || !call.IsError || call.StructuredContent == nil { + return result, nil + } + out := *call + if !leadsWithJSONObject(call.Content) { + raw, err := json.Marshal(call.StructuredContent) + if err != nil { + return nil, fmt.Errorf("render MCP error payload: %w", err) + } + out.Content = append([]sdkmcp.Content{&sdkmcp.TextContent{Text: string(raw)}}, call.Content...) + } + out.StructuredContent = nil + return &out, nil + } +} + +// leadsWithJSONObject reports whether content opens with a text block that +// holds a JSON object. +func leadsWithJSONObject(content []sdkmcp.Content) bool { + if len(content) == 0 { + return false + } + text, ok := content[0].(*sdkmcp.TextContent) + if !ok { + return false + } + trimmed := strings.TrimSpace(text.Text) + return strings.HasPrefix(trimmed, "{") && json.Valid([]byte(trimmed)) +} diff --git a/pkg/mcp/public_response_test.go b/pkg/mcp/public_response_test.go index ecb10fec..a7242cf6 100644 --- a/pkg/mcp/public_response_test.go +++ b/pkg/mcp/public_response_test.go @@ -15,10 +15,15 @@ import ( // These clients deliberately consume one wire representation only. Unlike // legacy behavior helpers, text decoding never consults StructuredContent. +// A failure has no structured content (its outputSchema describes success), +// so a structured client reads a failure's payload from text like any other. func wirePayload(t *testing.T, result *sdkmcp.CallToolResult, mode string) map[string]any { t.Helper() var raw []byte - if mode == "text" { + if result.IsError { + require.Nil(t, result.StructuredContent, "a failure carries no structured content") + } + if mode == "text" || result.IsError { for _, block := range result.Content { if text, ok := block.(*sdkmcp.TextContent); ok { raw = append(raw, text.Text...) @@ -85,7 +90,7 @@ func TestPublicContractIndependentNodeWorkflow(t *testing.T) { seen := []string{} offset := any(0) for page := 0; page < 10; page++ { - result := call("node_list", map[string]any{"query": "wire-contract", "id_only": true, "limit": 1, "offset": offset}, false) + result := call("node_list", map[string]any{"query": "wire-contract", "format": "%i", "limit": 1, "offset": offset}, false) for _, line := range result["lines"].([]any) { seen = append(seen, line.(string)) } diff --git a/pkg/mcp/relay_proxy_test.go b/pkg/mcp/relay_proxy_test.go index 2630437e..cf1adc66 100644 --- a/pkg/mcp/relay_proxy_test.go +++ b/pkg/mcp/relay_proxy_test.go @@ -139,3 +139,32 @@ func TestToolAllowlist_AdmitsRelayedToolsUnlessRefused(t *testing.T) { refused := &mcp.ToolAllowlist{Agent: "@alice/agent", Refused: "agent could not be loaded"} require.False(t, refused.Allows(relayEcho), "a failed agent load fails closed for relayed tools too") } + +// A relayed failure follows the same rule as a KEG tool's: no structured +// content, with any payload it carried kept as leading JSON text beside the +// relayed server's own text. +func TestRelayProxy_FailureCarriesPayloadAsText(t *testing.T) { + t.Parallel() + hub, remote, _ := fakeRelayHub(t) + const relayFail = "mcp__alice__everything__fail" + remote.AddTool(&sdkmcp.Tool{ + Name: relayFail, + InputSchema: map[string]any{"type": "object"}, + }, func(context.Context, *sdkmcp.CallToolRequest) (*sdkmcp.CallToolResult, error) { + return &sdkmcp.CallToolResult{ + IsError: true, + Content: []sdkmcp.Content{&sdkmcp.TextContent{Text: "relay: failed on purpose"}}, + StructuredContent: map[string]any{"code": "RELAY_FAILED"}, + }, nil + }) + session, ctx := relayLocalSession(t, hub.URL) + require.Eventually(t, func() bool { return slices.Contains(relayToolNames(t, ctx, session), relayFail) }, 5*time.Second, 20*time.Millisecond) + + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: relayFail, Arguments: map[string]any{}}) + require.NoError(t, err) + require.True(t, res.IsError) + require.Nil(t, res.StructuredContent) + require.Equal(t, "RELAY_FAILED", errorPayload(t, res)["code"]) + require.Len(t, res.Content, 2) + require.Equal(t, "relay: failed on purpose", res.Content[1].(*sdkmcp.TextContent).Text, "the relayed server's text is kept") +} diff --git a/pkg/mcp/repeat_failure.go b/pkg/mcp/repeat_failure.go new file mode 100644 index 00000000..82c5d4ac --- /dev/null +++ b/pkg/mcp/repeat_failure.go @@ -0,0 +1,188 @@ +package mcp + +import ( + "bytes" + "container/list" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "sync" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" +) + +const ( + // repeatFailureThreshold is the identical failure that earns the stop + // action. Two tries is an ordinary retry; a third is a loop. + repeatFailureThreshold = 3 + // repeatFailureKeysPerSession bounds the failing calls remembered for one + // session. Only the latest arguments per tool are kept, so this is also + // the number of distinct tools tracked at once. + repeatFailureKeysPerSession = 32 + // repeatFailureSessions bounds the sessions tracked at once. The least + // recently active session is forgotten first, so a long-lived Hub process + // never grows this without limit. + repeatFailureSessions = 256 +) + +// repeatFailureTracker flags a tool call that keeps failing with the same +// arguments. An agent that misreads an error tends to resend the identical +// call; the third identical failure keeps its original code and gains +// repeat_count and an action telling the agent to stop and read instead. +// +// A call is identified by session, tool, and a digest of its canonicalized +// arguments. A success, or a call with different arguments, resets that +// tool's count. +type repeatFailureTracker struct { + mu sync.Mutex + sessions map[string]*list.Element // of *repeatFailureSession + lru *list.List // most recently active session first +} + +type repeatFailureSession struct { + id string + tools map[string]*list.Element // tool name -> element of keys + keys *list.List // *repeatFailureEntry, most recent first +} + +type repeatFailureEntry struct { + tool string + digest [sha256.Size]byte + count int +} + +func newRepeatFailureTracker() *repeatFailureTracker { + return &repeatFailureTracker{sessions: map[string]*list.Element{}, lru: list.New()} +} + +// middleware must wrap publicResponseMiddleware, so that it sees SDK input +// validation failures in their public form, with their code already set. +// +// The digest is taken before the call runs. Inner layers rewrite the +// arguments in place (the flight gate strips `flight`, legacy names are +// renamed), and a digest of the rewritten form would count calls under +// different flights as one repeated call. +func (r *repeatFailureTracker) middleware(next sdkmcp.MethodHandler) sdkmcp.MethodHandler { + return func(ctx context.Context, method string, req sdkmcp.Request) (sdkmcp.Result, error) { + if method != "tools/call" { + return next(ctx, method, req) + } + params, _ := req.GetParams().(*sdkmcp.CallToolParamsRaw) + if params == nil || IsRelayedToolName(params.Name) { + return next(ctx, method, req) + } + tool, digest := params.Name, canonicalArgumentsDigest(params.Arguments) + result, err := next(ctx, method, req) + if err != nil { + return result, err + } + call, ok := result.(*sdkmcp.CallToolResult) + if !ok || call == nil { + return result, nil + } + sessionID := "" + if sess, ok := req.GetSession().(*sdkmcp.ServerSession); ok && sess != nil { + sessionID = sess.ID() + } + count := r.record(sessionID, tool, digest, call.IsError) + if count < repeatFailureThreshold { + return result, nil + } + return withRepeatAction(call, count), nil + } +} + +// record notes one call's outcome and returns how many times in a row this +// session has failed the same call, or 0 after a success. +func (r *repeatFailureTracker) record(sessionID, tool string, digest [sha256.Size]byte, failed bool) int { + r.mu.Lock() + defer r.mu.Unlock() + sess := r.session(sessionID, failed) + if sess == nil { + return 0 + } + if el, ok := sess.tools[tool]; ok { + entry := el.Value.(*repeatFailureEntry) + if failed && entry.digest == digest { + entry.count++ + sess.keys.MoveToFront(el) + return entry.count + } + sess.keys.Remove(el) + delete(sess.tools, tool) + } + if !failed { + return 0 + } + sess.tools[tool] = sess.keys.PushFront(&repeatFailureEntry{tool: tool, digest: digest, count: 1}) + for sess.keys.Len() > repeatFailureKeysPerSession { + oldest := sess.keys.Back() + delete(sess.tools, oldest.Value.(*repeatFailureEntry).tool) + sess.keys.Remove(oldest) + } + return 1 +} + +// session returns the state for id, marking it most recently active. A +// session with nothing recorded is only created for a failure, since a +// success has nothing to reset. +func (r *repeatFailureTracker) session(id string, create bool) *repeatFailureSession { + if el, ok := r.sessions[id]; ok { + r.lru.MoveToFront(el) + return el.Value.(*repeatFailureSession) + } + if !create { + return nil + } + sess := &repeatFailureSession{id: id, tools: map[string]*list.Element{}, keys: list.New()} + r.sessions[id] = r.lru.PushFront(sess) + for r.lru.Len() > repeatFailureSessions { + oldest := r.lru.Back() + delete(r.sessions, oldest.Value.(*repeatFailureSession).id) + r.lru.Remove(oldest) + } + return sess +} + +// canonicalArgumentsDigest hashes arguments independent of key order and +// whitespace, so a resent call matches however the client serialized it. +// Arguments that do not decode are hashed as sent. +func canonicalArgumentsDigest(raw json.RawMessage) [sha256.Size]byte { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err == nil { + if canonical, err := json.Marshal(value); err == nil { + return sha256.Sum256(canonical) + } + } + return sha256.Sum256(raw) +} + +// withRepeatAction adds the repeat guidance to a failure's public payload, +// keeping its code and its original action after the stop instruction. +func withRepeatAction(call *sdkmcp.CallToolResult, count int) sdkmcp.Result { + payload, ok := call.StructuredContent.(map[string]any) + if !ok { + return call + } + out := make(map[string]any, len(payload)+1) + for key, value := range payload { + out[key] = value + } + action := fmt.Sprintf("This identical call has failed %d times; stop retrying it. Read the target (node_read) or change the arguments named in the error.", count) + if original, _ := payload["action"].(string); original != "" { + action += " " + original + } + out["action"] = action + out["repeat_count"] = count + raw, err := json.Marshal(out) + if err != nil { + return call + } + rendered := *call + rendered.StructuredContent = out + rendered.Content = append([]sdkmcp.Content{&sdkmcp.TextContent{Text: string(raw)}}, call.Content[min(1, len(call.Content)):]...) + return &rendered +} diff --git a/pkg/mcp/repeat_failure_internal_test.go b/pkg/mcp/repeat_failure_internal_test.go new file mode 100644 index 00000000..47ee16e7 --- /dev/null +++ b/pkg/mcp/repeat_failure_internal_test.go @@ -0,0 +1,48 @@ +package mcp + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRepeatFailureTrackerStaysBounded(t *testing.T) { + t.Parallel() + tracker := newRepeatFailureTracker() + digest := canonicalArgumentsDigest(json.RawMessage(`{"a":1}`)) + + for i := range repeatFailureSessions + 10 { + tracker.record(fmt.Sprintf("s%d", i), "node_edit", digest, true) + } + require.Len(t, tracker.sessions, repeatFailureSessions) + require.Equal(t, repeatFailureSessions, tracker.lru.Len()) + require.NotContains(t, tracker.sessions, "s0", "the least recently active session is forgotten first") + + for i := range repeatFailureKeysPerSession + 10 { + tracker.record("busy", fmt.Sprintf("tool%d", i), digest, true) + } + busy := tracker.sessions["busy"].Value.(*repeatFailureSession) + require.Len(t, busy.tools, repeatFailureKeysPerSession) + require.Equal(t, repeatFailureKeysPerSession, busy.keys.Len()) + require.NotContains(t, busy.tools, "tool0") +} + +func TestRepeatFailureTrackerCountsCanonicalArguments(t *testing.T) { + t.Parallel() + tracker := newRepeatFailureTracker() + a := canonicalArgumentsDigest(json.RawMessage(`{"keg":"k","nodes":[{"node_id":"1"}]}`)) + b := canonicalArgumentsDigest(json.RawMessage(`{ "nodes" : [ {"node_id":"1"} ], "keg":"k" }`)) + require.Equal(t, a, b, "key order and whitespace do not make a different call") + + require.Equal(t, 1, tracker.record("s", "node_edit", a, true)) + require.Equal(t, 2, tracker.record("s", "node_edit", b, true)) + require.Equal(t, 1, tracker.record("s", "node_delete", a, true), "another tool counts separately") + require.Equal(t, 3, tracker.record("s", "node_edit", a, true)) + require.Equal(t, 1, tracker.record("other", "node_edit", a, true), "another session counts separately") + require.Equal(t, 0, tracker.record("s", "node_edit", a, false)) + require.Equal(t, 1, tracker.record("s", "node_edit", a, true), "a success resets the count") + require.Equal(t, 0, tracker.record("fresh", "node_read", a, false)) + require.NotContains(t, tracker.sessions, "fresh", "a success alone creates no state") +} diff --git a/pkg/mcp/server.go b/pkg/mcp/server.go index d4f261b5..71bbeb94 100644 --- a/pkg/mcp/server.go +++ b/pkg/mcp/server.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "math" "strings" "time" @@ -59,6 +60,10 @@ type ServerOptions struct { // orient, session_refresh, flight mutations, and any failed call always // discard the cached view. OrientationCacheTTL time.Duration + // DiscoveryCache, when set, persists each session's Hub API discovery + // successes (see apicontract.Session.Cache). `tap mcp` keeps them on + // disk; hosted servers leave it nil. + DiscoveryCache apicontract.DiscoveryCache } // NewServer builds an MCP server with all registered tools. @@ -97,6 +102,10 @@ func NewServer(tap *tapper.Tap, version string, defaults KegDefaults, opts ...Se defaults.gate.cacheTTL = opt.OrientationCacheTTL defaults.gate.now = tap.Runtime.Clock().Now } + defaults.gate.discoveryCache = opt.DiscoveryCache + if tap != nil && tap.Runtime != nil { + defaults.gate.discoveryNow = tap.Runtime.Clock().Now + } defaults.gate.logger = opt.Logger if defaults.gate.logger == nil && tap != nil && tap.Runtime != nil { defaults.gate.logger = tap.Runtime.Logger() @@ -120,6 +129,9 @@ func NewServer(tap *tapper.Tap, version string, defaults KegDefaults, opts ...Se defaults.gate.srv = srv srv.AddReceivingMiddleware(defaults.gate.middleware) } + // Wraps the gate, so old argument names are rewritten before anything + // reads the call; see legacy_arguments.go. + srv.AddReceivingMiddleware(legacyArgumentsMiddleware) // Node read/write tools. These all funnel // through Tap.resolveKegForRole, so a hub-injected KegResolver scopes them @@ -146,6 +158,12 @@ func NewServer(tap *tapper.Tap, version string, defaults KegDefaults, opts ...Se // Wrap the gate as well as handlers: recovery and SDK validation errors // obey the same public response contract as successful operations. srv.AddReceivingMiddleware(publicResponseMiddleware) + // Outside the public contract, so input validation failures are counted + // in their final form and a flagged repeat keeps its original code. + srv.AddReceivingMiddleware(newRepeatFailureTracker().middleware) + // Outside the tracker, which reads a failure's structured payload: a + // failed result leaves with its payload as JSON text only. + srv.AddReceivingMiddleware(errorResultMiddleware) if opt.Logger != nil || opt.Reporter != nil { var clk clock.Clock @@ -312,17 +330,70 @@ func errorResult(err error) *sdkmcp.CallToolResult { errors.Is(err, tapper.ErrTokenRejected) { return unauthorizedResult(err) } + var partHashes *keg.PartPreconditionRequiredError + if errors.As(err, &partHashes) { + action := partPreconditionAction(partHashes) + missing := make([]string, len(partHashes.Missing)) + for i, m := range partHashes.Missing { + missing[i] = m.Field + } + return &sdkmcp.CallToolResult{ + Content: []sdkmcp.Content{&sdkmcp.TextContent{Text: err.Error() + "\n\n" + action}}, + StructuredContent: map[string]any{ + "code": keg.RemoteCodePreconditionRequired, + "operationPerformed": false, + "action": action, + "node_id": partHashes.NodeID, + "missing": missing, + }, + IsError: true, + } + } if errors.Is(err, keg.ErrPreconditionRequired) { + action := "Nothing was written. Call the read tool the message names (node_read, schema_read, or keg_settings_read), then retry with the hash it returns in the field the message names." return &sdkmcp.CallToolResult{ - Content: []sdkmcp.Content{&sdkmcp.TextContent{Text: err.Error()}}, + Content: []sdkmcp.Content{&sdkmcp.TextContent{Text: err.Error() + "\n\n" + action}}, StructuredContent: map[string]any{ "code": keg.RemoteCodePreconditionRequired, "operationPerformed": false, - "action": "read the resource and retry with its returned hash", + "action": action, }, IsError: true, } } + var emptyEdit *tapper.EmptyEditError + if errors.As(err, &emptyEdit) { + action := fmt.Sprintf("Add content (or patch) to change node %s's body, or meta to change its metadata, with the matching hash from node_read. To make a new node, use node_create instead.", emptyEdit.NodeID) + return &sdkmcp.CallToolResult{ + Content: []sdkmcp.Content{&sdkmcp.TextContent{Text: err.Error() + "\n\n" + action}}, + StructuredContent: map[string]any{ + "code": keg.RemoteCodeInvalid, + "operationPerformed": false, + "action": action, + "node_id": emptyEdit.NodeID, + }, + IsError: true, + } + } + var limited *keg.RateLimitError + if errors.As(err, &limited) { + // A rate limit refuses the request before it runs, so the outcome is + // known; the wait is the Hub's Retry-After when it sent one. + action := "Too many requests; nothing was done. Wait retryAfterSeconds, when given, then retry the same call." + structured := map[string]any{ + "code": keg.RemoteCodeRateLimited, + "operationPerformed": false, + "action": action, + } + if limited.RetryAfter > 0 { + structured["retryAfterSeconds"] = int(math.Ceil(limited.RetryAfter.Seconds())) + } + return &sdkmcp.CallToolResult{ + Content: []sdkmcp.Content{&sdkmcp.TextContent{Text: err.Error() + "\n\n" + action}}, + StructuredContent: structured, + IsError: true, + } + } code, _ := keg.RemoteErrorCode(err) action, performed := errorGuidance(code) return &sdkmcp.CallToolResult{ @@ -338,6 +409,21 @@ func errorResult(err error) *sdkmcp.CallToolResult { } } +// partPreconditionAction names the exact read that supplies a node edit's +// missing tokens and the fields to pass them in, so the next call is +// copyable rather than inferred from prose. +func partPreconditionAction(e *keg.PartPreconditionRequiredError) string { + read := fmt.Sprintf("node_read(node_id=%q)", e.NodeID) + if e.Keg != "" { + read = fmt.Sprintf("node_read(keg=%q, node_id=%q)", e.Keg, e.NodeID) + } + pass := make([]string, len(e.Missing)) + for i, m := range e.Missing { + pass[i] = fmt.Sprintf("%s as %s", m.TokenField, m.Field) + } + return fmt.Sprintf("Nothing was written. Call %s and pass %s, then retry the same call with those fields set.", read, strings.Join(pass, " and ")) +} + // errorGuidance maps a wire code to the caller's next step and to whether the // operation changed state. // @@ -357,6 +443,10 @@ func errorGuidance(code string) (action string, operationPerformed any) { return "The target was not found or is not readable. Confirm its identity with `node_list`/`node_search`/`node_grep` for nodes, `schema_list` for schemas, `flight_list` for flights, or `keg_list` for KEGs before retrying.", false case keg.RemoteCodeExist, keg.RemoteCodeDestExists: return "Something already occupies that id or name. Choose another, or edit the existing node instead.", false + case keg.RemoteCodeConflict: + // A conflict is a refusal against current state, such as a flight or + // agent name already taken, decided before anything is written. + return "The request conflicts with current state. Read what exists (`flight_list`, `agent_list`, or the matching read tool), then choose another name or edit the existing one; do not repeat the same call.", false case keg.RemoteCodeSchemaInvalid: return "If the error says a schema selection is required, pass `schema` with one of the types it lists (`schema_list` lists them) and retry; type in `meta` alone does not count. Otherwise the node does not satisfy its schema: read the issues in this error, fix `content` or `meta`, and retry. `schema_read` shows the schema's requirements.", false case keg.RemoteCodeInvalid, keg.RemoteCodeBadRequest: @@ -371,6 +461,10 @@ func errorGuidance(code string) (action string, operationPerformed any) { return "The lock could not be acquired in time, usually because another write is in flight. Retry shortly; nothing was written.", false case keg.RemoteCodeNotSupported: return "This backend does not implement the operation. Do not retry unchanged; use a supported tool from tools/list or have the operator update the backend.", false + case keg.RemoteCodeKegRestoring: + return "The KEG is being restored from an archive and refuses writes until the restore ends. Reads still work; retry the write after the restore finishes.", false + case keg.RemoteCodeArchiveTooLarge: + return "The archive exceeds a size or count limit named in the error. Archive fewer nodes or smaller files and retry; the KEG was not changed.", false case keg.RemoteCodeKegVersionUnsupported: return "The KEG version does not offer the operation, or the Hub does not offer the requested KEG version. Do not retry unchanged: for keg_create, omit kegv to create a KEGv1 KEG; for node tools, keg_list labels KEGv2 preview KEGs, so pick a KEGv1 KEG.", false default: diff --git a/pkg/mcp/server_test.go b/pkg/mcp/server_test.go index 16849f7e..71de70f4 100644 --- a/pkg/mcp/server_test.go +++ b/pkg/mcp/server_test.go @@ -136,6 +136,18 @@ func newMemoryTap(t *testing.T, ctx context.Context, rt *toolkit.Runtime) *tappe } func newTestSessionWithOpts(t *testing.T, opts ...mcp.ServerOptions) (*sdkmcp.ClientSession, context.Context) { + t.Helper() + return connectTestSession(t, newTestClient(t, "test-client", nil), opts...) +} + +// newPlainTestSession connects without checking results against their +// outputSchemas: the listing those checks read comes from it. +func newPlainTestSession(t *testing.T, opts ...mcp.ServerOptions) (*sdkmcp.ClientSession, context.Context) { + t.Helper() + return connectTestSession(t, sdkmcp.NewClient(&sdkmcp.Implementation{Name: "test-client", Version: "0.1"}, nil), opts...) +} + +func connectTestSession(t *testing.T, client *sdkmcp.Client, opts ...mcp.ServerOptions) (*sdkmcp.ClientSession, context.Context) { t.Helper() ctx := context.Background() @@ -156,10 +168,6 @@ func newTestSessionWithOpts(t *testing.T, opts ...mcp.ServerOptions) (*sdkmcp.Cl <-done }) - client := sdkmcp.NewClient(&sdkmcp.Implementation{ - Name: "test-client", - Version: "0.1", - }, nil) session, err := client.Connect(ctx, clientTransport, nil) require.NoError(t, err) t.Cleanup(func() { @@ -203,10 +211,7 @@ func newTestSessionWithRuntime(t *testing.T, opts ...mcp.ServerOptions) (*sdkmcp <-done }) - client := sdkmcp.NewClient(&sdkmcp.Implementation{ - Name: "test-client", - Version: "0.1", - }, nil) + client := newTestClient(t, "test-client", nil) session, err := client.Connect(ctx, clientTransport, nil) require.NoError(t, err) t.Cleanup(func() { @@ -437,7 +442,7 @@ func TestMCP_ListDefaultLimit(t *testing.T) { res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ Name: "node_list", Arguments: map[string]any{ - "id_only": true, + "format": "%i", }, }) require.NoError(t, err) @@ -455,8 +460,8 @@ func TestMCP_ListUnlimitedWithNegativeOne(t *testing.T) { res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ Name: "node_list", Arguments: map[string]any{ - "id_only": true, - "limit": -1, + "format": "%i", + "limit": -1, }, }) require.NoError(t, err) @@ -474,8 +479,8 @@ func TestMCP_ListExplicitLimit(t *testing.T) { res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ Name: "node_list", Arguments: map[string]any{ - "id_only": true, - "limit": 1, + "format": "%i", + "limit": 1, }, }) require.NoError(t, err) @@ -853,6 +858,41 @@ func TestMCP_CreateBatchReturnsOrderedStructuredResults(t *testing.T) { require.NotEmpty(t, structured.Results[1].Hash) } +// Agents write a batch link as [Title]({{node:KEY}}); the stored link must be +// ../ID or it never registers. +func TestMCP_CreateBatchLinksBarePlaceholderTargets(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: "node_create", + Arguments: map[string]any{"nodes": []any{ + map[string]any{"key": "first", "content": "# First\n\n[Second]({{node:second}})\n"}, + map[string]any{"key": "second", "content": "# Second\n"}, + }}, + }) + require.NoError(t, err) + require.False(t, res.IsError, "batch create returned error: %s", extractText(t, res)) + raw, err := json.Marshal(res.StructuredContent) + require.NoError(t, err) + var structured struct { + Results []struct { + NodeID string `json:"node_id"` + } `json:"results"` + } + require.NoError(t, json.Unmarshal(raw, &structured)) + require.Len(t, structured.Results, 2) + + read, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: "node_read", + Arguments: map[string]any{"node_ids": []any{structured.Results[0].NodeID}, "content_only": true}, + }) + require.NoError(t, err) + require.False(t, read.IsError, "node_read returned error: %s", extractText(t, read)) + readRaw, err := json.Marshal(read.StructuredContent) + require.NoError(t, err) + require.Contains(t, string(readRaw), "[Second](../"+structured.Results[1].NodeID+")") +} + func TestMCP_CreateWithBody(t *testing.T) { t.Parallel() session, ctx := newTestSession(t) @@ -1054,7 +1094,19 @@ func TestMCP_EditRequiresContentOrMeta(t *testing.T) { }) require.NoError(t, err) require.True(t, res.IsError, "edit accepted an item with neither content nor meta") - require.Contains(t, extractText(t, res), "content, patch, or meta is required") + require.Contains(t, extractText(t, res), `item 0 (node "0") has nothing to change: supply content, patch, or meta`) + + // The action names the fields to add and the tool a create belongs in, + // not the generic malformed-request hint. + structured := structuredMap(t, res) + require.Equal(t, "INVALID", structured["code"]) + require.Equal(t, false, structured["operationPerformed"]) + require.Equal(t, "0", structured["node_id"]) + action, _ := structured["action"].(string) + require.Contains(t, action, "content") + require.Contains(t, action, "meta") + require.Contains(t, action, "node_create") + require.NotContains(t, action, "malformed") } func TestMCP_EditAppliesPatch(t *testing.T) { @@ -2194,6 +2246,17 @@ func extractText(t *testing.T, res *sdkmcp.CallToolResult) string { return message } } + // A failure carries its payload as the leading JSON text block only. + if len(res.Content) > 0 { + if tc, ok := res.Content[0].(*sdkmcp.TextContent); ok { + var payload map[string]any + if json.Unmarshal([]byte(tc.Text), &payload) == nil { + if message, ok := payload["message"].(string); ok { + return message + } + } + } + } var parts []string for _, c := range res.Content { if tc, ok := c.(*sdkmcp.TextContent); ok { diff --git a/pkg/mcp/session_bootstrap_test.go b/pkg/mcp/session_bootstrap_test.go index 84dedab3..2d12ebbf 100644 --- a/pkg/mcp/session_bootstrap_test.go +++ b/pkg/mcp/session_bootstrap_test.go @@ -231,6 +231,10 @@ func TestMCP_HubAgentWithoutMemoryCannotUseIdentityAuthority(t *testing.T) { result, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "session_refresh", Arguments: map[string]any{}}) require.NoError(t, err) require.False(t, result.IsError, extractText(t, result)) - require.Contains(t, callOrient(t, ctx, session), "Reader memory") + oriented := callOrient(t, ctx, session) + require.Contains(t, oriented, "Reader memory") + require.Contains(t, oriented, "Agent: `@local/reader`") + require.NotContains(t, oriented, "full identity authority") + require.NotContains(t, oriented, "identity-authorized full access") require.Contains(t, listedToolNames(t, ctx, session), "node_read") } diff --git a/pkg/mcp/session_flight.go b/pkg/mcp/session_flight.go index 290030fa..0a84a38b 100644 --- a/pkg/mcp/session_flight.go +++ b/pkg/mcp/session_flight.go @@ -18,7 +18,16 @@ import ( "github.com/jlrickert/tapper/pkg/tapper" ) -var errMCPFlightRequired = errors.New("the explicitly configured flight could not be activated; KEG tools are locked. Inspect flights with `flight_list` and `flight_read`, repair that exact selection outside MCP, then call `session_refresh` and `orient`") +// errMCPFlightRequired refuses a KEG tool while the session holds no flight +// authority. It wraps the denied sentinel because nothing was performed and +// the session's authority does not grant the call; orientationFailureResult +// gives it its own action, since naming another flight cannot help here. +var errMCPFlightRequired = fmt.Errorf("the explicitly configured flight could not be activated; KEG tools are locked. Inspect flights with `flight_list` and `flight_read`, repair that exact selection outside MCP, then call `session_refresh` and `orient`: %w", keg.ErrOrientationDenied) + +// flightRequiredAction is errMCPFlightRequired's next step. It differs from a +// denial's: the session has no active flight at all, so passing another one +// per call is refused the same way until the selection is repaired. +const flightRequiredAction = "KEG tools are locked because this session's configured flight could not be activated. Nothing was written. Passing a different flight will not unlock them: call `flight_list` to see what exists, ask the user to correct the selected flight outside MCP, then call `session_refresh` and `orient`." // ErrOrientationStale is returned without performing the requested operation. var ErrOrientationStale = fmt.Errorf("ORIENTATION_STALE: authority changed between per-call resolution and dispatch; retry the operation yourself after reviewing current authority. Mutations are never replayed automatically: %w", keg.ErrOrientationStale) @@ -150,6 +159,11 @@ type sessionFlightGate struct { cacheTTL time.Duration now func() time.Time + // discoveryCache and discoveryNow back each session's API contract + // discovery; see apicontract.Session.Cache. + discoveryCache apicontract.DiscoveryCache + discoveryNow func() time.Time + mu sync.Mutex states map[string]*flightSessionState resolved map[string]cachedOrientation @@ -181,7 +195,10 @@ func (g *sessionFlightGate) state(sessionID string) *flightSessionState { if state := g.states[sessionID]; state != nil { return state } - state := &flightSessionState{contract: apicontract.NewSession(g.clientVersion, g.logger)} + contract := apicontract.NewSession(g.clientVersion, g.logger) + contract.Cache = g.discoveryCache + contract.Now = g.discoveryNow + state := &flightSessionState{contract: contract} g.states[sessionID] = state return state } @@ -671,12 +688,12 @@ func (g *sessionFlightGate) middleware(next sdkmcp.MethodHandler) sdkmcp.MethodH } copyTool := *tool if authorityBearingTool(tool.Name) { - // Describe flight where the property is added, so the two - // can never drift. Agents read descriptions, not just - // schemas: an injected property nothing mentions reads as - // absent, and one has already been reported as missing. + // The property carries its own description, the one place + // flight selection is explained per tool; the operating + // rules and orient explain it once for the session. + // Repeating it in every tool description only buried each + // tool's own text under the same sentence. copyTool.InputSchema = schemaWithFlight(tool.InputSchema) - copyTool.Description = strings.TrimRight(copyTool.Description, " ") + flightParameterNote } copyResult.Tools = append(copyResult.Tools, ©Tool) } @@ -704,7 +721,9 @@ func (g *sessionFlightGate) middleware(next sdkmcp.MethodHandler) sdkmcp.MethodH return errorResult(validationErr), nil } if present { - return errorResult(errors.New(`validating "arguments": validating root: unexpected additional properties ["all"]`)), nil + // The SDK's own input-schema refusal, which the public + // response reports as INVALID_ARGUMENT. + return &sdkmcp.CallToolResult{IsError: true, Content: []sdkmcp.Content{&sdkmcp.TextContent{Text: `validating "arguments": validating root: unexpected additional properties ["all"]`}}}, nil } } selected, err := extractFlightArgument(params) @@ -721,7 +740,8 @@ func (g *sessionFlightGate) middleware(next sdkmcp.MethodHandler) sdkmcp.MethodH return errorResult(current.initializationError), nil } if selected != "" { - return errorResult(errors.New("cannot select a flight before this connection has an active pinned root; call `session_refresh`, then `orient`")), nil + return orientationFailureResultWithAction(fmt.Errorf("%w: cannot select a flight before this connection has an active pinned root", keg.ErrOrientationStale), + "Nothing was done. Call `session_refresh`, then `orient`; a flight can be selected once the connection has an active pinned root."), nil } callOrientation = current } else { @@ -732,7 +752,7 @@ func (g *sessionFlightGate) middleware(next sdkmcp.MethodHandler) sdkmcp.MethodH // A denial on a call that named no flight is reported in terms // of what the caller actually did, not in terms of a "requested // flight" they never requested. - if selected == "" && errors.Is(err, keg.ErrOrientationDenied) { + if selected == "" && errors.Is(err, keg.ErrOrientationDenied) && !errors.Is(err, errMCPFlightRequired) { return orientationFailureResultWithAction(err, bareCallDeniedAction), nil } return orientationFailureResult(err), nil @@ -793,7 +813,7 @@ func toolArgumentPresent(params *sdkmcp.CallToolParamsRaw, name string) (bool, e } var object map[string]json.RawMessage if err := json.Unmarshal(params.Arguments, &object); err != nil { - return false, fmt.Errorf("tool arguments must be an object: %w", err) + return false, fmt.Errorf("tool arguments must be an object: %w: %w", keg.ErrInvalid, err) } _, ok := object[name] return ok, nil @@ -824,10 +844,10 @@ func extractFlightArgument(params *sdkmcp.CallToolParamsRaw) (string, error) { return strings.TrimSpace(selected), nil } -// flightParameterNote documents the flight property injected by -// schemaWithFlight. It is appended to every authority-bearing tool's -// description at the same point the property is added. -const flightParameterNote = " Accepts an optional flight: omit it to use this connection's pinned authority, or name a flight listed by orient to run this call under that flight's cover and instructions." +// flightParameterDescription documents the flight property schemaWithFlight +// adds to every authority-bearing tool. It is the only per-tool explanation of +// flight selection; tool descriptions do not repeat it. +const flightParameterDescription = "Flight to run this call under, such as +work or @ns/+work; orient lists the ones you can name. Omit it to use the connection's flight." func schemaWithFlight(schema any) any { raw, err := json.Marshal(schema) @@ -845,7 +865,7 @@ func schemaWithFlight(schema any) any { } properties["flight"] = map[string]any{ "type": "string", - "description": "optional real flight available to current connection authority; omitted uses the connection-pinned authority", + "description": flightParameterDescription, } return object } @@ -880,6 +900,9 @@ func orientationFailureResultWithAction(err error, action string) *sdkmcp.CallTo override := action action = "Authority changed between resolution and dispatch. Nothing was written. Review current authority with `orient`, then reissue the call yourself — mutations are never replayed automatically." switch { + case errors.Is(err, errMCPFlightRequired): + code = "ORIENTATION_DENIED" + action = flightRequiredAction case errors.Is(err, keg.ErrOrientationRootUnavailable): code = "ORIENTATION_ROOT_UNAVAILABLE" action = "This session's pinned root flight is gone and cannot be replaced from inside MCP. Ask the user to repair or repin the flight outside MCP and start a new connection; `session_refresh` cannot recover this." diff --git a/pkg/mcp/session_flight_test.go b/pkg/mcp/session_flight_test.go index 97b44750..7b8bae50 100644 --- a/pkg/mcp/session_flight_test.go +++ b/pkg/mcp/session_flight_test.go @@ -481,7 +481,7 @@ func connectFlightSession(t *testing.T, ctx context.Context, srv *sdkmcp.Server, serverSession, err := srv.Connect(ctx, serverTransport, nil) require.NoError(t, err) t.Cleanup(func() { _ = serverSession.Close() }) - client := sdkmcp.NewClient(&sdkmcp.Implementation{Name: "flight-test", Version: "0.1"}, opts) + client := newTestClient(t, "flight-test", opts) session, err := client.Connect(ctx, clientTransport, nil) require.NoError(t, err) t.Cleanup(func() { _ = session.Close() }) diff --git a/pkg/mcp/session_transition_test.go b/pkg/mcp/session_transition_test.go index 741e6a29..8096f106 100644 --- a/pkg/mcp/session_transition_test.go +++ b/pkg/mcp/session_transition_test.go @@ -26,6 +26,7 @@ type perCallFlightBackend struct { created []string resolves int resolveErr error + updates []tapper.UpdateFlightOptions } type refreshFlightBackend struct { @@ -275,7 +276,7 @@ func (p *perCallFlightBackend) CreateFlight(_ context.Context, opts tapper.Creat return nil, err } flight := &tapper.Flight{Name: ref.Canonical(), Namespace: ref.Namespace, Slug: ref.Slug, Source: "test", - FlightManifest: tapper.FlightManifest{Title: opts.Title, Visibility: opts.Visibility, Cover: opts.Cover, Subflights: opts.Subflights, Instructions: opts.Instructions}} + FlightManifest: tapper.FlightManifest{Title: opts.Title, Visibility: opts.Visibility, Cover: opts.Cover, Subflights: opts.Subflights, Instructions: opts.Instructions, ManageKegs: opts.ManageKegs, ManageFlights: opts.ManageFlights}} p.mu.Lock() p.flights[flight.Name] = flight p.mu.Unlock() @@ -283,6 +284,9 @@ func (p *perCallFlightBackend) CreateFlight(_ context.Context, opts tapper.Creat } func (p *perCallFlightBackend) UpdateFlight(_ context.Context, opts tapper.UpdateFlightOptions) (*tapper.Flight, error) { + p.mu.Lock() + p.updates = append(p.updates, opts) + p.mu.Unlock() return p.GetFlight(context.Background(), opts.Ref) } @@ -547,7 +551,7 @@ func TestMCP_PerCallSelectionAdoptsGraphAuthorityChanges(t *testing.T) { delisted, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "keg_list", Arguments: map[string]any{"flight": "+grandchild"}}) require.NoError(t, err) require.True(t, delisted.IsError) - structured := delisted.StructuredContent.(map[string]any) + structured := errorPayload(t, delisted) require.Equal(t, "ORIENTATION_DENIED", structured["code"]) require.Equal(t, false, structured["operationPerformed"]) require.Equal(t, false, structured["reorientRequired"]) @@ -585,7 +589,7 @@ func TestMCP_PerCallDiscoveryAdoptsGraphAdditionDeletionAndTransientRecovery(t * denied, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "keg_list", Arguments: map[string]any{"flight": "+child"}}) require.NoError(t, err) require.True(t, denied.IsError) - require.Equal(t, "ORIENTATION_DENIED", denied.StructuredContent.(map[string]any)["code"]) + require.Equal(t, "ORIENTATION_DENIED", errorPayload(t, denied)["code"]) backend.mu.Lock() backend.resolveErr = errors.New("temporary Hub failure") @@ -593,7 +597,7 @@ func TestMCP_PerCallDiscoveryAdoptsGraphAdditionDeletionAndTransientRecovery(t * transient, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "keg_list", Arguments: map[string]any{}}) require.NoError(t, err) require.True(t, transient.IsError) - require.Equal(t, "ORIENTATION_UNAVAILABLE", transient.StructuredContent.(map[string]any)["code"]) + require.Equal(t, "ORIENTATION_UNAVAILABLE", errorPayload(t, transient)["code"]) backend.mu.Lock() backend.resolveErr = nil backend.mu.Unlock() @@ -608,7 +612,7 @@ func TestMCP_PerCallDiscoveryAdoptsGraphAdditionDeletionAndTransientRecovery(t * lost, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "keg_list", Arguments: map[string]any{}}) require.NoError(t, err) require.True(t, lost.IsError) - require.Equal(t, "ORIENTATION_ROOT_UNAVAILABLE", lost.StructuredContent.(map[string]any)["code"]) + require.Equal(t, "ORIENTATION_ROOT_UNAVAILABLE", errorPayload(t, lost)["code"]) } func TestMCP_AuthorityBearingSchemasExposeOptionalFlightAndRejectKegListAll(t *testing.T) { @@ -746,7 +750,7 @@ func TestMCP_FailedSessionRefreshPreservesRecoveryState(t *testing.T) { failed, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "session_refresh", Arguments: map[string]any{}}) require.NoError(t, err) require.True(t, failed.IsError) - structured := failed.StructuredContent.(map[string]any) + structured := errorPayload(t, failed) require.Equal(t, "SESSION_REFRESH_FAILED", structured["code"]) require.Equal(t, "recovery", structured["mode"]) require.Equal(t, false, structured["toolsChanged"]) @@ -788,7 +792,7 @@ func TestMCP_FailedSessionRefreshDoesNotBlockPublishedRecoveryOrientation(t *tes case failed := <-refreshDone: require.NotNil(t, failed) require.True(t, failed.IsError) - require.Equal(t, "SESSION_REFRESH_FAILED", failed.StructuredContent.(map[string]any)["code"]) + require.Equal(t, "SESSION_REFRESH_FAILED", errorPayload(t, failed)["code"]) case <-time.After(time.Second): t.Fatal("session refresh did not finish") } diff --git a/pkg/mcp/tools_agent.go b/pkg/mcp/tools_agent.go index 9c32df77..1f49eac1 100644 --- a/pkg/mcp/tools_agent.go +++ b/pkg/mcp/tools_agent.go @@ -11,35 +11,35 @@ import ( ) type agentListInput struct { - Namespace string `json:"namespace,omitempty" jsonschema:"namespace whose agents to list; omit for every namespace you belong to"` + Namespace string `json:"namespace,omitempty" jsonschema:"Namespace whose agents to list; omit for every namespace you belong to."` } type agentReadInput struct { - Agent string `json:"agent" jsonschema:"agent reference (@namespace/name)"` + Agent string `json:"agent" jsonschema:"Agent reference (@namespace/name)."` } type agentCreateInput struct { - Agent string `json:"agent" jsonschema:"agent reference (@namespace/name); the name is permanent"` - Title string `json:"title,omitempty" jsonschema:"agent title"` - Description string `json:"description,omitempty" jsonschema:"what the agent is for, in a sentence; a coordinator reads it to pick an agent"` - Instructions string `json:"instructions,omitempty" jsonschema:"markdown system prompt; the flight's orientation is added after it"` - Model string `json:"model,omitempty" jsonschema:"Hub model id (relay/provider/model or @namespace/model)"` - Tools []string `json:"tools,omitempty" jsonschema:"tool group ids (e.g. keg:read) and single tool names; omit or [] for every tool"` - Flight string `json:"flight,omitempty" jsonschema:"the agent's memory flight (@namespace/+slug); omit for none, which leaves the agent without KEG access"` + Agent string `json:"agent" jsonschema:"Agent reference (@namespace/name); the name is permanent."` + Title string `json:"title,omitempty" jsonschema:"Agent title."` + Description string `json:"description,omitempty" jsonschema:"What the agent is for, in a sentence; a coordinator reads it to pick an agent."` + Instructions string `json:"instructions,omitempty" jsonschema:"Markdown system prompt; the flight's orientation is added after it."` + Model string `json:"model,omitempty" jsonschema:"Hub model id (relay/provider/model or @namespace/model)."` + Tools []string `json:"tools,omitempty" jsonschema:"Tool group ids (such as keg:read) and single tool names; omit it or pass [] for every tool."` + Flight string `json:"flight,omitempty" jsonschema:"The agent's memory flight (@namespace/+slug); omit for none, which leaves the agent without KEG access."` } type agentEditInput struct { - Agent string `json:"agent" jsonschema:"agent reference (@namespace/name)"` - Title *string `json:"title,omitempty" jsonschema:"new title; omit to keep"` - Description *string `json:"description,omitempty" jsonschema:"new description; omit to keep"` - Instructions *string `json:"instructions,omitempty" jsonschema:"new markdown instructions; omit to keep"` - Model *string `json:"model,omitempty" jsonschema:"new Hub model id; omit to keep"` - Tools *[]string `json:"tools,omitempty" jsonschema:"replacement tool groups and names; omit to keep, [] for every tool"` - Flight *string `json:"flight,omitempty" jsonschema:"new memory flight (@namespace/+slug); omit to keep, \"\" to clear"` + Agent string `json:"agent" jsonschema:"Agent reference (@namespace/name)."` + Title *string `json:"title,omitempty" jsonschema:"New title; omit to keep."` + Description *string `json:"description,omitempty" jsonschema:"New description; omit to keep."` + Instructions *string `json:"instructions,omitempty" jsonschema:"New markdown instructions; omit to keep."` + Model *string `json:"model,omitempty" jsonschema:"New Hub model id; omit to keep."` + Tools *[]string `json:"tools,omitempty" jsonschema:"Replacement tool groups and names; omit to keep, [] for every tool."` + Flight *string `json:"flight,omitempty" jsonschema:"New memory flight (@namespace/+slug); omit to keep, \"\" to clear."` } type agentDeleteInput struct { - Agent string `json:"agent" jsonschema:"agent reference (@namespace/name)"` + Agent string `json:"agent" jsonschema:"Agent reference (@namespace/name)."` } // registerAgentTools exposes Hub agents over MCP at parity with @@ -48,9 +48,10 @@ type agentDeleteInput struct { // authority: its tools say what it may do and its flight is its memory. func registerAgentTools(srv *sdkmcp.Server, defaults KegDefaults, agents AgentProvider) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "agent_list", - Description: "Manage: List Hub agents (model, description, tool allowlist, flight) in a namespace, or in every namespace you belong to", - Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)}, + Name: "agent_list", + Description: "Manage: List Hub agents (model, description, tool allowlist, flight) in a namespace, or in every namespace you belong to.", + OutputSchema: outputSchema[agentListOutput](), + Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)}, }, func(ctx context.Context, _ *sdkmcp.CallToolRequest, in agentListInput) (*sdkmcp.CallToolResult, any, error) { rows, err := agents.ListAgents(ctx, in.Namespace) if err != nil { @@ -61,14 +62,15 @@ func registerAgentTools(srv *sdkmcp.Server, defaults KegDefaults, agents AgentPr lines = append(lines, strings.Join([]string{a.Ref, tsvField(a.Title), a.Model, tsvField(a.Description)}, "\t")) } res := linesResult(lines) - res.StructuredContent = map[string]any{"agents": publicAgents(rows)} + res.StructuredContent = agentListOutput{Agents: publicAgents(rows)} return res, nil, nil }) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "agent_read", - Description: "Manage: Read one Hub agent: model, description, instructions, tool allowlist (groups and names, plus effective_tools expanded), and its memory flight", - Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)}, + Name: "agent_read", + Description: "Manage: Read one Hub agent: model, description, instructions, tool allowlist (groups and names, plus effective_tools expanded), and its memory flight.", + OutputSchema: outputSchema[publicAgent](), + Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)}, }, func(ctx context.Context, _ *sdkmcp.CallToolRequest, in agentReadInput) (*sdkmcp.CallToolResult, any, error) { agent, err := agents.GetAgent(ctx, in.Agent) if err != nil { @@ -78,9 +80,10 @@ func registerAgentTools(srv *sdkmcp.Server, defaults KegDefaults, agents AgentPr }) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "agent_create", - Description: "Manage: Create a Hub agent in a namespace you own or administer. Its tools say what it may do; its flight is the memory it works in.", - Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: false, DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false)}, + Name: "agent_create", + Description: "Manage: Create a Hub agent in a namespace you own or administer. Its tools say what it may do; its flight is the memory it works in.", + OutputSchema: outputSchema[publicAgent](), + Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: false, DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false)}, }, func(ctx context.Context, _ *sdkmcp.CallToolRequest, in agentCreateInput) (*sdkmcp.CallToolResult, any, error) { if err := defaults.gate.authorizeMutation(ctx); err != nil { return errorResult(err), nil, nil @@ -96,9 +99,10 @@ func registerAgentTools(srv *sdkmcp.Server, defaults KegDefaults, agents AgentPr }) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "agent_edit", - Description: "Manage: Edit a Hub agent in a namespace you own or administer; omitted fields keep their current values. An edit applies to open sessions running the agent on their next call.", - Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: false, DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false)}, + Name: "agent_edit", + Description: "Manage: Edit a Hub agent in a namespace you own or administer; omitted fields keep their current values. An edit applies to open sessions running the agent on their next call.", + OutputSchema: outputSchema[publicAgent](), + Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: false, DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false)}, }, func(ctx context.Context, _ *sdkmcp.CallToolRequest, in agentEditInput) (*sdkmcp.CallToolResult, any, error) { if err := defaults.gate.authorizeMutation(ctx); err != nil { return errorResult(err), nil, nil @@ -114,9 +118,10 @@ func registerAgentTools(srv *sdkmcp.Server, defaults KegDefaults, agents AgentPr }) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "agent_delete", - Description: "Manage: Delete a Hub agent in a namespace you own or administer. Apps and tokens bound to it lose access.", - Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: false, DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false)}, + Name: "agent_delete", + Description: "Manage: Delete a Hub agent in a namespace you own or administer.", + OutputSchema: textOutputSchema(), + Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: false, DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false)}, }, func(ctx context.Context, _ *sdkmcp.CallToolRequest, in agentDeleteInput) (*sdkmcp.CallToolResult, any, error) { if err := defaults.gate.authorizeMutation(ctx); err != nil { return errorResult(err), nil, nil @@ -128,6 +133,11 @@ func registerAgentTools(srv *sdkmcp.Server, defaults KegDefaults, agents AgentPr }) } +// agentListOutput is agent_list's structured content. +type agentListOutput struct { + Agents []publicAgent `json:"agents" jsonschema:"The agents, without their instructions; agent_read returns those."` +} + // publicAgent is the MCP wire shape of an agent. Every slice is non-nil so // clients see [] rather than null. type publicAgent struct { diff --git a/pkg/mcp/tools_auth.go b/pkg/mcp/tools_auth.go index b1060c0b..a626102a 100644 --- a/pkg/mcp/tools_auth.go +++ b/pkg/mcp/tools_auth.go @@ -20,9 +20,10 @@ type authInfoOutput struct { // absent from both the structured and human-readable response. func registerAuthInfoTool(srv *sdkmcp.Server, _ KegDefaults, identities IdentityProvider, kegs KegDiscoveryProvider) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "session_info", - Description: "Session: Report authenticated hub identities and exact pinned-root KEG authority without exposing credentials or private account data. Use keg_list for the KEGs this session can work in and orient for the flight and its instructions", - Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)}, + Name: "session_info", + Description: "Session: Report authenticated hub identities and exact pinned-root KEG authority without exposing credentials or private account data. Use keg_list for the KEGs this session can work in and orient for the flight and its instructions.", + OutputSchema: outputSchema[authInfoOutput](), + Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)}, }, func(ctx context.Context, _ *sdkmcp.CallToolRequest, _ authInfoInput) (*sdkmcp.CallToolResult, any, error) { found, err := identities.Identities(ctx) if err != nil { diff --git a/pkg/mcp/tools_doctor.go b/pkg/mcp/tools_doctor.go index 38be343a..937820b7 100644 --- a/pkg/mcp/tools_doctor.go +++ b/pkg/mcp/tools_doctor.go @@ -17,13 +17,14 @@ func registerDoctorTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul // --- doctor --- type doctorInput struct { - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerDoctor(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "keg_check", - Description: "Manage: Check KEG health and report issues", + Name: "keg_check", + Description: "Manage: Check a KEG's health and report issues.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_explore.go b/pkg/mcp/tools_explore.go index 2c95a235..4f7634e5 100644 --- a/pkg/mcp/tools_explore.go +++ b/pkg/mcp/tools_explore.go @@ -35,7 +35,7 @@ func (p localExploreProvider) Explore(ctx context.Context, kind tapper.ExploreKi } type exploreInput struct { - Query string `json:"query" jsonschema:"literal text to match against public names, references, and descriptions"` + Query string `json:"query" jsonschema:"Literal text to match against public names, references, and descriptions."` } // registerExploreTools exposes the public directory, at parity with @@ -50,26 +50,27 @@ func registerExploreTools(srv *sdkmcp.Server, explore ExploreProvider) { }{ { name: "explore_kegs", kind: tapper.ExploreKegs, - desc: "Explore: search the hub's public KEGs by reference, title, or description. " + + desc: "Explore: Search the hub's public KEGs by reference, title, or description. " + "Discovery only: a result grants no access, and a KEG outside the flight still needs a flight that covers it. " + "Your own KEGs come from keg_list; notes come from node_search, never from here.", }, { name: "explore_flights", kind: tapper.ExploreFlights, - desc: "Explore: search the hub's public flights by reference, title, or description. " + + desc: "Explore: Search the hub's public flights by reference, title, or description. " + "Discovery only: returns metadata, never instructions, and grants no access. Your own flights come from flight_list.", }, { name: "explore_people", kind: tapper.ExplorePeople, - desc: "Explore: search the people and org namespaces on the hub by name or display name. " + + desc: "Explore: Search the people and org namespaces on the hub by name or display name. " + "Discovery only: role is set where you already belong. Your own namespaces come from namespace_list.", }, } for _, tool := range tools { kind := tool.kind sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: tool.name, - Description: tool.desc, + Name: tool.name, + Description: tool.desc, + OutputSchema: outputSchema[tapper.ExploreResult](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(true), diff --git a/pkg/mcp/tools_files.go b/pkg/mcp/tools_files.go index 05bb9566..cbee5ce5 100644 --- a/pkg/mcp/tools_files.go +++ b/pkg/mcp/tools_files.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "fmt" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -46,14 +47,15 @@ func registerFileTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults // --- file_list --- type listFilesInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to list files for"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to list files for."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerListFiles(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "file_list", - Description: "Memory: List file attachments for a node", + Name: "file_list", + Description: "Memory: List a node's file attachments.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -77,14 +79,15 @@ func registerListFiles(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults // --- image_list --- type listImagesInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to list images for"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to list images for."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerListImages(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "image_list", - Description: "Memory: List image attachments for a node", + Name: "image_list", + Description: "Memory: List a node's image attachments.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -132,16 +135,17 @@ func appendAttachmentResourceLink(result *sdkmcp.CallToolResult, nodeID string, // --- file_delete --- type deleteFileInput struct { - NodeID string `json:"node_id" jsonschema:"node ID containing the file"` - Filename *string `json:"filename,omitempty" jsonschema:"filename to delete; required unless name is supplied"` - Name *string `json:"name,omitempty" jsonschema:"compatibility alias for filename; must match when both are supplied"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID containing the file."` + Filename *string `json:"filename,omitempty" jsonschema:"Filename to delete; required unless name is supplied."` + Name *string `json:"name,omitempty" jsonschema:"Compatibility alias for filename; must match when both are supplied."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerDeleteFile(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "file_delete", - Description: "Memory: Delete a file attachment from a node", + Name: "file_delete", + Description: "Memory: Delete a file attachment from a node.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), @@ -166,16 +170,17 @@ func registerDeleteFile(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault // --- image_delete --- type deleteImageInput struct { - NodeID string `json:"node_id" jsonschema:"node ID containing the image"` - Filename *string `json:"filename,omitempty" jsonschema:"filename to delete; required unless name is supplied"` - Name *string `json:"name,omitempty" jsonschema:"compatibility alias for filename; must match when both are supplied"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID containing the image."` + Filename *string `json:"filename,omitempty" jsonschema:"Filename to delete; required unless name is supplied."` + Name *string `json:"name,omitempty" jsonschema:"Compatibility alias for filename; must match when both are supplied."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerDeleteImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "image_delete", - Description: "Memory: Delete an image attachment from a node", + Name: "image_delete", + Description: "Memory: Delete an image attachment from a node.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), @@ -200,32 +205,32 @@ func registerDeleteImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul // --- file_upload --- type uploadFileInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to attach the file to"` - Filename string `json:"filename,omitempty" jsonschema:"filename for the attachment; derived from a data URI or resource URI if empty"` - SourceURI string `json:"source_uri,omitempty" jsonschema:"data: URI for the source file"` - DataBase64 string `json:"data_base64,omitempty" jsonschema:"base64-encoded file bytes"` - MIMEType string `json:"mime_type,omitempty" jsonschema:"optional MIME type hint for raw file bytes"` - Resource *uploadResourceInput `json:"resource,omitempty" jsonschema:"embedded resource with uri, mime_type or mimeType, and blob or text"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to attach the file to."` + Filename string `json:"filename,omitempty" jsonschema:"Filename for the attachment; derived from a data URI or resource URI if empty."` + SourceURI string `json:"source_uri,omitempty" jsonschema:"Source file as a data: URI."` + DataBase64 string `json:"data_base64,omitempty" jsonschema:"Base64-encoded file bytes."` + MIMEType string `json:"mime_type,omitempty" jsonschema:"Optional MIME type hint for raw file bytes."` + Resource *uploadResourceInput `json:"resource,omitempty" jsonschema:"Embedded resource with uri, mime_type or mimeType, and blob or text."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } // localUploadFileInput is uploadFileInput plus the local-path source that only // a shared-filesystem transport can honour. Keeping it a separate type is what // keeps source_path out of the hosted schema. type localUploadFileInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to attach the file to"` - Filename string `json:"filename,omitempty" jsonschema:"filename for the attachment; derived from the source path, data URI, or resource URI if empty"` - SourcePath string `json:"source_path,omitempty" jsonschema:"absolute path to the source file on the machine running the server"` - SourceURI string `json:"source_uri,omitempty" jsonschema:"file: or data: URI for the source file"` - DataBase64 string `json:"data_base64,omitempty" jsonschema:"base64-encoded file bytes"` - MIMEType string `json:"mime_type,omitempty" jsonschema:"optional MIME type hint for raw file bytes"` - Resource *uploadResourceInput `json:"resource,omitempty" jsonschema:"embedded resource with uri, mime_type or mimeType, and blob or text"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to attach the file to."` + Filename string `json:"filename,omitempty" jsonschema:"Filename for the attachment; derived from the source path, data URI, or resource URI if empty."` + SourcePath string `json:"source_path,omitempty" jsonschema:"Regular file within the server working directory."` + SourceURI string `json:"source_uri,omitempty" jsonschema:"Source file as a file: or data: URI."` + DataBase64 string `json:"data_base64,omitempty" jsonschema:"Base64-encoded file bytes."` + MIMEType string `json:"mime_type,omitempty" jsonschema:"Optional MIME type hint for raw file bytes."` + Resource *uploadResourceInput `json:"resource,omitempty" jsonschema:"Embedded resource with uri, mime_type or mimeType, and blob or text."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerUploadFile(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, uploadFileTool( - "Memory: Upload a file attachment to a node from raw bytes, a data URI, or an embedded resource", + "Memory: Upload a file attachment to a node from raw bytes, a data URI, or an embedded resource.", ), func(ctx context.Context, req *sdkmcp.CallToolRequest, in uploadFileInput) (*sdkmcp.CallToolResult, any, error) { return handleFileUpload(ctx, tap, defaults, in.Keg, in.NodeID, in.Filename, uploadSourceInput{ SourceURI: in.SourceURI, @@ -237,7 +242,7 @@ func registerUploadFile(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault func registerLocalUploadFile(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, uploadFileTool( - "Memory: Upload a file attachment to a node from a local path, raw bytes, a data URI, or an embedded resource", + "Memory: Upload a file attachment to a node from a local path, raw bytes, a data URI, or an embedded resource.", ), func(ctx context.Context, req *sdkmcp.CallToolRequest, in localUploadFileInput) (*sdkmcp.CallToolResult, any, error) { return handleFileUpload(ctx, tap, defaults, in.Keg, in.NodeID, in.Filename, uploadSourceInput{ SourcePath: in.SourcePath, @@ -259,8 +264,9 @@ const ( func uploadFileTool(description string) *sdkmcp.Tool { return &sdkmcp.Tool{ - Name: "file_upload", - Description: description + uploadFileLinkHint, + Name: "file_upload", + Description: description + uploadFileLinkHint, + OutputSchema: outputSchema[uploadOutput](), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -288,10 +294,27 @@ func handleFileUpload(ctx context.Context, tap *tapper.Tap, defaults KegDefaults } result := textResult(fmt.Sprintf("uploaded file %q to node %s", storedName, nodeID)) appendAttachmentResourceLink(result, nodeID, keg.AttachmentFile, storedName, kegAlias) - result.StructuredContent = map[string]any{"node_id": nodeID, "filename": storedName, "link": "./assets/" + storedName} + result.StructuredContent = uploadOutput{NodeID: nodeID, Filename: storedName, Link: "./assets/" + storedName} return result, nil, nil } +// uploadOutput is file_upload's and image_upload's structured content. +type uploadOutput struct { + NodeID string `json:"node_id" jsonschema:"The node the attachment belongs to."` + Filename string `json:"filename" jsonschema:"The stored filename, which may differ from the one sent."` + Link string `json:"link" jsonschema:"The relative link to write in the node body."` +} + +// imageContentOutput describes the image image_download returns as MCP image +// content. Saving to dest_path (shared-filesystem servers only) returns text +// alone, so every field is optional. +type imageContentOutput struct { + NodeID string `json:"node_id,omitempty" jsonschema:"The node the image belongs to."` + Filename string `json:"filename,omitempty" jsonschema:"The image's filename."` + MIMEType string `json:"mime_type,omitempty" jsonschema:"The image's MIME type."` + Size int `json:"size,omitempty" jsonschema:"The image's size in bytes."` +} + // resolveUploadName prefers the caller's explicit filename and falls back to // one derived from the source. Byte sources such as data_base64 carry no name, // so an explicit one is required there. @@ -301,7 +324,7 @@ func resolveUploadName(explicit, derived string) (string, error) { name = derived } if name == "" { - return "", fmt.Errorf("filename is required when the upload source has no filename") + return "", fmt.Errorf("filename is required when the upload source has no filename: %w", keg.ErrInvalid) } return name, nil } @@ -309,28 +332,34 @@ func resolveUploadName(explicit, derived string) (string, error) { // --- file_download --- type downloadFileInput struct { - NodeID string `json:"node_id" jsonschema:"node ID containing the file"` - Filename string `json:"filename" jsonschema:"filename to download"` - DestPath string `json:"dest_path" jsonschema:"absolute path to write the downloaded file"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID containing the file."` + Filename string `json:"filename" jsonschema:"Filename to download."` + DestPath string `json:"dest_path" jsonschema:"New file path within the server working directory; existing files are refused."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerDownloadFile(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "file_download", - Description: "Memory: Download a file attachment from a node to a local file path", + Name: "file_download", + Description: "Memory: Download a file attachment from a node to a new local file.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ OpenWorldHint: boolPtr(false), }, }, func(ctx context.Context, req *sdkmcp.CallToolRequest, in downloadFileInput) (*sdkmcp.CallToolResult, any, error) { if in.DestPath == "-" { - return errorResult(fmt.Errorf("stdout mode is not supported over MCP")), nil, nil + return errorResult(fmt.Errorf("%w: stdout mode is not supported over MCP", keg.ErrInvalid)), nil, nil + } + destPath, pathErr := confinedTransferPath(tap.Runtime, in.DestPath, true) + if pathErr != nil { + return errorResult(pathErr), nil, nil } opts := tapper.DownloadFileOptions{ KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), NodeID: in.NodeID, Name: in.Filename, - Dest: in.DestPath, + Dest: destPath, + NoOverwrite: true, } dest, err := tap.DownloadFile(ctx, opts) if err != nil { @@ -343,31 +372,31 @@ func registerDownloadFile(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefau // --- image_upload --- type uploadImageInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to attach the image to"` - Filename string `json:"filename,omitempty" jsonschema:"image filename for the attachment; derived from a data URI or resource URI if empty"` - SourceURI string `json:"source_uri,omitempty" jsonschema:"data: URI for the source image"` - DataBase64 string `json:"data_base64,omitempty" jsonschema:"base64-encoded image bytes"` - MIMEType string `json:"mime_type,omitempty" jsonschema:"optional MIME type hint for raw image bytes"` - Resource *uploadResourceInput `json:"resource,omitempty" jsonschema:"embedded resource with uri, mime_type or mimeType, and blob"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to attach the image to."` + Filename string `json:"filename,omitempty" jsonschema:"Image filename for the attachment; derived from a data URI or resource URI if empty."` + SourceURI string `json:"source_uri,omitempty" jsonschema:"Source image as a data: URI."` + DataBase64 string `json:"data_base64,omitempty" jsonschema:"Base64-encoded image bytes."` + MIMEType string `json:"mime_type,omitempty" jsonschema:"Optional MIME type hint for raw image bytes."` + Resource *uploadResourceInput `json:"resource,omitempty" jsonschema:"Embedded resource with uri, mime_type or mimeType, and blob."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } // localUploadImageInput is uploadImageInput plus the local-path source. See // localUploadFileInput for why this is a separate type. type localUploadImageInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to attach the image to"` - Filename string `json:"filename,omitempty" jsonschema:"image filename for the attachment; derived from the source path, data URI, or resource URI if empty"` - SourcePath string `json:"source_path,omitempty" jsonschema:"absolute path to the source image on the machine running the server"` - SourceURI string `json:"source_uri,omitempty" jsonschema:"file: or data: URI for the source image"` - DataBase64 string `json:"data_base64,omitempty" jsonschema:"base64-encoded image bytes"` - MIMEType string `json:"mime_type,omitempty" jsonschema:"optional MIME type hint for raw image bytes"` - Resource *uploadResourceInput `json:"resource,omitempty" jsonschema:"embedded resource with uri, mime_type or mimeType, and blob"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to attach the image to."` + Filename string `json:"filename,omitempty" jsonschema:"Image filename for the attachment; derived from the source path, data URI, or resource URI if empty."` + SourcePath string `json:"source_path,omitempty" jsonschema:"Regular image file within the server working directory."` + SourceURI string `json:"source_uri,omitempty" jsonschema:"Source image as a file: or data: URI."` + DataBase64 string `json:"data_base64,omitempty" jsonschema:"Base64-encoded image bytes."` + MIMEType string `json:"mime_type,omitempty" jsonschema:"Optional MIME type hint for raw image bytes."` + Resource *uploadResourceInput `json:"resource,omitempty" jsonschema:"Embedded resource with uri, mime_type or mimeType, and blob."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerUploadImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, uploadImageTool( - "Memory: Upload an image attachment to a node from raw bytes, a data URI, or an embedded resource", + "Memory: Upload an image attachment to a node from raw bytes, a data URI, or an embedded resource.", ), func(ctx context.Context, req *sdkmcp.CallToolRequest, in uploadImageInput) (*sdkmcp.CallToolResult, any, error) { return handleImageUpload(ctx, tap, defaults, in.Keg, in.NodeID, in.Filename, uploadSourceInput{ SourceURI: in.SourceURI, @@ -379,7 +408,7 @@ func registerUploadImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul func registerLocalUploadImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, uploadImageTool( - "Memory: Upload an image attachment to a node from a local path, raw bytes, a data URI, or an embedded resource", + "Memory: Upload an image attachment to a node from a local path, raw bytes, a data URI, or an embedded resource.", ), func(ctx context.Context, req *sdkmcp.CallToolRequest, in localUploadImageInput) (*sdkmcp.CallToolResult, any, error) { return handleImageUpload(ctx, tap, defaults, in.Keg, in.NodeID, in.Filename, uploadSourceInput{ SourcePath: in.SourcePath, @@ -392,8 +421,9 @@ func registerLocalUploadImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegD func uploadImageTool(description string) *sdkmcp.Tool { return &sdkmcp.Tool{ - Name: "image_upload", - Description: description + uploadImageLinkHint, + Name: "image_upload", + Description: description + uploadImageLinkHint, + OutputSchema: outputSchema[uploadOutput](), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -421,52 +451,58 @@ func handleImageUpload(ctx context.Context, tap *tapper.Tap, defaults KegDefault } result := textResult(fmt.Sprintf("uploaded image %q to node %s", storedName, nodeID)) appendAttachmentResourceLink(result, nodeID, keg.AttachmentImage, storedName, kegAlias) - result.StructuredContent = map[string]any{"node_id": nodeID, "filename": storedName, "link": "./images/" + storedName} + result.StructuredContent = uploadOutput{NodeID: nodeID, Filename: storedName, Link: "./images/" + storedName} return result, nil, nil } // --- image_download --- type downloadImageInput struct { - NodeID string `json:"node_id" jsonschema:"node ID containing the image"` - Filename string `json:"filename" jsonschema:"image filename to download"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID containing the image."` + Filename string `json:"filename" jsonschema:"Image filename to download."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } // localDownloadImageInput adds an optional destination path. Omitting it keeps // the inline-content behaviour, so a shared-filesystem agent can either look at // the image or save it without needing two different tools. type localDownloadImageInput struct { - NodeID string `json:"node_id" jsonschema:"node ID containing the image"` - Filename string `json:"filename" jsonschema:"image filename to download"` - DestPath string `json:"dest_path,omitempty" jsonschema:"absolute path to write the image to; omit to receive the image as MCP content"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID containing the image."` + Filename string `json:"filename" jsonschema:"Image filename to download."` + DestPath string `json:"dest_path,omitempty" jsonschema:"New file path within the server working directory; existing files are refused; omit to receive the image as MCP content."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerDownloadImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, downloadImageTool( - "Memory: Return an image attachment from a node as MCP image content", + "Memory: Return an image attachment from a node as MCP image content.", ), func(ctx context.Context, req *sdkmcp.CallToolRequest, in downloadImageInput) (*sdkmcp.CallToolResult, any, error) { return readImageContent(ctx, tap, defaults, in.Keg, in.NodeID, in.Filename) }) } func registerLocalDownloadImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { - sdkmcp.AddTool(srv, downloadImageTool( - "Memory: Download an image attachment from a node, either to a local file path or as MCP image content", - ), func(ctx context.Context, req *sdkmcp.CallToolRequest, in localDownloadImageInput) (*sdkmcp.CallToolResult, any, error) { + tool := downloadImageTool("Memory: Return an image attachment as MCP image content, or save it to a new private file within the working directory; existing files are never overwritten.") + tool.Annotations.ReadOnlyHint = false + tool.Annotations.DestructiveHint = boolPtr(false) + sdkmcp.AddTool(srv, tool, func(ctx context.Context, req *sdkmcp.CallToolRequest, in localDownloadImageInput) (*sdkmcp.CallToolResult, any, error) { dest := strings.TrimSpace(in.DestPath) if dest == "" { return readImageContent(ctx, tap, defaults, in.Keg, in.NodeID, in.Filename) } if dest == "-" { - return errorResult(fmt.Errorf("stdout mode is not supported over MCP; omit dest_path to receive image content")), nil, nil + return errorResult(fmt.Errorf("%w: stdout mode is not supported over MCP; omit dest_path to receive image content", keg.ErrInvalid)), nil, nil + } + dest, pathErr := confinedTransferPath(tap.Runtime, dest, true) + if pathErr != nil { + return errorResult(pathErr), nil, nil } written, err := tap.DownloadImage(ctx, tapper.DownloadImageOptions{ KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), NodeID: in.NodeID, Name: in.Filename, Dest: dest, + NoOverwrite: true, }) if err != nil { return errorResult(err), nil, nil @@ -477,8 +513,9 @@ func registerLocalDownloadImage(srv *sdkmcp.Server, tap *tapper.Tap, defaults Ke func downloadImageTool(description string) *sdkmcp.Tool { return &sdkmcp.Tool{ - Name: "image_download", - Description: description, + Name: "image_download", + Description: description, + OutputSchema: outputSchema[imageContentOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -497,20 +534,17 @@ func readImageContent(ctx context.Context, tap *tapper.Tap, defaults KegDefaults } mimeType := keg.ImageContentType(format) return &sdkmcp.CallToolResult{ - Content: []sdkmcp.Content{&sdkmcp.ImageContent{Data: data, MIMEType: mimeType}}, - StructuredContent: map[string]any{ - "node_id": nodeID, "filename": filename, - "mime_type": mimeType, "size": len(data), - }, + Content: []sdkmcp.Content{&sdkmcp.ImageContent{Data: data, MIMEType: mimeType}}, + StructuredContent: imageContentOutput{NodeID: nodeID, Filename: filename, MIMEType: mimeType, Size: len(data)}, }, nil, nil } type uploadResourceInput struct { - URI string `json:"uri,omitempty" jsonschema:"resource URI"` - MIMEType string `json:"mime_type,omitempty" jsonschema:"resource MIME type"` - MIMETypeSpec string `json:"mimeType,omitempty" jsonschema:"resource MIME type using MCP field casing"` - Blob string `json:"blob,omitempty" jsonschema:"base64-encoded binary resource bytes"` - Text string `json:"text,omitempty" jsonschema:"text resource contents for file uploads"` + URI string `json:"uri,omitempty" jsonschema:"Resource URI."` + MIMEType string `json:"mime_type,omitempty" jsonschema:"Resource MIME type."` + MIMETypeSpec string `json:"mimeType,omitempty" jsonschema:"Resource MIME type using MCP field casing."` + Blob string `json:"blob,omitempty" jsonschema:"Base64-encoded binary resource bytes."` + Text string `json:"text,omitempty" jsonschema:"Text resource contents for file uploads."` } type uploadSourceInput struct { @@ -535,7 +569,7 @@ func resolveUploadSource(rt *toolkit.Runtime, in uploadSourceInput, allowLocalSo provided++ } if provided != 1 { - return nil, "", fmt.Errorf("provide exactly one upload source") + return nil, "", fmt.Errorf("provide exactly one upload source: %w", keg.ErrInvalid) } switch { @@ -553,7 +587,7 @@ func resolveUploadSource(rt *toolkit.Runtime, in uploadSourceInput, allowLocalSo func readUploadResource(rt *toolkit.Runtime, resource *uploadResourceInput, allowLocalSources bool) ([]byte, string, error) { if resource == nil { - return nil, "", fmt.Errorf("embedded resource is empty") + return nil, "", fmt.Errorf("embedded resource is empty: %w", keg.ErrInvalid) } name := uploadNameFromURI(resource.URI) if strings.TrimSpace(resource.Blob) != "" { @@ -564,7 +598,7 @@ func readUploadResource(rt *toolkit.Runtime, resource *uploadResourceInput, allo return []byte(resource.Text), name, nil } if strings.TrimSpace(resource.URI) == "" { - return nil, "", fmt.Errorf("embedded resource must include blob, text, or uri") + return nil, "", fmt.Errorf("embedded resource must include blob, text, or uri: %w", keg.ErrInvalid) } return readUploadURI(rt, strings.TrimSpace(resource.URI), allowLocalSources) } @@ -576,20 +610,24 @@ func readUploadURI(rt *toolkit.Runtime, raw string, allowLocalSources bool) ([]b } u, err := url.Parse(raw) if err != nil { - return nil, "", fmt.Errorf("invalid source_uri: %w", err) + return nil, "", fmt.Errorf("invalid source_uri: %w: %w", err, keg.ErrInvalid) } if u.Scheme == "file" { if u.Host != "" && u.Host != "localhost" { - return nil, "", fmt.Errorf("file URI host %q is not supported", u.Host) + return nil, "", fmt.Errorf("file URI host %q is not supported: %w", u.Host, keg.ErrInvalid) } return readLocalUploadSource(rt, u.Path, allowLocalSources) } - return nil, "", fmt.Errorf("unsupported source_uri scheme %q", u.Scheme) + return nil, "", fmt.Errorf("unsupported source_uri scheme %q: %w", u.Scheme, keg.ErrInvalid) } func readLocalUploadSource(rt *toolkit.Runtime, path string, allowLocalSources bool) ([]byte, string, error) { if !allowLocalSources { - return nil, "", fmt.Errorf("local file sources are not available on this MCP surface; use data_base64 or an embedded resource blob") + return nil, "", fmt.Errorf("local file sources are not available on this MCP surface; use data_base64 or an embedded resource blob: %w", keg.ErrInvalid) + } + path, err := confinedTransferPath(rt, path, false) + if err != nil { + return nil, "", fmt.Errorf("unable to read local file: %w", err) } data, err := rt.ReadFile(path) if err != nil { @@ -601,14 +639,14 @@ func readLocalUploadSource(rt *toolkit.Runtime, path string, allowLocalSources b func decodeDataURI(raw string) ([]byte, error) { prefix, payload, ok := strings.Cut(raw, ",") if !ok { - return nil, fmt.Errorf("invalid data URI") + return nil, fmt.Errorf("invalid data URI: %w", keg.ErrInvalid) } if strings.Contains(strings.ToLower(prefix), ";base64") { return decodeUploadBase64(payload) } decoded, err := url.PathUnescape(payload) if err != nil { - return nil, fmt.Errorf("invalid data URI payload: %w", err) + return nil, fmt.Errorf("invalid data URI payload: %w: %w", err, keg.ErrInvalid) } return []byte(decoded), nil } @@ -621,7 +659,7 @@ func decodeUploadBase64(raw string) ([]byte, error) { if data, rawErr := base64.RawStdEncoding.DecodeString(raw); rawErr == nil { return data, nil } - return nil, fmt.Errorf("invalid base64 upload data: %w", err) + return nil, fmt.Errorf("invalid base64 upload data: %w: %w", err, keg.ErrInvalid) } func uploadNameFromURI(raw string) string { @@ -654,3 +692,57 @@ func deletionFilename(filename, alias *string) (string, error) { } return *filename, nil } + +// Resolve symlinks and environment expansion before checking the workspace +// boundary. Downloads additionally use O_EXCL, so a final symlink cannot be +// used to overwrite an existing destination. +func confinedTransferPath(rt *toolkit.Runtime, raw string, writing bool) (string, error) { + if strings.TrimSpace(raw) == "" || raw == "-" { + return "", fmt.Errorf("an explicit workspace file path is required") + } + root, err := rt.ResolvePath(".", true) + if err != nil { + return "", err + } + path, err := rt.ResolvePath(raw, false) + // Runtime filesystem calls expand variables again. A resolved path must + // be literal so a nested variable cannot change it after confinement. + if strings.ContainsAny(root+path, "$") { + return "", fmt.Errorf("file transfer paths must resolve to literal paths") + } + if err == nil { + if writing { + var parent string + parent, err = rt.ResolvePath(filepath.Dir(path), true) + path = filepath.Join(parent, filepath.Base(path)) + } else { + path, err = rt.ResolvePath(path, true) + } + } + if err != nil { + return "", err + } + if strings.Contains(path, "$") { + return "", fmt.Errorf("file transfer paths must resolve to literal paths") + } + rel, err := filepath.Rel(root, path) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return "", fmt.Errorf("file transfers must stay within the server working directory") + } + if writing { + if _, err := rt.Stat(path, false); err == nil { + return "", fmt.Errorf("destination already exists") + } else if !os.IsNotExist(err) { + return "", err + } + } else { + info, err := rt.Stat(path, true) + if err != nil { + return "", err + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("source must be a regular file") + } + } + return path, nil +} diff --git a/pkg/mcp/tools_flight.go b/pkg/mcp/tools_flight.go index a820e1df..2ac089a1 100644 --- a/pkg/mcp/tools_flight.go +++ b/pkg/mcp/tools_flight.go @@ -13,33 +13,39 @@ import ( type listFlightsInput struct{} type flightShowInput struct { - Name string `json:"name" jsonschema:"flight name to inspect"` + Name string `json:"name" jsonschema:"Flight name to inspect."` } type flightCreateInput struct { - Ref string `json:"ref" jsonschema:"flight reference (@namespace/+slug; a bare slug uses the default namespace)"` - Title string `json:"title,omitempty" jsonschema:"flight title"` - Description string `json:"description,omitempty" jsonschema:"short description, separate from instructions"` - Visibility string `json:"visibility,omitempty" jsonschema:"flight visibility: private (default) or public"` - Instructions string `json:"instructions,omitempty" jsonschema:"markdown instructions"` - Cover []string `json:"cover,omitempty" jsonschema:"covered kegs with role caps, e.g. @ns/keg=viewer, @ns/keg=editor, or @ns/keg=admin (bare entries default to viewer). A covered keg also covers the kegs its notes link to, to depth 2; @ns/keg=none blocks a keg however it is reached"` - Subflights []string `json:"subflights,omitempty" jsonschema:"ordered child flight references; bare +slug references use this flight's namespace"` + Ref string `json:"ref" jsonschema:"Flight reference (@namespace/+slug; a bare slug uses the default namespace)."` + Title string `json:"title,omitempty" jsonschema:"Flight title."` + Description string `json:"description,omitempty" jsonschema:"Short description, separate from instructions."` + Visibility string `json:"visibility,omitempty" jsonschema:"Flight visibility: private (default) or public."` + Instructions string `json:"instructions,omitempty" jsonschema:"Markdown instructions."` + Cover []string `json:"cover,omitempty" jsonschema:"Covered KEGs with role caps, such as @ns/keg=viewer, @ns/keg=editor, or @ns/keg=admin (bare entries default to viewer). A covered keg also covers the kegs its notes link to, to depth 2; @ns/keg=none blocks a keg however it is reached."` + Subflights []string `json:"subflights,omitempty" jsonschema:"Ordered child flight references; bare +slug references use this flight's namespace."` + ManageKegs bool `json:"manage_kegs,omitempty" jsonschema:"Let a request rooted on this flight administer every KEG in the flight's namespace: create and delete KEGs, edit their settings, change their visibility, and manage their grants, even outside its cover (node content outside the cover stays out of reach), within your own role. Default false."` + ManageFlights bool `json:"manage_flights,omitempty" jsonschema:"Let a request rooted on this flight create, edit, and delete the flights of its namespace, within your own role. Default false."` + Manage bool `json:"manage,omitempty" jsonschema:"Deprecated: sets both manage_kegs and manage_flights."` } type flightEditInput struct { - Ref string `json:"ref" jsonschema:"flight reference (@namespace/+slug; a bare slug uses the default namespace)"` - Title *string `json:"title,omitempty" jsonschema:"new flight title; omit to keep the current title"` - Description *string `json:"description,omitempty" jsonschema:"short description, separate from instructions"` - Visibility *string `json:"visibility,omitempty" jsonschema:"new visibility: private or public; omit to keep current"` - Instructions *string `json:"instructions,omitempty" jsonschema:"new markdown instructions; omit to keep the current instructions"` - Cover []string `json:"cover,omitempty" jsonschema:"replacement declared cover, e.g. @ns/keg=viewer, or @ns/keg=none to block a keg its notes link to; entries use default depth 2. Custom depth cannot be set here. Omit to preserve all current cover entries and depths; [] clears"` - Subflights []string `json:"subflights,omitempty" jsonschema:"replacement ordered child flight references; omit to keep current"` - ExpectedHash string `json:"expected_hash" jsonschema:"precondition token returned by flight_read"` + Ref string `json:"ref" jsonschema:"Flight reference (@namespace/+slug; a bare slug uses the default namespace)."` + Title *string `json:"title,omitempty" jsonschema:"New flight title; omit to keep the current title."` + Description *string `json:"description,omitempty" jsonschema:"Short description, separate from instructions."` + Visibility *string `json:"visibility,omitempty" jsonschema:"New visibility: private or public; omit to keep current."` + Instructions *string `json:"instructions,omitempty" jsonschema:"New markdown instructions; omit to keep the current instructions."` + Cover []string `json:"cover,omitempty" jsonschema:"Replacement declared cover, such as @ns/keg=viewer, or @ns/keg=none to block a keg its notes link to; entries use default depth 2. Custom depth cannot be set here. Omit to preserve all current cover entries and depths; [] clears."` + Subflights []string `json:"subflights,omitempty" jsonschema:"Replacement ordered child flight references; omit to keep current."` + ManageKegs *bool `json:"manage_kegs,omitempty" jsonschema:"Whether to let a request rooted on this flight administer every KEG in the flight's namespace: create and delete KEGs, edit their settings, change their visibility, and manage their grants, even outside its cover (node content outside the cover stays out of reach), within your own role; omit to keep current. Turning it on takes a namespace owner or admin, unrooted or rooted on a flight that already manages the namespace's KEGs."` + ManageFlights *bool `json:"manage_flights,omitempty" jsonschema:"Whether to let a request rooted on this flight create, edit, and delete the flights of its namespace, within your own role; omit to keep current. Turning it on takes a namespace owner or admin, unrooted or rooted on a flight that already manages the namespace's flights."` + Manage *bool `json:"manage,omitempty" jsonschema:"Deprecated: sets both manage_kegs and manage_flights where they are omitted."` + ExpectedHash string `json:"expected_hash" jsonschema:"Precondition token returned by flight_read."` } type flightDeleteInput struct { - Ref string `json:"ref" jsonschema:"flight reference (@namespace/+slug; a bare slug uses the default namespace)"` - ExpectedHash string `json:"expected_hash" jsonschema:"precondition token returned by flight_read"` + Ref string `json:"ref" jsonschema:"Flight reference (@namespace/+slug; a bare slug uses the default namespace)."` + ExpectedHash string `json:"expected_hash" jsonschema:"Precondition token returned by flight_read."` } // registerFlightTools exposes flight discovery and management over MCP at @@ -49,8 +55,9 @@ type flightDeleteInput struct { // (omitted fields keep their current values) remains the MCP surface. func registerFlightTools(srv *sdkmcp.Server, defaults KegDefaults, flights FlightProvider) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "flight_list", - Description: "Memory: List available flights (keg restrictions + agent instructions)", + Name: "flight_list", + Description: "Memory: List the flights you can use, with their KEG cover and agent instructions.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -64,8 +71,9 @@ func registerFlightTools(srv *sdkmcp.Server, defaults KegDefaults, flights Fligh }) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "flight_read", - Description: "Memory: Inspect a permission-filtered flight manifest, including declared cover roles, depth, subflights, instructions, and hash. Inspection never activates a session or selects call authority. Cover is declared cover, not effective authority; use orient with flight for current authority. Pass hash as expected_hash to flight_edit or flight_delete.", + Name: "flight_read", + Description: "Memory: Inspect a permission-filtered flight manifest, including declared cover roles, depth, subflights, instructions, and hash. Inspection never activates a session or selects call authority. Cover is declared cover, not effective authority; use orient with flight for current authority. Pass hash as expected_hash to flight_edit or flight_delete.", + OutputSchema: outputSchema[PublicFlight](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -81,8 +89,9 @@ func registerFlightTools(srv *sdkmcp.Server, defaults KegDefaults, flights Fligh }) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "flight_create", - Description: "Manage: Create a Hub-backed flight (cover roles + agent instructions)", + Name: "flight_create", + Description: "Manage: Create a flight: its KEG cover with role caps, and its agent instructions.", + OutputSchema: outputSchema[PublicFlight](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: false, DestructiveHint: boolPtr(false), @@ -96,10 +105,12 @@ func registerFlightTools(srv *sdkmcp.Server, defaults KegDefaults, flights Fligh flight, err := flights.CreateFlight(ctx, tapper.CreateFlightOptions{ Ref: in.Ref, Title: in.Title, Description: in.Description, - Visibility: in.Visibility, - Instructions: in.Instructions, - Cover: cover, - Subflights: append([]string(nil), in.Subflights...), + Visibility: in.Visibility, + Instructions: in.Instructions, + Cover: cover, + Subflights: append([]string(nil), in.Subflights...), + ManageKegs: in.ManageKegs || in.Manage, + ManageFlights: in.ManageFlights || in.Manage, }) if err != nil { return errorResult(err), nil, nil @@ -114,8 +125,9 @@ func registerFlightTools(srv *sdkmcp.Server, defaults KegDefaults, flights Fligh }) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "flight_edit", - Description: "Manage: Call flight_read first, then edit a Hub-backed flight using its manifest hash as expected_hash; omitted fields keep their current values. On conflict, merge into the returned current flight (or refetch with flight_read) and retry with the returned current hash.", + Name: "flight_edit", + Description: "Manage: Call flight_read first, then edit a Hub-backed flight using its manifest hash as expected_hash; omitted fields keep their current values. On conflict, merge into the returned current flight (or refetch with flight_read) and retry with the returned current hash.", + OutputSchema: outputSchema[PublicFlight](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: false, DestructiveHint: boolPtr(true), @@ -132,9 +144,11 @@ func registerFlightTools(srv *sdkmcp.Server, defaults KegDefaults, flights Fligh opts := tapper.UpdateFlightOptions{ Ref: in.Ref, Title: in.Title, Description: in.Description, - Visibility: in.Visibility, - Instructions: in.Instructions, - ExpectedHash: in.ExpectedHash, + Visibility: in.Visibility, + Instructions: in.Instructions, + ManageKegs: switchOrAlias(in.ManageKegs, in.Manage), + ManageFlights: switchOrAlias(in.ManageFlights, in.Manage), + ExpectedHash: in.ExpectedHash, } if in.Cover != nil { cover, err := tapper.ParseFlightCoverSpecs(in.Cover) @@ -161,8 +175,9 @@ func registerFlightTools(srv *sdkmcp.Server, defaults KegDefaults, flights Fligh }) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "flight_delete", - Description: "Manage: Call flight_read first, then delete a Hub-backed flight using its manifest hash as expected_hash. On conflict, refetch with flight_read and retry with the returned current hash.", + Name: "flight_delete", + Description: "Manage: Call flight_read first, then delete a Hub-backed flight using its manifest hash as expected_hash. On conflict, refetch with flight_read and retry with the returned current hash.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: false, DestructiveHint: boolPtr(true), @@ -200,6 +215,12 @@ func renderFlight(f *tapper.Flight) string { } fmt.Fprintf(&b, "source: %s\n", f.Source) fmt.Fprintf(&b, "visibility: %s\n", f.Visibility) + if f.ManageKegs { + b.WriteString("manage_kegs: true (a request rooted on this flight administers its namespace's KEGs)\n") + } + if f.ManageFlights { + b.WriteString("manage_flights: true (a request rooted on this flight creates, edits, and deletes its namespace's flights)\n") + } if len(f.Cover) > 0 { b.WriteString("cover:\n") for _, c := range f.Cover { @@ -228,15 +249,17 @@ func renderFlight(f *tapper.Flight) string { // Do not serialize Flight itself: effective cover and internal identity fields // are not manifest fields and do not describe the authority of an inspection. type PublicFlight struct { - Name string `json:"name"` - Hash string `json:"hash"` - Title string `json:"title"` - Description string `json:"description"` - Source string `json:"source"` - Visibility string `json:"visibility"` - Cover []PublicFlightCover `json:"cover"` - Subflights []string `json:"subflights"` - Instructions string `json:"instructions"` + Name string `json:"name"` + Hash string `json:"hash"` + Title string `json:"title"` + Description string `json:"description"` + Source string `json:"source"` + Visibility string `json:"visibility"` + Cover []PublicFlightCover `json:"cover"` + Subflights []string `json:"subflights"` + ManageKegs bool `json:"manage_kegs"` + ManageFlights bool `json:"manage_flights"` + Instructions string `json:"instructions"` } type PublicFlightCover struct { Namespace string `json:"namespace"` @@ -256,5 +279,14 @@ func publicFlight(f *tapper.Flight) PublicFlight { } return PublicFlight{Name: f.Name, Hash: f.ManifestHash, Title: f.Title, Description: f.Description, Source: f.Source, Visibility: f.Visibility, - Cover: cover, Subflights: append([]string{}, f.Subflights...), Instructions: f.Instructions} + Cover: cover, Subflights: append([]string{}, f.Subflights...), ManageKegs: f.ManageKegs, ManageFlights: f.ManageFlights, Instructions: f.Instructions} +} + +// switchOrAlias is a flight_edit switch, or the deprecated manage alias when +// the switch itself is omitted. +func switchOrAlias(value, manage *bool) *bool { + if value != nil { + return value + } + return manage } diff --git a/pkg/mcp/tools_flight_lock_test.go b/pkg/mcp/tools_flight_lock_test.go index 0263136a..c9ee0bc6 100644 --- a/pkg/mcp/tools_flight_lock_test.go +++ b/pkg/mcp/tools_flight_lock_test.go @@ -120,10 +120,7 @@ func newFlightLockedSession(t *testing.T) (*sdkmcp.ClientSession, context.Contex <-done }) - client := sdkmcp.NewClient(&sdkmcp.Implementation{ - Name: "test-client", - Version: "0.1", - }, nil) + client := newTestClient(t, "test-client", nil) session, err := client.Connect(ctx, clientTransport, nil) require.NoError(t, err) t.Cleanup(func() { diff --git a/pkg/mcp/tools_index.go b/pkg/mcp/tools_index.go index 98c969ad..eca38a0d 100644 --- a/pkg/mcp/tools_index.go +++ b/pkg/mcp/tools_index.go @@ -18,13 +18,14 @@ func registerIndexTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault // --- index (rebuild) --- type indexInput struct { - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerIndex(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "index_rebuild", - Description: "Manage: Rebuild all KEG indexes from scratch (nodes, tags, links, backlinks)", + Name: "index_rebuild", + Description: "Manage: Rebuild every KEG index (nodes, tags, links, backlinks) from scratch.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), IdempotentHint: true, @@ -45,13 +46,14 @@ func registerIndex(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- index_list --- type listIndexesInput struct { - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerListIndexes(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "index_list", - Description: "Memory: List available index files for a KEG", + Name: "index_list", + Description: "Memory: List a KEG's index files.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -71,14 +73,15 @@ func registerListIndexes(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul // --- index_read --- type indexCatInput struct { - Name string `json:"name" jsonschema:"index file name (e.g. nodes.tsv, tags, links, backlinks, changes.md)"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Name string `json:"name" jsonschema:"Index file name, such as nodes.tsv, tags, links, backlinks, or changes.md."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerIndexCat(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "index_read", - Description: "Memory: Read the raw contents of a KEG index file", + Name: "index_read", + Description: "Memory: Read the raw contents of a KEG index file.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_keg.go b/pkg/mcp/tools_keg.go index 497962ea..eeb8062b 100644 --- a/pkg/mcp/tools_keg.go +++ b/pkg/mcp/tools_keg.go @@ -28,10 +28,10 @@ type kegListOutput struct { } type kegCreateInput struct { - Keg string `json:"keg" jsonschema:"required,qualified creation reference @namespace/keg; each component is 1-64 lowercase letters, digits, or hyphens and starts with a letter or digit"` - Title string `json:"title,omitempty" jsonschema:"human-readable keg title"` - Visibility string `json:"visibility,omitempty" jsonschema:"keg visibility: private (default) or public"` - KegVersion string `json:"kegv,omitempty" jsonschema:"immutable keg version: 2025-07 for a Markdown KEGv1 (the default) or 2026-09 for a KEGv2 preview, available only where the Hub enables it"` + Keg string `json:"keg" jsonschema:"Reference of the KEG to create, as @namespace/keg. Each part is 1-64 lowercase letters, digits, or hyphens, starting with a letter or digit."` + Title string `json:"title,omitempty" jsonschema:"Human-readable keg title."` + Visibility string `json:"visibility,omitempty" jsonschema:"Keg visibility: private (default) or public."` + KegVersion string `json:"kegv,omitempty" jsonschema:"Immutable keg version: 2025-07 for a Markdown KEGv1 (the default) or 2026-09 for a KEGv2 preview, available only where the Hub enables it."` } // registerKegTools exposes identity-authorized discovery, optionally narrowed @@ -42,8 +42,9 @@ type kegCreateInput struct { func registerKegTools(srv *sdkmcp.Server, defaults KegDefaults, kegs KegDiscoveryProvider) { registerKegDelete(srv, defaults, kegs) sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "keg_list", - Description: "Memory: List every KEG this session can work in, with role, title, description, and granting-flight provenance. The list is complete, so read it rather than searching; explore_kegs searches public KEGs you do not hold. Without a configured root, returns identity-accessible KEGs at their real roles. With a pinned root, omission aggregates its accessible transitive graph for discovery; a discovered child-only KEG still requires that child as flight on operational calls. Supplying flight returns exactly that flight projection", + Name: "keg_list", + Description: "Memory: List every KEG this session can work in, with role, title, description, and granting-flight provenance. The list is complete, so read it rather than searching; explore_kegs searches public KEGs you do not hold. Without a configured root, returns identity-accessible KEGs at their real roles. With a pinned root, omission aggregates its accessible transitive graph for discovery; a discovered child-only KEG still requires that child as flight on operational calls. Supplying flight returns exactly that flight's projection.", + OutputSchema: outputSchema[kegListOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -92,7 +93,9 @@ func registerKegTools(srv *sdkmcp.Server, defaults KegDefaults, kegs KegDiscover sdkmcp.AddTool(srv, &sdkmcp.Tool{ Name: "keg_create", Description: "Manage: Create a new KEG in a namespace you belong to. Creating a KEG " + - "never adds it to a flight's cover.", + "never adds it to a flight's cover. A session on a Hub flight creates KEGs only " + + "where that flight manages the namespace's KEGs (manage_kegs).", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: false, DestructiveHint: boolPtr(false), @@ -173,11 +176,17 @@ func filterKegRefs(ctx context.Context, refs []string) []string { } type kegDeleteInput struct { - Keg string `json:"keg" jsonschema:"explicit canonical @namespace/keg reference to permanently delete"` + Keg string `json:"keg" jsonschema:"Explicit canonical @namespace/keg reference to permanently delete."` +} + +// kegDeleteOutput is keg_delete's structured content. +type kegDeleteOutput struct { + Keg string `json:"keg" jsonschema:"The deleted KEG."` + Deleted bool `json:"deleted" jsonschema:"Always true: a delete that fails returns an error instead."` } func registerKegDelete(srv *sdkmcp.Server, defaults KegDefaults, provider KegDiscoveryProvider) { - sdkmcp.AddTool(srv, &sdkmcp.Tool{Name: "keg_delete", Description: "Manage: Permanently delete an empty or populated KEG and all its data, including snapshots. Requires identity admin permission; a selected flight also requires effective admin cover. No expected_hash: a settings hash does not cover a whole KEG.", Annotations: &sdkmcp.ToolAnnotations{DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false)}}, func(ctx context.Context, _ *sdkmcp.CallToolRequest, in kegDeleteInput) (*sdkmcp.CallToolResult, any, error) { + sdkmcp.AddTool(srv, &sdkmcp.Tool{Name: "keg_delete", OutputSchema: outputSchema[kegDeleteOutput](), Description: "Manage: Permanently delete an empty or populated KEG and all its data, including snapshots. Requires identity admin permission; a session on a Hub flight also needs that flight to manage the namespace's KEGs (manage_kegs), which reaches KEGs outside its cover. No expected_hash: a settings hash does not cover a whole KEG.", Annotations: &sdkmcp.ToolAnnotations{DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false)}}, func(ctx context.Context, _ *sdkmcp.CallToolRequest, in kegDeleteInput) (*sdkmcp.CallToolResult, any, error) { ns, alias, err := tapper.ParseCanonicalKegRef(in.Keg) if err != nil { return errorResult(fmt.Errorf("keg must be an explicit canonical @namespace/keg reference: %w", err)), nil, nil @@ -185,21 +194,21 @@ func registerKegDelete(srv *sdkmcp.Server, defaults KegDefaults, provider KegDis if err := defaults.gate.authorizeOriented(orientationFromContext(ctx)); err != nil { return errorResult(err), nil, nil } - if flight := SessionFlight(ctx); flight != nil { + deleter, ok := provider.(KegDeletionProvider) + if !ok { + return errorResult(keg.ErrNotSupported), nil, nil + } + authorizer, decides := provider.(KegDeletionAuthorizer) + if flight := SessionFlight(ctx); flight != nil && !(decides && authorizer.AuthorizesKegDeletion()) { role, covered := flight.RoleFor("", ns, alias) if !covered || role != tapper.FlightRoleAdmin { return errorResult(keg.ErrOrientationDenied), nil, nil } } - deleter, ok := provider.(KegDeletionProvider) - if !ok { - return errorResult(keg.ErrNotSupported), nil, nil - } if err := deleter.DeleteKeg(ctx, in.Keg); err != nil { return errorResult(err), nil, nil } - out := map[string]any{"keg": in.Keg, "deleted": true} - result := &sdkmcp.CallToolResult{StructuredContent: out} + result := &sdkmcp.CallToolResult{StructuredContent: kegDeleteOutput{Keg: in.Keg, Deleted: true}} return result, nil, nil }) } diff --git a/pkg/mcp/tools_license.go b/pkg/mcp/tools_license.go index 582cd061..27354e0b 100644 --- a/pkg/mcp/tools_license.go +++ b/pkg/mcp/tools_license.go @@ -16,8 +16,9 @@ type licenseInput struct{} func registerLicense(srv *sdkmcp.Server, licenseText string) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "license", - Description: "Return the full license text embedded in the binary", + Name: "license", + Description: "Return the full license text embedded in the binary", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_lock.go b/pkg/mcp/tools_lock.go index 437400c7..3054f36b 100644 --- a/pkg/mcp/tools_lock.go +++ b/pkg/mcp/tools_lock.go @@ -7,6 +7,7 @@ import ( sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/jlrickert/tapper/pkg/keg" "github.com/jlrickert/tapper/pkg/tapper" ) @@ -20,22 +21,29 @@ func registerLockTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults // --- lock_acquire --- type lockAcquireInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to lock"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to lock."` + WaitSeconds int `json:"wait_seconds,omitempty" jsonschema:"Seconds to wait for a lock another holder has before failing with LOCK_TIMEOUT. Default 10; values above 60 wait 60."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerLockAcquire(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "lock_acquire", - Description: "Memory: Acquire a cross-process lock on a node and return the token", + Name: "lock_acquire", + Description: "Memory: Acquire a Hub-held lock on a node and return its token. A held lock is waited for up to wait_seconds (default 10, max 60), then fails with LOCK_TIMEOUT.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), }, }, func(ctx context.Context, req *sdkmcp.CallToolRequest, in lockAcquireInput) (*sdkmcp.CallToolResult, any, error) { + if in.WaitSeconds < 0 { + return errorResult(fmt.Errorf("%w: wait_seconds must not be negative", keg.ErrInvalid)), nil, nil + } + // Tap.Lock applies the default and the 60s cap. opts := tapper.LockOptions{ KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), NodeID: in.NodeID, + Wait: time.Duration(in.WaitSeconds) * time.Second, } token, err := tap.Lock(ctx, opts) if err != nil { @@ -48,15 +56,16 @@ func registerLockAcquire(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul // --- lock_release --- type lockReleaseInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to unlock"` - Token string `json:"token" jsonschema:"lock token returned by lock_acquire"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to unlock."` + Token string `json:"token" jsonschema:"Lock token returned by lock_acquire."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerLockRelease(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "lock_release", - Description: "Memory: Release a cross-process lock on a node", + Name: "lock_release", + Description: "Memory: Release a lock on a node.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -77,14 +86,15 @@ func registerLockRelease(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul // --- lock_status --- type lockStatusInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to check"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to check."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerLockStatus(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "lock_status", - Description: "Memory: Check the lock state of a node", + Name: "lock_status", + Description: "Memory: Check a node's lock state.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -113,14 +123,15 @@ func registerLockStatus(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault // --- lock_force_release --- type lockForceReleaseInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to force-unlock"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to force-unlock."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerLockForceRelease(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "lock_force_release", - Description: "Manage: Unconditionally remove a lock on a node", + Name: "lock_force_release", + Description: "Manage: Remove a lock on a node unconditionally.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_lock_wait_test.go b/pkg/mcp/tools_lock_wait_test.go new file mode 100644 index 00000000..c34eae50 --- /dev/null +++ b/pkg/mcp/tools_lock_wait_test.go @@ -0,0 +1,155 @@ +package mcp_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/jlrickert/cli-toolkit/toolkit" + "github.com/jlrickert/tapper/internal/testapi" + "github.com/jlrickert/tapper/pkg/mcp" + "github.com/jlrickert/tapper/pkg/tapper" + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" +) + +// newLockWaitHubSession serves an unpinned session over a Hub whose lock +// endpoint records the ?wait= each acquisition sends. +func newLockWaitHubSession(t *testing.T) (*sdkmcp.ClientSession, context.Context, func() []string) { + t.Helper() + ctx := context.Background() + var ( + mu sync.Mutex + waits []string + ) + hub := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/v1/flights": + _ = json.NewEncoder(w).Encode([]tapper.HubFlight{}) + case r.URL.Path == "/api/v1/kegs": + _ = json.NewEncoder(w).Encode([]tapper.HubKeg{{Namespace: "root", Alias: "notes", Role: "admin", Visibility: "private"}}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/@root/kegs/notes/nodes/0/lock": + mu.Lock() + if r.URL.Query().Has("wait") { + waits = append(waits, r.URL.Query().Get("wait")) + } else { + waits = append(waits, "") + } + mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string]any{"token": "hub-token", "ttl_seconds": 300}) + case strings.HasSuffix(r.URL.Path, "/settings"): + _, _ = fmt.Fprint(w, "kegv: \"2025-07\"\ntitle: Root notes\n") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(hub.Close) + rt := newTestSandbox(t).Runtime() + tap, err := tapper.NewTap(tapper.TapOptions{Runtime: rt}) + require.NoError(t, err) + config := fmt.Sprintf("disableAtlasHub: true\nhub: home\nhubs:\n home: {url: '%s'}\n", hub.URL) + require.NoError(t, rt.AtomicWriteFile(tap.PathService.UserConfig(), []byte(config), 0o644)) + tap.ConfigService.Reload() + store, err := tapper.LoadAuthStore(ctx, rt, tap.PathService.AuthStorePath()) + require.NoError(t, err) + store.Set(hub.URL, tapper.AuthEntry{AccessToken: "root-token"}) + require.NoError(t, store.Save(ctx, rt, tap.PathService.AuthStorePath())) + tap.AuthValidateFn = func(context.Context, *toolkit.Runtime, string, string) (*tapper.WhoAmI, error) { + return &tapper.WhoAmI{UserID: 1, Username: "root", DefaultNamespace: "root", Namespaces: []string{"root"}}, nil + } + session := connectFlightSession(t, ctx, mcp.NewServer(tap, "test", mcp.KegDefaults{}), nil) + return session, ctx, func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), waits...) + } +} + +func TestMCP_LockAcquireSendsBoundedWaitToHub(t *testing.T) { + t.Parallel() + session, ctx, waits := newLockWaitHubSession(t) + acquire := func(args map[string]any) { + t.Helper() + args["keg"] = "@root/notes" + args["node_id"] = "0" + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "lock_acquire", Arguments: args}) + require.NoError(t, err) + require.False(t, res.IsError, extractText(t, res)) + require.Equal(t, "hub-token", extractText(t, res)) + } + + acquire(map[string]any{}) + acquire(map[string]any{"wait_seconds": 25}) + acquire(map[string]any{"wait_seconds": 600}) + + require.Equal(t, []string{"10", "25", "60"}, waits(), + "the default is 10s, a request inside the bound passes through, and anything above 60s is clamped") +} + +func TestMCP_LockAcquireRejectsNegativeWait(t *testing.T) { + t.Parallel() + session, ctx, waits := newLockWaitHubSession(t) + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "lock_acquire", Arguments: map[string]any{ + "keg": "@root/notes", "node_id": "0", "wait_seconds": -1, + }}) + require.NoError(t, err) + require.True(t, res.IsError) + require.Contains(t, extractText(t, res), "wait_seconds") + require.Empty(t, waits(), "an invalid bound must not reach the Hub") +} + +// A locally-backed KEG honors the same bound: a held lock fails with +// a lock timeout once wait_seconds pass instead of waiting out the lock's TTL. +func TestMCP_LockAcquireLocalHeldLockTimesOutAtWait(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + first, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "lock_acquire", Arguments: map[string]any{"node_id": "0"}}) + require.NoError(t, err) + require.False(t, first.IsError, extractText(t, first)) + + start := time.Now() + second, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "lock_acquire", Arguments: map[string]any{ + "node_id": "0", "wait_seconds": 1, + }}) + require.NoError(t, err) + elapsed := time.Since(start) + require.True(t, second.IsError, "a held lock must not be acquired twice") + require.Contains(t, extractText(t, second), "lock acquire timeout") + require.GreaterOrEqual(t, elapsed, 900*time.Millisecond, "the bound is waited out, not refused at once") + require.Less(t, elapsed, 5*time.Second) +} + +// Overshoot: the bound only caps the wait; a lock released while the caller +// waits is still acquired. +func TestMCP_LockAcquireLocalLockReleasedDuringWaitIsAcquired(t *testing.T) { + t.Parallel() + session, ctx := newTestSession(t) + first, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "lock_acquire", Arguments: map[string]any{"node_id": "0"}}) + require.NoError(t, err) + require.False(t, first.IsError, extractText(t, first)) + token := extractText(t, first) + + released := make(chan error, 1) + go func() { + time.Sleep(200 * time.Millisecond) + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "lock_release", Arguments: map[string]any{ + "node_id": "0", "token": token, + }}) + if err == nil && res.IsError { + err = fmt.Errorf("lock_release: %s", extractText(t, res)) + } + released <- err + }() + second, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "lock_acquire", Arguments: map[string]any{ + "node_id": "0", "wait_seconds": 5, + }}) + require.NoError(t, err) + require.NoError(t, <-released) + require.False(t, second.IsError, extractText(t, second)) + require.NotEqual(t, token, extractText(t, second)) +} diff --git a/pkg/mcp/tools_merge.go b/pkg/mcp/tools_merge.go index 4e24c6ff..b48414d0 100644 --- a/pkg/mcp/tools_merge.go +++ b/pkg/mcp/tools_merge.go @@ -13,16 +13,22 @@ import ( // --- node_merge --- type mergeInput struct { - SourceID string `json:"source_id" jsonschema:"id of the node merged away; it goes to the trash"` - ExpectedSourceHash string `json:"expected_source_hash" jsonschema:"the source's combined hash from node_read (not content_hash or meta_hash)"` - TargetID string `json:"target_id" jsonschema:"id of the node that survives and takes the merged body"` - Content *string `json:"content,omitempty" jsonschema:"the target's merged markdown body, replacing its content entirely. Must not start with a YAML frontmatter block. Send content or patch."` - Patch *string `json:"patch,omitempty" jsonschema:"unified diff against the target's current content, an alternative to content when the target already holds most of the merged body"` - Meta *string `json:"meta,omitempty" jsonschema:"the target's replacement metadata as YAML, for combining tags and fields; omit to keep it"` - ExpectedContentHash string `json:"expected_content_hash" jsonschema:"the target's content_hash from node_read"` - ExpectedMetaHash string `json:"expected_meta_hash,omitempty" jsonschema:"the target's meta_hash from node_read; required when meta is supplied"` - Schema string `json:"schema,omitempty" jsonschema:"schema for the target, as for node_edit: required on a strict keg when meta is supplied"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + SourceID string `json:"source_id" jsonschema:"ID of the node merged away; it goes to the trash."` + ExpectedSourceHash string `json:"expected_source_hash" jsonschema:"The source's combined hash from node_read, not content_hash or meta_hash."` + TargetID string `json:"target_id" jsonschema:"ID of the node that survives and takes the merged body."` + Content *string `json:"content,omitempty" jsonschema:"The target's merged markdown body, replacing its content entirely. Must not start with a YAML frontmatter block. Send content or patch."` + Patch *string `json:"patch,omitempty" jsonschema:"Unified diff against the target's current content, an alternative to content when the target already holds most of the merged body."` + Meta *string `json:"meta,omitempty" jsonschema:"The target's replacement metadata as YAML, for combining tags and fields; omit to keep it."` + ExpectedContentHash string `json:"expected_content_hash" jsonschema:"The target's content_hash from node_read."` + ExpectedMetaHash string `json:"expected_meta_hash,omitempty" jsonschema:"The target's meta_hash from node_read. Required with meta."` + Schema string `json:"schema,omitempty" jsonschema:"Schema for the target, as for node_edit: required on a strict keg when meta is supplied."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` +} + +// mergeOutput is node_merge's structured content. +type mergeOutput struct { + Target nodeUpdateOutput `json:"target" jsonschema:"The surviving node's new hashes."` + Rewritten []string `json:"rewritten" jsonschema:"IDs of the other nodes whose links to the source now point at the target."` } func registerMerge(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { @@ -33,6 +39,8 @@ func registerMerge(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { "Every ../SOURCE link in the keg, the target's own included, is repointed at the target, and the source goes to the trash, recorded as merged into the target, where trash_restore can bring it back. " + "Guard the source with its combined hash as expected_source_hash, and the target with its content_hash (and meta_hash when sending meta). " + "The result returns the target's new hashes and the other nodes whose links moved.", + InputSchema: constrainedInputSchema[mergeInput](), + OutputSchema: outputSchema[mergeOutput](), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), @@ -63,10 +71,7 @@ func registerMerge(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { rewritten[i] = id.Path() } res := textResult(fmt.Sprintf("merged %s into %s; %d other node(s) relinked", in.SourceID, result.Target.ID.Path(), len(rewritten))) - res.StructuredContent = map[string]any{ - "target": nodeUpdateOutputs([]keg.NodeUpdateResult{result.Target})[0], - "rewritten": rewritten, - } + res.StructuredContent = mergeOutput{Target: nodeUpdateOutputs([]keg.NodeUpdateResult{result.Target})[0], Rewritten: rewritten} return res, nil, nil }) } diff --git a/pkg/mcp/tools_namespace.go b/pkg/mcp/tools_namespace.go index 12758e01..f0fdb9d5 100644 --- a/pkg/mcp/tools_namespace.go +++ b/pkg/mcp/tools_namespace.go @@ -19,6 +19,16 @@ type InvitationLister interface { ListInvitations(context.Context) ([]tapper.HubInvitation, error) } +// invitationListOutput is invitation_list's structured content. +type invitationListOutput struct { + Invitations []tapper.HubInvitation `json:"invitations" jsonschema:"Pending invitations addressed to you."` +} + +// namespaceListOutput is namespace_list's structured content. +type namespaceListOutput struct { + Namespaces []tapper.HubNamespace `json:"namespaces" jsonschema:"The namespaces you belong to, with your role in each."` +} + // registerNamespaceTools exposes namespace discovery over MCP, at parity with // `tap namespace list`. The hub-wide people directory is explore_people. Listing never grants // authority. Member rosters and role changes stay UI-only, so those tools are @@ -28,8 +38,9 @@ type InvitationLister interface { func registerNamespaceTools(srv *sdkmcp.Server, namespaces NamespaceProvider) { if lister, ok := namespaces.(InvitationLister); ok { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "invitation_list", - Description: "Session: List pending invitations addressed to you: org memberships and keg grants that take effect only if you accept them. Read-only; accepting or declining is your user's decision, made in the Hub UI or with `tap invitation`, so ask them rather than acting.", + Name: "invitation_list", + Description: "Session: List pending invitations addressed to you: org memberships and keg grants that take effect only if you accept them. Read-only; accepting or declining is your user's decision, made in the Hub UI or with `tap invitation`, so ask them rather than acting.", + OutputSchema: outputSchema[invitationListOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -44,13 +55,14 @@ func registerNamespaceTools(srv *sdkmcp.Server, namespaces NamespaceProvider) { lines = append(lines, fmt.Sprintf("%d\t%s\t%s\t%s\t@%s", inv.ID, inv.Kind, inv.Target, inv.Role, inv.InvitedBy)) } res := linesResult(lines) - res.StructuredContent = map[string]any{"invitations": rows} + res.StructuredContent = invitationListOutput{Invitations: rows} return res, nil, nil }) } sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "namespace_list", - Description: "Session: List the namespaces you belong to, with your role in each", + Name: "namespace_list", + Description: "Session: List the namespaces you belong to, with your role in each.", + OutputSchema: outputSchema[namespaceListOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -65,7 +77,7 @@ func registerNamespaceTools(srv *sdkmcp.Server, namespaces NamespaceProvider) { lines = append(lines, fmt.Sprintf("@%s\t%s\t%s", ns.Name, ns.Kind, ns.Role)) } res := linesResult(lines) - res.StructuredContent = map[string]any{"namespaces": rows} + res.StructuredContent = namespaceListOutput{Namespaces: rows} return res, nil, nil }) } diff --git a/pkg/mcp/tools_orient.go b/pkg/mcp/tools_orient.go index 110b2621..ebf4f54a 100644 --- a/pkg/mcp/tools_orient.go +++ b/pkg/mcp/tools_orient.go @@ -43,6 +43,7 @@ func registerOrient(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { "changing shared session state. " + "If an explicitly configured root fails to load, the session fails closed with only recovery tools. " + "When no root is configured, no-flight full access stays pinned until the connection ends; pin a restrictive flight outside MCP and start a new connection.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -66,6 +67,7 @@ func registerSessionRefresh(srv *sdkmcp.Server, defaults KegDefaults) { Description: "Session: Retry MCP session activation after a failed configured flight becomes available. " + "Takes no arguments and never changes an already-active connection. A no-flight full-access " + "connection therefore requires a new MCP connection after a restrictive flight is pinned.", + OutputSchema: outputSchema[sessionRefreshOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: false, DestructiveHint: boolPtr(false), @@ -96,11 +98,11 @@ func registerSessionRefresh(srv *sdkmcp.Server, defaults KegDefaults) { } type guideInput struct { - Topic string `json:"topic" jsonschema:"operating, authoring, linking, snapshots, tools, or troubleshooting"` + Topic string `json:"topic" jsonschema:"Topic: operating, authoring, linking, snapshots, tools, or troubleshooting."` } func registerGuide(srv *sdkmcp.Server) { - sdkmcp.AddTool(srv, &sdkmcp.Tool{Name: "guide", Description: "Session: Read canonical KEG guidance on demand by topic.", Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)}}, func(ctx context.Context, req *sdkmcp.CallToolRequest, in guideInput) (*sdkmcp.CallToolResult, any, error) { + sdkmcp.AddTool(srv, &sdkmcp.Tool{Name: "guide", OutputSchema: textOutputSchema(), Description: "Session: Read canonical KEG guidance on demand by topic.", Annotations: &sdkmcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)}}, func(ctx context.Context, req *sdkmcp.CallToolRequest, in guideInput) (*sdkmcp.CallToolResult, any, error) { text, err := tapper.OrientationGuide(in.Topic) if err != nil { return errorResult(err), nil, nil diff --git a/pkg/mcp/tools_read.go b/pkg/mcp/tools_read.go index 478bc894..395025de 100644 --- a/pkg/mcp/tools_read.go +++ b/pkg/mcp/tools_read.go @@ -26,12 +26,12 @@ func registerReadTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults // --- node_read --- type catInput struct { - NodeIDs []string `json:"node_ids,omitempty" jsonschema:"node IDs to read"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` - ContentOnly bool `json:"content_only,omitempty" jsonschema:"return content without frontmatter"` - MetaOnly bool `json:"meta_only,omitempty" jsonschema:"return metadata only"` - StatsOnly bool `json:"stats_only,omitempty" jsonschema:"return stats only"` - Query string `json:"query,omitempty" jsonschema:"boolean expression to select nodes (alternative to node_ids)"` + NodeIDs []string `json:"node_ids,omitempty" jsonschema:"Node IDs to read."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` + ContentOnly bool `json:"content_only,omitempty" jsonschema:"Return content only, without metadata."` + MetaOnly bool `json:"meta_only,omitempty" jsonschema:"Return metadata only."` + StatsOnly bool `json:"stats_only,omitempty" jsonschema:"Return stats only."` + Query string `json:"query,omitempty" jsonschema:"Query expression selecting nodes, instead of node_ids."` } // nodeReadOutput is one self-contained read result: the node's document @@ -47,13 +47,18 @@ type catInput struct { // only for the parts the read mode returned. Hash is the combined token // node_delete and node_move take. type nodeReadOutput struct { - NodeID string `json:"node_id"` - Hash string `json:"hash"` - ContentHash string `json:"content_hash,omitempty"` - MetaHash string `json:"meta_hash,omitempty"` - Content string `json:"content,omitempty"` - Meta string `json:"meta,omitempty"` - Stats string `json:"stats,omitempty"` + NodeID string `json:"node_id" jsonschema:"The node's ID."` + Hash string `json:"hash" jsonschema:"The combined hash node_delete and node_move take as expected_hash."` + ContentHash string `json:"content_hash,omitempty" jsonschema:"Present when content was read; node_edit's expected_content_hash."` + MetaHash string `json:"meta_hash,omitempty" jsonschema:"Present when metadata was read; node_edit's expected_meta_hash."` + Content string `json:"content,omitempty" jsonschema:"The markdown body, unless meta_only or stats_only."` + Meta string `json:"meta,omitempty" jsonschema:"The metadata document as YAML, unless content_only or stats_only."` + Stats string `json:"stats,omitempty" jsonschema:"The node's stats, with stats_only."` +} + +// nodeReadResult is node_read's structured content. +type nodeReadResult struct { + Nodes []nodeReadOutput `json:"nodes" jsonschema:"One entry per node read, in request order."` } func nodeReadOutputs(ctx context.Context, views []keg.NodeView, opts tapper.CatOptions) []nodeReadOutput { @@ -93,6 +98,7 @@ func registerCat(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { "metadata was read): pass content_hash as expected_content_hash and meta_hash as " + "expected_meta_hash to node_edit. Each part's hash is independent, so editing one part " + "never invalidates the other's. The combined hash is for node_delete and node_move.", + OutputSchema: outputSchema[nodeReadResult](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -114,11 +120,11 @@ func registerCat(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { } if len(views) == 0 { res := textResult(tapper.FormatCatViews(ctx, views, opts)) - res.StructuredContent = map[string]any{"nodes": nodeReadOutputs(ctx, views, opts)} + res.StructuredContent = nodeReadResult{Nodes: nodeReadOutputs(ctx, views, opts)} return res, nil, nil } res := textResult(nodeReadSummary(len(views))) - res.StructuredContent = map[string]any{"nodes": nodeReadOutputs(ctx, views, opts)} + res.StructuredContent = nodeReadResult{Nodes: nodeReadOutputs(ctx, views, opts)} return res, nil, nil }) } @@ -126,20 +132,20 @@ func registerCat(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- list --- type listInput struct { - Query string `json:"query,omitempty" jsonschema:"boolean query expression to filter nodes. Supports tags ('golang'), key=value attributes ('entity=plan'), and dot-prefix stats fields ('.created>2026-01-01', '.accessCount>=5', '.hash=abc123'). Combine with 'and', 'or', 'not'."` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` - Format string `json:"format,omitempty" jsonschema:"output format template. Legacy verbs %i (id), %t (title), %d (updated), %c (created), %a (accessed); %% is a literal percent. Named selectors use %{...}: a bare word names a metadata key such as %{type} or %{status}, a leading dot names a statistics field such as %{.accessCount} or %{.omega}, and %{tags} is the node's tag list. Backslash escapes are interpreted: \\t tab, \\n newline, \\r return, \\\\ backslash. Selectors other than id, title, and the three dates are resolved by the Hub for the returned page in one batch."` - IdOnly bool `json:"id_only,omitempty" jsonschema:"return node IDs only"` - Reverse bool `json:"reverse,omitempty" jsonschema:"reverse output order"` - Sort string `json:"sort,omitempty" jsonschema:"sort order: 'id' (default), 'updated', 'created', or 'accessed'"` - Limit int `json:"limit,omitempty" jsonschema:"maximum results to return (default: 50, 0 in request means use default, -1 for unlimited)"` - Offset int `json:"offset,omitempty" jsonschema:"skip the first N results (for pagination)"` + Query string `json:"query,omitempty" jsonschema:"Query expression selecting nodes: tags (golang), key=value attributes (entity=plan), and stats fields with a leading dot (.created>2026-01-01, .accessCount>=5), combined with and, or, and not."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` + Format string `json:"format,omitempty" jsonschema:"Output template per node. Verbs: %i id, %t title, %d updated, %c created, %a accessed, %% a literal percent. Selectors: %{key} a metadata key such as %{type}, %{.field} a stats field such as %{.accessCount}, %{tags} the tag list. Escapes: \\t, \\n, \\r, \\\\."` + Reverse bool `json:"reverse,omitempty" jsonschema:"Reverse the order of each page."` + Sort string `json:"sort,omitempty" jsonschema:"Sort order: id (default), updated, created, or accessed."` + Limit int `json:"limit,omitempty" jsonschema:"Maximum results; 0 or omitted means 50, and -1 means no limit."` + Offset int `json:"offset,omitempty" jsonschema:"Results to skip; pass the previous page's next_offset to page."` } func registerList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "node_list", - Description: "Memory: List KEG nodes, optionally filtered by a query expression", + Name: "node_list", + Description: "Memory: List a KEG's nodes, optionally filtered by a query expression, one line per node rendered by format. Follow next_offset for more.", + OutputSchema: outputSchema[pagedLinesOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -149,7 +155,6 @@ func registerList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), Query: in.Query, Format: in.Format, - IdOnly: in.IdOnly, Reverse: in.Reverse, Sort: tapper.ListSortType(in.Sort), Limit: mcpDefaultLimit(in.Limit), @@ -166,21 +171,21 @@ func registerList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- node_grep --- type grepInput struct { - Query string `json:"query" jsonschema:"regex pattern to search node content, matched per line. Local KEGs use Go RE2 syntax; Hub KEGs use PostgreSQL regex, where \\y (not \\b) is a word boundary and lookahead is allowed"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` - Format string `json:"format,omitempty" jsonschema:"output format template. Legacy verbs %i (id), %t (title), %d (updated), %c (created), %a (accessed); %% is a literal percent. Named selectors use %{...}: a bare word names a metadata key such as %{type} or %{status}, a leading dot names a statistics field such as %{.accessCount} or %{.omega}, and %{tags} is the node's tag list. Backslash escapes are interpreted: \\t tab, \\n newline, \\r return, \\\\ backslash. Selectors other than id, title, and the three dates are resolved by the Hub for the returned page in one batch. Use id_only for compact MCP output."` - IdOnly bool `json:"id_only,omitempty" jsonschema:"return node IDs only (recommended for MCP to reduce token usage)"` - Reverse bool `json:"reverse,omitempty" jsonschema:"reverse output order"` - IgnoreCase bool `json:"ignore_case,omitempty" jsonschema:"case-insensitive matching"` - MaxLines int `json:"max_lines,omitempty" jsonschema:"max matched lines per node (default: unlimited for CLI, 3 for MCP)"` - Limit int `json:"limit,omitempty" jsonschema:"maximum results to return (default: 50, 0 in request means use default, -1 for unlimited)"` - Offset int `json:"offset,omitempty" jsonschema:"skip the first N results (for pagination)"` + Query string `json:"query" jsonschema:"PostgreSQL regular expression matched against each line of node content. \\y (not \\b) is a word boundary; lookahead is allowed."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` + Format string `json:"format,omitempty" jsonschema:"Output template per node. Verbs: %i id, %t title, %d updated, %c created, %a accessed, %% a literal percent. Selectors: %{key} a metadata key such as %{type}, %{.field} a stats field such as %{.accessCount}, %{tags} the tag list. Escapes: \\t, \\n, \\r, \\\\."` + Reverse bool `json:"reverse,omitempty" jsonschema:"Reverse the order of each page."` + IgnoreCase bool `json:"ignore_case,omitempty" jsonschema:"Match case-insensitively."` + MaxLines int `json:"max_lines,omitempty" jsonschema:"Matched lines to show per node; 0 or omitted means 3, and -1 means all."` + Limit int `json:"limit,omitempty" jsonschema:"Maximum results; 0 or omitted means 50, and -1 means no limit."` + Offset int `json:"offset,omitempty" jsonschema:"Results to skip; pass the previous page's next_offset to page."` } func registerGrep(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "node_grep", - Description: "Memory: Match KEG node content lines against a regex pattern in one KEG, in node id order. Use it for exact identifiers, symbols, and audits; use node_search to find the nodes about a topic. max_lines defaults to 3 matched lines per node; -1 returns all matched lines. Follow next_offset for more nodes; use node_read for complete bodies.", + Name: "node_grep", + Description: "Memory: Match node content lines in one KEG against a PostgreSQL regular expression, in node ID order. Use it for exact identifiers, symbols, and audits; use node_search to find the nodes about a topic. Follow next_offset for more nodes, and use node_read for complete bodies.", + OutputSchema: outputSchema[pagedLinesOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -190,7 +195,6 @@ func registerGrep(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), Query: in.Query, Format: in.Format, - IdOnly: in.IdOnly, Reverse: in.Reverse, IgnoreCase: in.IgnoreCase, MaxLines: mcpDefaultMaxLines(in.MaxLines), @@ -208,19 +212,19 @@ func registerGrep(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- tags --- type tagsInput struct { - Query string `json:"query,omitempty" jsonschema:"boolean expression to filter by tags, attributes, and dot-prefix stats fields (e.g. '.created>2026-01-01 and entity=plan')"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` - Format string `json:"format,omitempty" jsonschema:"output format template. Legacy verbs %i (id), %t (title), %d (updated), %c (created), %a (accessed); %% is a literal percent. Named selectors use %{...}: a bare word names a metadata key such as %{type} or %{status}, a leading dot names a statistics field such as %{.accessCount} or %{.omega}, and %{tags} is the node's tag list. Backslash escapes are interpreted: \\t tab, \\n newline, \\r return, \\\\ backslash. Selectors other than id, title, and the three dates are resolved by the Hub for the returned page in one batch."` - IdOnly bool `json:"id_only,omitempty" jsonschema:"return node IDs only"` - Reverse bool `json:"reverse,omitempty" jsonschema:"reverse output order"` - Limit int `json:"limit,omitempty" jsonschema:"maximum results to return (default: 50, 0 in request means use default, -1 for unlimited)"` - Offset int `json:"offset,omitempty" jsonschema:"skip the first N results (for pagination)"` + Query string `json:"query,omitempty" jsonschema:"Query expression selecting nodes, such as .created>2026-01-01 and entity=plan. Omit it to list the KEG's tags."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` + Format string `json:"format,omitempty" jsonschema:"Output template per node. Verbs: %i id, %t title, %d updated, %c created, %a accessed, %% a literal percent. Selectors: %{key} a metadata key such as %{type}, %{.field} a stats field such as %{.accessCount}, %{tags} the tag list. Escapes: \\t, \\n, \\r, \\\\."` + Reverse bool `json:"reverse,omitempty" jsonschema:"Reverse the order of each page."` + Limit int `json:"limit,omitempty" jsonschema:"Maximum results; 0 or omitted means 50, and -1 means no limit."` + Offset int `json:"offset,omitempty" jsonschema:"Results to skip; pass the previous page's next_offset to page."` } func registerTags(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "tag_list", - Description: "Memory: List tags or filter nodes by tag expression", + Name: "tag_list", + Description: "Memory: List a KEG's tags, or the nodes a query expression selects.", + OutputSchema: outputSchema[pagedLinesOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -230,7 +234,6 @@ func registerTags(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), Query: in.Query, Format: in.Format, - IdOnly: in.IdOnly, Reverse: in.Reverse, Limit: mcpDefaultLimit(in.Limit), Offset: in.Offset, @@ -246,19 +249,19 @@ func registerTags(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- backlinks --- type backlinksInput struct { - NodeIDs []string `json:"node_ids" jsonschema:"target node IDs to find incoming links for (results merged and deduplicated)"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` - Format string `json:"format,omitempty" jsonschema:"output format template. Legacy verbs %i (id), %t (title), %d (updated), %c (created), %a (accessed); %% is a literal percent. Named selectors use %{...}: a bare word names a metadata key such as %{type} or %{status}, a leading dot names a statistics field such as %{.accessCount} or %{.omega}, and %{tags} is the node's tag list. Backslash escapes are interpreted: \\t tab, \\n newline, \\r return, \\\\ backslash. Selectors other than id, title, and the three dates are resolved by the Hub for the returned page in one batch."` - IdOnly bool `json:"id_only,omitempty" jsonschema:"return node IDs only"` - Reverse bool `json:"reverse,omitempty" jsonschema:"reverse output order"` - Limit int `json:"limit,omitempty" jsonschema:"maximum results to return (default: 50, 0 in request means use default, -1 for unlimited)"` - Offset int `json:"offset,omitempty" jsonschema:"skip the first N results (for pagination)"` + NodeIDs []string `json:"node_ids" jsonschema:"Target node IDs to find incoming links for (results merged and deduplicated)."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` + Format string `json:"format,omitempty" jsonschema:"Output template per node. Verbs: %i id, %t title, %d updated, %c created, %a accessed, %% a literal percent. Selectors: %{key} a metadata key such as %{type}, %{.field} a stats field such as %{.accessCount}, %{tags} the tag list. Escapes: \\t, \\n, \\r, \\\\."` + Reverse bool `json:"reverse,omitempty" jsonschema:"Reverse the order of each page."` + Limit int `json:"limit,omitempty" jsonschema:"Maximum results; 0 or omitted means 50, and -1 means no limit."` + Offset int `json:"offset,omitempty" jsonschema:"Results to skip; pass the previous page's next_offset to page."` } func registerBacklinks(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "node_backlinks", - Description: "Memory: List nodes that link to a given node", + Name: "node_backlinks", + Description: "Memory: List the nodes that link to the given nodes.", + OutputSchema: outputSchema[pagedLinesOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -268,7 +271,6 @@ func registerBacklinks(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), NodeIDs: in.NodeIDs, Format: in.Format, - IdOnly: in.IdOnly, Reverse: in.Reverse, Limit: mcpDefaultLimit(in.Limit), Offset: in.Offset, @@ -284,19 +286,19 @@ func registerBacklinks(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults // --- links --- type linksInput struct { - NodeIDs []string `json:"node_ids" jsonschema:"source node IDs to find outgoing links for (results merged and deduplicated)"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` - Format string `json:"format,omitempty" jsonschema:"output format template. Legacy verbs %i (id), %t (title), %d (updated), %c (created), %a (accessed); %% is a literal percent. Named selectors use %{...}: a bare word names a metadata key such as %{type} or %{status}, a leading dot names a statistics field such as %{.accessCount} or %{.omega}, and %{tags} is the node's tag list. Backslash escapes are interpreted: \\t tab, \\n newline, \\r return, \\\\ backslash. Selectors other than id, title, and the three dates are resolved by the Hub for the returned page in one batch."` - IdOnly bool `json:"id_only,omitempty" jsonschema:"return node IDs only"` - Reverse bool `json:"reverse,omitempty" jsonschema:"reverse output order"` - Limit int `json:"limit,omitempty" jsonschema:"maximum results to return (default: 50, 0 in request means use default, -1 for unlimited)"` - Offset int `json:"offset,omitempty" jsonschema:"skip the first N results (for pagination)"` + NodeIDs []string `json:"node_ids" jsonschema:"Source node IDs to find outgoing links for (results merged and deduplicated)."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` + Format string `json:"format,omitempty" jsonschema:"Output template per node. Verbs: %i id, %t title, %d updated, %c created, %a accessed, %% a literal percent. Selectors: %{key} a metadata key such as %{type}, %{.field} a stats field such as %{.accessCount}, %{tags} the tag list. Escapes: \\t, \\n, \\r, \\\\."` + Reverse bool `json:"reverse,omitempty" jsonschema:"Reverse the order of each page."` + Limit int `json:"limit,omitempty" jsonschema:"Maximum results; 0 or omitted means 50, and -1 means no limit."` + Offset int `json:"offset,omitempty" jsonschema:"Results to skip; pass the previous page's next_offset to page."` } func registerLinks(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "node_links", - Description: "Memory: List outgoing links from a node", + Name: "node_links", + Description: "Memory: List the nodes the given nodes link to.", + OutputSchema: outputSchema[pagedLinesOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -306,7 +308,6 @@ func registerLinks(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), NodeIDs: in.NodeIDs, Format: in.Format, - IdOnly: in.IdOnly, Reverse: in.Reverse, Limit: mcpDefaultLimit(in.Limit), Offset: in.Offset, @@ -322,15 +323,16 @@ func registerLinks(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- keg_settings_read --- type kegSettingsInput struct { - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` - Kegs []string `json:"kegs,omitempty" jsonschema:"canonical keg references to read together (maximum 100; minimal mode only)"` - Minimal *bool `json:"minimal,omitempty" jsonschema:"return only core config fields (default true)"` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` + Kegs []string `json:"kegs,omitempty" jsonschema:"1-100 @namespace/keg references to read together; minimal mode only. Not combined with keg."` + Minimal *bool `json:"minimal,omitempty" jsonschema:"Return only the core settings fields (default true)."` } func registerKegSettings(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "keg_settings_read", - Description: "Memory: Read targeted KEG settings and instructions. Defaults to a minimal summary; set minimal=false for one complete YAML data document and its hash, required by keg_settings_edit. Use keg or 1-100 kegs, never both.", + Name: "keg_settings_read", + Description: "Memory: Read targeted KEG settings and instructions. Defaults to a minimal summary; set minimal=false for one complete YAML data document and its hash, required by keg_settings_edit. Use keg or 1-100 kegs, never both.", + OutputSchema: outputSchema[kegSettingsReadOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -341,22 +343,24 @@ func registerKegSettings(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul minimal = *in.Minimal } if in.Keg != "" && in.Kegs != nil { - return errorResult(fmt.Errorf("keg and kegs are mutually exclusive")), nil, nil + return errorResult(fmt.Errorf("keg and kegs are mutually exclusive: %w", keg.ErrInvalid)), nil, nil } if in.Kegs != nil { if len(in.Kegs) == 0 || len(in.Kegs) > 100 { - return errorResult(fmt.Errorf("kegs must contain 1 to 100 canonical references")), nil, nil + return errorResult(fmt.Errorf("kegs must contain 1 to 100 canonical references: %w", keg.ErrInvalid)), nil, nil } if !minimal && len(in.Kegs) != 1 { - return errorResult(fmt.Errorf("minimal=false requires exactly one keg")), nil, nil + return errorResult(fmt.Errorf("minimal=false requires exactly one keg: %w", keg.ErrInvalid)), nil, nil } } target := in.Keg if !minimal && len(in.Kegs) == 1 { target = in.Kegs[0] } + kegTarget := resolveKegTarget(ctx, target, defaults) + kegTarget.KegAdministration = true opts := tapper.KegSettingsOptions{ - KegTargetOptions: resolveKegTarget(ctx, target, defaults), + KegTargetOptions: kegTarget, Minimal: minimal, } if minimal { @@ -375,25 +379,30 @@ func registerKegSettings(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul // keg_settings_edit replaces. The minimal render is a cross-keg // summary, so hashing it would hand back a token for something // nobody can write. - res.StructuredContent = map[string]any{ - "hash": keg.DocumentHash([]byte(result)), - "data": result, - } + res.StructuredContent = kegSettingsReadOutput{Hash: keg.DocumentHash([]byte(result)), Data: result} } return res, nil, nil }) } +// kegSettingsReadOutput is keg_settings_read's structured content. The +// minimal summary is text only; minimal=false adds the document and its hash. +type kegSettingsReadOutput struct { + Hash string `json:"hash,omitempty" jsonschema:"With minimal=false, the settings document's hash: keg_settings_edit's expected_hash."` + Data string `json:"data,omitempty" jsonschema:"With minimal=false, the complete settings document as YAML."` +} + // --- info --- type infoInput struct { - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerInfo(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "keg_info", - Description: "Memory: Show concise path-free diagnostics for a resolved KEG (canonical ref, flight, description, node count, and capabilities)", + Name: "keg_info", + Description: "Memory: Show concise diagnostics for a KEG: canonical reference, flight, description, node count, and capabilities.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -413,14 +422,15 @@ func registerInfo(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- stats --- type statsInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to inspect"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to inspect."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerStats(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "node_stats", - Description: "Memory: Show stats (hash, timestamps, access count) for a node", + Name: "node_stats", + Description: "Memory: Show a node's stats: hash, timestamps, and access count.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_replace.go b/pkg/mcp/tools_replace.go index f9062612..353256f5 100644 --- a/pkg/mcp/tools_replace.go +++ b/pkg/mcp/tools_replace.go @@ -13,22 +13,28 @@ import ( // --- keg_replace --- type replaceInput struct { - Find string `json:"find" jsonschema:"text to find; with regex, a Go (RE2) regular expression"` - Replace string `json:"replace" jsonschema:"replacement text; with regex, $1 and ${name} insert captured groups"` - Regex bool `json:"regex,omitempty" jsonschema:"treat find as a regular expression instead of literal text"` - Query string `json:"query,omitempty" jsonschema:"limit the replace to the nodes this node_list query selects, e.g. a tag; empty means every node"` - IncludeMeta bool `json:"include_meta,omitempty" jsonschema:"replace in each node's metadata YAML as well as its body"` - DryRun bool `json:"dry_run,omitempty" jsonschema:"report what would change without writing"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Find string `json:"find" jsonschema:"Text to find; with regex, a Go RE2 regular expression (unlike node_grep, which uses PostgreSQL syntax)."` + Replace string `json:"replace" jsonschema:"Replacement text; with regex, $1 and ${name} insert captured groups."` + Regex bool `json:"regex,omitempty" jsonschema:"Treat find as a regular expression instead of literal text."` + Query string `json:"query,omitempty" jsonschema:"Limit the replace to the nodes this node_list query selects, such as a tag; empty means every node."` + IncludeMeta bool `json:"include_meta,omitempty" jsonschema:"Replace in each node's metadata YAML as well as its body."` + DryRun bool `json:"dry_run,omitempty" jsonschema:"Report what would change without writing."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } type replaceNodeOutput struct { - NodeID string `json:"node_id"` - Matches int `json:"matches"` - Error string `json:"error,omitempty"` - ContentHash string `json:"content_hash,omitempty"` - MetaHash string `json:"meta_hash,omitempty"` - Hash string `json:"hash,omitempty"` + NodeID string `json:"node_id" jsonschema:"The node's ID."` + Matches int `json:"matches" jsonschema:"How many matches the node has."` + Error string `json:"error,omitempty" jsonschema:"Why the node was left alone, when it was."` + ContentHash string `json:"content_hash,omitempty" jsonschema:"The node's new content_hash, after a write."` + MetaHash string `json:"meta_hash,omitempty" jsonschema:"The node's new meta_hash, after a write."` + Hash string `json:"hash,omitempty" jsonschema:"The node's new combined hash, after a write."` +} + +// replaceOutput is keg_replace's structured content. +type replaceOutput struct { + DryRun bool `json:"dry_run" jsonschema:"True when nothing was written."` + Nodes []replaceNodeOutput `json:"nodes" jsonschema:"Every node with a match, changed or left alone."` } func registerReplace(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { @@ -41,6 +47,7 @@ func registerReplace(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) fmt.Sprintf("At most %d matching nodes in one call, counting those left alone. ", keg.MaxReplaceNodes) + fmt.Sprintf("A node with more than %d matches, or that would grow past %d MiB, is left alone with its error; a replace that would grow the changed nodes past %d MiB in all fails. ", keg.MaxReplaceNodeMatches, keg.MaxReplaceNodeBytes>>20, keg.MaxReplaceTotalBytes>>20) + "Results return each changed node's new hashes.", + OutputSchema: outputSchema[replaceOutput](), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), @@ -76,7 +83,7 @@ func registerReplace(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) } lines = append([]string{fmt.Sprintf("%s %d node(s), %d match(es)", verb, changed, matches)}, lines...) res := linesResult(lines) - res.StructuredContent = map[string]any{"dry_run": in.DryRun, "nodes": nodes} + res.StructuredContent = replaceOutput{DryRun: in.DryRun, Nodes: nodes} return res, nil, nil }) } diff --git a/pkg/mcp/tools_repo.go b/pkg/mcp/tools_repo.go index 846eabfc..1362303c 100644 --- a/pkg/mcp/tools_repo.go +++ b/pkg/mcp/tools_repo.go @@ -7,6 +7,7 @@ import ( sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/jlrickert/tapper/pkg/keg" "github.com/jlrickert/tapper/pkg/tapper" ) @@ -18,14 +19,15 @@ func registerRepoTools(srv *sdkmcp.Server, tap *tapper.Tap) { // --- config --- type configInput struct { - Scope string `json:"scope,omitempty" jsonschema:"config scope: user, project, or empty for merged"` - Explain string `json:"explain,omitempty" jsonschema:"when set, return provenance for this field (or 'all' for all fields) instead of config dump"` + Scope string `json:"scope,omitempty" jsonschema:"Config scope: user, project, or empty for merged."` + Explain string `json:"explain,omitempty" jsonschema:"When set, return provenance for this field (or 'all' for all fields) instead of config dump."` } func registerConfig(srv *sdkmcp.Server, tap *tapper.Tap) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "config", - Description: "Read tap configuration (merged, user, or project scope)", + Name: "config", + Description: "Read tap configuration (merged, user, or project scope)", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -55,7 +57,7 @@ func registerConfig(srv *sdkmcp.Server, tap *tapper.Tap) { case "", "merged": // default: merged config default: - return errorResult(fmt.Errorf("invalid scope %q: must be user, project, or empty for merged", in.Scope)), nil, nil + return errorResult(fmt.Errorf("invalid scope %q: must be user, project, or empty for merged: %w", in.Scope, keg.ErrInvalid)), nil, nil } result, err := tap.Config(opts) @@ -82,13 +84,14 @@ func formatExplainResults(results []tapper.ConfigExplainResult) string { // --- config_template --- type configTemplateInput struct { - Scope string `json:"scope,omitempty" jsonschema:"template scope: user (default) or project"` + Scope string `json:"scope,omitempty" jsonschema:"Template scope: user (default) or project."` } func registerConfigTemplate(srv *sdkmcp.Server, tap *tapper.Tap) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "config_template", - Description: "Return starter YAML template for tap configuration", + Name: "config_template", + Description: "Return starter YAML template for tap configuration", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -101,7 +104,7 @@ func registerConfigTemplate(srv *sdkmcp.Server, tap *tapper.Tap) { case "", "user": // default: user template default: - return errorResult(fmt.Errorf("invalid scope %q: must be user or project", in.Scope)), nil, nil + return errorResult(fmt.Errorf("invalid scope %q: must be user or project: %w", in.Scope, keg.ErrInvalid)), nil, nil } result, err := tap.ConfigTemplate(opts) diff --git a/pkg/mcp/tools_schema.go b/pkg/mcp/tools_schema.go index dcf42f46..b5f6c06c 100644 --- a/pkg/mcp/tools_schema.go +++ b/pkg/mcp/tools_schema.go @@ -31,13 +31,14 @@ func registerSchemaTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul // --- schema_list --- type schemaListInput struct { - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerSchemaList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "schema_list", - Description: "Memory: List the schema type names defined for a keg", + Name: "schema_list", + Description: "Memory: List the schema type names a KEG defines.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -59,8 +60,15 @@ func registerSchemaList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault // --- schema_read --- type schemaReadInput struct { - Type string `json:"type" jsonschema:"schema type name to read"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Type string `json:"type" jsonschema:"Schema type name to read."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` +} + +// schemaReadOutput is schema_read's structured content. +type schemaReadOutput struct { + Type string `json:"type" jsonschema:"The schema type name."` + Hash string `json:"hash" jsonschema:"The schema's hash: schema_edit's and schema_delete's expected_hash."` + Data string `json:"data" jsonschema:"The schema definition as YAML."` } func registerSchemaRead(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { @@ -68,6 +76,7 @@ func registerSchemaRead(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault Name: "schema_read", Description: "Memory: Read one schema definition as YAML. The result carries the schema's " + "hash; pass it back as expected_hash when editing this schema.", + OutputSchema: outputSchema[schemaReadOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -81,11 +90,7 @@ func registerSchemaRead(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault return errorResult(err), nil, nil } res := textResult(string(data)) - res.StructuredContent = map[string]any{ - "type": in.Type, - "hash": keg.DocumentHash(data), - "data": string(data), - } + res.StructuredContent = schemaReadOutput{Type: in.Type, Hash: keg.DocumentHash(data), Data: string(data)} return res, nil, nil }) } @@ -93,14 +98,15 @@ func registerSchemaRead(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault // --- schema_create --- type schemaCreateInput struct { - Data string `json:"data" jsonschema:"full schema definition as YAML; the document must declare its type"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Data string `json:"data" jsonschema:"Full schema definition as YAML; the document must declare its type."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerSchemaCreate(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "schema_create", - Description: "Manage: Create a new keg schema from a YAML definition (fails if the type already exists). Requires admin access to the KEG, and admin cover when a flight is selected", + Name: "schema_create", + Description: "Manage: Create a new keg schema from a YAML definition (fails if the type already exists). Requires admin access to the KEG, and admin cover when a flight is selected.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -119,16 +125,17 @@ func registerSchemaCreate(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefau // --- schema_edit --- type schemaEditInput struct { - Type string `json:"type" jsonschema:"schema type name to replace"` - Data string `json:"data" jsonschema:"full schema definition as YAML; its declared type must match"` - ExpectedHash string `json:"expected_hash" jsonschema:"precondition token returned by schema_read"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Type string `json:"type" jsonschema:"Schema type name to replace."` + Data string `json:"data" jsonschema:"Full schema definition as YAML; its declared type must match."` + ExpectedHash string `json:"expected_hash" jsonschema:"Precondition token returned by schema_read."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerSchemaEdit(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "schema_edit", - Description: "Manage: Change how a KEG validates notes; writing a note never needs this, so pick an existing type from schema_list instead. Call schema_read first, then replace an existing keg schema with a new YAML definition using its hash as expected_hash. Requires admin access to the KEG, and admin cover when a flight is selected. On conflict, merge into the returned current schema (or refetch with schema_read) and retry with the returned current hash.", + Name: "schema_edit", + Description: "Manage: Change how a KEG validates notes; writing a note never needs this, so pick an existing type from schema_list instead. Call schema_read first, then replace an existing keg schema with a new YAML definition using its hash as expected_hash. Requires admin access to the KEG, and admin cover when a flight is selected. On conflict, merge into the returned current schema (or refetch with schema_read) and retry with the returned current hash.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -152,15 +159,16 @@ func registerSchemaEdit(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault // --- schema_delete --- type schemaDeleteInput struct { - Type string `json:"type" jsonschema:"schema type name to delete"` - ExpectedHash string `json:"expected_hash" jsonschema:"precondition token returned by schema_read"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Type string `json:"type" jsonschema:"Schema type name to delete."` + ExpectedHash string `json:"expected_hash" jsonschema:"Precondition token returned by schema_read."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerSchemaDelete(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "schema_delete", - Description: "Manage: Call schema_read first, then delete a keg schema using its hash as expected_hash. Requires admin access to the KEG, and admin cover when a flight is selected. On conflict, refetch with schema_read and retry with the returned current hash.", + Name: "schema_delete", + Description: "Manage: Call schema_read first, then delete a keg schema using its hash as expected_hash. Requires admin access to the KEG, and admin cover when a flight is selected. On conflict, refetch with schema_read and retry with the returned current hash.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), @@ -180,14 +188,15 @@ func registerSchemaDelete(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefau // --- validate --- type validateInput struct { - NodeIDs []string `json:"node_ids,omitempty" jsonschema:"node IDs to validate (validates every node if empty)"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeIDs []string `json:"node_ids,omitempty" jsonschema:"Node IDs to validate (validates every node if empty)."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerValidate(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "schema_validate", - Description: "Memory: Validate nodes against their declared schema type", + Name: "schema_validate", + Description: "Memory: Validate nodes against their declared schema type.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_search.go b/pkg/mcp/tools_search.go index 58112ddf..bb76609c 100644 --- a/pkg/mcp/tools_search.go +++ b/pkg/mcp/tools_search.go @@ -2,7 +2,6 @@ package mcp import ( "context" - "strings" sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" @@ -13,35 +12,36 @@ import ( // --- node_search --- type nodeSearchInput struct { - Query string `json:"query" jsonschema:"required,what to find: plain terms (any may match; more matches and title or metadata matches rank higher), \"quoted phrases\" that must appear, and -exclusions"` - Keg string `json:"keg,omitempty" jsonschema:"search only this KEG (@namespace/keg or alias). Omit to search every KEG this call may read; never combined with kegs"` - Kegs []string `json:"kegs,omitempty" jsonschema:"narrow the search to these @namespace/keg references (at most 50); omit to search every KEG this call may read"` - Cursor string `json:"cursor,omitempty" jsonschema:"next_cursor from a previous node_search response for the same query, passed back unchanged"` - Limit int `json:"limit,omitempty" jsonschema:"maximum hits to return, 0 or more; omit it or pass 0 to let the Hub choose its default. Unlike listing tools, -1 (unlimited) is not accepted"` + Query string `json:"query" jsonschema:"What to find: plain terms (any may match; more matches, and matches in titles or metadata, rank higher), \"quoted phrases\" that must appear, and -exclusions."` + Kegs []string `json:"kegs,omitempty" jsonschema:"KEGs to search, as @namespace/keg references (at most 50), each inside the call's flight cover. Omit it or pass [] to search every KEG the flight covers."` + Cursor string `json:"cursor,omitempty" jsonschema:"The next_cursor of a previous node_search response for the same query, passed back unchanged."` + Limit int `json:"limit,omitempty" jsonschema:"Maximum hits to return; 0 or omitted lets the Hub choose. Unlike listing tools, -1 is not accepted."` } type nodeSearchOutput struct { tapapi.SearchResponse - Pagination string `json:"pagination"` + Pagination string `json:"pagination" jsonschema:"How to page this search."` } func registerNodeSearch(srv *sdkmcp.Server, defaults KegDefaults, provider NodeSearchProvider) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ Name: "node_search", - Description: "Memory: Find notes about a topic: the KEG nodes most relevant to a query, ranked across every KEG this call may read (omit keg), one named keg, or the listed kegs. " + - "This is the only search over note content: explore_kegs finds public KEGs, never notes, and keg_list lists your own KEGs. " + - "Matches titles, metadata, tags, and bodies. Use node_grep for exact regex matching of identifiers or symbols. " + + Description: "Memory: Find the notes about a topic: KEG nodes ranked by relevance to a query, across every KEG the call's flight covers or only the kegs you name. " + + "It matches titles, metadata, tags, and bodies, and is the only search over note content: explore_kegs finds public KEGs, and keg_list lists your own. " + + "Use node_grep for exact regex matches of identifiers or symbols. " + "Each hit gives the node reference, score, title, matched fields, and a snippet with matches in **bold**; " + - "follow next_cursor for more and use node_read for complete bodies.", + "follow next_cursor for more, and use node_read for complete bodies.", + OutputSchema: outputSchema[nodeSearchOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), }, }, func(ctx context.Context, _ *sdkmcp.CallToolRequest, in nodeSearchInput) (*sdkmcp.CallToolResult, any, error) { - target := resolveKegTarget(ctx, in.Keg, defaults) - // An omitted keg means the whole readable scope, never the server's - // default KEG. - target.Keg = strings.TrimSpace(in.Keg) + // The scope is the call's flight cover, narrowed by kegs; never the + // server's default KEG. Only a pre-kegs caller's keg argument names + // one KEG through the keg target (see legacySearchKeg). + target := resolveKegTarget(ctx, "", defaults) + target.Keg = legacySearchKeg(ctx) opts := tapper.SearchOptions{ KegTargetOptions: target, Kegs: in.Kegs, @@ -49,7 +49,14 @@ func registerNodeSearch(srv *sdkmcp.Server, defaults KegDefaults, provider NodeS Cursor: in.Cursor, Limit: in.Limit, } - if _, err := opts.Validate(); err != nil { + kegs, err := opts.Validate() + if err != nil { + return errorResult(err), nil, nil + } + // Naming KEGs narrows the flight's search and never reaches past its + // cover. Refusing here names every KEG outside it, the same way on + // every Hub, instead of the Hub reporting the first as not found. + if err := tapper.CheckSearchCover(tapper.ComposedFlight(target.FlightContext), kegs); err != nil { return errorResult(err), nil, nil } var resp *tapapi.SearchResponse @@ -58,7 +65,6 @@ func registerNodeSearch(srv *sdkmcp.Server, defaults KegDefaults, provider NodeS // unscoped request would ask the Hub for everything. resp = &tapapi.SearchResponse{Hits: []tapapi.SearchHit{}} } else { - var err error resp, err = provider.SearchNodes(ctx, opts) if err != nil { return errorResult(err), nil, nil diff --git a/pkg/mcp/tools_search_test.go b/pkg/mcp/tools_search_test.go index 95088c59..363aa593 100644 --- a/pkg/mcp/tools_search_test.go +++ b/pkg/mcp/tools_search_test.go @@ -90,12 +90,11 @@ func TestMCP_NodeSearchOmittedLimitStaysZero(t *testing.T) { provider := &recordingNodeSearch{resp: &tapapi.SearchResponse{}} session, ctx := newTestSessionWithOpts(t, mcp.ServerOptions{NodeSearchProvider: provider}) - res := callNodeSearch(t, session, ctx, map[string]any{"query": "festival", "keg": "@acme/notes"}) + res := callNodeSearch(t, session, ctx, map[string]any{"query": "festival", "kegs": []string{"@acme/notes"}}) require.False(t, res.IsError, "node_search failed: %v", extractText(t, res)) calls := provider.recorded() require.Len(t, calls, 1) require.Zero(t, calls[0].Limit, "the Hub owns the default page size") - require.Equal(t, "@acme/notes", calls[0].Keg) payload := res.StructuredContent.(map[string]any) require.Equal(t, []any{}, payload["hits"], "a nil hit list renders as an empty array") @@ -117,7 +116,7 @@ func TestMCP_NodeSearchRefusesInvalidInputBeforeTheProvider(t *testing.T) { } { res := callNodeSearch(t, session, ctx, args) require.True(t, res.IsError, name) - require.Equal(t, keg.RemoteCodeInvalid, res.StructuredContent.(map[string]any)["code"], name) + require.Equal(t, keg.RemoteCodeInvalid, errorPayload(t, res)["code"], name) } require.Empty(t, provider.recorded()) } @@ -130,7 +129,7 @@ func TestMCP_NodeSearchUnavailableNamesNodeGrep(t *testing.T) { res := callNodeSearch(t, session, ctx, map[string]any{"query": "x"}) require.True(t, res.IsError) require.Contains(t, extractText(t, res), "ranked search unavailable") - payload := res.StructuredContent.(map[string]any) + payload := errorPayload(t, res) require.Equal(t, keg.RemoteCodeNotSupported, payload["code"], "a missing capability is not an unknown failure") require.Equal(t, false, payload["operationPerformed"], "a search never writes") require.Contains(t, payload["action"], "supported tool") @@ -140,7 +139,7 @@ func TestMCP_NodeGrepKeepsRegexBehavior(t *testing.T) { t.Parallel() session, ctx := newTestSession(t) - res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "node_grep", Arguments: map[string]any{"query": "Hel+o", "id_only": true}}) + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "node_grep", Arguments: map[string]any{"query": "Hel+o", "format": "%i"}}) require.NoError(t, err) require.False(t, res.IsError, "node_grep failed: %v", extractText(t, res)) require.NotEmpty(t, extractText(t, res)) @@ -184,3 +183,83 @@ func TestMCP_NodeSearchReachesNothingWithoutAuthority(t *testing.T) { } require.Empty(t, provider.recorded(), "a session that reaches no KEG must not search the Hub") } + +// node_search scopes to the call's flight: kegs only narrows it. One kegs +// list replaced the old keg/kegs pair, so the schema names no keg. +func TestMCP_NodeSearchSearchesTheFlightCoverAndNarrowsWithinIt(t *testing.T) { + t.Parallel() + provider := &recordingNodeSearch{resp: &tapapi.SearchResponse{}} + session, ctx := newFlightSearchSession(t, "active", provider) + + listed, err := session.ListTools(ctx, nil) + require.NoError(t, err) + for _, tool := range listed.Tools { + if tool.Name != "node_search" { + continue + } + props := schemaProperties(t, tool.InputSchema) + require.Contains(t, props, "kegs") + require.NotContains(t, props, "keg", "kegs is the one way to name KEGs") + } + + // Empty or omitted kegs searches the whole cover of the call's flight. + for _, args := range []map[string]any{ + {"query": "release"}, + {"query": "release", "kegs": []string{}}, + } { + res := callNodeSearch(t, session, ctx, args) + require.False(t, res.IsError, "node_search failed: %v", extractText(t, res)) + } + // An explicit flight selects that flight's cover instead. + res := callNodeSearch(t, session, ctx, map[string]any{"query": "release", "flight": "@team/+child"}) + require.False(t, res.IsError, "node_search failed: %v", extractText(t, res)) + calls := provider.recorded() + require.Len(t, calls, 3) + for i, call := range calls { + require.Emptyf(t, call.Kegs, "call %d named KEGs; the whole cover is searched", i) + require.Emptyf(t, call.Keg, "call %d named a single KEG", i) + } + require.Equal(t, "@team/+root", calls[0].FlightContext.Name) + require.Equal(t, "@team/+child", calls[2].FlightContext.Name) + + // Naming KEGs inside the cover narrows to them. + res = callNodeSearch(t, session, ctx, map[string]any{"query": "release", "kegs": []string{"@team/root-keg"}}) + require.False(t, res.IsError, "node_search failed: %v", extractText(t, res)) + require.Equal(t, []string{"@team/root-keg"}, provider.recorded()[3].Kegs) + + // Naming any outside it is refused, naming each, and searches nothing. + res = callNodeSearch(t, session, ctx, map[string]any{"query": "release", "kegs": []string{"@team/root-keg", "@team/child-keg", "@other/notes"}}) + require.True(t, res.IsError) + require.Equal(t, keg.RemoteCodeInvalid, errorPayload(t, res)["code"]) + text := extractText(t, res) + require.Contains(t, text, "@team/child-keg, @other/notes") + require.NotContains(t, text, "root-keg") + // The same KEG is in cover under the flight that covers it. + res = callNodeSearch(t, session, ctx, map[string]any{"query": "release", "flight": "@team/+child", "kegs": []string{"@team/child-keg"}}) + require.False(t, res.IsError, "node_search failed: %v", extractText(t, res)) + require.Len(t, provider.recorded(), 5) +} + +// A client written before kegs replaced keg still sends keg. It is searched +// as the one KEG named, bare names included, rather than refused by the +// schema. +func TestMCP_NodeSearchAcceptsLegacyKeg(t *testing.T) { + t.Parallel() + provider := &recordingNodeSearch{resp: &tapapi.SearchResponse{}} + session, ctx := newTestSessionWithOpts(t, mcp.ServerOptions{NodeSearchProvider: provider}) + + for _, ref := range []string{"@acme/notes", "notes"} { + res := callNodeSearch(t, session, ctx, map[string]any{"query": "festival", "keg": ref}) + require.False(t, res.IsError, "legacy keg %q: %v", ref, extractText(t, res)) + } + calls := provider.recorded() + require.Len(t, calls, 2) + require.Equal(t, "@acme/notes", calls[0].Keg) + require.Equal(t, "notes", calls[1].Keg) + require.Empty(t, calls[0].Kegs) + + // An empty legacy keg asks for nothing narrower than the whole scope. + res := callNodeSearch(t, session, ctx, map[string]any{"query": "festival", "keg": ""}) + require.False(t, res.IsError, extractText(t, res)) + require.Empty(t, provider.recorded()[2].Keg) +} diff --git a/pkg/mcp/tools_snapshot.go b/pkg/mcp/tools_snapshot.go index f6dddd10..fba885e3 100644 --- a/pkg/mcp/tools_snapshot.go +++ b/pkg/mcp/tools_snapshot.go @@ -20,14 +20,15 @@ func registerSnapshotTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefa // --- snapshot_list --- type nodeHistoryInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to show history for"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to show history for."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerNodeHistory(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "snapshot_list", - Description: "Memory: List snapshot history for a node", + Name: "snapshot_list", + Description: "Memory: List a node's snapshot history.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -59,8 +60,8 @@ func registerNodeHistory(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaul // --- snapshot_create --- type nodeSnapshotInput struct { - Nodes []nodeSnapshotItemInput `json:"nodes" jsonschema:"1-100 nodes to snapshot atomically"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Nodes []nodeSnapshotItemInput `json:"nodes" jsonschema:"1-100 nodes to snapshot atomically."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } type nodeSnapshotItemInput struct { NodeID string `json:"node_id"` @@ -68,16 +69,22 @@ type nodeSnapshotItemInput struct { } type nodeSnapshotOutput struct { - NodeID string `json:"node_id"` - Revision int64 `json:"revision"` - Message string `json:"message,omitempty"` + NodeID string `json:"node_id" jsonschema:"The node's ID."` + Revision int64 `json:"revision" jsonschema:"The new snapshot's revision number."` + Message string `json:"message,omitempty" jsonschema:"The snapshot's message, when one was given."` +} + +// snapshotCreateOutput is snapshot_create's structured content. +type snapshotCreateOutput struct { + Results []nodeSnapshotOutput `json:"results" jsonschema:"One snapshot per node, in request order."` } func registerNodeSnapshot(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "snapshot_create", - Description: "Memory: Atomically snapshot the current state of 1-100 nodes", - InputSchema: boundedMutationInputSchema[nodeSnapshotInput]("nodes"), + Name: "snapshot_create", + Description: "Memory: Snapshot the current state of 1-100 nodes atomically.", + InputSchema: boundedMutationInputSchema[nodeSnapshotInput]("nodes"), + OutputSchema: outputSchema[snapshotCreateOutput](), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -100,7 +107,7 @@ func registerNodeSnapshot(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefau for i, snap := range snaps { structured[i] = nodeSnapshotOutput{NodeID: snap.Node.Path(), Revision: int64(snap.ID), Message: snap.Message} } - res.StructuredContent = map[string]any{"results": structured} + res.StructuredContent = snapshotCreateOutput{Results: structured} return res, nil, nil }) } @@ -108,15 +115,16 @@ func registerNodeSnapshot(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefau // --- snapshot_read --- type nodeSnapshotViewInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to view"` - Rev string `json:"rev" jsonschema:"revision number to view"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to view."` + Rev string `json:"rev" jsonschema:"Revision number to view."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerNodeSnapshotView(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "snapshot_read", - Description: "Memory: View read-only content for a previous snapshot revision", + Name: "snapshot_read", + Description: "Memory: Read a node's content at a previous snapshot revision.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -138,15 +146,16 @@ func registerNodeSnapshotView(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegD // --- snapshot_restore --- type nodeRestoreInput struct { - NodeID string `json:"node_id" jsonschema:"node ID to restore"` - Rev string `json:"rev" jsonschema:"revision number to restore to"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeID string `json:"node_id" jsonschema:"Node ID to restore."` + Rev string `json:"rev" jsonschema:"Revision number to restore to."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerNodeRestore(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "snapshot_restore", - Description: "Memory: Restore a node to a previous snapshot revision", + Name: "snapshot_restore", + Description: "Memory: Restore a node to a previous snapshot revision.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_trash.go b/pkg/mcp/tools_trash.go index 28e62fbf..e69bc314 100644 --- a/pkg/mcp/tools_trash.go +++ b/pkg/mcp/tools_trash.go @@ -19,7 +19,7 @@ func registerTrashTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault // --- trash_list --- type trashListInput struct { - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } type trashEntryOutput struct { @@ -34,6 +34,11 @@ type trashEntryOutput struct { DisplacedFrom string `json:"displaced_from,omitempty"` } +// trashListOutput is trash_list's structured content. +type trashListOutput struct { + Trash []trashEntryOutput `json:"trash" jsonschema:"The KEG's removed nodes, newest first."` +} + func trashEntryOutputs(entries []keg.TrashEntry) []trashEntryOutput { out := make([]trashEntryOutput, len(entries)) for i, entry := range entries { @@ -57,6 +62,7 @@ func registerTrashList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults Description: "Memory: List a keg's removed nodes, newest first: id, title, why it was removed (manual, or merge with the node it was merged into), when, and when it will be purged for good. " + "Restore one with trash_restore. A trashed node's id stays reserved, and links to it are left in place until it is purged. " + "An archive restore that needs a trashed node's id moves the entry to a new id first; displaced_from names the id it was trashed at, and restoring puts it back at its new id.", + OutputSchema: outputSchema[trashListOutput](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), @@ -68,7 +74,7 @@ func registerTrashList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults } if len(entries) == 0 { res := textResult("trash is empty") - res.StructuredContent = map[string]any{"trash": []trashEntryOutput{}} + res.StructuredContent = trashListOutput{Trash: []trashEntryOutput{}} return res, nil, nil } structured := trashEntryOutputs(entries) @@ -87,7 +93,7 @@ func registerTrashList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults lines[i] = line } res := linesResult(lines) - res.StructuredContent = map[string]any{"trash": structured} + res.StructuredContent = trashListOutput{Trash: structured} return res, nil, nil }) } @@ -95,8 +101,8 @@ func registerTrashList(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults // --- trash_restore --- type trashRestoreInput struct { - NodeIDs []string `json:"node_ids" jsonschema:"1-100 trashed node ids to restore atomically"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + NodeIDs []string `json:"node_ids" jsonschema:"1-100 trashed node ids to restore atomically."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerTrashRestore(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { @@ -104,7 +110,8 @@ func registerTrashRestore(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefau Name: "trash_restore", Description: "Memory: Atomically restore 1-100 trashed nodes (see trash_list) at their own ids, with their content, metadata, history, files, and images. " + "Links that still name a restored node resolve again. A node merged into another comes back as it was before the merge; links the merge repointed stay on the node it was merged into.", - InputSchema: boundedMutationInputSchema[trashRestoreInput]("node_ids"), + InputSchema: boundedMutationInputSchema[trashRestoreInput]("node_ids"), + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_url.go b/pkg/mcp/tools_url.go index 1fcc9de2..c048bedd 100644 --- a/pkg/mcp/tools_url.go +++ b/pkg/mcp/tools_url.go @@ -11,8 +11,8 @@ import ( // --- url --- type urlInput struct { - Ref string `json:"ref,omitempty" jsonschema:"what to link: a node id in keg (42), a KEG (keg:alias, keg:~alias, @ns/keg), a note (keg:alias/42, keg:~alias/42, @ns/keg/42), a flight (+slug, @ns/+slug), or a namespace (@ns). Empty links keg itself."` - Keg string `json:"keg,omitempty" jsonschema:"keg a bare node id or keg-local ~alias resolves against (uses default if empty)"` + Ref string `json:"ref,omitempty" jsonschema:"What to link: a node id in keg (42), a KEG (@ns/keg, keg:name, keg:~alias), a note (@ns/keg/42, keg:name/42, keg:~alias/42), a flight (+slug, @ns/+slug), or a namespace (@ns). Empty links keg itself."` + Keg string `json:"keg,omitempty" jsonschema:"KEG a bare node id or ~alias resolves against, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerURL(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { @@ -20,6 +20,7 @@ func registerURL(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { Name: "url", Description: "Memory: Get the Hub web link for a KEG, note, flight, or namespace, to give the reader a clickable link. " + "Returns url, kind (namespace, keg, node, or flight), and the canonical ref. Building a link never reads the note.", + OutputSchema: outputSchema[tapper.WebURL](), Annotations: &sdkmcp.ToolAnnotations{ ReadOnlyHint: true, OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/tools_url_test.go b/pkg/mcp/tools_url_test.go index 4ae41058..004f916e 100644 --- a/pkg/mcp/tools_url_test.go +++ b/pkg/mcp/tools_url_test.go @@ -24,7 +24,7 @@ func newURLSession(t *testing.T, webBaseURL string) (*sdkmcp.ClientSession, cont done := make(chan error, 1) go func() { done <- srv.Run(ctx, serverTransport) }() t.Cleanup(func() { <-done }) - session, err := sdkmcp.NewClient(&sdkmcp.Implementation{Name: "test-client", Version: "0.1"}, nil). + session, err := newTestClient(t, "test-client", nil). Connect(ctx, clientTransport, nil) require.NoError(t, err) t.Cleanup(func() { _ = session.Close() }) diff --git a/pkg/mcp/tools_write.go b/pkg/mcp/tools_write.go index c45fb3d3..0d02e603 100644 --- a/pkg/mcp/tools_write.go +++ b/pkg/mcp/tools_write.go @@ -26,7 +26,36 @@ func boundedMutationInputSchema[T any](arrayFields ...string) *jsonschema.Schema } property.MinItems = &minItems property.MaxItems = &maxItems + if property.Items != nil { + constrainSchemaTypeField(property.Items) + } } + constrainSchemaTypeField(schema) + return schema +} + +// constrainSchemaTypeField gives an object's schema-selection property the +// same rule keg.ValidSchemaTypeName enforces. Struct tags cannot carry a +// pattern, so it is set on the derived schema; the SDK then refuses a +// sentence or a path as a schema before the handler runs. +func constrainSchemaTypeField(object *jsonschema.Schema) { + property := object.Properties["schema"] + if property == nil { + return + } + maxLength := keg.SchemaTypeNameMaxLength + property.Pattern = keg.SchemaTypeNamePattern + property.MaxLength = &maxLength +} + +// constrainedInputSchema derives T's input schema with its schema-selection +// field constrained, for tools that take one schema outside a batch. +func constrainedInputSchema[T any]() *jsonschema.Schema { + schema, err := jsonschema.For[T](nil) + if err != nil { + panic(fmt.Sprintf("derive MCP input schema: %v", err)) + } + constrainSchemaTypeField(schema) return schema } @@ -43,22 +72,25 @@ func registerWriteTools(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefault // --- keg_settings_edit --- type kegSettingsEditInput struct { - Data string `json:"data" jsonschema:"complete validated KEG YAML document"` - ExpectedHash string `json:"expected_hash" jsonschema:"precondition token returned by keg_settings_read"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Data string `json:"data" jsonschema:"Complete validated KEG YAML document."` + ExpectedHash string `json:"expected_hash" jsonschema:"Precondition token returned by keg_settings_read."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerKegSettingsEdit(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "keg_settings_edit", - Description: "Manage: Call keg_settings_read with minimal=false first, then replace the complete KEG settings with a validated YAML document using its hash as expected_hash. Requires admin access to the KEG itself, plus admin cover when a flight is selected. On conflict, merge into the returned current settings (or refetch with keg_settings_read) and retry with the returned current hash.", + Name: "keg_settings_edit", + Description: "Manage: Call keg_settings_read with minimal=false first, then replace the complete KEG settings with a validated YAML document using its hash as expected_hash. Requires admin access to the KEG itself, plus admin cover when a flight is selected, or a Hub flight that manages the namespace's KEGs (manage_kegs). On conflict, merge into the returned current settings (or refetch with keg_settings_read) and retry with the returned current hash.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), }, }, func(ctx context.Context, req *sdkmcp.CallToolRequest, in kegSettingsEditInput) (*sdkmcp.CallToolResult, any, error) { + target := resolveKegTarget(ctx, in.Keg, defaults) + target.KegAdministration = true opts := tapper.KegSettingsEditOptions{ - KegTargetOptions: resolveKegTarget(ctx, in.Keg, defaults), + KegTargetOptions: target, ExpectedHash: in.ExpectedHash, Stream: &toolkit.Stream{ IsPiped: true, @@ -75,21 +107,26 @@ func registerKegSettingsEdit(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDe // --- create --- type createInput struct { - Nodes []createNodeInput `json:"nodes" jsonschema:"1-100 nodes to create atomically"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Nodes []createNodeInput `json:"nodes" jsonschema:"1-100 nodes to create atomically."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } type createNodeInput struct { - Key string `json:"key" jsonschema:"caller-chosen label for this node within the batch, unique across the batch. Other nodes in the same batch link to it by writing {{node:KEY}} in their content; each placeholder is replaced with the id this node is assigned."` - Content string `json:"content" jsonschema:"the node's complete markdown body, opening with an H1 title. Must not start with a YAML frontmatter block: metadata goes in the meta field."` - Meta string `json:"meta,omitempty" jsonschema:"the node's complete metadata document as YAML, including keys such as type, tags, and any schema-defined attributes. Omit for no metadata."` - Schema string `json:"schema,omitempty" jsonschema:"schema selected for this write, one of the types schema_list returns. Writes meta.type; a different type declared in meta is a hard error, not an override. Required on a strict keg (the default for new kegs), where type in meta alone does not count as a selection."` + Key string `json:"key" jsonschema:"Caller-chosen label for this node within the batch, unique across the batch. Other nodes in the same batch refer to it by writing {{node:KEY}} in their content; each placeholder is replaced with the bare id this node is assigned, so link to it as [Title](../{{node:KEY}}) (a placeholder that is the whole link target gets the ../ added)."` + Content string `json:"content" jsonschema:"The node's complete markdown body, opening with an H1 title. Must not start with a YAML frontmatter block: metadata goes in the meta field."` + Meta string `json:"meta,omitempty" jsonschema:"The node's complete metadata document as YAML, including keys such as type, tags, and any schema-defined attributes. Omit for no metadata."` + Schema string `json:"schema,omitempty" jsonschema:"Schema selected for this write: a type name such as task, one of the types schema_list returns, never a description. Writes meta.type; a different type declared in meta is a hard error, not an override. Required on a strict keg (the default for new kegs), where type in meta alone does not count as a selection."` } type createNodeOutput struct { - Key string `json:"key"` - NodeID string `json:"node_id"` - Hash string `json:"hash"` - Validation *keg.SchemaValidationResult `json:"validation,omitempty"` + Key string `json:"key" jsonschema:"The key the node was sent with."` + NodeID string `json:"node_id" jsonschema:"The ID the node was assigned."` + Hash string `json:"hash" jsonschema:"The node's combined hash."` + Validation *keg.SchemaValidationResult `json:"validation,omitempty" jsonschema:"How the node validated against its schema, when it has one."` +} + +// nodeCreateOutput is node_create's structured content. +type nodeCreateOutput struct { + Results []createNodeOutput `json:"results" jsonschema:"One entry per node created, in request order."` } func createNodeOutputs(results []keg.CreateNodeResult) []createNodeOutput { @@ -107,9 +144,10 @@ func registerCreate(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { "Each node is a markdown content document plus an optional YAML metadata document. " + "content must not begin with a YAML frontmatter block — metadata goes in meta. " + "A node's title is its content H1 and its lead is the first paragraph, so there are no separate title, lead, tags, or attrs fields. " + - "Nodes in one batch can reference each other's not-yet-assigned ids by writing {{node:KEY}} in content. " + + "Nodes in one batch can reference each other's not-yet-assigned ids by writing {{node:KEY}} in content: it becomes the bare id, so a link is [Title](../{{node:KEY}}) and a cross-KEG reference keg:@namespace/keg/{{node:KEY}}. " + "On a strict keg (the default for new kegs) every node needs schema, and type in meta alone does not count; call schema_list once per keg before the first write to learn its types.", - InputSchema: boundedMutationInputSchema[createInput]("nodes"), + InputSchema: boundedMutationInputSchema[createInput]("nodes"), + OutputSchema: outputSchema[nodeCreateOutput](), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -129,7 +167,7 @@ func registerCreate(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { message = results[0].ID.String() } res := textResult(message) - res.StructuredContent = map[string]any{"results": createNodeOutputs(results)} + res.StructuredContent = nodeCreateOutput{Results: createNodeOutputs(results)} return res, nil, nil }) } @@ -137,28 +175,33 @@ func registerCreate(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- edit --- type editInput struct { - Nodes []editItemInput `json:"nodes" jsonschema:"1-100 nodes to update atomically"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Nodes []editItemInput `json:"nodes" jsonschema:"1-100 nodes to update atomically."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } type editItemInput struct { - NodeID string `json:"node_id" jsonschema:"id of the node to update"` - Content *string `json:"content,omitempty" jsonschema:"replacement markdown body, replacing the node's content entirely. Must not start with a YAML frontmatter block: metadata goes in the meta field. Omit to leave content unchanged."` - Patch *string `json:"patch,omitempty" jsonschema:"unified diff (the @@ hunks of diff -u) against the node's current content, an alternative to content. Each hunk's context and removed lines must match the current content exactly; header line numbers are a hint. Prefer it for small changes to long notes. The body keeps its line endings; a '\\ No newline at end of file' marker may only follow a hunk that reaches the end."` - Meta *string `json:"meta,omitempty" jsonschema:"replacement metadata document as YAML, replacing the node's metadata entirely. Omit to leave metadata unchanged."` - ExpectedContentHash string `json:"expected_content_hash,omitempty" jsonschema:"content_hash from node_read (or from a previous node_edit result); required when content or patch is supplied"` - ExpectedMetaHash string `json:"expected_meta_hash,omitempty" jsonschema:"meta_hash from node_read (or from a previous node_edit result); required when meta is supplied"` - Schema string `json:"schema,omitempty" jsonschema:"schema selected for this write, one of the types schema_list returns. Writes meta.type; a different type declared in meta is a hard error, not an override. On a strict keg (the default for new kegs) required whenever meta is supplied; a content-only edit keeps the node's current type and needs none."` + NodeID string `json:"node_id" jsonschema:"ID of the node to update."` + Content *string `json:"content,omitempty" jsonschema:"Replacement markdown body, replacing the node's content entirely. Must not start with a YAML frontmatter block: metadata goes in the meta field. Omit to leave content unchanged."` + Patch *string `json:"patch,omitempty" jsonschema:"Unified diff (the @@ hunks of diff -u) against the node's current content, an alternative to content. Each hunk's context and removed lines must match the current content exactly; header line numbers are a hint. Prefer it for small changes to long notes. The body keeps its line endings; a '\\ No newline at end of file' marker may only follow a hunk that reaches the end."` + Meta *string `json:"meta,omitempty" jsonschema:"Replacement metadata document as YAML, replacing the node's metadata entirely. Omit to leave metadata unchanged."` + ExpectedContentHash string `json:"expected_content_hash,omitempty" jsonschema:"The content_hash from node_read or a previous node_edit result. Required with content or patch."` + ExpectedMetaHash string `json:"expected_meta_hash,omitempty" jsonschema:"The meta_hash from node_read or a previous node_edit result. Required with meta."` + Schema string `json:"schema,omitempty" jsonschema:"Schema selected for this write: a type name such as task, one of the types schema_list returns, never a description. Writes meta.type; a different type declared in meta is a hard error, not an override. On a strict keg (the default for new kegs) required whenever meta is supplied; a content-only edit keeps the node's current type and needs none."` } // nodeUpdateOutput carries the written node's tokens. content_hash and // meta_hash are what a follow-up node_edit echoes back, so a second edit // needs no re-read; hash is the combined token node_delete and node_move take. type nodeUpdateOutput struct { - NodeID string `json:"node_id"` - Hash string `json:"hash"` - ContentHash string `json:"content_hash,omitempty"` - MetaHash string `json:"meta_hash,omitempty"` - Validation *keg.SchemaValidationResult `json:"validation,omitempty"` + NodeID string `json:"node_id" jsonschema:"The node's ID."` + Hash string `json:"hash" jsonschema:"The node's new combined hash."` + ContentHash string `json:"content_hash,omitempty" jsonschema:"The node's new content_hash, for the next edit."` + MetaHash string `json:"meta_hash,omitempty" jsonschema:"The node's new meta_hash, for the next edit."` + Validation *keg.SchemaValidationResult `json:"validation,omitempty" jsonschema:"How the node validated against its schema, when it has one."` +} + +// nodeEditOutput is node_edit's structured content. +type nodeEditOutput struct { + Results []nodeUpdateOutput `json:"results" jsonschema:"One entry per node updated, in request order."` } func nodeUpdateOutputs(results []keg.NodeUpdateResult) []nodeUpdateOutput { @@ -180,12 +223,13 @@ func registerEdit(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { "A patch keeps the body's line endings (CRLF stays CRLF), and '\\ No newline at end of file' may only follow a hunk that reaches the end of the body. " + "Each part has its own precondition token: pass node_read's content_hash as expected_content_hash when supplying content or patch, and its meta_hash as expected_meta_hash when supplying meta. " + "Editing content never invalidates meta_hash and editing meta never invalidates content_hash, so a concurrent edit to the other part does not conflict. " + - "Call node_read first for every node; node_read meta_only reads metadata. " + + "Call node_read first for every node; node_read meta_only reads metadata. A part sent without its token is refused, writing nothing, with the exact node_read call that supplies every missing token. " + "Each result returns the node's new content_hash and meta_hash, so a follow-up edit needs no re-read; its combined hash (for node_delete or node_move) changes with every edit. " + "Take a snapshot with snapshot_create before a large or destructive edit. " + "On a strict keg (the default for new kegs) an edit that supplies meta needs schema; a content-only edit keeps the node's current type and is validated against it. " + "On conflict, merge into the returned current content (or refetch with node_read) and retry with the returned current hash for the part you are writing: currentContentHash as expected_content_hash, currentMetaHash as expected_meta_hash.", - InputSchema: boundedMutationInputSchema[editInput]("nodes"), + InputSchema: boundedMutationInputSchema[editInput]("nodes"), + OutputSchema: outputSchema[nodeEditOutput](), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), @@ -211,7 +255,7 @@ func registerEdit(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { return errorResult(err), nil, nil } res := textResult(fmt.Sprintf("updated %d node(s)", len(results))) - res.StructuredContent = map[string]any{"results": nodeUpdateOutputs(results)} + res.StructuredContent = nodeEditOutput{Results: nodeUpdateOutputs(results)} return res, nil, nil }) } @@ -219,13 +263,13 @@ func registerEdit(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- remove --- type removeInput struct { - Nodes []removeNodeInput `json:"nodes" jsonschema:"1-100 nodes to remove atomically"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + Nodes []removeNodeInput `json:"nodes" jsonschema:"1-100 nodes to remove atomically."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } type removeNodeInput struct { NodeID string `json:"node_id"` - ExpectedHash string `json:"expected_hash" jsonschema:"the combined hash returned by node_read (not content_hash or meta_hash)"` + ExpectedHash string `json:"expected_hash" jsonschema:"The combined hash from node_read, not content_hash or meta_hash."` } func registerRemove(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { @@ -233,7 +277,8 @@ func registerRemove(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { Name: "node_delete", Description: "Memory: Call node_read first for every node, then atomically remove 1-100 nodes using the hash from node_read (the combined hash, not content_hash or meta_hash) as that node's expected_hash. On conflict, refetch with node_read and retry with the returned current hash. " + "Where the keg keeps a trash, a removed node goes there with its history and assets and trash_restore brings it back; links to it are left alone until it is purged. Otherwise the node and its snapshots are gone and links to it point at node 0.", - InputSchema: boundedMutationInputSchema[removeInput]("nodes"), + InputSchema: boundedMutationInputSchema[removeInput]("nodes"), + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), @@ -261,16 +306,17 @@ func registerRemove(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { // --- move --- type moveInput struct { - SourceID string `json:"source_id" jsonschema:"source node ID"` - DestID string `json:"dest_id" jsonschema:"destination node ID"` - ExpectedHash string `json:"expected_hash" jsonschema:"the combined hash returned by node_read (not content_hash or meta_hash)"` - Keg string `json:"keg,omitempty" jsonschema:"keg alias (uses default if empty)"` + SourceID string `json:"source_id" jsonschema:"Source node ID."` + DestID string `json:"dest_id" jsonschema:"Destination node ID."` + ExpectedHash string `json:"expected_hash" jsonschema:"The combined hash from node_read, not content_hash or meta_hash."` + Keg string `json:"keg,omitempty" jsonschema:"KEG to use, as @namespace/keg; a bare name uses your default namespace. Omit it to use the default KEG."` } func registerMove(srv *sdkmcp.Server, tap *tapper.Tap, defaults KegDefaults) { sdkmcp.AddTool(srv, &sdkmcp.Tool{ - Name: "node_move", - Description: "Memory: Call node_read first, then move (rename) a KEG node to a new ID using the hash from node_read (the combined hash, not content_hash or meta_hash) as expected_hash. The destination must be free: the id of a node in the trash stays reserved until it is restored or purged. On conflict, refetch with node_read and retry with the returned current hash.", + Name: "node_move", + Description: "Memory: Call node_read first, then move (rename) a KEG node to a new ID using the hash from node_read (the combined hash, not content_hash or meta_hash) as expected_hash. The destination must be free: the id of a node in the trash stays reserved until it is restored or purged. On conflict, refetch with node_read and retry with the returned current hash.", + OutputSchema: textOutputSchema(), Annotations: &sdkmcp.ToolAnnotations{ DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false), diff --git a/pkg/mcp/transfer_security_test.go b/pkg/mcp/transfer_security_test.go new file mode 100644 index 00000000..fb3b8277 --- /dev/null +++ b/pkg/mcp/transfer_security_test.go @@ -0,0 +1,47 @@ +package mcp_test + +import ( + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" + "testing" +) + +func TestMCPLocalTransfersStayInWorkspaceAndNeverOverwrite(t *testing.T) { + session, rt, ctx := newLocalTestSessionWithRuntime(t) + created, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "node_create", Arguments: batchCreateArgs(map[string]any{"title": "Transfer security"})}) + require.NoError(t, err) + id := extractText(t, created) + uploaded, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "file_upload", Arguments: map[string]any{"node_id": id, "filename": "safe.txt", "data_base64": "c2FmZQ=="}}) + require.NoError(t, err) + require.False(t, uploaded.IsError) + require.NoError(t, rt.Mkdir("/outside", 0700, true)) + require.NoError(t, rt.WriteFile("/outside/secret", []byte("secret"), 0600)) + require.NoError(t, rt.Symlink("/outside", "/home/testuser/link")) + require.NoError(t, rt.WriteFile("/home/testuser/present", []byte("keep"), 0600)) + for _, path := range []string{"/outside/secret", "../../outside/secret", "link/secret"} { + result, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "file_upload", Arguments: map[string]any{"node_id": id, "filename": "blocked.txt", "source_path": path}}) + require.NoError(t, err) + require.True(t, result.IsError, path) + } + for _, path := range []string{"/outside/new", "../../outside/new", "link/new", "present"} { + result, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "file_download", Arguments: map[string]any{"node_id": id, "filename": "safe.txt", "dest_path": path}}) + require.NoError(t, err) + require.True(t, result.IsError, path) + } + original, err := rt.ReadFile("/home/testuser/present") + require.NoError(t, err) + require.Equal(t, "keep", string(original)) + result, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "file_download", Arguments: map[string]any{"node_id": id, "filename": "safe.txt", "dest_path": "new"}}) + require.NoError(t, err) + require.False(t, result.IsError) + saved, err := rt.ReadFile("/home/testuser/new") + require.NoError(t, err) + require.Equal(t, "safe", string(saved)) + listed, err := session.ListTools(ctx, &sdkmcp.ListToolsParams{}) + require.NoError(t, err) + for _, tool := range listed.Tools { + if tool.Name == "image_download" { + require.False(t, tool.Annotations.ReadOnlyHint) + } + } +} diff --git a/pkg/parity/parity_coverage_test.go b/pkg/parity/parity_coverage_test.go index 2cc8b3b2..7b789b66 100644 --- a/pkg/parity/parity_coverage_test.go +++ b/pkg/parity/parity_coverage_test.go @@ -178,6 +178,7 @@ var tapMethodsExcluded = map[string]string{ "ContextInfo": "CLI-only `tap info` summary of the local hub, keg, and flight defaults; an MCP session learns its identity from session_info and its flight and KEGs from orient", "ActiveFlightName": "internal pure read of the explicit flight or the loaded cascade's selection; backs Orient and MCP session adoption rather than being an operation of its own", "ActiveLaunch": "internal pure read of the `tap launch` harness and model driving the process; reported in orientation rather than being an operation of its own", + "ActiveAgent": "internal pure read of the TAP_AGENT Hub agent a `tap mcp` session runs as; reported in orientation rather than being an operation of its own", "OrientationKegsForFlight": "internal authority projection used by MCP providers to compute a revision before rendering once", "IdentityKegCatalog": "internal identity metadata projection used by MCP providers for graph discovery and keg_list", // Dropped from MCP when the surface was unified behind providers: these diff --git a/pkg/parity/parity_read_test.go b/pkg/parity/parity_read_test.go index 3e703d99..1fe05846 100644 --- a/pkg/parity/parity_read_test.go +++ b/pkg/parity/parity_read_test.go @@ -100,8 +100,8 @@ func TestParity_ReadOperations(t *testing.T) { CLIArgs: []string{"node", "list", "--id-only"}, MCPTool: "node_list", MCPInput: map[string]any{ - "id_only": true, - "limit": -1, + "format": "%i", + "limit": -1, }, }, { @@ -109,7 +109,7 @@ func TestParity_ReadOperations(t *testing.T) { CLIArgs: []string{"node", "list", "--id-only", "--reverse"}, MCPTool: "node_list", MCPInput: map[string]any{ - "id_only": true, + "format": "%i", "reverse": true, "limit": -1, }, @@ -259,9 +259,9 @@ func TestParity_ReadOperations(t *testing.T) { CLIArgs: []string{"node", "list", "--id-only", "--sort", "updated"}, MCPTool: "node_list", MCPInput: map[string]any{ - "id_only": true, - "sort": "updated", - "limit": -1, + "format": "%i", + "sort": "updated", + "limit": -1, }, }, @@ -271,9 +271,9 @@ func TestParity_ReadOperations(t *testing.T) { CLIArgs: []string{"node", "grep", "Hello", "--id-only"}, MCPTool: "node_grep", MCPInput: map[string]any{ - "query": "Hello", - "id_only": true, - "limit": -1, + "query": "Hello", + "format": "%i", + "limit": -1, }, }, { @@ -290,9 +290,9 @@ func TestParity_ReadOperations(t *testing.T) { CLIArgs: []string{"node", "grep", "ZZZZNOTFOUND", "--id-only"}, MCPTool: "node_grep", MCPInput: map[string]any{ - "query": "ZZZZNOTFOUND", - "id_only": true, - "limit": -1, + "query": "ZZZZNOTFOUND", + "format": "%i", + "limit": -1, }, Compare: func(t *testing.T, cliOut, mcpOut string) { t.Helper() @@ -322,9 +322,9 @@ func TestParity_ReadOperations(t *testing.T) { CLIArgs: []string{"tag", "list", "--query", "test", "--id-only"}, MCPTool: "tag_list", MCPInput: map[string]any{ - "query": "test", - "id_only": true, - "limit": -1, + "query": "test", + "format": "%i", + "limit": -1, }, }, @@ -335,7 +335,7 @@ func TestParity_ReadOperations(t *testing.T) { MCPTool: "node_backlinks", MCPInput: map[string]any{ "node_ids": []string{"0"}, - "id_only": true, + "format": "%i", "limit": -1, }, }, @@ -347,7 +347,7 @@ func TestParity_ReadOperations(t *testing.T) { MCPTool: "node_links", MCPInput: map[string]any{ "node_ids": []string{"1"}, - "id_only": true, + "format": "%i", "limit": -1, }, }, @@ -358,9 +358,9 @@ func TestParity_ReadOperations(t *testing.T) { CLIArgs: []string{"node", "list", "--id-only", "--offset", "1"}, MCPTool: "node_list", MCPInput: map[string]any{ - "id_only": true, - "offset": 1, - "limit": -1, + "format": "%i", + "offset": 1, + "limit": -1, }, }, @@ -386,10 +386,10 @@ func TestParity_ReadOperations(t *testing.T) { CLIArgs: []string{"tag", "list", "--query", "test or overview", "--id-only", "--offset", "1"}, MCPTool: "tag_list", MCPInput: map[string]any{ - "query": "test or overview", - "id_only": true, - "offset": 1, - "limit": -1, + "query": "test or overview", + "format": "%i", + "offset": 1, + "limit": -1, }, }, @@ -400,7 +400,7 @@ func TestParity_ReadOperations(t *testing.T) { MCPTool: "node_grep", MCPInput: map[string]any{ "query": "node", - "id_only": true, + "format": "%i", "ignore_case": true, "offset": 1, "limit": -1, @@ -414,7 +414,7 @@ func TestParity_ReadOperations(t *testing.T) { MCPTool: "node_backlinks", MCPInput: map[string]any{ "node_ids": []string{"0"}, - "id_only": true, + "format": "%i", "offset": 1, "limit": -1, }, @@ -434,7 +434,7 @@ func TestParity_ReadOperations(t *testing.T) { MCPTool: "node_links", MCPInput: map[string]any{ "node_ids": []string{"1"}, - "id_only": true, + "format": "%i", "offset": 1, "limit": -1, }, @@ -518,7 +518,7 @@ func TestParity_ReadOperations(t *testing.T) { MCPTool: "node_backlinks", MCPInput: map[string]any{ "node_ids": []string{"0", "1"}, - "id_only": true, + "format": "%i", "limit": -1, }, }, @@ -530,7 +530,7 @@ func TestParity_ReadOperations(t *testing.T) { MCPTool: "node_links", MCPInput: map[string]any{ "node_ids": []string{"0", "1"}, - "id_only": true, + "format": "%i", "limit": -1, }, }, diff --git a/pkg/parity/parity_test.go b/pkg/parity/parity_test.go index e32430e4..02e9151d 100644 --- a/pkg/parity/parity_test.go +++ b/pkg/parity/parity_test.go @@ -23,11 +23,14 @@ import ( sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/stretchr/testify/require" + "encoding/json" + "github.com/google/jsonschema-go/jsonschema" "github.com/jlrickert/tapper/internal/testkegrepo" "github.com/jlrickert/tapper/pkg/cli" "github.com/jlrickert/tapper/pkg/keg" "github.com/jlrickert/tapper/pkg/mcp" "github.com/jlrickert/tapper/pkg/tapper" + "sync" ) //go:embed all:data/** @@ -102,8 +105,11 @@ func newParityEnv(t *testing.T) *parityEnv { Name: "parity-test", Version: "0.1", }, nil) + var connected *sdkmcp.ClientSession + client.AddSendingMiddleware(checkOutputSchemas(t, &connected)) session, err := client.Connect(ctx, clientTransport, nil) require.NoError(t, err) + connected = session t.Cleanup(func() { session.Close() }) @@ -471,3 +477,50 @@ func renderNodeReadRows(rows []any) string { } return buf.String() } + +// checkOutputSchemas fails t when a successful tool result does not match the +// outputSchema its tool declares, as a client that validates results would. +func checkOutputSchemas(t *testing.T, session **sdkmcp.ClientSession) sdkmcp.Middleware { + var ( + once sync.Once + schemas map[string]*jsonschema.Resolved + ) + load := func(ctx context.Context) { + schemas = map[string]*jsonschema.Resolved{} + listed, err := (*session).ListTools(ctx, nil) + if err != nil { + t.Errorf("list tools for their output schemas: %v", err) + return + } + for _, tool := range listed.Tools { + raw, _ := json.Marshal(tool.OutputSchema) + var schema jsonschema.Schema + if tool.OutputSchema == nil || json.Unmarshal(raw, &schema) != nil { + continue + } + if resolved, err := schema.Resolve(nil); err == nil { + schemas[tool.Name] = resolved + } + } + } + return func(next sdkmcp.MethodHandler) sdkmcp.MethodHandler { + return func(ctx context.Context, method string, req sdkmcp.Request) (sdkmcp.Result, error) { + result, err := next(ctx, method, req) + res, ok := result.(*sdkmcp.CallToolResult) + params, _ := req.GetParams().(*sdkmcp.CallToolParams) + if err != nil || method != "tools/call" || !ok || res == nil || res.IsError || params == nil || *session == nil { + return result, err + } + once.Do(func() { load(ctx) }) + if schema := schemas[params.Name]; schema != nil { + raw, _ := json.Marshal(res.StructuredContent) + var instance any + _ = json.Unmarshal(raw, &instance) + if verr := schema.Validate(instance); verr != nil { + t.Errorf("%s result does not match its outputSchema: %v", params.Name, verr) + } + } + return result, err + } + } +} diff --git a/pkg/parity/parity_write_test.go b/pkg/parity/parity_write_test.go index 9f22efea..836a379a 100644 --- a/pkg/parity/parity_write_test.go +++ b/pkg/parity/parity_write_test.go @@ -69,8 +69,8 @@ func TestParity_WriteOperations(t *testing.T) { require.Contains(t, cliTags, mcpNodeID, "CLI tags should list MCP-created node") mcpTags, err := env.runMCP("tag_list", map[string]any{ - "query": "parity-test", - "id_only": true, + "query": "parity-test", + "format": "%i", }) require.NoError(t, err, "MCP tags query should succeed") require.Contains(t, mcpTags, cliNodeID, "MCP tags should list CLI-created node") @@ -100,7 +100,7 @@ func TestParity_WriteOperations(t *testing.T) { // Both should appear in MCP list. mcpList, err := env.runMCP("node_list", map[string]any{ - "id_only": true, + "format": "%i", }) require.NoError(t, err) require.True(t, containsLine(mcpList, cliID), "MCP list should contain CLI-created node") @@ -163,7 +163,7 @@ func TestParity_WriteOperations(t *testing.T) { require.False(t, containsLine(cliList, nodeID), "CLI list should not contain removed node") mcpList, err := env.runMCP("node_list", map[string]any{ - "id_only": true, + "format": "%i", }) require.NoError(t, err) require.False(t, containsLine(mcpList, nodeID), "MCP list should not contain removed node") @@ -241,7 +241,7 @@ func TestParity_WriteOperations(t *testing.T) { require.False(t, containsLine(cliList, srcID), "CLI list should not contain old ID") mcpList, err := env.runMCP("node_list", map[string]any{ - "id_only": true, + "format": "%i", }) require.NoError(t, err) require.True(t, containsLine(mcpList, "777"), "MCP list should contain new ID") @@ -322,8 +322,8 @@ func TestParity_WriteOperations(t *testing.T) { require.Contains(t, cliTags, nodeID, "CLI tags should list node with updated-meta tag") mcpTags, err := env.runMCP("tag_list", map[string]any{ - "query": "updated-meta", - "id_only": true, + "query": "updated-meta", + "format": "%i", }) require.NoError(t, err) require.Contains(t, mcpTags, nodeID, "MCP tags should list node with updated-meta tag") diff --git a/pkg/relaycontract/contract.go b/pkg/relaycontract/contract.go index 4863c737..abc3cb6d 100644 --- a/pkg/relaycontract/contract.go +++ b/pkg/relaycontract/contract.go @@ -18,8 +18,10 @@ import ( // ProtocolVersion is the newest relay protocol this package speaks. It is // negotiated once, in register and registered, and gates which frames a // session may carry: version 2 adds tools and call, version 3 adds the -// attribution fields of infer. -const ProtocolVersion = 3 +// attribution fields of infer, version 4 adds in-band reauthentication +// (registered's credentialExpiresIn, reauth and reauthed) and the +// credential_revoked and going_away end codes. +const ProtocolVersion = 4 // EnvelopeVersion is the frame shape every envelope carries in V. It is // separate from ProtocolVersion because a peer checks V before it knows what @@ -49,6 +51,16 @@ const ( // like an infer: one chunk holding the CallToolResult, then done, or an // error. Cancel stops it. TypeCall = "call" + // TypeReauth presents a refreshed access token on a live session, so + // the session outlives the token that opened it. The relay sends it, + // with an id of its choosing, only on a protocol 4 session whose + // registered frame advertised credentialExpiresIn. Hub answers with + // reauthed carrying the same id, or with an error of that id and code + // CodeReauthRejected; either way the session goes on. + TypeReauth = "reauth" + // TypeReauthed accepts a reauth: the session now runs under the new + // token, which expires credentialExpiresIn seconds from this frame. + TypeReauthed = "reauthed" ) // APIOpenAIChatCompletions is the chat inference API. The request body is an @@ -108,8 +120,29 @@ const ( CodeInternal = "internal" // CodeDisconnected ends a session because the relay's owner disconnected // it from Hub. It is connection-level (no id) and the relay must not - // reconnect: the owner asked for it to go away. + // reconnect: the owner asked for it to go away. From protocol 4 it means + // only that; a revoked credential ends with CodeCredentialRevoked. CodeDisconnected = "disconnected" + // CodeCredentialExpired ends a session because the access token that + // opened it (or the last reauth) expired. It is connection-level (no id) + // and, unlike CodeDisconnected, retryable: the relay should reconnect at + // once with a refreshed token. Hub closes the WebSocket with status 1013 + // (try again later) after sending it. + CodeCredentialExpired = "credential_expired" + // CodeCredentialRevoked ends a session because the credential behind it + // was revoked: a sign-out, a revoked token, or a removed account. It is + // connection-level and final; only a new sign-in restores the relay. + // Hub sends it only on protocol 4 sessions. + CodeCredentialRevoked = "credential_revoked" + // CodeGoingAway ends a session because Hub is shutting down or + // restarting. It is connection-level and retryable at once: the relay + // reconnects after a short random delay, without a backoff penalty. + // Hub sends it only on protocol 4 sessions. + CodeGoingAway = "going_away" + // CodeReauthRejected answers a reauth Hub refused (the token is not + // valid, or not for the same account). It carries the reauth's id and + // does not end the session: the old credential stands until it expires. + CodeReauthRejected = "reauth_rejected" // CodeUnknownTool: the call names a server or tool the relay does not // currently offer. CodeUnknownTool = "unknown_tool" @@ -221,9 +254,52 @@ type Model struct { } // Registered is Hub's answer to Register. +// +// CredentialExpiresIn is how many seconds after this frame the access token +// that opened the session expires. Zero means Hub does not end the session +// at a token expiry (the token does not expire, or Hub predates protocol 4). +// Hub sends it only on protocol 4 sessions, since older relays decode +// registered strictly. The relay times its reauth from when it read this +// frame, never from its own wall clock, so clock skew cannot move it. type Registered struct { - Protocol int `json:"protocol"` - Models []CatalogBinding `json:"models"` + Protocol int `json:"protocol"` + Models []CatalogBinding `json:"models"` + CredentialExpiresIn int `json:"credentialExpiresIn,omitempty"` +} + +// Reauth is the payload of a reauth frame. +type Reauth struct { + Token string `json:"token"` +} + +// Reauthed is the payload of a reauthed frame. CredentialExpiresIn counts +// seconds from this frame, like Registered's. +type Reauthed struct { + CredentialExpiresIn int `json:"credentialExpiresIn"` +} + +// Validate checks the reauth frame. +func (r *Reauth) Validate() error { + if r.Token == "" { + return errors.New("token is empty") + } + return nil +} + +// Validate checks the reauthed frame. +func (r *Reauthed) Validate() error { + if r.CredentialExpiresIn < 0 { + return errors.New("credentialExpiresIn must not be negative") + } + return nil +} + +// Validate checks the registered frame. +func (r *Registered) Validate() error { + if r.CredentialExpiresIn < 0 { + return errors.New("credentialExpiresIn must not be negative") + } + return nil } // CatalogBinding maps a relay's model to the catalog id Hub assigned it. diff --git a/pkg/relaycontract/contract_test.go b/pkg/relaycontract/contract_test.go index 7a602d72..672a71fa 100644 --- a/pkg/relaycontract/contract_test.go +++ b/pkg/relaycontract/contract_test.go @@ -151,13 +151,59 @@ func TestTranscriptionRequestValidate(t *testing.T) { } func TestSelectProtocol(t *testing.T) { - if got := SelectProtocol([]int{1, 2, 3, 4}); got != 3 { - t.Fatalf("SelectProtocol = %d, want 3", got) + if got := SelectProtocol([]int{1, 2, 3, 4, 5}); got != 4 { + t.Fatalf("SelectProtocol = %d, want 4", got) + } + if got := SelectProtocol([]int{1, 2, 3}); got != 3 { + t.Fatalf("SelectProtocol = %d, want 3 for a relay that predates reauth", got) } if got := SelectProtocol([]int{1}); got != 1 { t.Fatalf("SelectProtocol = %d, want 1", got) } - if got := SelectProtocol([]int{4}); got != 0 { + if got := SelectProtocol([]int{5}); got != 0 { t.Fatalf("SelectProtocol = %d, want 0", got) } } + +// A relay that predates protocol 4 decodes registered strictly, so a hub +// that leaves credentialExpiresIn at zero must not put the field on the wire. +func TestRegisteredOmitsZeroCredentialExpiry(t *testing.T) { + env, err := NewEnvelope(TypeRegistered, "", Registered{Protocol: 3, Models: []CatalogBinding{}}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(env.Payload), "credentialExpiresIn") { + t.Fatalf("payload = %s, want no credentialExpiresIn", env.Payload) + } + env, err = NewEnvelope(TypeRegistered, "", Registered{Protocol: 4, Models: []CatalogBinding{}, CredentialExpiresIn: 3600}) + if err != nil { + t.Fatal(err) + } + var reg Registered + if err := env.Decode(®); err != nil || reg.CredentialExpiresIn != 3600 { + t.Fatalf("registered = %+v, %v", reg, err) + } +} + +func TestReauthFramesValidate(t *testing.T) { + ok, err := NewEnvelope(TypeReauth, "reauth-1", Reauth{Token: "tok"}) + if err != nil { + t.Fatal(err) + } + var r Reauth + if err := ok.Decode(&r); err != nil || r.Token != "tok" { + t.Fatalf("reauth = %+v, %v", r, err) + } + empty, _ := NewEnvelope(TypeReauth, "reauth-1", Reauth{}) + if err := empty.Decode(&Reauth{}); err == nil { + t.Fatal("reauth with no token accepted") + } + neg, _ := NewEnvelope(TypeReauthed, "reauth-1", Reauthed{CredentialExpiresIn: -1}) + if err := neg.Decode(&Reauthed{}); err == nil { + t.Fatal("reauthed with a negative lifetime accepted") + } + zero, _ := NewEnvelope(TypeReauthed, "reauth-1", Reauthed{}) + if err := zero.Decode(&Reauthed{}); err != nil { + t.Fatalf("reauthed with no expiry rejected: %v", err) + } +} diff --git a/pkg/relaycontract/tools.go b/pkg/relaycontract/tools.go index 57fbf55d..a2d0d21b 100644 --- a/pkg/relaycontract/tools.go +++ b/pkg/relaycontract/tools.go @@ -39,14 +39,17 @@ type ToolServer struct { } // Tool is one tool as its MCP server lists it. InputSchema is the tool's JSON -// Schema for its arguments; Annotations are the server's own hints, passed -// through as the server reported them. +// Schema for its arguments, and OutputSchema the one for its structured +// result when the server declares one; a relay that predates OutputSchema, or +// a server that declares none, leaves it out. Annotations are the server's own +// hints, passed through as the server reported them. type Tool struct { - Name string `json:"name"` - Title string `json:"title,omitempty"` - Description string `json:"description,omitempty"` - InputSchema json.RawMessage `json:"inputSchema"` - Annotations *ToolAnnotations `json:"annotations,omitempty"` + Name string `json:"name"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + InputSchema json.RawMessage `json:"inputSchema"` + OutputSchema json.RawMessage `json:"outputSchema,omitempty"` + Annotations *ToolAnnotations `json:"annotations,omitempty"` } // ToolAnnotations mirrors MCP's tool annotations. Nil means the server did not @@ -136,6 +139,14 @@ func (t *Tool) Validate() error { if !isJSONObject(t.InputSchema) { return errors.New("inputSchema must be a JSON object") } + if len(t.OutputSchema) > 0 { + if len(t.OutputSchema) > MaxToolSchemaBytes { + return fmt.Errorf("outputSchema is larger than %d bytes", MaxToolSchemaBytes) + } + if !isJSONObject(t.OutputSchema) { + return errors.New("outputSchema must be a JSON object") + } + } return nil } diff --git a/pkg/relaycontract/tools_test.go b/pkg/relaycontract/tools_test.go index 13769875..2fd0ed0b 100644 --- a/pkg/relaycontract/tools_test.go +++ b/pkg/relaycontract/tools_test.go @@ -54,6 +54,10 @@ func TestToolsValidate(t *testing.T) { "schema too large": func(x *Tools) { x.Servers[0].Tools[0].InputSchema = json.RawMessage(`{"d":"` + strings.Repeat("a", MaxToolSchemaBytes) + `"}`) }, + "output schema not object": func(x *Tools) { x.Servers[0].Tools[0].OutputSchema = json.RawMessage(`"object"`) }, + "output schema too large": func(x *Tools) { + x.Servers[0].Tools[0].OutputSchema = json.RawMessage(`{"d":"` + strings.Repeat("a", MaxToolSchemaBytes) + `"}`) + }, "too many servers": func(x *Tools) { for i := 0; i < MaxToolServers; i++ { s := x.Servers[0] @@ -83,6 +87,18 @@ func TestToolsValidate(t *testing.T) { if err := tools.Validate(); err != nil { t.Fatalf("valid tools rejected: %v", err) } + // An output schema is optional; a relay that predates it sends none. + tools.Servers[0].Tools[0].OutputSchema = json.RawMessage(`{"type":"object"}`) + if err := tools.Validate(); err != nil { + t.Fatalf("tool with an output schema rejected: %v", err) + } + raw, err := json.Marshal(validTools()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "outputSchema") { + t.Fatalf("a tool without an output schema must not send one: %s", raw) + } } func TestToolsValidateTotalLimit(t *testing.T) { diff --git a/pkg/tapapi/tapapi.go b/pkg/tapapi/tapapi.go index 1b8b4a90..2c37862a 100644 --- a/pkg/tapapi/tapapi.go +++ b/pkg/tapapi/tapapi.go @@ -122,8 +122,8 @@ type SearchRequest struct { // Span is a highlighted range of a hit's Snippet in rune offsets: Start is // inclusive and End exclusive. type Span struct { - Start int `json:"start"` - End int `json:"end"` + Start int `json:"start" jsonschema:"First highlighted rune of the snippet."` + End int `json:"end" jsonschema:"Rune just past the highlight."` } // Search hit fields name where a hit matched. @@ -150,38 +150,38 @@ const ( // descending, then Keg, then NodeID. type SearchHit struct { // Ref is the qualified node reference, "keg:@namespace/alias/id". - Ref string `json:"ref"` + Ref string `json:"ref" jsonschema:"The node reference, keg:@namespace/keg/id."` // Keg is the qualified KEG reference, "@namespace/alias". - Keg string `json:"keg"` - NodeID string `json:"node_id"` - Title string `json:"title"` - Score float64 `json:"score"` + Keg string `json:"keg" jsonschema:"The KEG, as @namespace/keg."` + NodeID string `json:"node_id" jsonschema:"The node's ID in its KEG."` + Title string `json:"title" jsonschema:"The node's title."` + Score float64 `json:"score" jsonschema:"Relevance; hits arrive highest first."` // Fields lists where the node matched: SearchFieldTitle, SearchFieldMeta, // SearchFieldBody. - Fields []string `json:"fields"` + Fields []string `json:"fields" jsonschema:"Where the node matched: title, meta, or body."` // Sources lists the signals that found the node: SearchSourceKeyword, // SearchSourceSemantic. - Sources []string `json:"sources,omitempty"` - Snippet string `json:"snippet,omitempty"` + Sources []string `json:"sources,omitempty" jsonschema:"The signals that found the node: keyword, semantic."` + Snippet string `json:"snippet,omitempty" jsonschema:"An excerpt around the match."` // Highlights are the matched ranges of Snippet. - Highlights []Span `json:"highlights,omitempty"` + Highlights []Span `json:"highlights,omitempty" jsonschema:"The matched ranges of snippet."` } // TermExpansion reports how the Hub interpreted one query term: the lexemes it // searched for, and whether they came from typo expansion. type TermExpansion struct { - Term string `json:"term"` - Lexemes []string `json:"lexemes"` - Typo bool `json:"typo"` + Term string `json:"term" jsonschema:"A term of the query."` + Lexemes []string `json:"lexemes" jsonschema:"The lexemes searched for it."` + Typo bool `json:"typo" jsonschema:"True when the lexemes came from typo expansion."` } // SearchResponse is one page of ranked hits. NextCursor is empty on the last // page. Mode is SearchModeKeyword or SearchModeHybrid. Warnings report // partial results the caller should know about. type SearchResponse struct { - Hits []SearchHit `json:"hits"` - NextCursor string `json:"next_cursor,omitempty"` - Terms []TermExpansion `json:"terms,omitempty"` - Mode string `json:"mode,omitempty"` - Warnings []string `json:"warnings,omitempty"` + Hits []SearchHit `json:"hits" jsonschema:"The page's hits, most relevant first."` + NextCursor string `json:"next_cursor,omitempty" jsonschema:"Pass as cursor, with the same query, for the next page; absent on the last page."` + Terms []TermExpansion `json:"terms,omitempty" jsonschema:"How the Hub interpreted each query term."` + Mode string `json:"mode,omitempty" jsonschema:"How the page was ranked: keyword or hybrid."` + Warnings []string `json:"warnings,omitempty" jsonschema:"Partial results or degraded ranking to know about."` } diff --git a/pkg/tapper/api_discovery_cache.go b/pkg/tapper/api_discovery_cache.go new file mode 100644 index 00000000..3e6d3b87 --- /dev/null +++ b/pkg/tapper/api_discovery_cache.go @@ -0,0 +1,96 @@ +// Package tapper — on-disk cache of Hub API discovery. +// +// Every CLI invocation used to start with GET /api/version before its first +// REST request. apiDiscoveryCache remembers each success under CacheRoot so +// the next invocation within the TTL skips it (see apicontract.Session.Cache). +// Only successes are written, and a stale one is harmless: the Hub still +// validates the contract on every request, and its typed rejection evicts +// the entry. +package tapper + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "sync" + "time" + + "github.com/jlrickert/cli-toolkit/toolkit" + "github.com/jlrickert/tapper/pkg/apicontract" +) + +const apiDiscoveryCacheFile = "api-discovery.json" + +// apiDiscoveryCache is an apicontract.DiscoveryCache over one JSON file +// mapping cache keys to the time discovery last succeeded. Every error reads +// as a miss or a skipped write; the cache only ever saves a request. +type apiDiscoveryCache struct { + rt *toolkit.Runtime + path string + mu sync.Mutex +} + +// NewAPIDiscoveryCache returns the discovery cache under paths' CacheRoot, +// or nil when there is nowhere to keep it. +func NewAPIDiscoveryCache(rt *toolkit.Runtime, paths *PathService) apicontract.DiscoveryCache { + if rt == nil || paths == nil || paths.AppPaths == nil || paths.CacheRoot == "" { + return nil + } + return &apiDiscoveryCache{rt: rt, path: filepath.Join(paths.CacheRoot, apiDiscoveryCacheFile)} +} + +func (c *apiDiscoveryCache) Get(key string) (time.Time, bool) { + c.mu.Lock() + defer c.mu.Unlock() + at, ok := c.load()[key] + return at, ok +} + +func (c *apiDiscoveryCache) Put(key string, checked time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + entries := c.load() + entries[key] = checked + c.save(entries) +} + +func (c *apiDiscoveryCache) Delete(key string) { + c.mu.Lock() + defer c.mu.Unlock() + entries := c.load() + if _, ok := entries[key]; !ok { + return + } + delete(entries, key) + c.save(entries) +} + +func (c *apiDiscoveryCache) load() map[string]time.Time { + entries := map[string]time.Time{} + raw, err := c.rt.ReadFile(c.path) + if err != nil || len(raw) == 0 { + return entries + } + if json.Unmarshal(raw, &entries) != nil { + return map[string]time.Time{} + } + return entries +} + +// save mirrors AuthStore.Save: create the directory 0700 only when missing, +// then replace the file atomically. +func (c *apiDiscoveryCache) save(entries map[string]time.Time) { + data, err := json.Marshal(entries) + if err != nil { + return + } + if parent := filepath.Dir(c.path); parent != "" { + if _, err := c.rt.Stat(parent, false); errors.Is(err, os.ErrNotExist) { + if err := c.rt.Mkdir(parent, 0o700, true); err != nil { + return + } + } + } + _ = c.rt.AtomicWriteFile(c.path, data, 0o600) +} diff --git a/pkg/tapper/api_discovery_cache_test.go b/pkg/tapper/api_discovery_cache_test.go new file mode 100644 index 00000000..54f21241 --- /dev/null +++ b/pkg/tapper/api_discovery_cache_test.go @@ -0,0 +1,55 @@ +package tapper_test + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/jlrickert/tapper/pkg/tapper" +) + +func TestAPIDiscoveryCache_PersistsUnderCacheRoot(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + tap := newTestTap(t, sb) + at := sb.Runtime().Clock().Now().UTC() + + cache := tapper.NewAPIDiscoveryCache(sb.Runtime(), tap.PathService) + require.NotNil(t, cache) + _, ok := cache.Get("rev https://hub.example.com") + require.False(t, ok, "an absent file is a miss") + + cache.Put("rev https://hub.example.com", at) + path := filepath.Join(tap.PathService.CacheRoot, "api-discovery.json") + info, err := sb.Runtime().Stat(path, false) + require.NoError(t, err) + require.Equal(t, "-rw-------", info.Mode().Perm().String()) + + // A new cache over the same file sees the entry, as the next invocation does. + reopened := tapper.NewAPIDiscoveryCache(sb.Runtime(), tap.PathService) + got, ok := reopened.Get("rev https://hub.example.com") + require.True(t, ok) + require.True(t, got.Equal(at), "got %v want %v", got, at) + + reopened.Delete("rev https://hub.example.com") + _, ok = cache.Get("rev https://hub.example.com") + require.False(t, ok) +} + +func TestAPIDiscoveryCache_CorruptFileIsAMiss(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + tap := newTestTap(t, sb) + path := filepath.Join(tap.PathService.CacheRoot, "api-discovery.json") + require.NoError(t, sb.Runtime().Mkdir(tap.PathService.CacheRoot, 0o700, true)) + require.NoError(t, sb.Runtime().AtomicWriteFile(path, []byte("not json"), 0o600)) + + cache := tapper.NewAPIDiscoveryCache(sb.Runtime(), tap.PathService) + _, ok := cache.Get("rev https://hub.example.com") + require.False(t, ok) + cache.Put("rev https://hub.example.com", time.Unix(0, 0)) + _, ok = cache.Get("rev https://hub.example.com") + require.True(t, ok, "a write replaces the corrupt file") +} diff --git a/pkg/tapper/auth_refresh.go b/pkg/tapper/auth_refresh.go index f612cc57..f35cb4fa 100644 --- a/pkg/tapper/auth_refresh.go +++ b/pkg/tapper/auth_refresh.go @@ -10,6 +10,7 @@ package tapper import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -32,6 +33,15 @@ const ( authRefreshTimeout = 10 * time.Second ) +// ErrRefreshRejected means the hub refused the refresh token itself: its +// token endpoint answered with an OAuth error body whose `error` is +// invalid_grant or invalid_client, whatever the HTTP status. The sign-in is +// gone and only a new `tap auth login` restores it. Every other failure is +// not a refusal and may succeed on a later attempt: an unreachable hub, a +// 5xx, a server_error or invalid_request answer, or a body that does not +// parse as an OAuth error. +var ErrRefreshRejected = errors.New("auth refresh: hub rejected refresh token") + // authEntryNeedsRefresh reports whether entry has a refresh token and an access // token that is expired or within refreshSkew of expiring. func authEntryNeedsRefresh(rt *toolkit.Runtime, entry *AuthEntry) bool { @@ -43,9 +53,15 @@ func authEntryNeedsRefresh(rt *toolkit.Runtime, entry *AuthEntry) bool { // refreshAuthStoreEntryIfNeeded renews one stored hub credential when its // access token is expired or close to expiry. It also handles the cross-process -// rotation race: before contacting the hub, and again after a rejected refresh, -// it adopts a fresher on-disk entry when another process has already persisted -// one. +// rotation race: the refresh runs under an OS lock on the store file +// (lockAuthStore), so of several processes sharing one store only the first +// spends the refresh token, and the rest adopt the pair it persisted when they +// get the lock. A process that cannot get the lock in time never refreshes +// unlocked: it adopts a fresher on-disk entry if there is one, and otherwise +// fails so the caller keeps its cached token. Under the lock the store file is +// authoritative: a refresh spends the refresh token stored there, not a stale +// in-memory one, does not run at all once a logout removed the entry, and +// persists only over the entry it started from. func refreshAuthStoreEntryIfNeeded(ctx context.Context, rt *toolkit.Runtime, store *AuthStore, storePath, hubURL, key string, entry *AuthEntry) (*AuthEntry, error) { if !authEntryNeedsRefresh(rt, entry) { return entry, nil @@ -67,6 +83,35 @@ func refreshAuthStoreEntryIfNeeded(ctx context.Context, rt *toolkit.Runtime, sto return disk, nil } + if storePath != "" { + unlock, err := lockAuthStore(ctx, rt, storePath, authStoreLockWait) + if err != nil { + if disk := adoptFresherAuthEntryFromDisk(ctx, rt, store, storePath, key); disk != nil { + return disk, nil + } + return nil, fmt.Errorf("auth refresh: %w", err) + } + defer unlock() + // Another process may have refreshed while this one waited. + if disk := adoptFresherAuthEntryFromDisk(ctx, rt, store, storePath, key); disk != nil { + return disk, nil + } + // The store file, not this process's copy, holds the live login. A + // logout may have removed it, and another process may have rotated + // the refresh token this copy holds: presenting the spent one is a + // replay, and the hub revokes the whole sign-in for it. + if disk, err := LoadAuthStore(ctx, rt, storePath); err == nil { + cur, ok := disk.Get(key) + if !ok || cur == nil { + store.Delete(key) + return nil, fmt.Errorf("auth refresh: login for %s was removed", key) + } + if cur.RefreshToken != entry.RefreshToken { + entry = cur + } + } + } + rctx, cancel := context.WithTimeout(ctx, authRefreshTimeout) next, err := RefreshHubToken(rctx, rt, hubURL, entry) cancel() @@ -77,13 +122,111 @@ func refreshAuthStoreEntryIfNeeded(ctx context.Context, rt *toolkit.Runtime, sto return nil, err } + keepSpoolAcrossRotation(ctx, rt, storePath, key, hubURL, entry, next) + if err := persistAuthStoreEntry(ctx, rt, storePath, key, entry.RefreshToken, next); err != nil { + if errors.Is(err, errAuthStoreEntryChanged) { + // A logout or a new login replaced the entry this refresh + // started from; that write wins over the rotated pair. + if disk := adoptFresherAuthEntryFromDisk(ctx, rt, store, storePath, key); disk != nil { + return disk, nil + } + if disk, lerr := LoadAuthStore(ctx, rt, storePath); lerr == nil { + if cur, ok := disk.Get(key); ok && cur != nil { + store.Set(key, *cur) + } else { + store.Delete(key) + } + } + return nil, fmt.Errorf("auth refresh: %w", err) + } + store.Set(key, *next) + return next, fmt.Errorf("auth refresh: persist: %w", err) + } store.Set(key, *next) - if err := persistAuthStoreEntry(ctx, rt, storePath, key, next); err != nil { + return next, nil +} + +// forceRefreshAuthStoreEntry renews the stored credential for key whatever +// its recorded expiry says. A long-lived caller (the relay) uses it when the +// hub, not the local clock, says the access token is spent or soon will be: +// the hub refused it, ended a session for its expiry, or announced when it +// will. authEntryNeedsRefresh is not consulted, so a skewed local clock +// cannot keep a dead token in use. +// +// It follows refreshAuthStoreEntryIfNeeded's discipline: the refresh runs +// under lockAuthStore, spends the refresh token stored on disk, and persists +// only over the entry it started from. "Already refreshed" means the stored +// access token differs from the one this process held: another process +// rotated past it, and that pair is adopted instead. A stored entry without a +// refresh token (a pasted API token) has nothing to renew and is returned as +// it is. +func forceRefreshAuthStoreEntry(ctx context.Context, rt *toolkit.Runtime, store *AuthStore, storePath, hubURL, key string) (*AuthEntry, error) { + if store == nil { + return nil, fmt.Errorf("auth refresh: store is nil") + } + entry, ok := store.Get(key) + if !ok || entry == nil { + return nil, fmt.Errorf("auth refresh: no login stored for %s", key) + } + if entry.RefreshToken == "" { + return entry, nil + } + held := entry.AccessToken + + if storePath != "" { + unlock, err := lockAuthStore(ctx, rt, storePath, authStoreLockWait) + if err != nil { + return nil, fmt.Errorf("auth refresh: %w", err) + } + defer unlock() + if disk, err := LoadAuthStore(ctx, rt, storePath); err == nil { + cur, ok := disk.Get(key) + if !ok || cur == nil { + store.Delete(key) + return nil, fmt.Errorf("auth refresh: login for %s was removed", key) + } + if cur.AccessToken != held { + store.Set(key, *cur) + return cur, nil + } + entry = cur + } + } + + rctx, cancel := context.WithTimeout(ctx, authRefreshTimeout) + next, err := RefreshHubToken(rctx, rt, hubURL, entry) + cancel() + if err != nil { + return nil, err + } + + keepSpoolAcrossRotation(ctx, rt, storePath, key, hubURL, entry, next) + if storePath == "" { + store.Set(key, *next) + return next, nil + } + if err := persistAuthStoreEntry(ctx, rt, storePath, key, entry.RefreshToken, next); err != nil { + if errors.Is(err, errAuthStoreEntryChanged) { + if disk, lerr := LoadAuthStore(ctx, rt, storePath); lerr == nil { + if cur, ok := disk.Get(key); ok && cur != nil { + store.Set(key, *cur) + } else { + store.Delete(key) + } + } + return nil, fmt.Errorf("auth refresh: %w", err) + } + store.Set(key, *next) return next, fmt.Errorf("auth refresh: persist: %w", err) } + store.Set(key, *next) return next, nil } +// errAuthStoreEntryChanged means the stored entry a refresh started from was +// removed or replaced before the refresh could persist its rotated pair. +var errAuthStoreEntryChanged = errors.New("auth store entry changed during refresh") + // adoptFresherAuthEntryFromDisk re-reads the auth store file and, when it holds // an entry for key that does not itself need refreshing, copies it into store // and returns it. This lets long-running processes and parallel commands pick @@ -107,7 +250,11 @@ func adoptFresherAuthEntryFromDisk(ctx context.Context, rt *toolkit.Runtime, sto return entry } -func persistAuthStoreEntry(ctx context.Context, rt *toolkit.Runtime, storePath, key string, entry *AuthEntry) error { +// persistAuthStoreEntry writes entry under key only while the stored entry is +// still the one the refresh started from, the one holding refreshToken. +// Otherwise it writes nothing and returns errAuthStoreEntryChanged, so a +// refresh never brings back a logged-out entry or replaces a newer login. +func persistAuthStoreEntry(ctx context.Context, rt *toolkit.Runtime, storePath, key, refreshToken string, entry *AuthEntry) error { if rt == nil { return fmt.Errorf("auth refresh: runtime is nil") } @@ -121,6 +268,11 @@ func persistAuthStoreEntry(ctx context.Context, rt *toolkit.Runtime, storePath, if err != nil { return err } + prev, ok := disk.Get(key) + if !ok || prev == nil || prev.RefreshToken != refreshToken { + return errAuthStoreEntryChanged + } + carryAuthAccountKey(entry, prev) disk.Set(key, *entry) return disk.Save(ctx, rt, storePath) } @@ -176,9 +328,7 @@ func RefreshHubToken(ctx context.Context, rt *toolkit.Runtime, hubURL string, en return nil, fmt.Errorf("auth refresh: read response: %w", err) } if resp.StatusCode != http.StatusOK { - // A rejected/expired/replayed refresh token returns invalid_grant. - // Surface the status so callers can fall back to a full re-login. - return nil, fmt.Errorf("auth refresh: hub rejected refresh token (%s)", resp.Status) + return nil, refreshFailure(resp.Status, body) } var tok tokenResponse @@ -208,8 +358,29 @@ func RefreshHubToken(ctx context.Context, rt *toolkit.Runtime, hubURL string, en if next.Scope == "" { next.Scope = entry.Scope } + carryAuthAccountKey(next, entry) if tok.ExpiresIn > 0 { next.ExpiresAt = rt.Clock().Now().Add(time.Duration(tok.ExpiresIn) * time.Second) } return next, nil } + +// refreshFailure classifies a non-200 token-endpoint answer. Only an OAuth +// error body naming invalid_grant (the refresh token is expired, revoked, or +// spent) or invalid_client (the client is gone) is ErrRefreshRejected; the +// status code alone decides nothing, because a hub whose store is failing can +// answer 400 or 401 for a credential that is still good. Everything else is +// an ordinary error the caller may retry. +func refreshFailure(status string, body []byte) error { + var oauthErr tokenErrorResponse + if json.Unmarshal(body, &oauthErr) == nil { + switch oauthErr.Error { + case "invalid_grant", "invalid_client": + return fmt.Errorf("%w (%s: %s)", ErrRefreshRejected, status, oauthErr.Error) + case "": + default: + return fmt.Errorf("auth refresh: hub answered %s: %s", status, oauthErr.Error) + } + } + return fmt.Errorf("auth refresh: hub answered %s", status) +} diff --git a/pkg/tapper/auth_refresh_test.go b/pkg/tapper/auth_refresh_test.go index 3540b073..c9438d64 100644 --- a/pkg/tapper/auth_refresh_test.go +++ b/pkg/tapper/auth_refresh_test.go @@ -1,13 +1,17 @@ package tapper_test import ( + "errors" + "fmt" "net/http" "net/http/httptest" "net/url" + "sync" "sync/atomic" "testing" "time" + "github.com/jlrickert/cli-toolkit/sandbox" "github.com/jlrickert/tapper/internal/testapi" "github.com/stretchr/testify/require" @@ -46,9 +50,11 @@ func TestRefreshHubToken_RotatesAndRenews(t *testing.T) { RefreshToken: "rt-old", ClientID: "tapper-cli", TokenEndpoint: srv.URL, + AccountKey: "account-key", } next, err := tapper.RefreshHubToken(fx.Context(), fx.Runtime(), "https://hub.example.com", entry) require.NoError(t, err) + require.Equal(t, "account-key", next.AccountKey, "a refresh renews the credential, not the login") require.Equal(t, "thub_refreshednew99", next.AccessToken) require.Equal(t, "rt-rotated", next.RefreshToken) require.Equal(t, "tapper-cli", next.ClientID) @@ -61,18 +67,45 @@ func TestRefreshHubToken_RotatesAndRenews(t *testing.T) { require.Equal(t, "tapper-cli", got.Get("client_id")) } -func TestRefreshHubToken_RejectedRefreshErrors(t *testing.T) { +// TestRefreshHubToken_ClassifiesHubAnswers proves only an OAuth +// invalid_grant or invalid_client answer reads as a refused refresh token, +// whatever the status; any other failure stays retryable, so an outage on +// the hub never looks like a lost sign-in. +func TestRefreshHubToken_ClassifiesHubAnswers(t *testing.T) { t.Parallel() - fx := NewSandbox(t) - srv := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"invalid_grant"}`)) - })) - t.Cleanup(srv.Close) + cases := []struct { + name string + status int + body string + rejected bool + }{ + {"400 invalid_grant", http.StatusBadRequest, `{"error":"invalid_grant"}`, true}, + {"401 invalid_client", http.StatusUnauthorized, `{"error":"invalid_client"}`, true}, + {"503 invalid_grant", http.StatusServiceUnavailable, `{"error":"invalid_grant","error_description":"spent"}`, true}, + {"503 server_error", http.StatusServiceUnavailable, `{"error":"server_error"}`, false}, + {"500 empty body", http.StatusInternalServerError, ``, false}, + {"400 invalid_request", http.StatusBadRequest, `{"error":"invalid_request"}`, false}, + {"400 not json", http.StatusBadRequest, `bad request`, false}, + {"401 no error field", http.StatusUnauthorized, `{"message":"unauthorized"}`, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + srv := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + t.Cleanup(srv.Close) - entry := &tapper.AuthEntry{RefreshToken: "spent", ClientID: "tapper-cli", TokenEndpoint: srv.URL} - _, err := tapper.RefreshHubToken(fx.Context(), fx.Runtime(), "https://hub", entry) - require.Error(t, err) + entry := &tapper.AuthEntry{RefreshToken: "rt", ClientID: "tapper-cli", TokenEndpoint: srv.URL} + next, err := tapper.RefreshHubToken(fx.Context(), fx.Runtime(), "https://hub", entry) + require.Error(t, err) + require.Nil(t, next) + require.Equal(t, tc.rejected, errors.Is(err, tapper.ErrRefreshRejected), "err: %v", err) + }) + } } func TestRefreshHubToken_NoRefreshTokenErrors(t *testing.T) { @@ -124,6 +157,59 @@ func TestAuthStoreTokenResolver_RefreshesExpiredToken(t *testing.T) { require.False(t, got2.ExpiresAt.IsZero()) } +// TestAuthStoreTokenResolver_ConcurrentResolvesDuringRefresh resolves tokens +// from many goroutines while one of them refreshes an expired entry, so the +// race detector sees the store's reads and the refresh's write overlap. +func TestAuthStoreTokenResolver_ConcurrentResolvesDuringRefresh(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + path := authStorePath(t, fx) + srv := rotatingTokenHub(t, nil) + + expiredKey := tapper.CanonicalHubURL(srv.URL) + const freshHub = "https://fresh.example.com" + store := &tapper.AuthStore{} + store.Set(expiredKey, tapper.AuthEntry{ + AccessToken: "thub_expiredtok00", + TokenType: "Bearer", + ExpiresAt: fx.Runtime().Clock().Now().Add(-time.Hour), + RefreshToken: "rt-old", + ClientID: "tapper-cli", + TokenEndpoint: srv.URL, + }) + store.Set(tapper.CanonicalHubURL(freshHub), tapper.AuthEntry{AccessToken: "thub_freshtok000"}) + require.NoError(t, store.Save(fx.Context(), fx.Runtime(), path)) + + resolver := tapper.NewAuthStoreTokenResolver(store, fx.Runtime(), path) + expiredTarget := keg.Target{Url: srv.URL + "/api/v1/@me/kegs/@demo/nodes"} + freshTarget := keg.Target{Url: freshHub + "/api/v1/@me/kegs/@demo/nodes"} + + const workers = 16 + tokens := make([]string, workers) + var wg sync.WaitGroup + for i := range workers { + wg.Add(1) + go func(i int) { + defer wg.Done() + if i%2 == 0 { + tokens[i] = resolver.ResolveToken(&expiredTarget) + return + } + tokens[i] = resolver.ResolveToken(&freshTarget) + _ = store.Hubs() + }(i) + } + wg.Wait() + + for i, tok := range tokens { + if i%2 == 0 { + require.Equal(t, "thub_refreshednew99", tok, "worker %d", i) + continue + } + require.Equal(t, "thub_freshtok000", tok, "worker %d", i) + } +} + func TestTap_AuthRefreshAll_RefreshesOnlyRenewableExpiringEntries(t *testing.T) { t.Parallel() sb := NewSandbox(t) @@ -234,3 +320,220 @@ func TestTap_AuthRefreshAll_AdoptsDiskAfterRefreshRejected(t *testing.T) { require.Equal(t, "thub_siblingwin00", entry.AccessToken) require.Equal(t, "rt-sibling", entry.RefreshToken) } + +// singleUseTokenHub is a hub that rotates refresh tokens single-use, the way +// a real hub without a grace window does: each POST spends the presented +// token and mints a new pair, and a spent token is refused. It answers after +// delay, so concurrent refreshers overlap. +func singleUseTokenHub(t *testing.T, first string, delay time.Duration, posts *atomic.Int64) *httptest.Server { + t.Helper() + var mu sync.Mutex + live := first + srv := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + n := posts.Add(1) + time.Sleep(delay) + mu.Lock() + defer mu.Unlock() + if r.PostForm.Get("refresh_token") != live { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"invalid_grant"}`)) + return + } + live = fmt.Sprintf("rt-%d", n) + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"access_token":"thub_access%04d","token_type":"Bearer","expires_in":900,"refresh_token":%q}`, n, live) + })) + t.Cleanup(srv.Close) + return srv +} + +// Two processes sharing one auth store, each with its own in-memory copy, +// find the stored token expired at once. Only one of them spends the refresh +// token; the other adopts the pair it persisted. +func TestRefreshAcrossStoresSpendsRefreshTokenOnce(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + path := authStorePath(t, fx) + var posts atomic.Int64 + srv := singleUseTokenHub(t, "rt-old", 200*time.Millisecond, &posts) + hubKey := tapper.CanonicalHubURL(srv.URL) + seed := &tapper.AuthStore{} + seed.Set(hubKey, tapper.AuthEntry{ + AccessToken: "thub_expiredtok00", + TokenType: "Bearer", + ExpiresAt: fx.Runtime().Clock().Now().Add(-time.Hour), + RefreshToken: "rt-old", + ClientID: "tapper-cli", + TokenEndpoint: srv.URL, + }) + require.NoError(t, seed.Save(fx.Context(), fx.Runtime(), path)) + + const processes = 2 + target := keg.Target{Url: srv.URL + "/api/v1/@me/kegs/@demo/nodes"} + tokens := make([]string, processes) + var wg sync.WaitGroup + for i := range processes { + store, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + resolver := tapper.NewAuthStoreTokenResolver(store, fx.Runtime(), path) + wg.Add(1) + go func() { + defer wg.Done() + tokens[i] = resolver.ResolveToken(&target) + }() + } + wg.Wait() + + require.Equal(t, int64(1), posts.Load(), "the refresh token is spent once") + for i, tok := range tokens { + require.Equal(t, "thub_access0001", tok, "process %d uses the one refreshed token", i) + } + stored, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + entry, ok := stored.Get(hubKey) + require.True(t, ok) + require.Equal(t, "rt-1", entry.RefreshToken) +} + +// A process whose token is still fresh never waits on another process's +// refresh of the same store. +func TestRefreshLockDoesNotBlockFreshTokens(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + path := authStorePath(t, fx) + var posts atomic.Int64 + srv := singleUseTokenHub(t, "rt-old", 2*time.Second, &posts) + expiredKey := tapper.CanonicalHubURL(srv.URL) + const freshHub = "https://fresh.example.com" + seed := &tapper.AuthStore{} + seed.Set(expiredKey, tapper.AuthEntry{ + AccessToken: "thub_expiredtok00", + ExpiresAt: fx.Runtime().Clock().Now().Add(-time.Hour), + RefreshToken: "rt-old", + ClientID: "tapper-cli", + TokenEndpoint: srv.URL, + }) + seed.Set(tapper.CanonicalHubURL(freshHub), tapper.AuthEntry{AccessToken: "thub_freshtok000"}) + require.NoError(t, seed.Save(fx.Context(), fx.Runtime(), path)) + + slow, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + refreshing := make(chan struct{}) + go func() { + defer close(refreshing) + tapper.NewAuthStoreTokenResolver(slow, fx.Runtime(), path). + ResolveToken(&keg.Target{Url: srv.URL + "/api/v1/@me/kegs/@demo/nodes"}) + }() + require.Eventually(t, func() bool { return posts.Load() == 1 }, 2*time.Second, 5*time.Millisecond) + + other, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + start := time.Now() + tok := tapper.NewAuthStoreTokenResolver(other, fx.Runtime(), path). + ResolveToken(&keg.Target{Url: freshHub + "/api/v1/@me/kegs/@demo/nodes"}) + require.Equal(t, "thub_freshtok000", tok) + require.Less(t, time.Since(start), time.Second, "a fresh token does not wait for the refresh lock") + <-refreshing +} + +// expiredLoginForTest is a stored login whose access token has expired, so +// the next resolve refreshes it at tokenEndpoint. +func expiredLoginForTest(fx *sandbox.Sandbox, refreshToken, tokenEndpoint string) tapper.AuthEntry { + return tapper.AuthEntry{ + AccessToken: "thub_expiredtok00", + TokenType: "Bearer", + ExpiresAt: fx.Runtime().Clock().Now().Add(-time.Hour), + RefreshToken: refreshToken, + ClientID: "tapper-cli", + TokenEndpoint: tokenEndpoint, + } +} + +// A running process loaded the login before `tap auth logout` removed it. +// When its token expires it does not sign the user back in. +func TestRefreshAfterLogoutLeavesUserLoggedOut(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + tap := newTestTap(t, fx) + var posts atomic.Int64 + srv := singleUseTokenHub(t, "rt-old", 0, &posts) + hubKey := seedStore(t, fx, tap, map[string]tapper.AuthEntry{srv.URL: expiredLoginForTest(fx, "rt-old", srv.URL)}) + path := tap.PathService.AuthStorePath() + + running, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + res, err := tap.AuthLogout(fx.Context(), tapper.AuthLogoutOptions{Hub: hubKey}) + require.NoError(t, err) + require.True(t, res.Removed) + + tok := tapper.NewAuthStoreTokenResolver(running, fx.Runtime(), path). + ResolveToken(&keg.Target{Url: srv.URL + "/api/v1/@me/kegs/@demo/nodes"}) + require.NotEqual(t, "thub_access0001", tok, "the logged-out process gets no new token") + require.Zero(t, posts.Load(), "the removed login's refresh token is not spent") + + stored, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + _, ok := stored.Get(hubKey) + require.False(t, ok, "the logout stays in effect") +} + +// `tap auth logout` while another process is mid-refresh: the refresh does +// not write the login back after the logout removed it. +func TestLogoutDuringRefreshStaysLoggedOut(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + tap := newTestTap(t, fx) + var posts atomic.Int64 + srv := singleUseTokenHub(t, "rt-old", 300*time.Millisecond, &posts) + hubKey := seedStore(t, fx, tap, map[string]tapper.AuthEntry{srv.URL: expiredLoginForTest(fx, "rt-old", srv.URL)}) + path := tap.PathService.AuthStorePath() + + running, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + refreshed := make(chan struct{}) + go func() { + defer close(refreshed) + tapper.NewAuthStoreTokenResolver(running, fx.Runtime(), path). + ResolveToken(&keg.Target{Url: srv.URL + "/api/v1/@me/kegs/@demo/nodes"}) + }() + require.Eventually(t, func() bool { return posts.Load() == 1 }, 2*time.Second, 5*time.Millisecond) + + res, err := tap.AuthLogout(fx.Context(), tapper.AuthLogoutOptions{Hub: hubKey}) + require.NoError(t, err) + require.True(t, res.Removed) + <-refreshed + + stored, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + _, ok := stored.Get(hubKey) + require.False(t, ok, "the refresh did not undo the logout") +} + +// This process still holds a refresh token another process already spent; +// the stored pair has since expired too. The refresh presents the stored +// token, so the hub sees no replay. +func TestRefreshUsesStoredTokenOverStaleInMemoryToken(t *testing.T) { + t.Parallel() + fx := NewSandbox(t) + path := authStorePath(t, fx) + var posts atomic.Int64 + srv := singleUseTokenHub(t, "rt-stored", 0, &posts) + hubKey := tapper.CanonicalHubURL(srv.URL) + disk := &tapper.AuthStore{} + disk.Set(hubKey, expiredLoginForTest(fx, "rt-stored", srv.URL)) + require.NoError(t, disk.Save(fx.Context(), fx.Runtime(), path)) + + stale := &tapper.AuthStore{} + stale.Set(hubKey, expiredLoginForTest(fx, "rt-spent", srv.URL)) + tok := tapper.NewAuthStoreTokenResolver(stale, fx.Runtime(), path). + ResolveToken(&keg.Target{Url: srv.URL + "/api/v1/@me/kegs/@demo/nodes"}) + require.Equal(t, "thub_access0001", tok) + require.Equal(t, int64(1), posts.Load()) + + stored, err := tapper.LoadAuthStore(fx.Context(), fx.Runtime(), path) + require.NoError(t, err) + entry, ok := stored.Get(hubKey) + require.True(t, ok) + require.Equal(t, "rt-1", entry.RefreshToken) +} diff --git a/pkg/tapper/auth_resolver.go b/pkg/tapper/auth_resolver.go index ef4292a3..fd8d5139 100644 --- a/pkg/tapper/auth_resolver.go +++ b/pkg/tapper/auth_resolver.go @@ -41,24 +41,33 @@ func NewAuthStoreTokenResolver(store *AuthStore, rt *toolkit.Runtime, storePath // any failure falls back to the cached token, leaving the hub's 401 as the // backstop signal. func (r *authStoreTokenResolver) ResolveToken(target *keg.Target) string { + token, _ := r.resolveToken(target) + return token +} + +// resolveToken is ResolveToken that also reports a failed refresh, alongside +// the cached token it fell back to. +func (r *authStoreTokenResolver) resolveToken(target *keg.Target) (string, error) { if r == nil || r.store == nil || target == nil { - return "" + return "", nil } hubRoot := hubRootFromTarget(target) if hubRoot == "" { - return "" + return "", nil } key := CanonicalHubURL(hubRoot) entry, ok := r.store.Get(key) if !ok || entry == nil { - return "" + return "", nil } if r.shouldRefresh(entry) { - if next := r.refresh(hubRoot, key, entry); next != nil { - return next.AccessToken + next, err := r.refresh(hubRoot, key, entry) + if next != nil { + return next.AccessToken, nil } + return entry.AccessToken, err } - return entry.AccessToken + return entry.AccessToken, nil } // shouldRefresh reports whether entry has a refresh token and an access token @@ -67,9 +76,10 @@ func (r *authStoreTokenResolver) shouldRefresh(entry *AuthEntry) bool { return authEntryNeedsRefresh(r.rt, entry) } -// refresh renews the token under a lock and persists the rotated pair. Returns -// the fresh entry, or nil on failure (callers fall back to the cached token). -func (r *authStoreTokenResolver) refresh(hubURL, key string, entry *AuthEntry) *AuthEntry { +// refresh renews the token under a lock and persists the rotated pair. It +// returns the fresh entry, or nil and the refresh error on failure (callers +// fall back to the cached token). +func (r *authStoreTokenResolver) refresh(hubURL, key string, entry *AuthEntry) (*AuthEntry, error) { r.mu.Lock() defer r.mu.Unlock() next, err := refreshAuthStoreEntryIfNeeded(context.Background(), r.rt, r.store, r.storePath, hubURL, key, entry) @@ -79,9 +89,9 @@ func (r *authStoreTokenResolver) refresh(hubURL, key string, entry *AuthEntry) * } } if next != nil { - return next + return next, nil } - return nil + return nil, err } // ErrAtlasHubDisabled is returned by ResolveLoginHubURL when the chain diff --git a/pkg/tapper/auth_store.go b/pkg/tapper/auth_store.go index 46ec6fb4..2c366ff5 100644 --- a/pkg/tapper/auth_store.go +++ b/pkg/tapper/auth_store.go @@ -14,10 +14,13 @@ package tapper import ( "context" + "crypto/rand" + "encoding/hex" "errors" "fmt" "os" "sort" + "sync" "time" "github.com/jlrickert/cli-toolkit/toolkit" @@ -44,6 +47,81 @@ type AuthEntry struct { RefreshToken string `yaml:"refresh_token,omitempty"` ClientID string `yaml:"client_id,omitempty"` TokenEndpoint string `yaml:"token_endpoint,omitempty"` + + // AccountKey is a random, local-only name for the login this entry + // holds. It is minted at login and kept across token refreshes, so state + // that belongs to the login (the telemetry spool) stays put while its + // credentials rotate. It is never sent to a hub. + AccountKey string `yaml:"account_key,omitempty"` +} + +// NewAuthAccountKey mints a fresh AuthEntry.AccountKey. +func NewAuthAccountKey() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +// carryAuthAccountKey keeps prev's AccountKey on next when next has none, so +// replacing an entry's credentials never renames the login. +func carryAuthAccountKey(next, prev *AuthEntry) { + if next != nil && prev != nil && next.AccountKey == "" { + next.AccountKey = prev.AccountKey + } +} + +// SaveAuthLogin stores entry as the login for hubKey in the auth store at +// storePath, replacing any earlier login for that hub. It holds the auth store +// lock while it reads and saves, so a refresh running in another tap process +// neither loses the write nor overwrites it with the earlier login's tokens. +func SaveAuthLogin(ctx context.Context, rt *toolkit.Runtime, storePath, hubKey string, entry AuthEntry) error { + unlock, err := lockAuthStore(ctx, rt, storePath, authStoreLockWait) + if err != nil { + return err + } + defer unlock() + store, err := LoadAuthStore(ctx, rt, storePath) + if err != nil { + return err + } + store.Set(hubKey, entry) + return store.Save(ctx, rt, storePath) +} + +// ensureAuthAccountKey returns the AccountKey of the stored entry for hubKey, +// minting and saving one when the entry predates account keys. It holds the +// auth store lock while it reads, mints, and saves, so two processes naming +// the same login at once agree on one key, and neither writes back a token +// pair that a concurrent refresh has already rotated. +func ensureAuthAccountKey(ctx context.Context, rt *toolkit.Runtime, storePath, hubKey string) (string, error) { + unlock, err := lockAuthStore(ctx, rt, storePath, authStoreLockWait) + if err != nil { + return "", err + } + defer unlock() + return ensureAuthAccountKeyLocked(ctx, rt, storePath, hubKey) +} + +// ensureAuthAccountKeyLocked is ensureAuthAccountKey for a caller that already +// holds the auth store lock, such as a refresh. +func ensureAuthAccountKeyLocked(ctx context.Context, rt *toolkit.Runtime, storePath, hubKey string) (string, error) { + store, err := LoadAuthStore(ctx, rt, storePath) + if err != nil { + return "", err + } + entry, ok := store.Get(hubKey) + if !ok { + return "", fmt.Errorf("auth store: no login for %s", hubKey) + } + if entry.AccountKey != "" { + return entry.AccountKey, nil + } + entry.AccountKey = NewAuthAccountKey() + store.Set(hubKey, *entry) + if err := store.Save(ctx, rt, storePath); err != nil { + return "", err + } + return entry.AccountKey, nil } // authStoreDTO is the on-disk shape. Private so callers can't couple to @@ -60,7 +138,11 @@ type authStoreDTO struct { // authStoreDTO. All methods are nil-safe on the receiver: a (*AuthStore)(nil) // reads as an empty store and writes are no-ops. This lets callers skip // the "did Load return something?" dance when the file didn't exist. +// +// mu guards data: the token resolver reads the store from concurrent hub +// requests while a refresh may be writing a rotated entry. type AuthStore struct { + mu sync.RWMutex data *authStoreDTO } @@ -101,7 +183,12 @@ func LoadAuthStore(ctx context.Context, rt *toolkit.Runtime, path string) (*Auth // return a value (not a pointer into the map) so callers can't mutate // the stored entry by accident — all edits must go through Set. func (s *AuthStore) Get(hubURL string) (*AuthEntry, bool) { - if s == nil || s.data == nil || s.data.Hubs == nil { + if s == nil { + return nil, false + } + s.mu.RLock() + defer s.mu.RUnlock() + if s.data == nil || s.data.Hubs == nil { return nil, false } entry, ok := s.data.Hubs[hubURL] @@ -120,6 +207,8 @@ func (s *AuthStore) Set(hubURL string, entry AuthEntry) { if s == nil { return } + s.mu.Lock() + defer s.mu.Unlock() if s.data == nil { s.data = &authStoreDTO{} } @@ -134,7 +223,12 @@ func (s *AuthStore) Set(hubURL string, entry AuthEntry) { // callers can use this to decide whether to log "logged out" vs "was // already logged out". func (s *AuthStore) Delete(hubURL string) bool { - if s == nil || s.data == nil || s.data.Hubs == nil { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if s.data == nil || s.data.Hubs == nil { return false } if _, ok := s.data.Hubs[hubURL]; !ok { @@ -147,7 +241,12 @@ func (s *AuthStore) Delete(hubURL string) bool { // Hubs returns the hub URL keys, sorted, for enumeration. Sorted output // makes CLI listings stable without callers having to re-sort. func (s *AuthStore) Hubs() []string { - if s == nil || s.data == nil || s.data.Hubs == nil { + if s == nil { + return []string{} + } + s.mu.RLock() + defer s.mu.RUnlock() + if s.data == nil || s.data.Hubs == nil { return []string{} } keys := make([]string, 0, len(s.data.Hubs)) @@ -161,7 +260,12 @@ func (s *AuthStore) Hubs() []string { // IsEmpty reports whether the store has no hubs. Used by Save to decide // whether to delete the file rather than write an empty document. func (s *AuthStore) IsEmpty() bool { - if s == nil || s.data == nil || s.data.Hubs == nil { + if s == nil { + return true + } + s.mu.RLock() + defer s.mu.RUnlock() + if s.data == nil || s.data.Hubs == nil { return true } return len(s.data.Hubs) == 0 @@ -205,7 +309,9 @@ func (s *AuthStore) Save(ctx context.Context, rt *toolkit.Runtime, path string) } } + s.mu.RLock() data, err := yaml.Marshal(s.data) + s.mu.RUnlock() if err != nil { return fmt.Errorf("unable to marshal auth store: %w", err) } diff --git a/pkg/tapper/auth_store_lock.go b/pkg/tapper/auth_store_lock.go new file mode 100644 index 00000000..251d5ee7 --- /dev/null +++ b/pkg/tapper/auth_store_lock.go @@ -0,0 +1,81 @@ +package tapper + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/jlrickert/cli-toolkit/toolkit" +) + +// authStoreLockPoll is how often a blocked refresher retries the store lock. +const authStoreLockPoll = 20 * time.Millisecond + +// authStoreLockWait bounds how long a refresher waits for another process's +// refresh of the same store: that refresh's own timeout plus room to persist. +const authStoreLockWait = authRefreshTimeout + 5*time.Second + +// errAuthStoreLockTimeout reports that another process held the store lock for +// longer than the wait allowed. +var errAuthStoreLockTimeout = errors.New("auth store: timed out waiting for another process's refresh") + +// lockFile is the part of an opened lock file the platform lock needs. +type lockFile interface { + io.Closer + Fd() uintptr +} + +// lockAuthStore takes the cross-process lock that serializes refreshes of the +// store at storePath, waiting up to wait. The lock is an advisory OS lock on +// ".lock", opened through the runtime so the jail applies; the OS +// drops it if the process dies. When the runtime's filesystem cannot provide +// a lockable file, the returned unlock is a no-op and refreshes are serialized +// within the process only. +func lockAuthStore(ctx context.Context, rt *toolkit.Runtime, storePath string, wait time.Duration) (func(), error) { + // A login on a new machine takes the lock before the store exists. + // Create the store's directory owner-only here, as Save would, rather + // than let opening the lock file create it with wider permissions. + if parent := parentDir(storePath); parent != "" { + if _, err := rt.Stat(parent, false); errors.Is(err, os.ErrNotExist) { + if err := rt.Mkdir(parent, 0o700, true); err != nil { + return nil, fmt.Errorf("auth store: create directory: %w", err) + } + } + } + w, err := rt.OpenFile(storePath+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("auth store: open lock: %w", err) + } + f, ok := w.(lockFile) + if !ok { + _ = w.Close() + return func() {}, nil + } + ctx, cancel := context.WithTimeout(ctx, wait) + defer cancel() + for { + locked, err := tryLockFile(f) + if err != nil { + _ = f.Close() + return nil, fmt.Errorf("auth store: lock: %w", err) + } + if locked { + return func() { + _ = unlockFile(f) + _ = f.Close() + }, nil + } + select { + case <-ctx.Done(): + _ = f.Close() + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return nil, errAuthStoreLockTimeout + } + return nil, ctx.Err() + case <-time.After(authStoreLockPoll): + } + } +} diff --git a/pkg/tapper/auth_store_lock_other.go b/pkg/tapper/auth_store_lock_other.go new file mode 100644 index 00000000..a011897d --- /dev/null +++ b/pkg/tapper/auth_store_lock_other.go @@ -0,0 +1,9 @@ +//go:build !(darwin || linux || freebsd || netbsd || openbsd || dragonfly || windows) + +package tapper + +// Platforms without a supported advisory lock serialize refreshes within the +// process only. +func tryLockFile(lockFile) (bool, error) { return true, nil } + +func unlockFile(lockFile) error { return nil } diff --git a/pkg/tapper/auth_store_lock_unix.go b/pkg/tapper/auth_store_lock_unix.go new file mode 100644 index 00000000..4cc3e188 --- /dev/null +++ b/pkg/tapper/auth_store_lock_unix.go @@ -0,0 +1,22 @@ +//go:build darwin || linux || freebsd || netbsd || openbsd || dragonfly + +package tapper + +import ( + "errors" + "syscall" +) + +// tryLockFile takes an exclusive flock on f without blocking, reporting false +// when another open file holds it. +func tryLockFile(f lockFile) (bool, error) { + err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if errors.Is(err, syscall.EWOULDBLOCK) { + return false, nil + } + return err == nil, err +} + +func unlockFile(f lockFile) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) +} diff --git a/pkg/tapper/auth_store_lock_windows.go b/pkg/tapper/auth_store_lock_windows.go new file mode 100644 index 00000000..a6fe1965 --- /dev/null +++ b/pkg/tapper/auth_store_lock_windows.go @@ -0,0 +1,26 @@ +//go:build windows + +package tapper + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +// tryLockFile takes an exclusive lock on f's first byte without blocking, +// reporting false when another handle holds it. +func tryLockFile(f lockFile) (bool, error) { + var ol windows.Overlapped + err := windows.LockFileEx(windows.Handle(f.Fd()), + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &ol) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return false, nil + } + return err == nil, err +} + +func unlockFile(f lockFile) error { + var ol windows.Overlapped + return windows.UnlockFileEx(windows.Handle(f.Fd()), 0, 1, 0, &ol) +} diff --git a/pkg/tapper/auth_whoami.go b/pkg/tapper/auth_whoami.go index 8b1fe4b1..5a16d57a 100644 --- a/pkg/tapper/auth_whoami.go +++ b/pkg/tapper/auth_whoami.go @@ -114,9 +114,5 @@ func ValidateToken(ctx context.Context, rt *toolkit.Runtime, hubURL, token strin // SelectedHubIdentity probes only the selected Hub using its live credentials. func (t *Tap) SelectedHubIdentity(ctx context.Context) (*WhoAmI, error) { - _, entry, err := t.ConfigService.SelectedHub("") - if err != nil { - return nil, err - } - return t.AuthValidateFn(ctx, t.Runtime, entry.URL, t.hubToken(entry)) + return t.hubIdentity(ctx, "") } diff --git a/pkg/tapper/flight.go b/pkg/tapper/flight.go index a37c61c2..468739a5 100644 --- a/pkg/tapper/flight.go +++ b/pkg/tapper/flight.go @@ -98,6 +98,18 @@ type FlightManifest struct { Subflights []string `yaml:"subflights,omitempty" json:"subflights,omitempty"` AllowedKegs []string `yaml:"allowedKegs,omitempty" json:"allowedKegs,omitempty"` Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` + // ManageKegs lets a request rooted on this flight administer every keg + // in the flight's own namespace: create and delete kegs, edit their + // settings, change their visibility, and manage their grants, even for + // kegs outside its cover. It never reaches node content outside the + // cover and never gives more than the user's own role. A subflight in + // another namespace that sets it adds that namespace for its roots. + ManageKegs bool `yaml:"manageKegs,omitempty" json:"manageKegs,omitempty"` + // ManageFlights lets a request rooted on this flight create, edit, and + // delete the flights of its own namespace, widening them up to the + // user's own role. A subflight in another namespace that sets it adds + // that namespace for its roots. + ManageFlights bool `yaml:"manageFlights,omitempty" json:"manageFlights,omitempty"` } // Flight is a discovered flight: its manifest plus provenance. @@ -819,10 +831,12 @@ func flightFromHub(hf HubFlight, hubName string) *Flight { } m := FlightManifest{ Title: hf.Title, Description: hf.Description, - Visibility: hf.Visibility, - Cover: cover, - Subflights: append([]string(nil), hf.Subflights...), - Instructions: hf.Instructions, + Visibility: hf.Visibility, + Cover: cover, + Subflights: append([]string(nil), hf.Subflights...), + Instructions: hf.Instructions, + ManageKegs: hubSwitch(hf.ManageKegs, hf.Manage), + ManageFlights: hubSwitch(hf.ManageFlights, hf.Manage), } normalizeFlightManifest(&m) ref := FlightRef{Namespace: hf.Namespace, Slug: hf.Slug} diff --git a/pkg/tapper/hub_context.go b/pkg/tapper/hub_context.go new file mode 100644 index 00000000..2b8f8ab6 --- /dev/null +++ b/pkg/tapper/hub_context.go @@ -0,0 +1,353 @@ +// Package tapper — the Hub's one-call caller context. +// +// GET /api/v1/info answers, in one round trip, what `tap info` otherwise +// asks the selected Hub in three: who the token belongs to, the active flight, +// and the identity KEG catalog. Hubs that predate the endpoint answer 404; the +// client then issues the three requests concurrently and remembers the 404 for +// the life of the Tap. +package tapper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + + "github.com/jlrickert/tapper/pkg/apicontract" + "github.com/jlrickert/tapper/pkg/keg" +) + +const hubContextPath = "/api/v1/info" + +// errHubContextUnsupported reports a Hub that has no /api/v1/info. +var errHubContextUnsupported = errors.New("hub has no context endpoint") + +// hubContextResponse is the body of GET /api/v1/info. Decoding ignores +// unknown fields so the Hub can extend it without a contract revision. +type hubContextResponse struct { + User *WhoAmI `json:"user"` + Flight *hubContextFlight `json:"flight"` + FlightError *string `json:"flight_error"` + Kegs []hubContextKeg `json:"kegs"` +} + +// hubContextFlight is the requested flight as the Hub resolved it. Name is +// the canonical @namespace/+slug, and Cover is the Hub-composed cover: the +// rows GET /@{namespace}/+{flight} reports as effective_cover. +type hubContextFlight struct { + Name string `json:"name"` + Title string `json:"title"` + Cover []HubEffectiveKeg `json:"cover"` +} + +// hubContextKeg is one row of the identity KEG catalog. FlightCap and +// EffectiveRole are decoded for completeness; the client projects the +// composed cover itself so the result matches the fallback path exactly. +type hubContextKeg struct { + Ref string `json:"ref"` + Namespace string `json:"namespace"` + Alias string `json:"alias"` + Title string `json:"title"` + Description string `json:"description"` + Visibility string `json:"visibility"` + KegVersion string `json:"keg_version"` + LegacyVersion string `json:"kegv"` + Role string `json:"role"` + FlightCap string `json:"flight_cap"` + EffectiveRole string `json:"effective_role"` +} + +// getHubContext calls GET /api/v1/info. flight, when non-empty, is an +// @namespace/+slug ref, or a bare +slug the Hub resolves among the flights +// GET /api/v1/flights lists for the caller: one match is that flight, none +// is not found, and several are ambiguous. A flight the Hub cannot load or +// resolve comes back in FlightError, not as a failure. +// A 404 wraps errHubContextUnsupported. +func getHubContext(ctx context.Context, hubURL, token, flight string) (*hubContextResponse, error) { + base, err := normalizeHubURL(hubURL) + if err != nil { + return nil, err + } + endpoint := base + hubContextPath + if flight != "" { + endpoint += "?flight=" + url.QueryEscape(flight) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("hub: build context request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/json") + + resp, err := apicontract.Do(hubHTTPClient(), req) + if err != nil { + return nil, fmt.Errorf("hub: contact hub: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + switch resp.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, fmt.Errorf("hub: %w", errHubContextUnsupported) + case http.StatusForbidden: + return nil, fmt.Errorf("hub: %w (%s)%s", keg.ErrForbidden, resp.Status, readHubError(resp)) + case http.StatusUnauthorized: + return nil, fmt.Errorf("hub: %w (%s)", ErrTokenRejected, resp.Status) + default: + return nil, fmt.Errorf("hub: info returned %s for %s", resp.Status, hubContextPath) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("hub: read context response: %w", err) + } + var out hubContextResponse + if err := json.Unmarshal(body, &out); err != nil { + return nil, fmt.Errorf("hub: parse context response: %w", err) + } + return &out, nil +} + +// hubContextProbe remembers which Hubs answered 404 for /api/v1/info, so a +// long-lived Tap asks an older Hub only once. +type hubContextProbe struct { + mu sync.Mutex + unsupported map[string]bool +} + +func (p *hubContextProbe) supported(hubURL string) bool { + if p == nil { + return true + } + p.mu.Lock() + defer p.mu.Unlock() + return !p.unsupported[CanonicalConfiguredHubURL(hubURL)] +} + +func (p *hubContextProbe) markUnsupported(hubURL string) { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + if p.unsupported == nil { + p.unsupported = map[string]bool{} + } + p.unsupported[CanonicalConfiguredHubURL(hubURL)] = true +} + +// hubContextRequest selects what loadHubContext resolves. +type hubContextRequest struct { + // hub is the explicit hub selector; "" uses the selected hub. + hub string + // flight is the flight to resolve; "" resolves none. + flight string + // kegs requests the identity KEG catalog. The context endpoint always + // returns it; the flag only spares the separate requests. + kegs bool +} + +// hubContext is the caller's context on the selected hub. Each part carries +// the error its standalone request would have produced, so callers keep their +// own reporting order: +// - whoErr is the identity probe's error. +// - flight, flightNote, and flightErr match resolveOrientFlight. +// - kegs, warnings, and kegsErr match identityKegCatalog. +type hubContext struct { + who *WhoAmI + whoErr error + + flight *Flight + flightNote string + flightErr error + + kegs []OrientationKeg + warnings []string + kegsErr error +} + +// loadHubContext resolves the selected hub's identity, the requested flight, +// and the identity KEG catalog. The selected hub answers all three with one +// GET /api/v1/info; a hub without it gets the three requests concurrently. +func (t *Tap) loadHubContext(ctx context.Context, req hubContextRequest) hubContext { + if t == nil || t.ConfigService == nil { + return t.loadHubContextSeparately(ctx, req) + } + cfg, _, err := t.ConfigService.Load() + if err != nil { + return t.loadHubContextSeparately(ctx, req) + } + name, entry, err := t.ConfigService.SelectedHub(req.hub) + if err != nil || !t.hubContextProbe.supported(entry.URL) { + return t.loadHubContextSeparately(ctx, req) + } + token := t.hubToken(entry) + if token == "" { + return t.loadHubContextSeparately(ctx, req) + } + + // The Hub resolves the flight itself. It resolves a bare +slug the way + // orient does (see FlightService.getFlight): across the flights the user + // can read, so one match is that flight, none is "flight not found", and + // several are an ambiguous flight_error. A ref it cannot parse comes back + // as flight_error like any other unreadable one. + flightName := strings.TrimSpace(req.flight) + contextFlight := flightName + if ref, err := ParseFlightRef(flightName, ""); err == nil { + contextFlight = ref.Canonical() + } + + var ( + resp *hubContextResponse + respErr error + out hubContext + others []hubKegsResult + wg sync.WaitGroup + ) + wg.Add(2) + go func() { + defer wg.Done() + resp, respErr = getHubContext(ctx, entry.URL, token, contextFlight) + }() + go func() { + defer wg.Done() + if req.kegs { + others = t.orientKegsForHubs(ctx, cfg, name) + } + }() + wg.Wait() + + if errors.Is(respErr, errHubContextUnsupported) { + t.hubContextProbe.markUnsupported(entry.URL) + return t.loadHubContextSeparately(ctx, req) + } + + // One failed request stands in for all three: each part reports it the + // way its own request would have. + selected := hubKegsResult{name: name, err: respErr} + if respErr != nil { + out.whoErr = respErr + } else { + out.who = resp.User + if out.who == nil { + out.whoErr = fmt.Errorf("auth: hub returned no user for %s", hubContextPath) + } + selected.rows = hubContextKegRows(resp.Kegs, name) + } + if flightName != "" { + out.flight, out.flightNote, out.flightErr = hubContextFlightResult(flightName, name, resp, respErr) + } + for i := range others { + if others[i].name == name { + others[i] = selected + } + } + out.kegs, out.warnings, out.kegsErr = mergeHubKegs(others) + return out +} + +// loadHubContextSeparately is the fallback for hubs without /api/v1/info: +// the identity probe, the flight, and the catalog, issued concurrently. +func (t *Tap) loadHubContextSeparately(ctx context.Context, req hubContextRequest) hubContext { + var out hubContext + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + out.who, out.whoErr = t.hubIdentity(ctx, req.hub) + }() + go func() { + defer wg.Done() + out.flight, out.flightNote, out.flightErr = t.resolveOrientFlight(ctx, req.flight) + }() + if req.kegs { + wg.Add(1) + go func() { + defer wg.Done() + out.kegs, out.warnings, out.kegsErr = t.identityKegCatalog(ctx) + }() + } + wg.Wait() + return out +} + +// hubContextFlightResult reads the flight part of a context response the way +// resolveOrientFlight reports a standalone flight load: a compatibility error +// fails, and any other problem becomes a note. +func hubContextFlightResult(name, hubName string, resp *hubContextResponse, respErr error) (*Flight, string, error) { + unavailable := func(reason any) (*Flight, string, error) { + return &Flight{Name: name}, fmt.Sprintf("Flight %q is unavailable: %v", name, reason), nil + } + switch { + case respErr != nil: + if apicontract.IsCompatibility(respErr) { + return nil, "", respErr + } + return unavailable(respErr) + case resp.FlightError != nil && strings.TrimSpace(*resp.FlightError) != "": + return unavailable(strings.TrimSpace(*resp.FlightError)) + case resp.Flight == nil: + return unavailable("hub returned no flight") + } + f := resp.Flight + ref, err := ParseFlightRef(f.Name, "") + if err != nil || ref.Namespace == "" { + return unavailable(fmt.Sprintf("hub returned flight %q", f.Name)) + } + cover := f.Cover + if cover == nil { + cover = []HubEffectiveKeg{} + } + hf := HubFlight{Namespace: ref.Namespace, Slug: ref.Slug, Title: f.Title, EffectiveCover: cover} + if err := validateHubFlight(hf); err != nil { + return unavailable(fmt.Errorf("hub: invalid flight %q: %w", hf.Slug, err)) + } + // The cover is already composed, so it is the flight's cover here. + return ComposedFlight(flightFromHub(hf, hubName)), "", nil +} + +// hubContextKegRows converts the context catalog to orientation rows shaped +// exactly as orientKegsForHub builds them from /api/v1/kegs. +func hubContextKegRows(kegs []hubContextKeg, hubName string) []OrientationKeg { + out := make([]OrientationKeg, 0, len(kegs)) + for _, k := range kegs { + ns, alias := k.Namespace, k.Alias + if ns == "" || alias == "" { + var ok bool + if ns, alias, ok = splitKegRef(k.Ref); !ok { + continue + } + } + version := k.KegVersion + if version == "" { + version = k.LegacyVersion + } + out = append(out, OrientationKeg{ + Ref: "@" + ns + "/" + alias, + Namespace: ns, + Alias: alias, + Title: k.Title, + Description: k.Description, + Role: k.Role, + Source: hubName, + Visibility: k.Visibility, + KegVersion: version, + }) + } + return out +} + +// hubIdentity probes the hub selected by explicit ("" for the active hub) +// with its live credentials. +func (t *Tap) hubIdentity(ctx context.Context, explicit string) (*WhoAmI, error) { + _, entry, err := t.ConfigService.SelectedHub(explicit) + if err != nil { + return nil, err + } + return t.AuthValidateFn(ctx, t.Runtime, entry.URL, t.hubToken(entry)) +} diff --git a/pkg/tapper/hub_context_test.go b/pkg/tapper/hub_context_test.go new file mode 100644 index 00000000..f39d230b --- /dev/null +++ b/pkg/tapper/hub_context_test.go @@ -0,0 +1,323 @@ +package tapper_test + +import ( + "encoding/json" + "fmt" + "net/http" + "sort" + "strings" + "sync" + "testing" + + "github.com/jlrickert/cli-toolkit/sandbox" + "github.com/jlrickert/tapper/internal/testapi" + "github.com/stretchr/testify/require" + + "github.com/jlrickert/tapper/pkg/tapper" +) + +// contextHub is a mock Hub that records the paths it serves. With +// serveContext false it answers /api/v1/info with 404, as a Hub that +// predates the endpoint does. +type contextHub struct { + mu sync.Mutex + paths []string +} + +func (h *contextHub) record(path string) { + h.mu.Lock() + defer h.mu.Unlock() + h.paths = append(h.paths, path) +} + +func (h *contextHub) seen() []string { + h.mu.Lock() + defer h.mu.Unlock() + out := append([]string(nil), h.paths...) + sort.Strings(out) + return out +} + +// workEffectiveCover is the composed cover testapi.HubContextWork reports, as +// an older Hub's flight read carries it. +var workEffectiveCover = []tapper.HubEffectiveKeg{ + {Ref: "@foldwise/dev", Role: tapper.FlightRoleEditor, Source: "@foldwise/dev", Distance: 0, Path: []string{"@foldwise/dev"}}, + {Ref: "@foldwise/linked", Role: tapper.FlightRoleViewer, Source: "@foldwise/dev", Distance: 1, Path: []string{"@foldwise/dev", "@foldwise/linked"}}, +} + +// workFlight is the one flight named work the default mock Hub serves. +var workFlight = tapper.HubFlight{ + Namespace: "foldwise", Slug: "work", Title: "Work", + Cover: []tapper.HubFlightCover{{Namespace: "foldwise", Keg: "dev", Role: "editor"}}, +} + +type contextHubOptions struct { + serveContext bool + flightNotFound bool + // flights is what the user can read, as GET /api/v1/flights lists it. + // Nil means just workFlight. + flights []tapper.HubFlight +} + +func newContextHub(t *testing.T, serveContext, flightNotFound bool) (*contextHub, string) { + t.Helper() + return newContextHubWith(t, contextHubOptions{serveContext: serveContext, flightNotFound: flightNotFound}) +} + +// newContextHubWith serves a mock Hub. Its /api/v1/info resolves a bare +// +slug the way the real Hub and orient do: across the flights the user can +// read, so +work resolves to @foldwise/+work only because that is the user's +// one readable flight named work, even though jared is the personal +// namespace. +func newContextHubWith(t *testing.T, opts contextHubOptions) (*contextHub, string) { + t.Helper() + flights := opts.flights + if flights == nil { + flights = []tapper.HubFlight{workFlight} + } + serveContext, flightNotFound := opts.serveContext, opts.flightNotFound + rec := &contextHub{} + srv := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.record(r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/info": + if !serveContext { + http.NotFound(w, r) + return + } + flight := r.URL.Query().Get("flight") + if r.Header.Get("Authorization") != "Bearer tok" { + http.Error(w, "unexpected context request", http.StatusTeapot) + return + } + if flightNotFound { + _, _ = w.Write([]byte(testapi.HubContextFlightNotFound)) + return + } + resolved := flight + if slug, bare := strings.CutPrefix(flight, "+"); bare { + var matches []string + for _, f := range flights { + if f.Slug == slug { + matches = append(matches, "@"+f.Namespace+"/+"+f.Slug) + } + } + switch len(matches) { + case 0: + _, _ = w.Write([]byte(testapi.HubContextFlightNotFound)) + return + case 1: + resolved = matches[0] + default: + _, _ = w.Write([]byte(testapi.HubContextFlightAmbiguous)) + return + } + } + if resolved != "@foldwise/+work" { + http.Error(w, "unexpected context request", http.StatusTeapot) + return + } + _, _ = w.Write([]byte(testapi.HubContextWork)) + case "GET /api/v1/whoami": + _ = json.NewEncoder(w).Encode(tapper.WhoAmI{UserID: 1, Username: "jared", DefaultNamespace: "foldwise"}) + case "GET /api/v1/@foldwise/+work": + _ = json.NewEncoder(w).Encode(tapper.HubFlight{ + Namespace: "foldwise", Slug: "work", Title: "Work", + Cover: []tapper.HubFlightCover{{Namespace: "foldwise", Keg: "dev", Role: "editor"}}, + EffectiveCover: workEffectiveCover, + }) + case "GET /api/v1/flights": + _ = json.NewEncoder(w).Encode(flights) + case "GET /api/v1/kegs": + _ = json.NewEncoder(w).Encode([]tapper.HubKeg{ + {Namespace: "foldwise", Alias: "dev", Title: "Development", Role: "admin"}, + {Namespace: "foldwise", Alias: "linked", Title: "Linked", Role: "admin"}, + {Namespace: "foldwise", Alias: "private", Title: "Outside the flight", Role: "admin"}, + }) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return rec, srv.URL +} + +func writeContextConfig(t *testing.T, sb *sandbox.Sandbox, hubURL string) { + t.Helper() + cfg := fmt.Sprintf("hub: cloud\nkeg: \"@foldwise/dev\"\nflight: \"@foldwise/+work\"\nhubs:\n cloud: {url: %s, token: tok}\n", hubURL) + require.NoError(t, sb.Runtime().AtomicWriteFile("/home/testuser/.config/tapper/config.yaml", []byte(cfg), 0o600)) +} + +func wantWorkInfo(t *testing.T, info tapper.ContextInfoResult) { + t.Helper() + require.Equal(t, "jared", info.Hub.User) + require.Equal(t, "@foldwise/+work", info.Flight.Ref) + require.Equal(t, "Work", info.Flight.Title) + require.Equal(t, []tapper.ContextInfoFlightKeg{ + {Ref: "@foldwise/dev", Role: "editor", Title: "Development", Source: "@foldwise/dev"}, + {Ref: "@foldwise/linked", Role: "viewer", Title: "Linked", Source: "@foldwise/dev", Distance: 1, Path: []string{"@foldwise/dev", "@foldwise/linked"}}, + }, info.Kegs, "the composed cover, at the role the flight caps each KEG to, with the path that decided it") + require.Empty(t, info.Warnings) +} + +func TestContextInfo_UsesOneContextRequest(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + rec, url := newContextHub(t, true, false) + writeContextConfig(t, sb, url) + tap := newTestTap(t, sb) + + info, err := tap.ContextInfo(sb.Context(), tapper.KegTargetOptions{}) + require.NoError(t, err) + wantWorkInfo(t, info) + require.Equal(t, []string{"/api/v1/info"}, rec.seen(), "the context endpoint replaces whoami, flight, and kegs") +} + +func TestContextInfo_SendsBareSlugToContext(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + rec, url := newContextHub(t, true, false) + cfg := fmt.Sprintf("hub: cloud\nkeg: \"@foldwise/dev\"\nflight: \"+work\"\nhubs:\n cloud: {url: %s, token: tok}\n", url) + require.NoError(t, sb.Runtime().AtomicWriteFile("/home/testuser/.config/tapper/config.yaml", []byte(cfg), 0o600)) + tap := newTestTap(t, sb) + + info, err := tap.ContextInfo(sb.Context(), tapper.KegTargetOptions{}) + require.NoError(t, err) + wantWorkInfo(t, info) + require.Equal(t, []string{"/api/v1/info"}, rec.seen(), "the Hub resolves a bare slug itself") +} + +func TestContextInfo_FallsBackOnOlderHubAndRemembersIt(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + rec, url := newContextHub(t, false, false) + writeContextConfig(t, sb, url) + tap := newTestTap(t, sb) + + info, err := tap.ContextInfo(sb.Context(), tapper.KegTargetOptions{}) + require.NoError(t, err) + wantWorkInfo(t, info) + require.Equal(t, []string{"/api/v1/@foldwise/+work", "/api/v1/info", "/api/v1/kegs", "/api/v1/whoami"}, rec.seen()) + + info, err = tap.ContextInfo(sb.Context(), tapper.KegTargetOptions{}) + require.NoError(t, err) + wantWorkInfo(t, info) + seen := rec.seen() + contextCalls := 0 + for _, p := range seen { + if p == "/api/v1/info" { + contextCalls++ + } + } + require.Equal(t, 1, contextCalls, "a 404 is remembered for the life of the Tap") +} + +func TestContextInfo_FlightErrorIsAWarning(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + _, url := newContextHub(t, true, true) + writeContextConfig(t, sb, url) + tap := newTestTap(t, sb) + + info, err := tap.ContextInfo(sb.Context(), tapper.KegTargetOptions{}) + require.NoError(t, err) + require.Equal(t, "jared", info.Hub.User) + require.Equal(t, "@foldwise/+work", info.Flight.Ref) + require.Empty(t, info.Kegs) + require.Equal(t, []string{`Flight "@foldwise/+work" is unavailable: flight not found`}, info.Warnings) +} + +func TestContextInfo_UnreachableHubWarnsForEachPart(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + srv := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "down", http.StatusServiceUnavailable) + })) + t.Cleanup(srv.Close) + writeContextConfig(t, sb, srv.URL) + tap := newTestTap(t, sb) + + info, err := tap.ContextInfo(sb.Context(), tapper.KegTargetOptions{}) + require.NoError(t, err) + require.Len(t, info.Warnings, 2) + require.Contains(t, info.Warnings[0], "hub identity: ") + require.Contains(t, info.Warnings[1], `Flight "@foldwise/+work" is unavailable: `) +} + +func TestOrient_NoFlightListsKegsOnce(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + rec, url := newContextHub(t, true, false) + cfg := fmt.Sprintf("hub: cloud\nhubs:\n cloud: {url: %s, token: tok}\n", url) + require.NoError(t, sb.Runtime().AtomicWriteFile("/home/testuser/.config/tapper/config.yaml", []byte(cfg), 0o600)) + tap := newTestTap(t, sb) + + out, err := tap.Orient(sb.Context(), tapper.OrientOptions{}) + require.NoError(t, err) + require.Contains(t, out, "No flight is active.") + require.Equal(t, []string{"/api/v1/kegs"}, rec.seen(), "the identity catalog is fetched once") +} + +func TestOrient_FlightLoadsGraphAndCatalog(t *testing.T) { + t.Parallel() + sb := NewSandbox(t) + rec, url := newContextHub(t, true, false) + writeContextConfig(t, sb, url) + tap := newTestTap(t, sb) + + out, err := tap.Orient(sb.Context(), tapper.OrientOptions{}) + require.NoError(t, err) + require.Contains(t, out, "Selected flight: `@foldwise/+work`") + require.Contains(t, out, "`@foldwise/dev`") + require.NotContains(t, out, "`@foldwise/private`") + require.Equal(t, []string{"/api/v1/@foldwise/+work", "/api/v1/flights", "/api/v1/kegs"}, rec.seen()) +} + +func writeBareSlugConfig(t *testing.T, sb *sandbox.Sandbox, hubURL string) { + t.Helper() + cfg := fmt.Sprintf("hub: cloud\nkeg: \"@foldwise/dev\"\nflight: \"+work\"\nhubs:\n cloud: {url: %s, token: tok}\n", hubURL) + require.NoError(t, sb.Runtime().AtomicWriteFile("/home/testuser/.config/tapper/config.yaml", []byte(cfg), 0o600)) +} + +// A bare +slug means the same flight whether the Hub resolves it through +// /api/v1/info or Tapper resolves it itself against an older Hub. +func TestContextInfo_BareSlugAgreesWithFallback(t *testing.T) { + t.Parallel() + var infos []tapper.ContextInfoResult + for _, serveContext := range []bool{true, false} { + sb := NewSandbox(t) + _, url := newContextHub(t, serveContext, false) + writeBareSlugConfig(t, sb, url) + tap := newTestTap(t, sb) + info, err := tap.ContextInfo(sb.Context(), tapper.KegTargetOptions{}) + require.NoError(t, err) + require.Empty(t, info.Warnings, "serveContext=%v", serveContext) + infos = append(infos, info) + } + require.Equal(t, infos[0].Flight, infos[1].Flight) + require.Equal(t, "@foldwise/+work", infos[0].Flight.Ref) + require.Equal(t, "Work", infos[0].Flight.Title) +} + +// When the user can read two flights named work, a bare +work is ambiguous +// on both paths, and both report it in the same words. +func TestContextInfo_AmbiguousBareSlugAgreesWithFallback(t *testing.T) { + t.Parallel() + personal := workFlight + personal.Namespace = "jared" + var warnings [][]string + for _, serveContext := range []bool{true, false} { + sb := NewSandbox(t) + _, url := newContextHubWith(t, contextHubOptions{serveContext: serveContext, flights: []tapper.HubFlight{workFlight, personal}}) + writeBareSlugConfig(t, sb, url) + tap := newTestTap(t, sb) + info, err := tap.ContextInfo(sb.Context(), tapper.KegTargetOptions{}) + require.NoError(t, err) + require.Empty(t, info.Kegs) + warnings = append(warnings, info.Warnings) + } + want := []string{`Flight "+work" is unavailable: flight "+work" is ambiguous; use @namespace/+work`} + require.Equal(t, want, warnings[0], "through /api/v1/info") + require.Equal(t, want, warnings[1], "through the fallback") +} diff --git a/pkg/tapper/hub_flights.go b/pkg/tapper/hub_flights.go index 0ee4d2aa..be606656 100644 --- a/pkg/tapper/hub_flights.go +++ b/pkg/tapper/hub_flights.go @@ -27,6 +27,9 @@ type HubEffectiveKeg struct { Role FlightRole `json:"role"` Source string `json:"source"` Distance int `json:"distance"` + // Path is the shortest link path that decided Role, from Source to Ref + // (Distance+1 KEGs). A Hub that predates it leaves it empty. + Path []string `json:"path,omitempty"` } type HubFlight struct { @@ -39,7 +42,23 @@ type HubFlight struct { Visibility string `json:"visibility"` Cover []HubFlightCover `json:"cover"` Subflights []string `json:"subflights"` - Hash string `json:"hash,omitempty"` + // ManageKegs and ManageFlights are the manifest's switches. A write + // that leaves one nil keeps the flight's current value. + ManageKegs *bool `json:"manage_kegs,omitempty"` + ManageFlights *bool `json:"manage_flights,omitempty"` + // Manage is the single flag an older Hub answers with in place of the + // two switches; it is read, never sent. + Manage *bool `json:"manage,omitempty"` + Hash string `json:"hash,omitempty"` +} + +// hubSwitch is one switch as a Hub answered it, falling back to an older +// Hub's single manage flag. +func hubSwitch(value, legacy *bool) bool { + if value != nil { + return *value + } + return legacy != nil && *legacy } func ListUserFlights(ctx context.Context, hubURL, token string) ([]HubFlight, error) { diff --git a/pkg/tapper/invocation_telemetry.go b/pkg/tapper/invocation_telemetry.go index 68111766..a635f419 100644 --- a/pkg/tapper/invocation_telemetry.go +++ b/pkg/tapper/invocation_telemetry.go @@ -3,9 +3,17 @@ package tapper import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "errors" + "fmt" "io" "net/http" + "os" + "path/filepath" + "slices" + "strconv" "strings" "sync" "sync/atomic" @@ -20,9 +28,45 @@ const ( invocationTelemetryBatchSize = 50 invocationTelemetryFlushInterval = time.Second invocationTelemetryRequestTimeout = 2 * time.Second - invocationTelemetryShutdownFlush = 500 * time.Millisecond + + // InvocationTelemetryInFlightWait bounds how long Close waits for a send + // that is already in flight. Close never starts a send of its own. + InvocationTelemetryInFlightWait = 100 * time.Millisecond + + // invocationTelemetrySpoolLimit bounds the spool; the oldest events are + // dropped first. + invocationTelemetrySpoolLimit = 500 + // invocationTelemetrySpoolDir holds one spool directory per destination + // (hub and account), so events are only ever resent where they belong. + invocationTelemetrySpoolDir = "telemetry-spool" + // invocationTelemetryLegacySpoolFile is the single shared spool file of + // earlier releases. Its events carry no destination, so it is deleted + // unsent. + invocationTelemetryLegacySpoolFile = "telemetry-spool.json" + // invocationTelemetryClaimMarker separates a spool file's name from the + // claim suffix a reporter adds when it takes the file. + invocationTelemetryClaimMarker = ".claimed-" + // invocationTelemetryStaleClaim is how long a claimed file may linger + // before it is treated as left behind by a crashed process. + invocationTelemetryStaleClaim = 24 * time.Hour + // invocationTelemetryRejectionMarker, in a destination's spool directory, + // records that the hub refused this client's events. It has no .json + // suffix, so it is never taken for a spool file. + invocationTelemetryRejectionMarker = "rejected" + // invocationTelemetryRejectionTTL is how long a rejection keeps processes + // of the same client version from sending. + invocationTelemetryRejectionTTL = 24 * time.Hour + // invocationTelemetrySpoolDirMaxAge is how long another login's spool + // directory is kept after its last write, in case that login returns. + invocationTelemetrySpoolDirMaxAge = 7 * 24 * time.Hour + // invocationTelemetrySpoolDirLimit bounds how many spool directories, + // the current login's included, are kept at once. + invocationTelemetrySpoolDirLimit = 8 ) +// invocationSpoolSeq keeps spool file names unique within one process. +var invocationSpoolSeq atomic.Uint64 + // InvocationEvent is the privacy-minimized shape accepted by the Tapper Hub // invocation telemetry endpoint. Callers must set exactly one of Command or // Tool according to Surface. ClientVersion and Agent are injected by the @@ -43,11 +87,12 @@ type InvocationEvent struct { } // InvocationReporter accepts best-effort telemetry without blocking the -// caller. Close gives queued events a bounded final flush. +// caller. Events still unsent at Close are spooled for the next process. type InvocationReporter interface { // Report queues an event without blocking and may drop it under pressure. Report(InvocationEvent) - // Close waits for a bounded final flush until ctx is done. + // Close waits for a send already in flight, bounded by ctx and + // InvocationTelemetryInFlightWait, then spools what is left unsent. Close(context.Context) } @@ -62,8 +107,23 @@ type httpInvocationReporter struct { queue chan InvocationEvent done chan struct{} + // sendCtx parents every send; Close cancels it once its wait is over so + // an in-flight request stops and its batch is spooled instead. + sendCtx context.Context + cancelSend context.CancelFunc + + // rt and spoolDir locate this destination's spool. An empty spoolDir + // disables it: unsent events are then dropped at Close. + rt *toolkit.Runtime + spoolDir string + // spooled holds the previous process's events, sent at startup. + spooled []InvocationEvent + mu sync.Mutex closed bool + // unsent collects events that were never delivered: failed sends and + // whatever the queue held at Close. Written by run, read after done. + unsent []InvocationEvent disabled atomic.Bool dropped atomic.Uint64 @@ -73,6 +133,8 @@ type httpInvocationReporter struct { } type invocationReporterOptions struct { + rt *toolkit.Runtime + spoolDir string contract *apicontract.Session client *http.Client queueSize int @@ -86,20 +148,116 @@ type invocationReporterOptions struct { // existing AuthStore token. It returns nil when telemetry is disabled or the // client is not bootstrapped, authenticated, or configured with a remote hub. func NewInvocationReporter(rt *toolkit.Runtime, configService *ConfigService, version string, sessions ...*apicontract.Session) InvocationReporter { - endpoint, token, ok := resolveInvocationTelemetryTarget(rt, configService) + endpoint, token, key, ok := resolveInvocationTelemetryTarget(rt, configService) if !ok { return nil } + if legacy := invocationTelemetryLegacySpoolPath(configService); legacy != "" { + _ = rt.Remove(legacy, false) + } session := apicontract.NewSession(version, rt.Logger()) if len(sessions) > 0 && sessions[0] != nil { session = sessions[0] } + spoolDir := invocationTelemetrySpoolDirFor(configService, key) + if invocationTelemetryRejected(rt, spoolDir, version) { + return nil + } + if key != "" { + pruneInvocationSpoolDirs(rt, filepath.Join(configService.PathService.StateRoot, invocationTelemetrySpoolDir), key) + } return newHTTPInvocationReporter(endpoint, token, version, invocationReporterOptions{ + rt: rt, + spoolDir: spoolDir, contract: session, agent: resolveTelemetryAgent(configService), }) } +// invocationTelemetryRejection is the rejection marker's content. +type invocationTelemetryRejection struct { + ClientVersion string `json:"client_version"` + Status int `json:"status"` + At time.Time `json:"at"` +} + +// invocationTelemetryRejected reports whether this destination refused +// events from this client version within invocationTelemetryRejectionTTL. A +// rejection says nothing about another version, so an upgrade tries again. +func invocationTelemetryRejected(rt *toolkit.Runtime, spoolDir, version string) bool { + if rt == nil || spoolDir == "" { + return false + } + raw, err := rt.ReadFile(filepath.Join(spoolDir, invocationTelemetryRejectionMarker)) + if err != nil { + return false + } + var marker invocationTelemetryRejection + if json.Unmarshal(raw, &marker) != nil || marker.ClientVersion != version { + return false + } + age := rt.Clock().Now().Sub(marker.At) + return age >= 0 && age < invocationTelemetryRejectionTTL +} + +// writeRejection records that the hub refused this client's events, so the +// processes that follow do not each send a request bound to be refused. +func (r *httpInvocationReporter) writeRejection(status int) { + if r.rt == nil || r.spoolDir == "" { + return + } + data, err := json.Marshal(invocationTelemetryRejection{ClientVersion: r.version, Status: status, At: r.rt.Clock().Now().UTC()}) + if err != nil { + return + } + if _, err := r.rt.Stat(r.spoolDir, false); errors.Is(err, os.ErrNotExist) { + if err := r.rt.Mkdir(r.spoolDir, 0o700, true); err != nil { + return + } + } + _ = r.rt.AtomicWriteFile(filepath.Join(r.spoolDir, invocationTelemetryRejectionMarker), data, 0o600) +} + +// invocationTelemetrySpoolDirFor places a destination's spool beside the auth +// store under StateRoot. It holds events only, never credentials; key names +// the destination without revealing it. +func invocationTelemetrySpoolDirFor(configService *ConfigService, key string) string { + if configService == nil || configService.PathService == nil || key == "" { + return "" + } + return filepath.Join(configService.PathService.StateRoot, invocationTelemetrySpoolDir, key) +} + +func invocationTelemetryLegacySpoolPath(configService *ConfigService) string { + if configService == nil || configService.PathService == nil { + return "" + } + return filepath.Join(configService.PathService.StateRoot, invocationTelemetryLegacySpoolFile) +} + +// invocationTelemetryKey names the spool directory for the hub and the login +// signed in to it. It is built from the entry's AccountKey, which survives +// token refreshes, never from a credential: a rotated refresh token must not +// strand what was spooled before it. A new login mints a new AccountKey, and +// another hub hashes differently, so events are never sent to a destination +// that did not produce them. +func invocationTelemetryKey(hubURL, accountKey string) string { + sum := sha256.Sum256([]byte(CanonicalHubURL(hubURL) + "\naccount:" + accountKey)) + return hex.EncodeToString(sum[:])[:16] +} + +// invocationTelemetryCredentialKey is the spool key earlier releases derived +// from the stored credential. The reporter claims that directory for the +// login while the credential is still the one that keyed it. +func invocationTelemetryCredentialKey(hubURL string, entry *AuthEntry) string { + credential := entry.RefreshToken + if strings.TrimSpace(credential) == "" { + credential = entry.AccessToken + } + sum := sha256.Sum256([]byte(CanonicalHubURL(hubURL) + "\n" + credential)) + return hex.EncodeToString(sum[:])[:16] +} + // resolveTelemetryAgent reads the launching harness from the merged config, // which is where TAP_HARNESS lands. It is resolved once at construction rather // than per event: the harness driving a process cannot change while it runs. @@ -114,32 +272,166 @@ func resolveTelemetryAgent(configService *ConfigService) string { return cfg.LaunchHarness() } -func resolveInvocationTelemetryTarget(rt *toolkit.Runtime, configService *ConfigService) (string, string, bool) { +// resolveInvocationTelemetryTarget returns the endpoint, the bearer token, +// and the spool identity key (see invocationTelemetryKey). It names a login +// from an earlier release and claims that login's credential-keyed spool. An +// empty key means the reporter keeps no spool. +func resolveInvocationTelemetryTarget(rt *toolkit.Runtime, configService *ConfigService) (endpoint, token, key string, ok bool) { if rt == nil || configService == nil || !configService.UserConfigExists() { - return "", "", false + return "", "", "", false } if parseEnvBool(rt.Env().Get("TAP_DISABLE_TELEMETRY")) { - return "", "", false + return "", "", "", false } cfg, err := configService.Config() if err != nil || cfg == nil || cfg.DisableTelemetry() { - return "", "", false + return "", "", "", false } hubURL, err := ResolveLoginHubURL(cfg, "") if err != nil || strings.TrimSpace(hubURL) == "" { - return "", "", false + return "", "", "", false } store, err := LoadAuthStore(context.Background(), rt, configService.PathService.AuthStorePath()) if err != nil { - return "", "", false + return "", "", "", false + } + entry, found := store.Get(hubURL) + if !found || strings.TrimSpace(entry.AccessToken) == "" { + return "", "", "", false } - entry, ok := store.Get(hubURL) - if !ok || strings.TrimSpace(entry.AccessToken) == "" { - return "", "", false + accountKey := entry.AccountKey + if accountKey == "" { + // A login from an earlier release: name it now. When that fails the + // reporter still sends, it just keeps no spool. + accountKey, _ = ensureAuthAccountKey(context.Background(), rt, configService.PathService.AuthStorePath(), hubURL) + } + if accountKey != "" { + key = invocationTelemetryKey(hubURL, accountKey) + claimInvocationSpoolDir(rt, filepath.Join(configService.PathService.StateRoot, invocationTelemetrySpoolDir), invocationTelemetryCredentialKey(hubURL, entry), key) + } + return strings.TrimRight(hubURL, "/") + "/api/v1/telemetry/invocations", entry.AccessToken, key, true +} + +// claimInvocationSpoolDir moves the spool files of directory from into +// directory to, then removes from. Earlier releases keyed the spool by the +// current credential; this hands those events to the login that produced +// them. A file another process claims first is simply skipped. +func claimInvocationSpoolDir(rt *toolkit.Runtime, root, from, to string) { + if rt == nil || root == "" || from == "" || to == "" || from == to { + return + } + src, dst := filepath.Join(root, from), filepath.Join(root, to) + entries, err := rt.ReadDir(src) + if err != nil { + return + } + if _, err := rt.Stat(dst, false); errors.Is(err, os.ErrNotExist) { + if err := rt.Mkdir(dst, 0o700, true); err != nil { + return + } + } + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".json") || strings.Contains(name, invocationTelemetryClaimMarker) { + continue + } + _ = rt.Rename(filepath.Join(src, name), filepath.Join(dst, name)) + } + _ = rt.Remove(src, true) +} + +// keepSpoolAcrossRotation runs when a refresh is about to replace prev's +// credential with next's. A login from an earlier release has no AccountKey +// yet and its spool is keyed by the credential being spent, which nothing +// could name afterwards, so the login is named now (persisted, and carried +// onto next) and that spool is claimed for it. The spool lives beside the +// auth store. The caller holds the auth store lock. +func keepSpoolAcrossRotation(ctx context.Context, rt *toolkit.Runtime, storePath, storeKey, hubURL string, prev, next *AuthEntry) { + if rt == nil || storePath == "" || prev == nil || next == nil || next.AccountKey != "" { + return + } + // The refresh already holds the auth store lock. + accountKey, err := ensureAuthAccountKeyLocked(ctx, rt, storePath, storeKey) + if err != nil { + return + } + next.AccountKey = accountKey + root := filepath.Join(filepath.Dir(storePath), invocationTelemetrySpoolDir) + claimInvocationSpoolDir(rt, root, invocationTelemetryCredentialKey(hubURL, prev), invocationTelemetryKey(hubURL, accountKey)) +} + +// pruneInvocationSpoolDirs removes the spool directories of other logins that +// have spooled nothing for invocationTelemetrySpoolDirMaxAge, then the oldest +// of the rest until at most invocationTelemetrySpoolDirLimit remain. A +// directory's age is that of its newest file name, which is stamped from the +// runtime clock. The current login's directory is never removed. +func pruneInvocationSpoolDirs(rt *toolkit.Runtime, root, keep string) { + entries, err := rt.ReadDir(root) + if err != nil { + return + } + now := rt.Clock().Now() + type spoolDir struct { + name string + newest time.Time + } + var dirs []spoolDir + for _, entry := range entries { + if !entry.IsDir() { + continue + } + name := entry.Name() + if name == keep { + continue + } + newest := invocationSpoolDirNewest(rt, filepath.Join(root, name)) + if now.Sub(newest) > invocationTelemetrySpoolDirMaxAge { + _ = rt.Remove(filepath.Join(root, name), true) + continue + } + dirs = append(dirs, spoolDir{name: name, newest: newest}) + } + slices.SortFunc(dirs, func(a, b spoolDir) int { return a.newest.Compare(b.newest) }) + // One place is always held for the current login's directory, which + // may not exist until this process spools. + for len(dirs)+1 > invocationTelemetrySpoolDirLimit && len(dirs) > 0 { + _ = rt.Remove(filepath.Join(root, dirs[0].name), true) + dirs = dirs[1:] } - return strings.TrimRight(hubURL, "/") + "/api/v1/telemetry/invocations", entry.AccessToken, true +} + +// invocationSpoolDirNewest reports when the newest file in a spool directory +// was written or claimed. A directory with no recognizable file reads as +// ancient. +func invocationSpoolDirNewest(rt *toolkit.Runtime, dir string) time.Time { + entries, err := rt.ReadDir(dir) + if err != nil { + return time.Time{} + } + var newest time.Time + for _, entry := range entries { + at, ok := invocationSpoolFileTime(entry.Name()) + if ok && at.After(newest) { + newest = at + } + } + return newest +} + +// invocationSpoolFileTime reads the time a spool file name carries: its claim +// time when claimed, otherwise the leading write timestamp. +func invocationSpoolFileTime(name string) (time.Time, bool) { + if at, ok := invocationSpoolClaimTime(name); ok { + return at, !at.IsZero() + } + stamp, _, _ := strings.Cut(name, "-") + nanos, err := strconv.ParseInt(stamp, 10, 64) + if err != nil { + return time.Time{}, false + } + return time.Unix(0, nanos), true } func newHTTPInvocationReporter(endpoint, token, version string, opts invocationReporterOptions) *httpInvocationReporter { @@ -161,7 +453,12 @@ func newHTTPInvocationReporter(endpoint, token, version string, opts invocationR if opts.contract == nil { opts.contract = apicontract.NewSession(version, nil) } + sendCtx, cancelSend := context.WithCancel(context.Background()) r := &httpInvocationReporter{ + sendCtx: sendCtx, + cancelSend: cancelSend, + rt: opts.rt, + spoolDir: opts.spoolDir, endpoint: endpoint, token: token, version: version, @@ -173,6 +470,7 @@ func newHTTPInvocationReporter(endpoint, token, version string, opts invocationR flushInterval: opts.flushInterval, requestTimeout: opts.requestTimeout, } + r.spooled = r.takeSpool() go r.run(opts.batchSize) return r } @@ -206,79 +504,298 @@ func (r *httpInvocationReporter) Close(ctx context.Context) { } r.mu.Unlock() + // Give a send already in flight a short window, then cancel it so its + // batch joins the spool rather than holding up exit. + wait := time.NewTimer(InvocationTelemetryInFlightWait) + defer wait.Stop() select { case <-r.done: + case <-wait.C: case <-ctx.Done(): } + if r.cancelSend != nil { + r.cancelSend() + } + select { + case <-r.done: + case <-ctx.Done(): + return + } + r.writeSpool() +} + +func (r *httpInvocationReporter) isClosed() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.closed } func (r *httpInvocationReporter) run(batchSize int) { defer close(r.done) ticker := time.NewTicker(r.flushInterval) defer ticker.Stop() + + // The previous process's events go first, overlapping the command's own + // requests instead of delaying its exit. + for len(r.spooled) > 0 && !r.disabled.Load() && !r.isClosed() { + n := min(batchSize, len(r.spooled)) + r.flush(r.spooled[:n]) + r.spooled = r.spooled[n:] + } + r.keepUnsent(r.spooled) + r.spooled = nil + batch := make([]InvocationEvent, 0, batchSize) for { select { case event, ok := <-r.queue: if !ok { - if len(batch) > 0 && !r.disabled.Load() { - ctx, cancel := context.WithTimeout(context.Background(), invocationTelemetryShutdownFlush) - r.send(ctx, batch) - cancel() - } + r.keepUnsent(batch) return } if r.disabled.Load() { continue } batch = append(batch, event) - if len(batch) >= batchSize { - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - r.send(ctx, batch) - cancel() + // Close never starts a send; the drained remainder is spooled. + if len(batch) >= batchSize && !r.isClosed() { + r.flush(batch) batch = batch[:0] } case <-ticker.C: - if len(batch) == 0 || r.disabled.Load() { + if len(batch) == 0 || r.disabled.Load() || r.isClosed() { continue } - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - r.send(ctx, batch) - cancel() + r.flush(batch) batch = batch[:0] } } } -func (r *httpInvocationReporter) send(ctx context.Context, batch []InvocationEvent) { - payload, err := json.Marshal(batch) +// flush sends one batch and keeps it for the spool when delivery failed for a +// reason a later process might not hit. +func (r *httpInvocationReporter) flush(batch []InvocationEvent) { + parent := r.sendCtx + if parent == nil { + parent = context.Background() + } + ctx, cancel := context.WithTimeout(parent, r.requestTimeout) + defer cancel() + if !r.send(ctx, batch) && !r.disabled.Load() { + r.keepUnsent(batch) + } +} + +// keepUnsent holds events for the spool, bounded so a long-lived MCP server +// with an unreachable Hub does not grow without limit. +func (r *httpInvocationReporter) keepUnsent(events []InvocationEvent) { + r.unsent = append(r.unsent, events...) + if over := len(r.unsent) - invocationTelemetrySpoolLimit; over > 0 { + r.unsent = append([]InvocationEvent(nil), r.unsent[over:]...) + } +} + +// takeSpool claims and removes every spooled file for this destination. A +// file is claimed by renaming it, so when several processes start at once +// each file goes to exactly one of them. Any problem reading the spool just +// means there is nothing to resend. +func (r *httpInvocationReporter) takeSpool() []InvocationEvent { + if r.rt == nil || r.spoolDir == "" { + return nil + } + entries, err := r.rt.ReadDir(r.spoolDir) + if err != nil { + return nil + } + now := r.rt.Clock().Now() + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if claimedAt, ok := invocationSpoolClaimTime(name); ok { + if now.Sub(claimedAt) > invocationTelemetryStaleClaim { + _ = r.rt.Remove(filepath.Join(r.spoolDir, name), false) + } + continue + } + if strings.HasSuffix(name, ".json") { + names = append(names, name) + } + } + slices.Sort(names) + + var events []InvocationEvent + for _, name := range names { + path := filepath.Join(r.spoolDir, name) + claimed := fmt.Sprintf("%s%s%d-%d", path, invocationTelemetryClaimMarker, now.UnixNano(), r.pid()) + if err := r.rt.Rename(path, claimed); err != nil { + // Another process claimed it first. + continue + } + events = append(events, readInvocationSpool(r.rt, claimed)...) + _ = r.rt.Remove(claimed, false) + } + return events +} + +// invocationSpoolClaimTime reports when a claimed spool file was claimed. +func invocationSpoolClaimTime(name string) (time.Time, bool) { + i := strings.LastIndex(name, invocationTelemetryClaimMarker) + if i < 0 { + return time.Time{}, false + } + stamp, _, _ := strings.Cut(name[i+len(invocationTelemetryClaimMarker):], "-") + nanos, err := strconv.ParseInt(stamp, 10, 64) + if err != nil { + // Unparseable: treat it as ancient so it is cleaned up. + return time.Time{}, true + } + return time.Unix(0, nanos), true +} + +func (r *httpInvocationReporter) pid() int { + if p := r.rt.Process(); p != nil { + return p.PID + } + return 0 +} + +// writeSpool writes the unsent events to a new file of their own, so +// concurrent processes never read-modify-write a shared file, then prunes +// the oldest files so the spool keeps at most invocationTelemetrySpoolLimit +// events. A disabled reporter writes nothing: the Hub will not accept those +// events from this client. +func (r *httpInvocationReporter) writeSpool() { + if r.rt == nil || r.spoolDir == "" || len(r.unsent) == 0 || r.disabled.Load() { + return + } + data, err := json.Marshal(r.unsent) if err != nil { return } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.endpoint, bytes.NewReader(payload)) + if _, err := r.rt.Stat(r.spoolDir, false); errors.Is(err, os.ErrNotExist) { + if err := r.rt.Mkdir(r.spoolDir, 0o700, true); err != nil { + return + } + } + name := fmt.Sprintf("%020d-%d-%d.json", r.rt.Clock().Now().UnixNano(), r.pid(), invocationSpoolSeq.Add(1)) + if err := r.rt.AtomicWriteFile(filepath.Join(r.spoolDir, name), data, 0o600); err != nil { + return + } + r.pruneSpool() +} + +// pruneSpool drops the oldest spool files, by name, until the rest hold at +// most invocationTelemetrySpoolLimit events. Whole files go, never part of +// one, so a file another process is claiming is never rewritten. +func (r *httpInvocationReporter) pruneSpool() { + entries, err := r.rt.ReadDir(r.spoolDir) if err != nil { return } + var names []string + for _, entry := range entries { + name := entry.Name() + if !entry.IsDir() && strings.HasSuffix(name, ".json") && !strings.Contains(name, invocationTelemetryClaimMarker) { + names = append(names, name) + } + } + slices.Sort(names) + total := 0 + for i := len(names) - 1; i >= 0; i-- { + path := filepath.Join(r.spoolDir, names[i]) + if total >= invocationTelemetrySpoolLimit { + _ = r.rt.Remove(path, false) + continue + } + n := len(readInvocationSpool(r.rt, path)) + if total+n > invocationTelemetrySpoolLimit { + _ = r.rt.Remove(path, false) + total = invocationTelemetrySpoolLimit + continue + } + total += n + } +} + +func readInvocationSpool(rt *toolkit.Runtime, path string) []InvocationEvent { + raw, err := rt.ReadFile(path) + if err != nil || len(raw) == 0 { + return nil + } + var events []InvocationEvent + if json.Unmarshal(raw, &events) != nil { + return nil + } + return events +} + +// send posts one batch. It reports false when the batch was not delivered; +// a response that disables the reporter counts as delivered, since resending +// it would only be rejected again. +func (r *httpInvocationReporter) send(ctx context.Context, batch []InvocationEvent) bool { + payload, err := json.Marshal(batch) + if err != nil { + return true + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.endpoint, bytes.NewReader(payload)) + if err != nil { + return true + } req.Header.Set("Authorization", "Bearer "+r.token) req.Header.Set("Content-Type", "application/json") req = req.WithContext(apicontract.WithSession(req.Context(), r.contract)) resp, err := apicontract.Do(r.client, req) if err != nil { if apicontract.IsCompatibility(err) { - r.disabled.Store(true) + r.disable(0) + return true } - return + return false } _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) _ = resp.Body.Close() if disablesInvocationTelemetry(resp.StatusCode) { - r.disabled.Store(true) + r.disable(resp.StatusCode) + return true + } + return resp.StatusCode < 300 || resp.StatusCode == http.StatusRequestEntityTooLarge +} + +// disable stops this reporter and, once per process, records the rejection +// for the processes that follow. status is 0 when the API contract, not a +// response status, ruled this client out. A credential failure stops only +// this process: it is not recorded, so a later process holding a refreshed +// or different credential sends again. +func (r *httpInvocationReporter) disable(status int) { + if r.disabled.Swap(true) { + return + } + if invocationTelemetryCredentialFailure(status) { + return } + r.writeRejection(status) +} + +// invocationTelemetryCredentialFailure reports whether a status refuses the +// credential the request carried rather than this client. 401 is an expired +// or revoked access token, which tap's auth refresh replaces. 403 is the hub +// denying that credential — on the telemetry route, a token bound to an +// orientation the hub no longer honors — and another credential may be +// accepted. Neither says anything about this client version, so neither may +// leave a rejection marker that would silence the next process for a day. +func invocationTelemetryCredentialFailure(status int) bool { + return status == http.StatusUnauthorized || status == http.StatusForbidden } // disablesInvocationTelemetry reports whether a status means "stop trying" as -// opposed to "try again later". Every code here says the hub will never accept -// this client's events, so retrying only wastes requests. +// opposed to "try again later". Retrying any of these within this process +// only wastes requests. Most say the hub will never accept this client's +// events and are remembered across processes; 401 and 403 instead refuse the +// credential this process holds, so they stop this process only (see +// invocationTelemetryCredentialFailure). // // 400 is in the list because a hub older than the client rejects any field it // does not know — its decoder disallows unknown fields — and the client cannot diff --git a/pkg/tapper/invocation_telemetry_account_test.go b/pkg/tapper/invocation_telemetry_account_test.go new file mode 100644 index 00000000..d9363cf2 --- /dev/null +++ b/pkg/tapper/invocation_telemetry_account_test.go @@ -0,0 +1,298 @@ +package tapper + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/jlrickert/cli-toolkit/sandbox" + "github.com/stretchr/testify/require" +) + +func newTelemetryAccountFixture(t *testing.T) (*sandbox.Sandbox, *Tap) { + t.Helper() + fx := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + tap, err := NewTap(TapOptions{Root: "/home/testuser", Runtime: fx.Runtime()}) + require.NoError(t, err) + return fx, tap +} + +// rotatingRefreshServer answers every refresh grant with a new token pair, as +// a hub that rotates refresh tokens single-use does. +func rotatingRefreshServer(t *testing.T) *httptest.Server { + t.Helper() + n := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n++ + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"access_token":"access-rotated-%d","token_type":"Bearer","expires_in":900,"refresh_token":"refresh-rotated-%d"}`, n, n) + })) + t.Cleanup(srv.Close) + return srv +} + +func storedEntryForTest(t *testing.T, fx *sandbox.Sandbox, tap *Tap, hubURL string) *AuthEntry { + t.Helper() + store, err := LoadAuthStore(fx.Context(), fx.Runtime(), tap.PathService.AuthStorePath()) + require.NoError(t, err) + entry, ok := store.Get(hubURL) + require.True(t, ok) + return entry +} + +// rotateStoredLoginForTest spends the stored refresh token the way a command +// does when its access token has expired: RefreshHubToken against a rotating +// token endpoint, persisted through the auth store. +func rotateStoredLoginForTest(t *testing.T, fx *sandbox.Sandbox, tap *Tap, hubURL, tokenEndpoint string) { + t.Helper() + storePath := tap.PathService.AuthStorePath() + store, err := LoadAuthStore(fx.Context(), fx.Runtime(), storePath) + require.NoError(t, err) + entry, ok := store.Get(hubURL) + require.True(t, ok) + before := entry.RefreshToken + entry.TokenEndpoint = tokenEndpoint + entry.ExpiresAt = fx.Runtime().Clock().Now().Add(-time.Minute) + store.Set(hubURL, *entry) + require.NoError(t, store.Save(fx.Context(), fx.Runtime(), storePath)) + _, err = refreshAuthStoreEntryIfNeeded(fx.Context(), fx.Runtime(), store, storePath, hubURL, hubURL, entry) + require.NoError(t, err) + require.NotEqual(t, before, storedEntryForTest(t, fx, tap, hubURL).RefreshToken, "the refresh token rotated") +} + +// Events spooled while the hub was unreachable are still the same account's +// after its refresh token rotates, so the next process sends them. +func TestInvocationSpoolSurvivesRefreshTokenRotation(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + hub := newFakeTelemetryHub(t) + tokens := rotatingRefreshServer(t) + + hub.failing.Store(true) + signInForTelemetry(t, fx, tap, hub.URL, "refresh-before") + runTelemetryReporter(t, fx, tap, "tap offline") + require.Empty(t, hub.commands()) + + rotateStoredLoginForTest(t, fx, tap, hub.URL, tokens.URL) + + hub.failing.Store(false) + r := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "test") + require.NotNil(t, r) + require.Eventually(t, func() bool { return len(hub.commands()) == 1 }, 2*time.Second, 5*time.Millisecond) + r.Close(context.Background()) + require.Equal(t, []string{"tap offline"}, hub.commands()) + require.Len(t, telemetrySpoolDirsForTest(t, fx, tap), 1, "no directory is left behind for the spent credential") +} + +// A refresh that started from an entry read before the account key was +// minted must not erase the key from disk. +func TestAuthRefreshKeepsAccountKeyMintedMeanwhile(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + tokens := rotatingRefreshServer(t) + hubURL := "https://hub.example.com" + storePath := tap.PathService.AuthStorePath() + signInForTelemetry(t, fx, tap, hubURL, "refresh-old") + expired := storedEntryForTest(t, fx, tap, hubURL) + expired.TokenEndpoint = tokens.URL + expired.ExpiresAt = fx.Runtime().Clock().Now().Add(-time.Minute) + require.NoError(t, persistAuthStoreEntry(fx.Context(), fx.Runtime(), storePath, hubURL, expired.RefreshToken, expired)) + + stale, err := LoadAuthStore(fx.Context(), fx.Runtime(), storePath) + require.NoError(t, err) + entry, _ := stale.Get(hubURL) + require.Empty(t, entry.AccountKey) + + key, err := ensureAuthAccountKey(fx.Context(), fx.Runtime(), storePath, hubURL) + require.NoError(t, err) + require.NotEmpty(t, key) + + _, err = refreshAuthStoreEntryIfNeeded(fx.Context(), fx.Runtime(), stale, storePath, hubURL, hubURL, entry) + require.NoError(t, err) + stored := storedEntryForTest(t, fx, tap, hubURL) + require.Equal(t, "refresh-rotated-1", stored.RefreshToken) + require.Equal(t, key, stored.AccountKey) +} + +// An entry written by an earlier release gets a key once, persisted, and +// every later process uses that same key. +func TestEnsureAuthAccountKeyIsMintedOnceAndPersisted(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + hubURL := "https://hub.example.com" + signInForTelemetry(t, fx, tap, hubURL, "refresh") + + first, err := ensureAuthAccountKey(fx.Context(), fx.Runtime(), tap.PathService.AuthStorePath(), hubURL) + require.NoError(t, err) + second, err := ensureAuthAccountKey(fx.Context(), fx.Runtime(), tap.PathService.AuthStorePath(), hubURL) + require.NoError(t, err) + require.NotEmpty(t, first) + require.Equal(t, first, second) + require.Equal(t, first, storedEntryForTest(t, fx, tap, hubURL).AccountKey) + require.Equal(t, "access-refresh", storedEntryForTest(t, fx, tap, hubURL).AccessToken, "minting leaves the credential alone") +} + +// A spool directory from an earlier release, keyed by the then-current +// credential, is claimed by the account and sent. +func TestInvocationLegacyCredentialSpoolIsClaimed(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + hub := newFakeTelemetryHub(t) + signInForTelemetry(t, fx, tap, hub.URL, "refresh-legacy") + legacy := invocationTelemetrySpoolDirFor(tap.ConfigService, invocationTelemetryCredentialKey(hub.URL, storedEntryForTest(t, fx, tap, hub.URL))) + writeSpoolForTest(t, fx, legacy, []InvocationEvent{{Surface: "cli", Command: "tap from-legacy"}}) + + r := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "test") + require.NotNil(t, r) + require.Eventually(t, func() bool { return len(hub.commands()) == 1 }, 2*time.Second, 5*time.Millisecond) + r.Close(context.Background()) + require.Equal(t, []string{"tap from-legacy"}, hub.commands()) + _, err := fx.Runtime().Stat(legacy, false) + require.Error(t, err, "the legacy directory is gone once claimed") +} + +// A spool an earlier release keyed by the credential is not stranded when the +// first command of the new release rotates that credential before telemetry +// starts. +func TestInvocationLegacyCredentialSpoolSurvivesRotationOnUpgrade(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + hub := newFakeTelemetryHub(t) + tokens := rotatingRefreshServer(t) + signInForTelemetry(t, fx, tap, hub.URL, "refresh-legacy") + legacy := invocationTelemetrySpoolDirFor(tap.ConfigService, invocationTelemetryCredentialKey(hub.URL, storedEntryForTest(t, fx, tap, hub.URL))) + writeSpoolForTest(t, fx, legacy, []InvocationEvent{{Surface: "cli", Command: "tap before-upgrade"}}) + + rotateStoredLoginForTest(t, fx, tap, hub.URL, tokens.URL) + + r := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "test") + require.NotNil(t, r) + require.Eventually(t, func() bool { return len(hub.commands()) == 1 }, 2*time.Second, 5*time.Millisecond) + r.Close(context.Background()) + require.Equal(t, []string{"tap before-upgrade"}, hub.commands()) + _, err := fx.Runtime().Stat(legacy, false) + require.Error(t, err, "the legacy directory is gone once claimed") +} + +// Another account's spool is never sent, and is kept while it is recent. +func TestInvocationSpoolOfAnotherLoginIsKeptUnsent(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + hub := newFakeTelemetryHub(t) + + hub.failing.Store(true) + signInForTelemetry(t, fx, tap, hub.URL, "refresh-one") + setAccountKeyForTest(t, fx, tap, hub.URL, "account-one") + runTelemetryReporter(t, fx, tap, "tap account-one") + + hub.failing.Store(false) + signInForTelemetry(t, fx, tap, hub.URL, "refresh-two") + setAccountKeyForTest(t, fx, tap, hub.URL, "account-two") + runTelemetryReporter(t, fx, tap, "tap account-two") + time.Sleep(100 * time.Millisecond) + require.NotContains(t, hub.commands(), "tap account-one") + + one := invocationTelemetrySpoolDirFor(tap.ConfigService, invocationTelemetryKey(hub.URL, "account-one")) + require.Len(t, readSpoolForTest(t, fx, one), 1, "the other account's recent events are retained") +} + +// Directories of accounts that have not spooled anything for a week are +// pruned, and at most invocationTelemetrySpoolDirLimit directories remain. +func TestInvocationSpoolPrunesStaleAndExcessDirectories(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + hub := newFakeTelemetryHub(t) + hub.failing.Store(true) + signInForTelemetry(t, fx, tap, hub.URL, "refresh") + setAccountKeyForTest(t, fx, tap, hub.URL, "current") + root := filepath.Join(tap.PathService.StateRoot, invocationTelemetrySpoolDir) + now := fx.Runtime().Clock().Now() + + writeAgedSpoolForTest(t, fx, filepath.Join(root, "stale"), now.Add(-8*24*time.Hour)) + for i := range invocationTelemetrySpoolDirLimit + 2 { + writeAgedSpoolForTest(t, fx, filepath.Join(root, fmt.Sprintf("recent-%02d", i)), now.Add(-time.Duration(20-i)*time.Hour)) + } + + runTelemetryReporter(t, fx, tap, "tap now") + + dirs := telemetrySpoolDirsForTest(t, fx, tap) + require.Len(t, dirs, invocationTelemetrySpoolDirLimit) + require.NotContains(t, dirs, "stale") + require.NotContains(t, dirs, "recent-00", "the oldest directories go first") + require.Contains(t, dirs, invocationTelemetryKey(hub.URL, "current"), "the current account's spool is kept") + require.Contains(t, dirs, fmt.Sprintf("recent-%02d", invocationTelemetrySpoolDirLimit+1)) +} + +func setAccountKeyForTest(t *testing.T, fx *sandbox.Sandbox, tap *Tap, hubURL, key string) { + t.Helper() + storePath := tap.PathService.AuthStorePath() + store, err := LoadAuthStore(fx.Context(), fx.Runtime(), storePath) + require.NoError(t, err) + entry, ok := store.Get(hubURL) + require.True(t, ok) + entry.AccountKey = key + store.Set(hubURL, *entry) + require.NoError(t, store.Save(fx.Context(), fx.Runtime(), storePath)) +} + +func writeAgedSpoolForTest(t *testing.T, fx *sandbox.Sandbox, dir string, at time.Time) { + t.Helper() + raw, err := json.Marshal([]InvocationEvent{{Surface: "cli", Command: "tap aged"}}) + require.NoError(t, err) + require.NoError(t, fx.Runtime().Mkdir(dir, 0o700, true)) + require.NoError(t, fx.Runtime().AtomicWriteFile(filepath.Join(dir, fmt.Sprintf("%020d-1-1.json", at.UnixNano())), raw, 0o600)) +} + +func telemetrySpoolDirsForTest(t *testing.T, fx *sandbox.Sandbox, tap *Tap) []string { + t.Helper() + entries, err := fx.Runtime().ReadDir(filepath.Join(tap.PathService.StateRoot, invocationTelemetrySpoolDir)) + require.NoError(t, err) + var dirs []string + for _, e := range entries { + if e.IsDir() { + dirs = append(dirs, e.Name()) + } + } + return dirs +} + +// Two processes that both find a login from an earlier release name it at +// the same time. The auth store lock makes them agree: each one returns the +// key that ends up on disk, so neither spools under a key nothing will read. +func TestEnsureAuthAccountKeyConcurrentMintsAgree(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + hubURL := "https://hub.example.com" + storePath := tap.PathService.AuthStorePath() + signInForTelemetry(t, fx, tap, hubURL, "refresh") + + const processes = 8 + for round := range 10 { + entry := storedEntryForTest(t, fx, tap, hubURL) + entry.AccountKey = "" + store, err := LoadAuthStore(fx.Context(), fx.Runtime(), storePath) + require.NoError(t, err) + store.Set(hubURL, *entry) + require.NoError(t, store.Save(fx.Context(), fx.Runtime(), storePath)) + + keys := make([]string, processes) + errs := make([]error, processes) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range processes { + wg.Add(1) + go func() { + defer wg.Done() + <-start + keys[i], errs[i] = ensureAuthAccountKey(fx.Context(), fx.Runtime(), storePath, hubURL) + }() + } + close(start) + wg.Wait() + + stored := storedEntryForTest(t, fx, tap, hubURL).AccountKey + require.NotEmpty(t, stored) + for i := range processes { + require.NoError(t, errs[i]) + require.Equal(t, stored, keys[i], "round %d: process %d returned the key on disk", round, i) + } + } +} diff --git a/pkg/tapper/invocation_telemetry_rejection_test.go b/pkg/tapper/invocation_telemetry_rejection_test.go new file mode 100644 index 00000000..36d769b4 --- /dev/null +++ b/pkg/tapper/invocation_telemetry_rejection_test.go @@ -0,0 +1,145 @@ +package tapper + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/jlrickert/tapper/pkg/apicontract" + "github.com/stretchr/testify/require" +) + +// rejectingTelemetryHub answers every telemetry post with status, the way a +// hub that will never accept this client's events does, and counts them. +func rejectingTelemetryHub(t *testing.T, status *atomic.Int32) (*httptest.Server, *atomic.Int32) { + t.Helper() + var posts atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/api/version" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"server_version":"test","api_versions":["` + apicontract.Revision + `"]}`)) + return + } + posts.Add(1) + w.WriteHeader(int(status.Load())) + })) + t.Cleanup(srv.Close) + return srv, &posts +} + +func runVersionedReporter(t *testing.T, r InvocationReporter, command string) { + t.Helper() + if r == nil { + return + } + r.Report(InvocationEvent{Surface: "cli", Command: command}) + time.Sleep(1200 * time.Millisecond) // one flush interval + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + r.Close(ctx) +} + +// A hub that refused this client's events once is not asked again by every +// later process of the same version for a day. +func TestInvocationTelemetryRejectionIsRememberedPerVersion(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + var status atomic.Int32 + status.Store(http.StatusNotFound) + hub, posts := rejectingTelemetryHub(t, &status) + signInForTelemetry(t, fx, tap, hub.URL, "refresh") + + runVersionedReporter(t, NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0"), "tap first") + require.EqualValues(t, 1, posts.Load()) + + again := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0") + runVersionedReporter(t, again, "tap second") + require.Nil(t, again, "the same version skips telemetry after a rejection") + require.EqualValues(t, 1, posts.Load(), "no request is sent to a hub that rejected this version") +} + +// The rejection expires after a day, and never applies to another version: +// an upgraded client, or the hub after it upgrades, gets to try again. +func TestInvocationTelemetryRejectionExpiresAndIsVersionScoped(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + var status atomic.Int32 + status.Store(http.StatusBadRequest) + hub, posts := rejectingTelemetryHub(t, &status) + signInForTelemetry(t, fx, tap, hub.URL, "refresh") + + runVersionedReporter(t, NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0"), "tap first") + require.EqualValues(t, 1, posts.Load()) + + status.Store(http.StatusNoContent) + runVersionedReporter(t, NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.1.0"), "tap upgraded") + require.EqualValues(t, 2, posts.Load(), "another version is not covered by the rejection") + + fx.Advance(25 * time.Hour) + runVersionedReporter(t, NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0"), "tap next-day") + require.EqualValues(t, 3, posts.Load(), "the rejection lapses after a day") +} + +// A retryable failure is not a rejection: the next process tries again. +func TestInvocationTelemetryTransientFailureIsNotRemembered(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + var status atomic.Int32 + status.Store(http.StatusServiceUnavailable) + hub, posts := rejectingTelemetryHub(t, &status) + signInForTelemetry(t, fx, tap, hub.URL, "refresh") + + runVersionedReporter(t, NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0"), "tap first") + first := posts.Load() + require.Positive(t, first) + r := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0") + require.NotNil(t, r) + runVersionedReporter(t, r, "tap second") + require.Greater(t, posts.Load(), first) +} + +// A refused credential is not a refused client: an expired or revoked token +// stops this process's reporter, but the next process, holding a refreshed +// credential for the same login and version, reports again. +func TestInvocationTelemetryCredentialFailureIsNotRemembered(t *testing.T) { + for _, code := range []int32{http.StatusUnauthorized, http.StatusForbidden} { + t.Run(http.StatusText(int(code)), func(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + var status atomic.Int32 + status.Store(code) + hub, posts := rejectingTelemetryHub(t, &status) + signInForTelemetry(t, fx, tap, hub.URL, "refresh") + + first := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0") + require.NotNil(t, first) + first.Report(InvocationEvent{Surface: "cli", Command: "tap first"}) + runVersionedReporter(t, first, "tap first-again") + require.EqualValues(t, 1, posts.Load(), "a refused credential is not retried within the process") + + status.Store(http.StatusNoContent) + again := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0") + require.NotNil(t, again, "a credential failure must not suppress the next process") + runVersionedReporter(t, again, "tap second") + require.EqualValues(t, 2, posts.Load(), "the next process sends with its credential") + }) + } +} + +// A client-version refusal is still remembered for the same login and +// version, alongside the credential case above. +func TestInvocationTelemetryBadRequestIsStillRemembered(t *testing.T) { + fx, tap := newTelemetryAccountFixture(t) + var status atomic.Int32 + status.Store(http.StatusBadRequest) + hub, posts := rejectingTelemetryHub(t, &status) + signInForTelemetry(t, fx, tap, hub.URL, "refresh") + + runVersionedReporter(t, NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0"), "tap first") + require.EqualValues(t, 1, posts.Load()) + + status.Store(http.StatusNoContent) + again := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "v1.0.0") + runVersionedReporter(t, again, "tap second") + require.Nil(t, again, "a 400 still suppresses the same version") + require.EqualValues(t, 1, posts.Load()) +} diff --git a/pkg/tapper/invocation_telemetry_spool_test.go b/pkg/tapper/invocation_telemetry_spool_test.go new file mode 100644 index 00000000..137fb3d4 --- /dev/null +++ b/pkg/tapper/invocation_telemetry_spool_test.go @@ -0,0 +1,264 @@ +package tapper + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jlrickert/cli-toolkit/sandbox" + "github.com/jlrickert/tapper/pkg/apicontract" + "github.com/stretchr/testify/require" +) + +// fakeTelemetryHub records the events a Hub received. While failing is set it +// answers 503, so a reporter keeps its events for the spool. +type fakeTelemetryHub struct { + *httptest.Server + failing atomic.Bool + mu sync.Mutex + got []string +} + +func newFakeTelemetryHub(t *testing.T) *fakeTelemetryHub { + t.Helper() + h := &fakeTelemetryHub{} + h.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/api/version" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"server_version":"test","api_versions":["`+apicontract.Revision+`"]}`) + return + } + if h.failing.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + var batch []InvocationEvent + _ = json.NewDecoder(req.Body).Decode(&batch) + h.mu.Lock() + for _, e := range batch { + h.got = append(h.got, e.Command) + } + h.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(h.Close) + return h +} + +func (h *fakeTelemetryHub) commands() []string { + h.mu.Lock() + defer h.mu.Unlock() + return append([]string(nil), h.got...) +} + +// signInForTelemetry points the user config at hubURL and stores a login +// whose refresh token is refresh. It writes the entry as an earlier release +// did, with no AccountKey, unless the same login is already stored. +func signInForTelemetry(t *testing.T, fx *sandbox.Sandbox, tap *Tap, hubURL, refresh string) { + t.Helper() + cfg := fmt.Sprintf("hub: primary\nhubs:\n primary:\n url: %s\n", hubURL) + require.NoError(t, fx.Runtime().AtomicWriteFile(tap.PathService.UserConfig(), []byte(cfg), 0o644)) + store, err := LoadAuthStore(fx.Context(), fx.Runtime(), tap.PathService.AuthStorePath()) + require.NoError(t, err) + entry := AuthEntry{AccessToken: "access-" + refresh, RefreshToken: refresh} + if prev, ok := store.Get(hubURL); ok && prev.RefreshToken == refresh { + // Signing in again with the same credential is the same login. + entry.AccountKey = prev.AccountKey + } + store.Set(hubURL, entry) + require.NoError(t, store.Save(fx.Context(), fx.Runtime(), tap.PathService.AuthStorePath())) + tap.ConfigService.Reload() +} + +// runTelemetryReporter starts a reporter for the current login, reports +// commands, and closes it. +func runTelemetryReporter(t *testing.T, fx *sandbox.Sandbox, tap *Tap, commands ...string) { + t.Helper() + r := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "test") + require.NotNil(t, r) + for _, c := range commands { + r.Report(InvocationEvent{Surface: "cli", Command: c}) + } + // Let a startup resend finish before Close, which never starts a send. + time.Sleep(50 * time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + r.Close(ctx) +} + +// Spooled events belong to the hub and account that produced them: another +// hub never receives them, and the original hub still does later. +func TestInvocationSpoolIsNeverSentToAnotherHub(t *testing.T) { + fx := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + tap, err := NewTap(TapOptions{Root: "/home/testuser", Runtime: fx.Runtime()}) + require.NoError(t, err) + hubA, hubB := newFakeTelemetryHub(t), newFakeTelemetryHub(t) + + hubA.failing.Store(true) + signInForTelemetry(t, fx, tap, hubA.URL, "refresh-a") + runTelemetryReporter(t, fx, tap, "tap on-a") + require.Empty(t, hubA.commands()) + + signInForTelemetry(t, fx, tap, hubB.URL, "refresh-b") + runTelemetryReporter(t, fx, tap) + require.Empty(t, hubB.commands(), "hub B must not receive hub A's events") + + hubA.failing.Store(false) + signInForTelemetry(t, fx, tap, hubA.URL, "refresh-a") + r := NewInvocationReporter(fx.Runtime(), tap.ConfigService, "test") + require.NotNil(t, r) + require.Eventually(t, func() bool { return len(hubA.commands()) == 1 }, 2*time.Second, 5*time.Millisecond) + r.Close(context.Background()) + require.Equal(t, []string{"tap on-a"}, hubA.commands()) + require.Empty(t, hubB.commands()) +} + +// A new login on the same hub is another identity: the previous login's +// spool is not sent with the new credential. +func TestInvocationSpoolIsNotSentUnderAnotherLogin(t *testing.T) { + fx := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + tap, err := NewTap(TapOptions{Root: "/home/testuser", Runtime: fx.Runtime()}) + require.NoError(t, err) + hub := newFakeTelemetryHub(t) + + hub.failing.Store(true) + signInForTelemetry(t, fx, tap, hub.URL, "refresh-one") + runTelemetryReporter(t, fx, tap, "tap first-login") + + hub.failing.Store(false) + signInForTelemetry(t, fx, tap, hub.URL, "refresh-two") + runTelemetryReporter(t, fx, tap, "tap second-login") + time.Sleep(100 * time.Millisecond) + require.NotContains(t, hub.commands(), "tap first-login") +} + +// The single shared spool file of earlier releases names no destination, so +// it is deleted without being sent. +func TestInvocationLegacySpoolIsDiscarded(t *testing.T) { + fx := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + tap, err := NewTap(TapOptions{Root: "/home/testuser", Runtime: fx.Runtime()}) + require.NoError(t, err) + hub := newFakeTelemetryHub(t) + signInForTelemetry(t, fx, tap, hub.URL, "refresh") + legacy := invocationTelemetryLegacySpoolPath(tap.ConfigService) + raw, err := json.Marshal([]InvocationEvent{{Surface: "cli", Command: "tap legacy"}}) + require.NoError(t, err) + require.NoError(t, fx.Runtime().AtomicWriteFile(legacy, raw, 0o600)) + + runTelemetryReporter(t, fx, tap) + time.Sleep(100 * time.Millisecond) + require.Empty(t, hub.commands()) + _, err = fx.Runtime().Stat(legacy, false) + require.Error(t, err, "the legacy spool is removed") +} + +func recordingTelemetryClient(status int, mu *sync.Mutex, got *[]string) *http.Client { + return &http.Client{Transport: telemetryRoundTripFunc(func(req *http.Request) (*http.Response, error) { + if status >= 300 { + return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil + } + var batch []InvocationEvent + _ = json.NewDecoder(req.Body).Decode(&batch) + mu.Lock() + for _, e := range batch { + *got = append(*got, e.Command) + } + mu.Unlock() + return noContent() + })} +} + +const testTelemetryEndpoint = "https://hub.example.com/api/v1/telemetry/invocations" + +// Processes that close at the same time each spool their own events, and the +// next process sends every one of them exactly once. +func TestInvocationSpoolConcurrentClosesLoseNothing(t *testing.T) { + fx, spool := spoolFixture(t) + var want []string + var wg sync.WaitGroup + for p := range 4 { + r := newHTTPInvocationReporter(testTelemetryEndpoint, "token", "test", invocationReporterOptions{ + rt: fx.Runtime(), spoolDir: spool, client: recordingTelemetryClient(http.StatusServiceUnavailable, nil, nil), + flushInterval: time.Hour, + }) + for i := range 5 { + c := fmt.Sprintf("p%d-%d", p, i) + want = append(want, c) + r.Report(InvocationEvent{Surface: "cli", Command: c}) + } + wg.Add(1) + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + r.Close(ctx) + }() + } + wg.Wait() + + var mu sync.Mutex + var got []string + r := newHTTPInvocationReporter(testTelemetryEndpoint, "token", "test", invocationReporterOptions{ + rt: fx.Runtime(), spoolDir: spool, client: recordingTelemetryClient(http.StatusNoContent, &mu, &got), + flushInterval: time.Hour, + }) + require.Eventually(t, func() bool { mu.Lock(); defer mu.Unlock(); return len(got) >= len(want) }, 2*time.Second, 5*time.Millisecond) + r.Close(context.Background()) + mu.Lock() + defer mu.Unlock() + sort.Strings(got) + sort.Strings(want) + require.Equal(t, want, got) + require.Empty(t, spoolFilesForTest(t, fx, spool)) +} + +// Processes that start at the same time split the spool between them: each +// spooled event is sent exactly once. +func TestInvocationSpoolConcurrentStartsSendEachEventOnce(t *testing.T) { + fx, spool := spoolFixture(t) + var want []string + for f := range 20 { + var batch []InvocationEvent + for i := range 3 { + c := fmt.Sprintf("f%d-%d", f, i) + want = append(want, c) + batch = append(batch, InvocationEvent{Surface: "cli", Command: c}) + } + writeSpoolForTest(t, fx, spool, batch) + } + + var mu sync.Mutex + var got []string + reporters := make([]*httpInvocationReporter, 4) + var wg sync.WaitGroup + for i := range reporters { + wg.Add(1) + go func() { + defer wg.Done() + reporters[i] = newHTTPInvocationReporter(testTelemetryEndpoint, "token", "test", invocationReporterOptions{ + rt: fx.Runtime(), spoolDir: spool, client: recordingTelemetryClient(http.StatusNoContent, &mu, &got), + flushInterval: time.Hour, + }) + }() + } + wg.Wait() + require.Eventually(t, func() bool { mu.Lock(); defer mu.Unlock(); return len(got) >= len(want) }, 2*time.Second, 5*time.Millisecond) + for _, r := range reporters { + r.Close(context.Background()) + } + mu.Lock() + defer mu.Unlock() + sort.Strings(got) + sort.Strings(want) + require.Equal(t, want, got) + require.Empty(t, spoolFilesForTest(t, fx, spool)) +} diff --git a/pkg/tapper/invocation_telemetry_test.go b/pkg/tapper/invocation_telemetry_test.go index f08bdf86..42422dc7 100644 --- a/pkg/tapper/invocation_telemetry_test.go +++ b/pkg/tapper/invocation_telemetry_test.go @@ -3,8 +3,13 @@ package tapper import ( "context" "encoding/json" + "errors" + "fmt" "io" "net/http" + "os" + "path/filepath" + "sort" "strings" "sync" "sync/atomic" @@ -12,6 +17,7 @@ import ( "time" "github.com/jlrickert/cli-toolkit/sandbox" + "github.com/jlrickert/tapper/pkg/apicontract" "github.com/stretchr/testify/require" ) @@ -19,7 +25,7 @@ type telemetryRoundTripFunc func(*http.Request) (*http.Response, error) func (f telemetryRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { if req.URL.Path == "/api/version" { - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"server_version":"test","api_versions":["2026-10-12"]}`))}, nil + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"server_version":"test","api_versions":["` + apicontract.Revision + `"]}`))}, nil } return f(req) @@ -42,7 +48,7 @@ func TestResolveInvocationTelemetryTargetDefaultOnAndOptOut(t *testing.T) { t.Run("default on", func(t *testing.T) { fx, tap := telemetryTargetFixture(t, base) - endpoint, token, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) + endpoint, token, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) require.True(t, ok) require.Equal(t, "https://hub.example.com/api/v1/telemetry/invocations", endpoint) require.Equal(t, "stored-token", token) @@ -50,14 +56,14 @@ func TestResolveInvocationTelemetryTargetDefaultOnAndOptOut(t *testing.T) { t.Run("user config opt out", func(t *testing.T) { fx, tap := telemetryTargetFixture(t, "disableTelemetry: true\n"+base) - _, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) + _, _, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) require.False(t, ok) }) t.Run("environment opt out", func(t *testing.T) { fx, tap := telemetryTargetFixture(t, base) require.NoError(t, fx.Runtime().Env().Set("TAP_DISABLE_TELEMETRY", "1")) - _, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) + _, _, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) require.False(t, ok) }) @@ -66,7 +72,7 @@ func TestResolveInvocationTelemetryTargetDefaultOnAndOptOut(t *testing.T) { projectConfig := "/home/testuser/.tapper/config.yaml" require.NoError(t, fx.Runtime().Mkdir("/home/testuser/.tapper", 0o755, true)) require.NoError(t, fx.Runtime().AtomicWriteFile(projectConfig, []byte("hub: attacker\n"), 0o644)) - endpoint, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) + endpoint, _, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) require.False(t, ok) require.Empty(t, endpoint) }) @@ -76,17 +82,17 @@ func TestResolveInvocationTelemetryTargetSilentlySkipsUnavailableState(t *testin fx := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) tap, err := NewTap(TapOptions{Root: "/home/testuser", Runtime: fx.Runtime()}) require.NoError(t, err) - _, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) + _, _, _, ok := resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) require.False(t, ok, "unbootstrapped client must skip") require.NoError(t, fx.Runtime().AtomicWriteFile(tap.PathService.UserConfig(), []byte("hub: local\nhubs:\n local: {}\n"), 0o644)) tap.ConfigService.Reload() - _, _, ok = resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) + _, _, _, ok = resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) require.False(t, ok, "client whose selected hub has no URL must skip") require.NoError(t, fx.Runtime().AtomicWriteFile(tap.PathService.UserConfig(), []byte("hub: remote\nhubs:\n remote:\n url: https://hub.example.com\n"), 0o644)) tap.ConfigService.Reload() - _, _, ok = resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) + _, _, _, ok = resolveInvocationTelemetryTarget(fx.Runtime(), tap.ConfigService) require.False(t, ok, "unauthenticated client must skip") } @@ -111,6 +117,11 @@ func TestHTTPInvocationReporterPayloadIsMinimalAndBatched(t *testing.T) { interactive := true r.Report(InvocationEvent{Surface: "cli", Command: "tap keg list", DurationMS: 10, Success: true, Interactive: &interactive}) r.Report(InvocationEvent{Surface: "mcp", Tool: "keg_list", DurationMS: 20, Success: false}) + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(requests) > 0 + }, time.Second, time.Millisecond, "a full batch is sent without waiting for Close") ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() r.Close(ctx) @@ -212,14 +223,16 @@ func TestHTTPInvocationReporterInjectsAgent(t *testing.T) { client: client, batchSize: 1, flushInterval: time.Hour, agent: "qwen", }) r.Report(InvocationEvent{Surface: "mcp", Tool: "list"}) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - r.Close(ctx) + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r.Close(ctx) + }() select { case body := <-bodies: require.Contains(t, string(body), `"agent":"qwen"`) - default: + case <-time.After(time.Second): t.Fatal("no telemetry batch was sent") } } @@ -251,3 +264,174 @@ func TestHTTPInvocationReporterConcurrentReportAndClose(t *testing.T) { defer cancel() r.Close(ctx) } + +func spoolFixture(t *testing.T) (*sandbox.Sandbox, string) { + t.Helper() + fx := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + tap, err := NewTap(TapOptions{Root: "/home/testuser", Runtime: fx.Runtime()}) + require.NoError(t, err) + return fx, invocationTelemetrySpoolDirFor(tap.ConfigService, "testkey") +} + +// spoolFilesForTest lists the unclaimed spool files in dir, oldest first. +func spoolFilesForTest(t *testing.T, fx *sandbox.Sandbox, dir string) []string { + t.Helper() + entries, err := fx.Runtime().ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + require.NoError(t, err) + var names []string + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".json") && !strings.Contains(entry.Name(), invocationTelemetryClaimMarker) { + names = append(names, entry.Name()) + } + } + sort.Strings(names) + return names +} + +// readSpoolForTest returns every event spooled in dir, oldest first. +func readSpoolForTest(t *testing.T, fx *sandbox.Sandbox, dir string) []InvocationEvent { + t.Helper() + var events []InvocationEvent + for _, name := range spoolFilesForTest(t, fx, dir) { + raw, err := fx.Runtime().ReadFile(filepath.Join(dir, name)) + require.NoError(t, err) + var batch []InvocationEvent + require.NoError(t, json.Unmarshal(raw, &batch)) + events = append(events, batch...) + } + return events +} + +var spoolTestSeq atomic.Int64 + +// writeSpoolForTest adds one spool file holding events to dir. +func writeSpoolForTest(t *testing.T, fx *sandbox.Sandbox, dir string, events []InvocationEvent) { + t.Helper() + raw, err := json.Marshal(events) + require.NoError(t, err) + require.NoError(t, fx.Runtime().Mkdir(dir, 0o700, true)) + name := fmt.Sprintf("%020d-test.json", spoolTestSeq.Add(1)) + require.NoError(t, fx.Runtime().AtomicWriteFile(filepath.Join(dir, name), raw, 0o600)) +} + +func noContent() (*http.Response, error) { + return &http.Response{StatusCode: http.StatusNoContent, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil +} + +// A slow Hub must not hold up exit: Close waits only briefly for the send in +// flight, then spools it with everything still queued. +func TestHTTPInvocationReporterCloseSpoolsInsteadOfWaiting(t *testing.T) { + fx, spool := spoolFixture(t) + started := make(chan struct{}, 1) + client := &http.Client{Transport: telemetryRoundTripFunc(func(req *http.Request) (*http.Response, error) { + started <- struct{}{} + <-req.Context().Done() + return nil, req.Context().Err() + })} + r := newHTTPInvocationReporter("https://hub.example.com/api/v1/telemetry/invocations", "token", "test", invocationReporterOptions{ + rt: fx.Runtime(), spoolDir: spool, client: client, batchSize: 1, flushInterval: time.Hour, requestTimeout: time.Minute, + }) + r.Report(InvocationEvent{Surface: "cli", Command: "tap first"}) + <-started + r.Report(InvocationEvent{Surface: "cli", Command: "tap info"}) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + begin := time.Now() + r.Close(ctx) + require.Less(t, time.Since(begin), time.Second, "Close must not wait out the request timeout") + + events := readSpoolForTest(t, fx, spool) + require.Len(t, events, 2) + require.Equal(t, "tap first", events[0].Command) + require.Equal(t, "tap info", events[1].Command) +} + +func TestHTTPInvocationReporterSendsAndClearsSpool(t *testing.T) { + fx, spool := spoolFixture(t) + writeSpoolForTest(t, fx, spool, []InvocationEvent{ + {Surface: "cli", Command: "tap one", ClientVersion: "v0.old"}, + {Surface: "cli", Command: "tap two", ClientVersion: "v0.old"}, + }) + bodies := make(chan []InvocationEvent, 4) + client := &http.Client{Transport: telemetryRoundTripFunc(func(req *http.Request) (*http.Response, error) { + var batch []InvocationEvent + _ = json.NewDecoder(req.Body).Decode(&batch) + bodies <- batch + return noContent() + })} + r := newHTTPInvocationReporter("https://hub.example.com/api/v1/telemetry/invocations", "token", "test", invocationReporterOptions{ + rt: fx.Runtime(), spoolDir: spool, client: client, flushInterval: time.Hour, + }) + + select { + case batch := <-bodies: + require.Len(t, batch, 2) + require.Equal(t, "tap one", batch[0].Command) + require.Equal(t, "v0.old", batch[0].ClientVersion, "spooled events keep the version that produced them") + case <-time.After(time.Second): + t.Fatal("the spool was not sent at startup") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r.Close(ctx) + require.Empty(t, spoolFilesForTest(t, fx, spool), "a delivered spool is cleared") +} + +func TestHTTPInvocationReporterSpoolIsBounded(t *testing.T) { + fx, spool := spoolFixture(t) + old := make([]InvocationEvent, invocationTelemetrySpoolLimit) + for i := range old { + old[i] = InvocationEvent{Surface: "cli", Command: fmt.Sprintf("old-%d", i)} + } + writeSpoolForTest(t, fx, spool, old) + client := &http.Client{Transport: telemetryRoundTripFunc(func(_ *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusServiceUnavailable, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil + })} + r := newHTTPInvocationReporter("https://hub.example.com/api/v1/telemetry/invocations", "token", "test", invocationReporterOptions{ + rt: fx.Runtime(), spoolDir: spool, client: client, flushInterval: time.Hour, + }) + for i := range 3 { + r.Report(InvocationEvent{Surface: "cli", Command: fmt.Sprintf("new-%d", i)}) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r.Close(ctx) + + events := readSpoolForTest(t, fx, spool) + require.Len(t, events, invocationTelemetrySpoolLimit) + require.Equal(t, "old-3", events[0].Command, "the oldest events are dropped first") + require.Equal(t, "new-2", events[len(events)-1].Command) +} + +func TestInvocationTelemetryDisabledLeavesSpoolAlone(t *testing.T) { + t.Run("opted out", func(t *testing.T) { + fx, tap := telemetryTargetFixture(t, "disableTelemetry: true\nhub: primary\nhubs:\n primary:\n url: https://hub.example.com\n") + spool := invocationTelemetrySpoolDirFor(tap.ConfigService, "testkey") + writeSpoolForTest(t, fx, spool, []InvocationEvent{{Surface: "cli", Command: "tap kept"}}) + before := readSpoolForTest(t, fx, spool) + + require.Nil(t, NewInvocationReporter(fx.Runtime(), tap.ConfigService, "test")) + require.Equal(t, before, readSpoolForTest(t, fx, spool), "disabled telemetry neither sends nor rewrites the spool") + }) + + t.Run("rejected by the Hub", func(t *testing.T) { + fx, spool := spoolFixture(t) + client := &http.Client{Transport: telemetryRoundTripFunc(func(_ *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusNotFound, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil + })} + r := newHTTPInvocationReporter("https://old.example.com/api/v1/telemetry/invocations", "token", "test", invocationReporterOptions{ + rt: fx.Runtime(), spoolDir: spool, client: client, batchSize: 1, flushInterval: time.Hour, + }) + r.Report(InvocationEvent{Surface: "cli", Command: "tap first"}) + require.Eventually(t, r.disabled.Load, time.Second, time.Millisecond) + r.Report(InvocationEvent{Surface: "cli", Command: "tap second"}) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r.Close(ctx) + require.Empty(t, spoolFilesForTest(t, fx, spool), "a Hub that rejects telemetry gets no spool") + }) +} diff --git a/pkg/tapper/node_ref_resolve.go b/pkg/tapper/node_ref_resolve.go index d0b943da..96b42eeb 100644 --- a/pkg/tapper/node_ref_resolve.go +++ b/pkg/tapper/node_ref_resolve.go @@ -105,7 +105,9 @@ func (t *Tap) ResolveNodeRef(ctx context.Context, ref *keg.NodeRef, rc RefContex func (t *Tap) resolveNodeArg(ctx context.Context, currentKeg keg.Keg, raw string) (keg.Keg, keg.NodeId, error) { ref, err := keg.ParseNodeRef(raw) if err != nil { - return nil, keg.NodeId{}, fmt.Errorf("invalid node ID %q: %w", raw, err) + // raw is the caller's argument; the parser stays unclassified because + // it also reads stored ids. + return nil, keg.NodeId{}, fmt.Errorf("invalid node ID %q: %w: %w", raw, err, keg.ErrInvalid) } return t.ResolveNodeRef(ctx, ref, RefContext{CurrentKeg: currentKeg}) } diff --git a/pkg/tapper/relationship_observer_test.go b/pkg/tapper/relationship_observer_test.go index 4a82b1f0..1d3480e7 100644 --- a/pkg/tapper/relationship_observer_test.go +++ b/pkg/tapper/relationship_observer_test.go @@ -33,9 +33,9 @@ func TestRelationshipObserverOnlyActualPairsInResultPage(t *testing.T) { }) var err error if back { - _, err = tap.Backlinks(ctx, BacklinksOptions{NodeIDs: []string{"1", "2"}, IdOnly: true, Offset: 1, Limit: 1}) + _, err = tap.Backlinks(ctx, BacklinksOptions{NodeIDs: []string{"1", "2"}, Offset: 1, Limit: 1}) } else { - _, err = tap.Links(ctx, LinksOptions{NodeIDs: []string{"1", "2"}, IdOnly: true, Offset: 1, Limit: 1}) + _, err = tap.Links(ctx, LinksOptions{NodeIDs: []string{"1", "2"}, Offset: 1, Limit: 1}) } if err != nil { t.Fatal(err) @@ -85,9 +85,9 @@ func TestRelationshipObserverFullIdentitiesAndFailures(t *testing.T) { ctx := WithRelationshipResultObserver(context.Background(), func(_ context.Context, _ keg.Keg, p []Relationship) { got = p }) var err error if back { - _, err = tap.Backlinks(ctx, BacklinksOptions{NodeIDs: []string{"1", "2"}, IdOnly: true}) + _, err = tap.Backlinks(ctx, BacklinksOptions{NodeIDs: []string{"1", "2"}}) } else { - _, err = tap.Links(ctx, LinksOptions{NodeIDs: []string{"1", "2"}, IdOnly: true}) + _, err = tap.Links(ctx, LinksOptions{NodeIDs: []string{"1", "2"}}) } if fail != "" { if err == nil || len(got) != 0 { diff --git a/pkg/tapper/tap.go b/pkg/tapper/tap.go index 78924a8e..c627c5a8 100644 --- a/pkg/tapper/tap.go +++ b/pkg/tapper/tap.go @@ -56,6 +56,10 @@ type Tap struct { // manifest it extracts, so an installed plugin always reports the tap // that installed it rather than whatever was committed in the repo. Version string + + // hubContextProbe remembers Hubs that predate GET /api/v1/info. Nil + // (a Tap built without NewTap) asks every time. + hubContextProbe *hubContextProbe } // DevPluginVersion is the version an unreleased binary reports. Manifests must @@ -124,14 +128,15 @@ func NewTap(opts TapOptions) (*Tap, error) { KegService: kegService, } return &Tap{ - Runtime: rt, - Root: opts.Root, - PathService: pathService, - ConfigService: configService, - KegService: kegService, - FlightService: flightService, - AuthValidateFn: ValidateToken, - Version: normalizeVersion(opts.Version), + Runtime: rt, + Root: opts.Root, + PathService: pathService, + ConfigService: configService, + KegService: kegService, + FlightService: flightService, + AuthValidateFn: ValidateToken, + Version: normalizeVersion(opts.Version), + hubContextProbe: &hubContextProbe{}, }, nil } @@ -165,6 +170,14 @@ type KegTargetOptions struct { // orient. Leave false for MCP and other agent-facing surfaces. BypassFlightRestrictions bool + // KegAdministration marks a call that administers the KEG itself (its + // settings) rather than its notes. A Hub resolver (KegResolver) decides + // it with the session flight's full authority, which may reach past the + // flight's cover when the flight manages the namespace's KEGs, so the + // client-side cover check below defers to it there. Without a resolver + // the check applies as for any call. + KegAdministration bool + // RequireBootstrap makes config-driven resolution fail with // ErrNotBootstrapped when no user config exists. Set by the full `tap` // surface and the MCP server; direct Tap API callers may leave it false. @@ -204,7 +217,7 @@ func (t *Tap) resolveKegForRoles(ctx context.Context, opts KegTargetOptions, ide if err != nil { return nil, err } - if !opts.BypassFlightRestrictions && opts.FlightContext != nil { + if !opts.BypassFlightRestrictions && !opts.KegAdministration && opts.FlightContext != nil { if err := t.enforceFlightSnapshot(opts.FlightContext, k, flightRole); err != nil { return nil, err } diff --git a/pkg/tapper/tap_archive.go b/pkg/tapper/tap_archive.go index a8d252fe..cd269c0e 100644 --- a/pkg/tapper/tap_archive.go +++ b/pkg/tapper/tap_archive.go @@ -1,18 +1,14 @@ package tapper import ( - "bytes" "context" "fmt" "io" - "net/http" - "os" "path/filepath" "slices" "strings" "github.com/jlrickert/cli-toolkit/toolkit" - "github.com/jlrickert/tapper/pkg/apicontract" "github.com/jlrickert/tapper/pkg/keg" ) @@ -23,11 +19,6 @@ type ExportOptions struct { OutputPath string } -type ImportOptions struct { - KegTargetOptions - Input string -} - // Export writes a keg-archive of the selected nodes (all nodes when none are // named) to opts.OutputPath and returns the resolved output path. func (t *Tap) Export(ctx context.Context, opts ExportOptions) (string, error) { @@ -74,31 +65,6 @@ func (t *Tap) Export(ctx context.Context, opts ExportOptions) (string, error) { return output, nil } -// Import loads a keg-archive from a file path or http(s) URL into the -// resolved keg and returns the imported node ids. -func (t *Tap) Import(ctx context.Context, opts ImportOptions) ([]keg.NodeId, error) { - k, err := t.resolveKegForRole(ctx, opts.KegTargetOptions, FlightRoleEditor) - if err != nil { - return nil, fmt.Errorf("unable to open keg: %w", err) - } - - archiveBytes, err := readArchiveInput(ctx, t.Runtime, opts.Input) - if err != nil { - return nil, err - } - - imported, err := k.ImportNodes(ctx, bytes.NewReader(archiveBytes), keg.ImportNodesOptions{}) - if err != nil { - return nil, err - } - - ids := make([]keg.NodeId, 0, len(imported)) - for _, node := range imported { - ids = append(ids, node.ID) - } - return ids, nil -} - // exportNodeIDs parses explicit export node-id arguments. An empty list means // "all nodes" and is passed through as nil. func exportNodeIDs(raw []string) ([]keg.NodeId, error) { @@ -132,45 +98,3 @@ func expandArchivePath(rt *toolkit.Runtime, raw string) (string, error) { } return filepath.Clean(path), nil } - -func readArchiveInput(ctx context.Context, rt *toolkit.Runtime, input string) ([]byte, error) { - if strings.HasPrefix(input, "http://") || strings.HasPrefix(input, "https://") { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, input, nil) - if err != nil { - return nil, fmt.Errorf("unable to create archive request: %w", err) - } - resp, err := apicontract.Do(http.DefaultClient, req) - if err != nil { - return nil, fmt.Errorf("unable to download archive: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("unable to download archive: status %d", resp.StatusCode) - } - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("unable to read archive download: %w", err) - } - return data, nil - } - - path, err := expandArchivePath(rt, input) - if err != nil { - return nil, err - } - resolved, err := rt.ResolvePath(path, false) - if err != nil { - return nil, fmt.Errorf("unable to resolve archive path %s: %w", path, err) - } - if _, err := rt.Stat(resolved, false); err != nil { - if os.IsNotExist(err) { - return nil, fmt.Errorf("archive not found: %s: %w", resolved, err) - } - return nil, fmt.Errorf("unable to stat archive %s: %w", resolved, err) - } - data, err := rt.ReadFile(resolved) - if err != nil { - return nil, fmt.Errorf("unable to read archive %s: %w", resolved, err) - } - return data, nil -} diff --git a/pkg/tapper/tap_auth.go b/pkg/tapper/tap_auth.go index 3e330e62..6a13e4d4 100644 --- a/pkg/tapper/tap_auth.go +++ b/pkg/tapper/tap_auth.go @@ -623,6 +623,13 @@ type AuthLogoutResult struct { func (t *Tap) AuthLogout(ctx context.Context, opts AuthLogoutOptions) (*AuthLogoutResult, error) { rt := t.Runtime storePath := t.PathService.AuthStorePath() + // Hold the store lock so a refresh running in another tap process + // cannot write the entry back after it is removed. + unlock, err := lockAuthStore(ctx, rt, storePath, authStoreLockWait) + if err != nil { + return nil, fmt.Errorf("auth logout: %w", err) + } + defer unlock() store, err := LoadAuthStore(ctx, rt, storePath) if err != nil { return nil, err diff --git a/pkg/tapper/tap_batch.go b/pkg/tapper/tap_batch.go index b99002e3..5330f917 100644 --- a/pkg/tapper/tap_batch.go +++ b/pkg/tapper/tap_batch.go @@ -70,7 +70,7 @@ func (t *Tap) EditBatch(ctx context.Context, opts BatchEditOptions) ([]keg.NodeU ctx = keg.WithDefaultValidationActor(ctx, keg.ValidationActorHuman) updates := make([]keg.NodeUpdateOptions, len(opts.Edits)) for i, item := range opts.Edits { - update, err := batchEditUpdate(ctx, i, item) + update, err := batchEditUpdate(ctx, i, opts.Keg, item) if err != nil { return nil, err } @@ -80,18 +80,20 @@ func (t *Tap) EditBatch(ctx context.Context, opts BatchEditOptions) ([]keg.NodeU } // batchEditUpdate checks one edit item and turns it into the keg update. -func batchEditUpdate(ctx context.Context, i int, item BatchEditItem) (keg.NodeUpdateOptions, error) { +// kegAlias is the keg the caller targeted, carried into a missing-hash error +// so it can name the node_read that fixes it. +func batchEditUpdate(ctx context.Context, i int, kegAlias string, item BatchEditItem) (keg.NodeUpdateOptions, error) { id, err := keg.ParseNode(item.NodeID) if err != nil { - return keg.NodeUpdateOptions{}, fmt.Errorf("edit %d node %q: %w", i, item.NodeID, err) + return keg.NodeUpdateOptions{}, fmt.Errorf("edit %d node %q: %w: %w", i, item.NodeID, err, keg.ErrInvalid) } if !item.HasContent && !item.HasContentPatch && !item.HasMeta { - return keg.NodeUpdateOptions{}, fmt.Errorf("edit %d node %q: content, patch, or meta is required: %w", i, item.NodeID, keg.ErrInvalid) + return keg.NodeUpdateOptions{}, &EmptyEditError{Item: i, NodeID: item.NodeID} } if item.HasContent && item.HasContentPatch { return keg.NodeUpdateOptions{}, fmt.Errorf("edit %d node %q: content and patch are alternatives; send one: %w", i, item.NodeID, keg.ErrInvalid) } - if err := requireEditPartHashes(i, item); err != nil { + if err := requireEditPartHashes(i, kegAlias, item); err != nil { return keg.NodeUpdateOptions{}, err } update := keg.NodeUpdateOptions{ @@ -120,6 +122,21 @@ func batchEditUpdate(ctx context.Context, i int, item BatchEditItem) (keg.NodeUp return update, nil } +// EmptyEditError reports an edit item that supplies none of content, patch, +// or meta, so it has nothing to change. It unwraps to keg.ErrInvalid; the +// typed form lets a surface say which field to add instead of a generic +// malformed-request hint. +type EmptyEditError struct { + Item int + NodeID string +} + +func (e *EmptyEditError) Error() string { + return fmt.Sprintf("item %d (node %q) has nothing to change: supply content, patch, or meta: %s", e.Item, e.NodeID, keg.ErrInvalid) +} + +func (e *EmptyEditError) Unwrap() error { return keg.ErrInvalid } + type BatchSnapshotItem struct { NodeID string Message string @@ -138,7 +155,7 @@ func (t *Tap) NodeSnapshotBatch(ctx context.Context, opts BatchSnapshotOptions) for i, item := range opts.Nodes { id, err := keg.ParseNode(item.NodeID) if err != nil { - return nil, fmt.Errorf("snapshot %d node %q: %w", i, item.NodeID, err) + return nil, fmt.Errorf("snapshot %d node %q: %w: %w", i, item.NodeID, err, keg.ErrInvalid) } nodes[i] = keg.NodeSnapshotRequest{ID: *id, Message: item.Message} } @@ -149,21 +166,28 @@ func (t *Tap) NodeSnapshotBatch(ctx context.Context, opts BatchSnapshotOptions) // part's token. It applies whenever the caller holds no combined ExpectedHash // (node_edit never sends one), so an unguarded part can never fall through to // the keg layer's combined-hash path. It runs after keg resolution so -// authorization failures still surface first. -func requireEditPartHashes(i int, item BatchEditItem) error { +// authorization failures still surface first. Every missing token is reported +// at once, so one node_read is enough to fix the call. +func requireEditPartHashes(i int, kegAlias string, item BatchEditItem) error { if item.ExpectedHash != "" && item.ExpectedContentHash == "" && item.ExpectedMetaHash == "" { return nil } - if item.HasContentPatch && item.ExpectedContentHash == "" { - return fmt.Errorf("edit %d node %q: patch requires expected_content_hash (the content_hash from node_read): %w", i, item.NodeID, keg.ErrPreconditionRequired) - } - if item.HasContent && item.ExpectedContentHash == "" { - return fmt.Errorf("edit %d node %q: content requires expected_content_hash (the content_hash from node_read): %w", i, item.NodeID, keg.ErrPreconditionRequired) + var missing []keg.MissingPartHash + if item.ExpectedContentHash == "" { + switch { + case item.HasContentPatch: + missing = append(missing, keg.MissingPartHash{Part: "patch", Field: "expected_content_hash", TokenField: "content_hash"}) + case item.HasContent: + missing = append(missing, keg.MissingPartHash{Part: "content", Field: "expected_content_hash", TokenField: "content_hash"}) + } } if item.HasMeta && item.ExpectedMetaHash == "" { - return fmt.Errorf("edit %d node %q: meta requires expected_meta_hash (the meta_hash from node_read): %w", i, item.NodeID, keg.ErrPreconditionRequired) + missing = append(missing, keg.MissingPartHash{Part: "meta", Field: "expected_meta_hash", TokenField: "meta_hash"}) + } + if len(missing) == 0 { + return nil } - return nil + return &keg.PartPreconditionRequiredError{Item: i, Keg: kegAlias, NodeID: item.NodeID, Missing: missing} } type MergeOptions struct { @@ -184,12 +208,12 @@ func (t *Tap) Merge(ctx context.Context, opts MergeOptions) (keg.MergeNodesResul ctx = keg.WithDefaultValidationActor(ctx, keg.ValidationActorHuman) source, err := keg.ParseNode(opts.SourceID) if err != nil { - return keg.MergeNodesResult{}, fmt.Errorf("merge source %q: %w", opts.SourceID, err) + return keg.MergeNodesResult{}, fmt.Errorf("merge source %q: %w: %w", opts.SourceID, err, keg.ErrInvalid) } if !opts.Target.HasContent && !opts.Target.HasContentPatch { return keg.MergeNodesResult{}, fmt.Errorf("merge into %q: content or patch is required, since a merge rewrites the target: %w", opts.Target.NodeID, keg.ErrInvalid) } - update, err := batchEditUpdate(ctx, 0, opts.Target) + update, err := batchEditUpdate(ctx, 0, opts.Keg, opts.Target) if err != nil { return keg.MergeNodesResult{}, err } diff --git a/pkg/tapper/tap_cat.go b/pkg/tapper/tap_cat.go index 4ae06dbb..91b0cb97 100644 --- a/pkg/tapper/tap_cat.go +++ b/pkg/tapper/tap_cat.go @@ -62,7 +62,7 @@ func (t *Tap) Cat(ctx context.Context, opts CatOptions) (string, error) { nodeIDs := opts.NodeIDs if opts.Query != "" { if len(nodeIDs) > 0 { - return "", fmt.Errorf("cannot specify both node IDs and --query") + return "", fmt.Errorf("cannot specify both node IDs and --query: %w", keg.ErrInvalid) } } @@ -149,7 +149,7 @@ func FormatCatViews(ctx context.Context, views []keg.NodeView, opts CatOptions) func (t *Tap) CatViews(ctx context.Context, opts CatOptions) ([]keg.NodeView, error) { nodeIDs := opts.NodeIDs if opts.Query != "" && len(nodeIDs) > 0 { - return nil, fmt.Errorf("cannot specify both node IDs and --query") + return nil, fmt.Errorf("cannot specify both node IDs and --query: %w", keg.ErrInvalid) } if len(nodeIDs) == 0 && opts.Query == "" { return nil, nil diff --git a/pkg/tapper/tap_files.go b/pkg/tapper/tap_files.go index 673b45b1..c791f616 100644 --- a/pkg/tapper/tap_files.go +++ b/pkg/tapper/tap_files.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "path/filepath" "github.com/jlrickert/tapper/pkg/keg" @@ -29,6 +30,8 @@ type DownloadFileOptions struct { NodeID string Name string Dest string + // NoOverwrite creates a private new file and refuses existing destinations. + NoOverwrite bool } // DeleteFileOptions configures behavior for Tap.DeleteFile. @@ -59,6 +62,8 @@ type DownloadImageOptions struct { NodeID string Name string Dest string + // NoOverwrite creates a private new file and refuses existing destinations. + NoOverwrite bool } // ReadImageOptions configures behavior for Tap.ReadImage. @@ -111,7 +116,7 @@ func (t *Tap) UploadFile(ctx context.Context, opts UploadFileOptions) (string, e return "", fmt.Errorf("unable to inspect node: %w", err) } if !exists { - return "", fmt.Errorf("node %s not found in %s", id.Path(), describeKeg(k)) + return "", fmt.Errorf("node %s not found in %s: %w", id.Path(), describeKeg(k), keg.ErrNotExist) } data := opts.Data if data == nil { @@ -163,7 +168,7 @@ func (t *Tap) DownloadFile(ctx context.Context, opts DownloadFileOptions) (strin } dest = filepath.Join(cwd, opts.Name) } - if err := t.Runtime.WriteFile(dest, data, 0o644); err != nil { + if err := t.writeDownload(dest, data, opts.NoOverwrite); err != nil { return "", fmt.Errorf("unable to write file to %q: %w", dest, err) } return dest, nil @@ -221,7 +226,7 @@ func (t *Tap) UploadImage(ctx context.Context, opts UploadImageOptions) (string, return "", fmt.Errorf("unable to inspect node: %w", err) } if !exists { - return "", fmt.Errorf("node %s not found in %s", id.Path(), describeKeg(k)) + return "", fmt.Errorf("node %s not found in %s: %w", id.Path(), describeKeg(k), keg.ErrNotExist) } data := opts.Data if data == nil { @@ -294,7 +299,7 @@ func (t *Tap) DownloadImage(ctx context.Context, opts DownloadImageOptions) (str } dest = filepath.Join(cwd, opts.Name) } - if err := t.Runtime.WriteFile(dest, data, 0o644); err != nil { + if err := t.writeDownload(dest, data, opts.NoOverwrite); err != nil { return "", fmt.Errorf("unable to write image to %q: %w", dest, err) } return dest, nil @@ -341,3 +346,18 @@ func (t *Tap) ReadAttachment(ctx context.Context, opts ReadAttachmentOptions) ([ } return data, keg.AttachmentContentType(opts.Kind, opts.Name, data), nil } + +func (t *Tap) writeDownload(dest string, data []byte, exclusive bool) error { + if !exclusive { + return t.Runtime.WriteFile(dest, data, 0o644) + } + f, err := t.Runtime.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/pkg/tapper/tap_flight.go b/pkg/tapper/tap_flight.go index 434da970..d511bf9b 100644 --- a/pkg/tapper/tap_flight.go +++ b/pkg/tapper/tap_flight.go @@ -33,6 +33,10 @@ type CreateFlightOptions struct { Instructions string Cover []FlightCover Subflights []string + // ManageKegs and ManageFlights are the flight's switches + // (FlightManifest.ManageKegs, FlightManifest.ManageFlights). + ManageKegs bool + ManageFlights bool } // UpdateFlightOptions is a partial update: nil fields keep the flight's @@ -41,14 +45,16 @@ type CreateFlightOptions struct { // slug that exists in several places cannot read one flight and // overwrite another. type UpdateFlightOptions struct { - Ref string - Title *string - Description *string `json:"description,omitempty" jsonschema:"short description, separate from instructions"` - Visibility *string - Instructions *string - Cover *[]FlightCover - Subflights *[]string - ExpectedHash string + Ref string + Title *string + Description *string `json:"description,omitempty" jsonschema:"short description, separate from instructions"` + Visibility *string + Instructions *string + Cover *[]FlightCover + Subflights *[]string + ManageKegs *bool + ManageFlights *bool + ExpectedHash string } type DeleteFlightOptions struct { @@ -62,9 +68,13 @@ func (t *Tap) ListFlights(ctx context.Context, opts ListFlightsOptions) ([]strin return t.FlightService.ListFlights(ctx, opts.Hub, opts.Warnings) } -// GetFlight loads a single flight by name. +// GetFlight loads a single flight by name, as the Hub holds it now. It is +// the inspection read behind `tap flight show` and MCP flight_read, and its +// hash is the precondition for a guarded edit or delete, so it never serves +// the process cache that per-operation flight gating uses: a long-running +// `tap mcp` would otherwise show a flight as it was when first read. func (t *Tap) GetFlight(ctx context.Context, opts GetFlightOptions) (*Flight, error) { - return t.FlightService.GetFlight(ctx, opts.Name) + return t.FlightService.GetFlightFresh(ctx, opts.Name) } func (t *Tap) CreateFlight(ctx context.Context, opts CreateFlightOptions) (*Flight, error) { @@ -85,6 +95,12 @@ func (t *Tap) CreateFlight(ctx context.Context, opts CreateFlightOptions) (*Flig Cover: hubCoverFromFlightCover(opts.Cover), Subflights: append([]string(nil), opts.Subflights...), } + if opts.ManageKegs { + flight.ManageKegs = boolRef(true) + } + if opts.ManageFlights { + flight.ManageFlights = boolRef(true) + } hf, err := CreateHubFlight(ctx, entry.URL, t.FlightService.hubToken(entry), ref.Namespace, flight) if err != nil { return nil, err @@ -135,6 +151,16 @@ func (t *Tap) UpdateFlight(ctx context.Context, opts UpdateFlightOptions) (*Flig if opts.Subflights != nil { next.Subflights = append([]string(nil), (*opts.Subflights)...) } + // The switches are sent only when the caller changes them: leaving one + // out keeps the flight's current value, so an edit of anything else + // never touches what the flight lets a request manage. + next.Manage, next.ManageKegs, next.ManageFlights = nil, nil, nil + if opts.ManageKegs != nil { + next.ManageKegs = boolRef(*opts.ManageKegs) + } + if opts.ManageFlights != nil { + next.ManageFlights = boolRef(*opts.ManageFlights) + } hf, err := UpdateHubFlight(ctx, entry.URL, token, ref.Namespace, ref.Slug, next, opts.ExpectedHash) if err != nil { return nil, err @@ -143,6 +169,8 @@ func (t *Tap) UpdateFlight(ctx context.Context, opts UpdateFlightOptions) (*Flig return flightFromHub(*hf, hubName), nil } +func boolRef(v bool) *bool { return &v } + func normalizeFlightVisibility(visibility string) string { visibility = strings.TrimSpace(visibility) if visibility == "" { diff --git a/pkg/tapper/tap_flight_edit.go b/pkg/tapper/tap_flight_edit.go index 03648107..1b0e7796 100644 --- a/pkg/tapper/tap_flight_edit.go +++ b/pkg/tapper/tap_flight_edit.go @@ -72,6 +72,14 @@ func (t *Tap) EditFlight(ctx context.Context, opts EditFlightOptions) (*Flight, Cover: hubCoverFromFlightCover(m.Cover), Subflights: append([]string(nil), m.Subflights...), } + // The document always shows the switches, so each is sent only + // when the edit changed it; an unchanged one stays the Hub's to keep. + if m.ManageKegs != lastManifest.ManageKegs { + next.ManageKegs = boolRef(m.ManageKegs) + } + if m.ManageFlights != lastManifest.ManageFlights { + next.ManageFlights = boolRef(m.ManageFlights) + } hf, err := UpdateHubFlight(ctx, entry.URL, token, ref.Namespace, ref.Slug, next, expectedHash) if err != nil { return nil, err @@ -125,22 +133,26 @@ func flightEditorTempFilePrefix(ref FlightRef) string { } type flightManifestEditorDocument struct { - Description string `yaml:"description"` - Title string `yaml:"title"` - Visibility string `yaml:"visibility"` - Cover []FlightCover `yaml:"cover"` - Subflights []string `yaml:"subflights"` - Instructions string `yaml:"instructions"` + Description string `yaml:"description"` + Title string `yaml:"title"` + Visibility string `yaml:"visibility"` + Cover []FlightCover `yaml:"cover"` + Subflights []string `yaml:"subflights"` + ManageKegs bool `yaml:"manageKegs"` + ManageFlights bool `yaml:"manageFlights"` + Instructions string `yaml:"instructions"` } func renderFlightManifestEditorDocument(rt *toolkit.Runtime, ref FlightRef, m FlightManifest) ([]byte, error) { canonical := canonicalFlightManifest(m) doc := flightManifestEditorDocument{ Title: canonical.Title, Description: canonical.Description, - Visibility: canonical.Visibility, - Cover: canonical.Cover, - Subflights: canonical.Subflights, - Instructions: canonical.Instructions, + Visibility: canonical.Visibility, + Cover: canonical.Cover, + Subflights: canonical.Subflights, + ManageKegs: canonical.ManageKegs, + ManageFlights: canonical.ManageFlights, + Instructions: canonical.Instructions, } body, err := yaml.Marshal(doc) if err != nil { @@ -149,7 +161,7 @@ func renderFlightManifestEditorDocument(rt *toolkit.Runtime, ref FlightRef, m Fl var out bytes.Buffer out.WriteString(schemas.Modeline(rt, schemas.FlightManifest)) - fmt.Fprintf(&out, "# Flight %s. Ref is immutable; edit title, visibility, cover, and instructions.\n", ref.Canonical()) + fmt.Fprintf(&out, "# Flight %s. Ref is immutable; edit title, visibility, cover, manageKegs, manageFlights, and instructions.\n", ref.Canonical()) out.Write(body) if !bytes.HasSuffix(out.Bytes(), []byte("\n")) { out.WriteByte('\n') @@ -158,12 +170,14 @@ func renderFlightManifestEditorDocument(rt *toolkit.Runtime, ref FlightRef, m Fl } type comparableFlightManifest struct { - Description string - Title string - Visibility string - Cover []FlightCover - Subflights []string - Instructions string + Description string + Title string + Visibility string + Cover []FlightCover + Subflights []string + ManageKegs bool + ManageFlights bool + Instructions string } func canonicalFlightManifest(m FlightManifest) comparableFlightManifest { @@ -184,17 +198,19 @@ func canonicalFlightManifest(m FlightManifest) comparableFlightManifest { } return comparableFlightManifest{ Title: m.Title, Description: m.Description, - Visibility: m.Visibility, - Cover: cover, - Subflights: append([]string(nil), m.Subflights...), - Instructions: m.Instructions, + Visibility: m.Visibility, + Cover: cover, + Subflights: append([]string(nil), m.Subflights...), + ManageKegs: m.ManageKegs, + ManageFlights: m.ManageFlights, + Instructions: m.Instructions, } } func flightManifestSemanticallyEqual(a, b FlightManifest) bool { ca := canonicalFlightManifest(a) cb := canonicalFlightManifest(b) - if ca.Description != cb.Description || ca.Title != cb.Title || ca.Visibility != cb.Visibility || ca.Instructions != cb.Instructions || len(ca.Cover) != len(cb.Cover) || len(ca.Subflights) != len(cb.Subflights) { + if ca.Description != cb.Description || ca.Title != cb.Title || ca.Visibility != cb.Visibility || ca.Instructions != cb.Instructions || ca.ManageKegs != cb.ManageKegs || ca.ManageFlights != cb.ManageFlights || len(ca.Cover) != len(cb.Cover) || len(ca.Subflights) != len(cb.Subflights) { return false } for i := range ca.Cover { diff --git a/pkg/tapper/tap_flight_edit_test.go b/pkg/tapper/tap_flight_edit_test.go index 45c44f63..06e695cc 100644 --- a/pkg/tapper/tap_flight_edit_test.go +++ b/pkg/tapper/tap_flight_edit_test.go @@ -28,6 +28,9 @@ type flightEditFixture struct { mu sync.Mutex current tapper.HubFlight puts []tapper.HubFlight + // received holds each PUT body exactly as sent, before the fake Hub + // fills in what a write left out. + received []tapper.HubFlight } func newFlightEditFixture(t *testing.T) *flightEditFixture { @@ -55,6 +58,17 @@ func newFlightEditFixture(t *testing.T) *flightEditFixture { body.Namespace = "foldwise" body.Slug = "agent-work" fx.mu.Lock() + fx.received = append(fx.received, body) + // The Hub keeps a switch when a write leaves it out. + if body.ManageKegs == nil { + body.ManageKegs = fx.current.ManageKegs + } + if body.ManageFlights == nil { + body.ManageFlights = fx.current.ManageFlights + } + if body.ManageKegs == nil && body.ManageFlights == nil { + body.Manage = fx.current.Manage + } fx.puts = append(fx.puts, body) fx.current = body fx.mu.Unlock() @@ -298,7 +312,9 @@ func TestEditFlight_EditorStartsWithSchemaBackedManifest(t *testing.T) { // is published on main. require.True(t, strings.HasPrefix(opened, schemas.ModelinePrefix+schemas.ModelineURI(fx.Runtime(), schemas.FlightManifest)+"\n"), "got: %s", opened) - require.Contains(t, opened, "# Flight @foldwise/+agent-work. Ref is immutable; edit title, visibility, cover, and instructions.") + require.Contains(t, opened, "# Flight @foldwise/+agent-work. Ref is immutable; edit title, visibility, cover, manageKegs, manageFlights, and instructions.") + require.Contains(t, opened, "manageKegs: false") + require.Contains(t, opened, "manageFlights: false") require.Contains(t, opened, `title: ""`) require.Contains(t, opened, `visibility: private`) require.NotContains(t, opened, "capabilities") diff --git a/pkg/tapper/tap_flight_manage_test.go b/pkg/tapper/tap_flight_manage_test.go new file mode 100644 index 00000000..895c8686 --- /dev/null +++ b/pkg/tapper/tap_flight_manage_test.go @@ -0,0 +1,151 @@ +package tapper_test + +import ( + "encoding/json" + "net/http" + "sync" + "testing" + + "github.com/jlrickert/tapper/internal/testapi" + "github.com/jlrickert/tapper/pkg/tapper" + "github.com/stretchr/testify/require" +) + +func (fx *flightEditFixture) lastReceived(t *testing.T) tapper.HubFlight { + t.Helper() + fx.mu.Lock() + defer fx.mu.Unlock() + require.NotEmpty(t, fx.received) + return fx.received[len(fx.received)-1] +} + +// A piped manifest that sets a switch sends it, and the flight it returns +// carries it. +func TestEditFlight_PipedManifestSetsSwitches(t *testing.T) { + t.Parallel() + hub := newFlightEditFixture(t) + tap, fx := newFlightEditTap(t, hub.srv.URL) + + flight, err := tap.EditFlight(fx.Context(), tapper.EditFlightOptions{ + Ref: "@foldwise/+agent-work", + Stream: pipedStream("title: Agent Work\nmanageKegs: true\ncover:\n - keg: docs\ninstructions: Stay inside the cover.\n"), + }) + require.NoError(t, err) + require.True(t, flight.ManageKegs) + require.False(t, flight.ManageFlights) + sent := hub.lastReceived(t) + require.NotNil(t, sent.ManageKegs) + require.True(t, *sent.ManageKegs) + require.Nil(t, sent.ManageFlights, "an unchanged switch is not sent") +} + +// A manifest that leaves the switches as they are does not send them, so +// editing a managing flight's title cannot change what it lets a request +// manage; a manifest that drops one clears it. +func TestEditFlight_SwitchesSentOnlyWhenChanged(t *testing.T) { + t.Parallel() + hub := newFlightEditFixture(t) + on := true + hub.setCurrent(tapper.HubFlight{Namespace: "foldwise", Slug: "agent-work", Title: "Agent Work", ManageKegs: &on, ManageFlights: &on, + Cover: []tapper.HubFlightCover{{Namespace: "foldwise", Keg: "docs", Role: "viewer"}}}) + tap, fx := newFlightEditTap(t, hub.srv.URL) + + flight, err := tap.EditFlight(fx.Context(), tapper.EditFlightOptions{ + Ref: "@foldwise/+agent-work", + Stream: pipedStream("title: Retitled\nmanageKegs: true\nmanageFlights: true\ncover:\n - namespace: foldwise\n keg: docs\n"), + }) + require.NoError(t, err) + sent := hub.lastReceived(t) + require.Nil(t, sent.ManageKegs, "an unchanged switch is not sent") + require.Nil(t, sent.ManageFlights) + require.True(t, flight.ManageKegs) + + flight, err = tap.EditFlight(fx.Context(), tapper.EditFlightOptions{ + Ref: "@foldwise/+agent-work", + Stream: pipedStream("title: Retitled\nmanageFlights: true\ncover:\n - namespace: foldwise\n keg: docs\n"), + }) + require.NoError(t, err) + sent = hub.lastReceived(t) + require.NotNil(t, sent.ManageKegs) + require.False(t, *sent.ManageKegs, "a full manifest without a switch clears it") + require.Nil(t, sent.ManageFlights) + require.False(t, flight.ManageKegs) + require.True(t, flight.ManageFlights) +} + +// A partial update sends a switch only when the caller sets it. +func TestUpdateFlight_SwitchesOnlyWhenSet(t *testing.T) { + t.Parallel() + hub := newFlightEditFixture(t) + tap, fx := newFlightEditTap(t, hub.srv.URL) + + title := "Retitled" + _, err := tap.UpdateFlight(fx.Context(), tapper.UpdateFlightOptions{Ref: "@foldwise/+agent-work", Title: &title, ExpectedHash: "h"}) + require.NoError(t, err) + require.Nil(t, hub.lastReceived(t).ManageKegs) + require.Nil(t, hub.lastReceived(t).ManageFlights) + + on := true + flight, err := tap.UpdateFlight(fx.Context(), tapper.UpdateFlightOptions{Ref: "@foldwise/+agent-work", ManageFlights: &on, ExpectedHash: "h"}) + require.NoError(t, err) + sent := hub.lastReceived(t) + require.Nil(t, sent.ManageKegs) + require.NotNil(t, sent.ManageFlights) + require.True(t, *sent.ManageFlights) + require.True(t, flight.ManageFlights) +} + +// Creating a flight sends only the switches it sets; the create body leaves +// the others out. +func TestCreateFlight_Switches(t *testing.T) { + t.Parallel() + var mu sync.Mutex + var bodies []map[string]any + srv := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST /api/v1/@foldwise/flights", r.Method+" "+r.URL.Path) + var raw map[string]any + require.NoError(t, json.NewDecoder(r.Body).Decode(&raw)) + mu.Lock() + bodies = append(bodies, raw) + mu.Unlock() + kegs, _ := raw["manage_kegs"].(bool) + flights, _ := raw["manage_flights"].(bool) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(tapper.HubFlight{Namespace: "foldwise", Slug: raw["slug"].(string), Visibility: "private", ManageKegs: &kegs, ManageFlights: &flights}) + })) + t.Cleanup(srv.Close) + tap, fx := newFlightEditTap(t, srv.URL) + + managing, err := tap.CreateFlight(fx.Context(), tapper.CreateFlightOptions{Ref: "@foldwise/+desk", ManageKegs: true}) + require.NoError(t, err) + require.True(t, managing.ManageKegs) + require.False(t, managing.ManageFlights) + plain, err := tap.CreateFlight(fx.Context(), tapper.CreateFlightOptions{Ref: "@foldwise/+plain"}) + require.NoError(t, err) + require.False(t, plain.ManageKegs) + + mu.Lock() + defer mu.Unlock() + require.Equal(t, true, bodies[0]["manage_kegs"]) + _, sent := bodies[0]["manage_flights"] + require.False(t, sent, "a switch not set is left out") + _, sent = bodies[1]["manage_kegs"] + require.False(t, sent) +} + +// An older Hub answers with the single manage flag; it reads as both +// switches. +func TestFlightFromAnOlderHubReadsManageAsBothSwitches(t *testing.T) { + t.Parallel() + hub := newFlightEditFixture(t) + on := true + hub.setCurrent(tapper.HubFlight{Namespace: "foldwise", Slug: "agent-work", Title: "Agent Work", Manage: &on}) + tap, fx := newFlightEditTap(t, hub.srv.URL) + + title := "Retitled" + flight, err := tap.UpdateFlight(fx.Context(), tapper.UpdateFlightOptions{Ref: "@foldwise/+agent-work", Title: &title, ExpectedHash: "h"}) + require.NoError(t, err) + require.True(t, flight.ManageKegs) + require.True(t, flight.ManageFlights) + require.Nil(t, hub.lastReceived(t).Manage, "the old flag is never sent") +} diff --git a/pkg/tapper/tap_hub.go b/pkg/tapper/tap_hub.go index 37cb7301..fd0b28b5 100644 --- a/pkg/tapper/tap_hub.go +++ b/pkg/tapper/tap_hub.go @@ -87,41 +87,60 @@ func (t *Tap) HubRelayTools(agent string) (endpoint string, token func() string, // hubToken resolves the bearer token for a configured remote hub. It builds a // synthetic target from the hub entry and defers to hubTokenForTarget. func (t *Tap) hubToken(entry HubEntry) string { + target, ok := hubEntryTarget(entry) + if !ok { + return "" + } + return t.hubTokenForTarget(&target) +} + +// hubEntryTarget is the synthetic target a configured hub entry resolves its +// credential through; ok is false for an entry with no URL. +func hubEntryTarget(entry HubEntry) (keg.Target, bool) { url := hubURLWithScheme(strings.TrimSpace(entry.URL)) if url == "" { - return "" + return keg.Target{}, false } - target := keg.Target{ + return keg.Target{ Url: strings.TrimRight(url, "/"), HubURL: url, Token: entry.Token, TokenEnv: entry.TokenEnv, - } - return t.hubTokenForTarget(&target) + }, true } // hubTokenForTarget resolves the bearer token for a remote target: TokenEnv // (environment variable) → inline Token → the AuthStore keyed by the hub URL // (the token `tap auth login` persists). Returns "" when no credential applies. func (t *Tap) hubTokenForTarget(target *keg.Target) string { + token, _ := t.resolveHubToken(target) + return token +} + +// resolveHubToken is hubTokenForTarget that also reports a failed refresh of +// a stored OAuth token, alongside the cached token it fell back to. +func (t *Tap) resolveHubToken(target *keg.Target) (string, error) { if target == nil { - return "" + return "", nil } if target.TokenEnv != "" { // Explicit configuration owns credential selection, even when missing. - return t.Runtime.Get(target.TokenEnv) + return t.Runtime.Get(target.TokenEnv), nil } if target.Token != "" { - return target.Token + return target.Token, nil } if t.KegService == nil { - return "" + return "", nil } resolver := t.KegService.tokenResolver() if resolver == nil { - return "" + return "", nil + } + if store, ok := resolver.(*authStoreTokenResolver); ok { + return store.resolveToken(target) } - return resolver.ResolveToken(target) + return resolver.ResolveToken(target), nil } // HubInfo describes a configured hub for `tap hub list`. diff --git a/pkg/tapper/tap_info.go b/pkg/tapper/tap_info.go index f7c57c2d..19e62b5a 100644 --- a/pkg/tapper/tap_info.go +++ b/pkg/tapper/tap_info.go @@ -530,11 +530,16 @@ type ContextInfoFlight struct { } // ContextInfoFlightKeg is one KEG the active flight covers, at the role the -// caller holds through it. +// caller holds through it. Source is the KEG the flight lists whose role it +// took, Distance the links between them, and Path, for a KEG reached through +// links, the KEGs along that shortest path from Source to Ref. type ContextInfoFlightKeg struct { - Ref string `yaml:"ref" json:"ref"` - Role string `yaml:"role,omitempty" json:"role,omitempty"` - Title string `yaml:"title,omitempty" json:"title,omitempty"` + Ref string `yaml:"ref" json:"ref"` + Role string `yaml:"role,omitempty" json:"role,omitempty"` + Title string `yaml:"title,omitempty" json:"title,omitempty"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Distance int `yaml:"distance,omitempty" json:"distance,omitempty"` + Path []string `yaml:"path,omitempty" json:"path,omitempty"` } // ContextInfoResult summarizes the CLI's context for `tap info`. @@ -549,50 +554,67 @@ type ContextInfoResult struct { // ContextInfo reports the hub, the default KEG, the active flight, and the // KEGs that flight covers. It is best-effort: a hub that cannot be reached or // a flight that cannot be loaded becomes a warning, and the locally -// configured defaults are still reported. +// configured defaults are still reported. The hub's part comes from one +// GET /api/v1/info, or from concurrent requests on a hub without it. func (t *Tap) ContextInfo(ctx context.Context, opts KegTargetOptions) (ContextInfoResult, error) { out := ContextInfoResult{Kegs: []ContextInfoFlightKeg{}} slots := t.configSlots(opts) + out.Keg = ContextInfoKeg{Ref: slots.Keg.Value, Scope: slots.Keg.Scope} + if ref := t.resolveIdentity(opts).Ref; ref != "" { + out.Keg.Ref = ref + } + out.Flight = ContextInfoFlight{Ref: t.ActiveFlightName(opts.Flight), Scope: slots.Flight.Scope} + out.Hub = ContextInfoHub{Name: slots.Hub.Value, Scope: slots.Hub.Scope} - if name, entry, err := t.ConfigService.SelectedHub(opts.Hub); err != nil { - out.Warnings = append(out.Warnings, fmt.Sprintf("hub: %v", err)) + name, entry, hubErr := t.ConfigService.SelectedHub(opts.Hub) + hub := t.loadHubContext(ctx, hubContextRequest{hub: opts.Hub, flight: out.Flight.Ref, kegs: out.Flight.Ref != ""}) + if hubErr != nil { + out.Warnings = append(out.Warnings, fmt.Sprintf("hub: %v", hubErr)) } else { out.Hub.Name, out.Hub.URL = name, entry.URL - if who, err := t.SelectedHubIdentity(ctx); err != nil { - out.Warnings = append(out.Warnings, fmt.Sprintf("hub identity: %v", err)) - } else if who != nil { - out.Hub.User = who.Username + if hub.whoErr != nil { + out.Warnings = append(out.Warnings, fmt.Sprintf("hub identity: %v", hub.whoErr)) + } else if hub.who != nil { + out.Hub.User = hub.who.Username } } - out.Keg = ContextInfoKeg{Ref: slots.Keg.Value, Scope: slots.Keg.Scope} - if ref := t.resolveIdentity(opts).Ref; ref != "" { - out.Keg.Ref = ref - } - - out.Flight = ContextInfoFlight{Ref: t.ActiveFlightName(opts.Flight), Scope: slots.Flight.Scope} if out.Flight.Ref == "" { return out, nil } - flight, note, err := t.resolveOrientFlight(ctx, out.Flight.Ref) - if err != nil { - out.Warnings = append(out.Warnings, fmt.Sprintf("flight: %v", err)) + if hub.flightErr != nil { + out.Warnings = append(out.Warnings, fmt.Sprintf("flight: %v", hub.flightErr)) return out, nil } - if note != "" { - out.Warnings = append(out.Warnings, note) + if hub.flightNote != "" { + out.Warnings = append(out.Warnings, hub.flightNote) return out, nil } + // Project the Hub-composed cover, the authority MCP enforces. A flight + // read from an older Hub carries it as EffectiveCover. + flight := ComposedFlight(hub.flight) out.Flight.Ref, out.Flight.Title = flight.Name, flight.Title - rows, warnings, err := t.OrientationKegsForFlight(ctx, flight) - out.Warnings = append(out.Warnings, warnings...) - if err != nil { - out.Warnings = append(out.Warnings, fmt.Sprintf("flight kegs: %v", err)) + out.Warnings = append(out.Warnings, hub.warnings...) + if hub.kegsErr != nil { + out.Warnings = append(out.Warnings, fmt.Sprintf("flight kegs: %v", hub.kegsErr)) return out, nil } - for _, row := range rows { - out.Kegs = append(out.Kegs, ContextInfoFlightKeg{Ref: row.Ref, Role: EffectiveOrientationRole(row), Title: row.Title}) + decided := map[string]HubEffectiveKeg{} + if hub.flight != nil { + for _, c := range hub.flight.EffectiveCover { + decided[c.Ref] = c + } + } + for _, row := range ProjectOrientationKegs(flight, hub.kegs) { + k := ContextInfoFlightKeg{Ref: row.Ref, Role: EffectiveOrientationRole(row), Title: row.Title} + if c, ok := decided[row.Ref]; ok { + k.Source, k.Distance = c.Source, c.Distance + if c.Distance > 0 { + k.Path = c.Path + } + } + out.Kegs = append(out.Kegs, k) } return out, nil } diff --git a/pkg/tapper/tap_list.go b/pkg/tapper/tap_list.go index 541f7668..923eab13 100644 --- a/pkg/tapper/tap_list.go +++ b/pkg/tapper/tap_list.go @@ -23,6 +23,10 @@ const ( SortByAccessed ListSortType = "accessed" // ascending by last-accessed timestamp ) +// IDOnlyFormat is the listing format that prints only node IDs, which every +// surface's id-only shorthand expands to. +const IDOnlyFormat = "%i" + type ListOptions struct { KegTargetOptions @@ -40,8 +44,6 @@ type ListOptions struct { // returned page in one batch. Format string - IdOnly bool - Reverse bool // Sort selects the sort order. Empty string means sort by node ID (default). @@ -70,8 +72,6 @@ type BacklinksOptions struct { // returned page in one batch. Format string - IdOnly bool - Reverse bool // Limit caps the number of results returned. 0 means no limit. @@ -97,8 +97,6 @@ type LinksOptions struct { // returned page in one batch. Format string - IdOnly bool - Reverse bool // Limit caps the number of results returned. 0 means no limit. @@ -123,8 +121,6 @@ type GrepOptions struct { // returned page in one batch. Format string - IdOnly bool - Reverse bool // IgnoreCase enables case-insensitive regex matching. @@ -159,8 +155,6 @@ type TagsOptions struct { // returned page in one batch. Format string - IdOnly bool - Reverse bool // Limit caps the number of results returned. 0 means no limit. @@ -199,21 +193,21 @@ func (t *Tap) List(ctx context.Context, opts ListOptions) ([]string, error) { // costs the same as displaying a title. view, err := k.ListView(ctx, keg.ListViewOptions{ Query: opts.Query, - Fields: compiled.selectorTexts(opts.IdOnly), + Fields: compiled.selectorTexts(), Sort: sortSelector, Limit: opts.Limit, Offset: opts.Offset, }) switch { case err == nil: - case strings.TrimSpace(opts.Query) != "": + case strings.TrimSpace(opts.Query) != "" && errors.Is(err, keg.ErrInvalid): return []string{}, fmt.Errorf("invalid query expression: %w", err) default: return []string{}, fmt.Errorf("unable to list keg: %w", err) } t.warnStaleIndex(view.IndexedCount, view.NodeCount) return renderListView(compiled, view.Rows, renderOptions{ - Format: opts.Format, IdOnly: opts.IdOnly, Reverse: opts.Reverse, + Format: opts.Format, Reverse: opts.Reverse, }), nil } @@ -287,7 +281,7 @@ func (t *Tap) Backlinks(ctx context.Context, opts BacklinksOptions) ([]string, e return t.resolveAndLookupLinks(ctx, relatedListOptions{ KegTargetOptions: opts.KegTargetOptions, NodeIDs: opts.NodeIDs, - Render: renderOptions{Format: opts.Format, IdOnly: opts.IdOnly, Reverse: opts.Reverse}, + Render: renderOptions{Format: opts.Format, Reverse: opts.Reverse}, Limit: opts.Limit, Offset: opts.Offset, Direction: keg.RelatedBacklinks, @@ -301,7 +295,7 @@ func (t *Tap) Links(ctx context.Context, opts LinksOptions) ([]string, error) { return t.resolveAndLookupLinks(ctx, relatedListOptions{ KegTargetOptions: opts.KegTargetOptions, NodeIDs: opts.NodeIDs, - Render: renderOptions{Format: opts.Format, IdOnly: opts.IdOnly, Reverse: opts.Reverse}, + Render: renderOptions{Format: opts.Format, Reverse: opts.Reverse}, Limit: opts.Limit, Offset: opts.Offset, Direction: keg.RelatedLinks, @@ -358,7 +352,7 @@ func (t *Tap) resolveAndLookupLinks(ctx context.Context, opts relatedListOptions } related, err := t.operations().Related(ctx, k, opts.Direction, tapapi.RelatedRequest{ NodeIDs: ids, - Fields: compiled.selectorTexts(opts.Render.IdOnly), + Fields: compiled.selectorTexts(), Offset: opts.Offset, Limit: opts.Limit, }) @@ -367,7 +361,7 @@ func (t *Tap) resolveAndLookupLinks(ctx context.Context, opts relatedListOptions return []string{}, err } if errors.Is(err, keg.ErrNotExist) { - return []string{}, fmt.Errorf("node %s not found in %s", ids[0], describeKeg(k)) + return []string{}, fmt.Errorf("node %s not found in %s: %w", ids[0], describeKeg(k), keg.ErrNotExist) } return []string{}, err } @@ -412,7 +406,7 @@ func (t *Tap) Grep(ctx context.Context, opts GrepOptions) ([]string, error) { return []string{}, fmt.Errorf("unable to open keg: %w", err) } - formatted := opts.IdOnly || opts.Format != "" + formatted := opts.Format != "" var compiled compiledFormat if formatted { if compiled, err = compileListFormat(opts.Format); err != nil { @@ -425,12 +419,15 @@ func (t *Tap) Grep(ctx context.Context, opts GrepOptions) ([]string, error) { Pattern: opts.Query, IgnoreCase: opts.IgnoreCase, MaxLines: opts.MaxLines, - Fields: compiled.selectorTexts(opts.IdOnly), + Fields: compiled.selectorTexts(), Offset: opts.Offset, Limit: opts.Limit, }) if err != nil { - return []string{}, fmt.Errorf("invalid query regex %q: %w", opts.Query, err) + if errors.Is(err, keg.ErrInvalid) { + return []string{}, fmt.Errorf("invalid query regex %q: %w", opts.Query, err) + } + return []string{}, fmt.Errorf("unable to search keg: %w", err) } if formatted { rows := make([]keg.ListViewRow, len(kegMatches)) @@ -438,7 +435,7 @@ func (t *Tap) Grep(ctx context.Context, opts GrepOptions) ([]string, error) { rows[i] = keg.ListViewRow{Entry: m.Entry, Fields: m.Fields} } return renderListView(compiled, rows, renderOptions{ - Format: opts.Format, IdOnly: opts.IdOnly, Reverse: opts.Reverse, + Format: opts.Format, Reverse: opts.Reverse, }), nil } matches := make([]grepMatch, 0, len(kegMatches)) @@ -464,12 +461,12 @@ func (t *Tap) Tags(ctx context.Context, opts TagsOptions) ([]string, error) { queryExpr := strings.TrimSpace(opts.Query) result, err := t.operations().Tags(ctx, k, tapapi.TagsRequest{ Query: queryExpr, - Fields: compiled.selectorTexts(opts.IdOnly), + Fields: compiled.selectorTexts(), Offset: opts.Offset, Limit: opts.Limit, }) if err != nil { - if queryExpr != "" { + if queryExpr != "" && errors.Is(err, keg.ErrInvalid) { return []string{}, fmt.Errorf("invalid query expression: %w", err) } return []string{}, fmt.Errorf("unable to list entries: %w", err) @@ -485,7 +482,7 @@ func (t *Tap) Tags(ctx context.Context, opts TagsOptions) ([]string, error) { return tags, nil } return renderListView(compiled, result.Rows, renderOptions{ - Format: opts.Format, IdOnly: opts.IdOnly, Reverse: opts.Reverse, + Format: opts.Format, Reverse: opts.Reverse, }), nil } @@ -541,7 +538,6 @@ func renderGrepMatches(matches []grepMatch, reverse bool) []string { // renderOptions carries the presentation knobs shared by every listing surface. type renderOptions struct { Format string - IdOnly bool Reverse bool } diff --git a/pkg/tapper/tap_list_format.go b/pkg/tapper/tap_list_format.go index 1bbd9a29..28c23895 100644 --- a/pkg/tapper/tap_list_format.go +++ b/pkg/tapper/tap_list_format.go @@ -152,10 +152,7 @@ func compileListFormat(format string) (compiledFormat, error) { // server to resolve. Intrinsics and index timestamps are omitted: they come // from the index entry that every listing already carries, so asking for them // would make the server do work the client can do for free. -func (c compiledFormat) selectorTexts(idOnly bool) []string { - if idOnly { - return nil - } +func (c compiledFormat) selectorTexts() []string { seen := make(map[string]struct{}, len(c.segments)) out := make([]string, 0, len(c.segments)) for _, seg := range c.segments { @@ -181,10 +178,6 @@ func renderListView(compiled compiledFormat, rows []keg.ListViewRow, opts render lines := make([]string, 0, len(rows)) start, end, step := iterationBounds(len(rows), opts.Reverse) for i := start; i != end; i += step { - if opts.IdOnly { - lines = append(lines, rows[i].Entry.ID) - continue - } lines = append(lines, expandFormat(compiled, nodeFieldSource{ entry: rows[i].Entry, resolved: rows[i].Fields, diff --git a/pkg/tapper/tap_lock.go b/pkg/tapper/tap_lock.go index b9ccc337..8e45e24a 100644 --- a/pkg/tapper/tap_lock.go +++ b/pkg/tapper/tap_lock.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/jlrickert/tapper/pkg/keg" ) @@ -11,6 +12,10 @@ import ( // LockOptions configures behavior for Tap.Lock. type LockOptions struct { NodeID string + // Wait bounds how long to wait for a lock another holder has, clamped by + // keg.ClampNodeLockWait: zero means the 10s default, and anything above 60s + // waits 60s. See keg.WithNodeLockWait. + Wait time.Duration KegTargetOptions } @@ -45,6 +50,10 @@ func (t *Tap) Lock(ctx context.Context, opts LockOptions) (keg.LockToken, error) return "", err } + if opts.Wait < 0 { + return "", fmt.Errorf("%w: lock wait must not be negative", keg.ErrInvalid) + } + ctx = keg.WithNodeLockWait(ctx, keg.ClampNodeLockWait(opts.Wait)) info, err := k.Lock(ctx, id) if err != nil { if errors.Is(err, keg.ErrNotExist) { diff --git a/pkg/tapper/tap_move.go b/pkg/tapper/tap_move.go index 5874ea9d..dcae938d 100644 --- a/pkg/tapper/tap_move.go +++ b/pkg/tapper/tap_move.go @@ -56,7 +56,7 @@ func (t *Tap) Move(ctx context.Context, opts MoveOptions) error { if _, err := k.Move(ctx, keg.NodeMoveOptions{Source: srcID, Destination: dstID, ExpectedHash: opts.ExpectedHash}); err != nil { if errors.Is(err, keg.ErrNotExist) { - return fmt.Errorf("node %s not found in %s", srcID.Path(), describeKeg(k)) + return fmt.Errorf("node %s not found in %s: %w", srcID.Path(), describeKeg(k), keg.ErrNotExist) } if errors.Is(err, keg.ErrDestinationExists) { return fmt.Errorf("destination node %s already exists", dstID.Path()) diff --git a/pkg/tapper/tap_orient.go b/pkg/tapper/tap_orient.go index d9386f4d..48f9d69b 100644 --- a/pkg/tapper/tap_orient.go +++ b/pkg/tapper/tap_orient.go @@ -7,20 +7,22 @@ import ( "io/fs" "sort" "strings" + "sync" "github.com/jlrickert/tapper/pkg/apicontract" "github.com/jlrickert/tapper/pkg/integrations" "github.com/jlrickert/tapper/pkg/keg" ) -const orientPurpose = "Tapper provides an MCP interface for KEG (Knowledge Exchange Graph) systems. A KEG is a numbered collection of markdown nodes with metadata, links, tags, and snapshot history. Agents operate on a KEG through the `mcp__tapper__*` tools; reading or writing node files directly bypasses indexing, locking, and snapshots." +const orientPurpose = "Tapper provides an MCP interface for KEG (Knowledge Exchange Graph) systems. A KEG is a numbered collection of markdown nodes with metadata, links, tags, and snapshot history, hosted on a Tapper Hub. Agents operate on a KEG only through the `mcp__tapper__*` tools, which enforce access, locking, indexing, and snapshots." const orientRulesSummary = "Rules:\n" + "- Call `orient` first in every session, before any other tool and before replying. The active flight carries this session's instructions, so until you orient you do not know what the session is for.\n" + "- Call `orient` again after any context reset such as a clear or a compact. These instructions were delivered into the conversation and are discarded with it; the connection survives, so nothing re-sends them on its own. If you cannot tell whether you have oriented in the current context, you have not.\n" + "- These operating rules also ship once at initialization and do not change. The flight, its cover, and its instructions do change, and this payload is the only current source for them: a compaction summary may paraphrase a previous orientation, and an older copy can still be present. If anything you remember about the flight, its cover, or its instructions disagrees with what you are reading here, this is current and that is stale. Do not merge them; replace.\n" + - "- Use the `mcp__tapper__*` tools for every KEG operation; never read or write node files directly.\n" + - "- The target keg resolves from the working directory unless the `keg` parameter overrides it.\n" + + "- Use the `mcp__tapper__*` tools for every KEG operation. KEGs live on the Hub, not in local files, so there are no node files to read or write.\n" + + "- Pass `keg` as `@namespace/keg` to choose a tool's target KEG. A call that omits it uses the configured default KEG and fails when none is configured.\n" + + "- Every KEG tool takes an optional `flight`, such as `+work` or `@namespace/+work`. Omit it to run under this connection's flight; name a flight `orient` lists to run that one call under its cover and instructions. Naming a flight never widens what your identity may do.\n" + "- Take a snapshot before non-trivial edits. Snapshots do not protect against `node_delete`; preserve content some other way before deletion.\n" + "- Writes may invalidate the hash you were holding; not every mutation returns a replacement. Re-read with `node_read`, `schema_read`, or `keg_settings_read` before each guarded write; a hash never covers two writes, so an edit followed by a delete needs two reads. `node_edit` uses per-part hashes: pass `node_read`'s `content_hash` as `expected_content_hash` and `meta_hash` as `expected_meta_hash`; editing one part never invalidates the other's hash, and each edit result returns the new part hashes. `node_delete` and `node_move` take the combined `hash`.\n" + "- Node ids are per-keg counters. Node 4 in one keg has nothing to do with node 4 in another, ids are never reused after a removal, and a create takes the next free id rather than filling a gap.\n" + @@ -53,30 +55,53 @@ func (t *Tap) Orient(ctx context.Context, opts OrientOptions) (string, error) { t.KegService.ReloadAuthStore() } flightName := t.ActiveFlightName(opts.Flight) - flight, flightNote, err := t.resolveOrientFlight(ctx, flightName) - if err != nil { - return "", err + + // The flight (then its graph) and the identity KEG catalog are + // independent, so they load together. Results are checked in the order + // the requests used to run, so the same error wins. + var ( + flight *Flight + flightNote string + flightErr error + graph *FlightGraph + graphErr error + wg sync.WaitGroup + ) + wg.Add(1) + go func() { + defer wg.Done() + flight, flightNote, flightErr = t.resolveOrientFlight(ctx, flightName) + if flightErr == nil && flight != nil && flightNote == "" { + graph, graphErr = t.FlightService.ResolveFlightGraph(ctx, flight) + } + }() + catalog, warnings, err := t.IdentityKegCatalog(ctx) + wg.Wait() + if flightErr != nil { + return "", flightErr } - available, warnings, err := t.OrientationKegsForFlight(ctx, flight) if err != nil { return "", err } + var available []OrientationKeg var authority *OrientationAuthority if strings.TrimSpace(flightName) == "" { - available, warnings, err = t.IdentityKegCatalog(ctx) - if err != nil { - return "", err - } + // No flight: the identity catalog is the listing. + available = catalog + authority = &OrientationAuthority{FullAccess: true, Agent: t.ActiveAgent()} flightNote = "No flight is configured, so normal identity-authorized full access applies. Pin a least-privilege flight outside MCP and start a new connection to narrow it." - authority = &OrientationAuthority{FullAccess: true} + if authority.Agent != "" { + flightNote = "No flight is configured for this agent session." + } + } else if flight != nil { + available = ProjectOrientationKegs(flight, catalog) } if flight != nil && flightNote == "" { - graph, err := t.FlightService.ResolveFlightGraph(ctx, flight) - if err != nil { - return "", err + if graphErr != nil { + return "", graphErr } rows := append([]*Flight{graph.Root}, graph.Available...) - authority = &OrientationAuthority{Root: graph.Root, Active: flight, Children: ImmediateFlightChildren(flight, rows)} + authority = &OrientationAuthority{Root: graph.Root, Active: flight, Children: ImmediateFlightChildren(flight, rows), Agent: t.ActiveAgent()} } payload, err := BuildOrientationPayload(flight, flightNote, t.ActiveLaunch(), available, warnings, authority) if err != nil { @@ -179,6 +204,15 @@ func (t *Tap) ActiveLaunch() string { } } +// ActiveAgent returns the Hub agent this process runs as (TAP_AGENT, set by +// `tap launch` for a `tap mcp` session), or "" when a person is driving it. +func (t *Tap) ActiveAgent() string { + if t == nil || t.Runtime == nil { + return "" + } + return strings.TrimSpace(t.Runtime.Env().Get("TAP_AGENT")) +} + func (t *Tap) resolveOrientFlight(ctx context.Context, name string) (*Flight, string, error) { name = strings.TrimSpace(name) if name == "" || t == nil || t.FlightService == nil { @@ -223,6 +257,10 @@ type OrientationAuthority struct { AvailableFlights []string Revision string FullAccess bool + // Agent names the Hub agent a `tap mcp` session runs as (TAP_AGENT), or + // "" when a person is. An agent session never claims identity authority: + // the agent's tool allowlist and the flight govern what it may do. + Agent string } func (t *Tap) identityKegCatalog(ctx context.Context) ([]OrientationKeg, []string, error) { @@ -233,25 +271,60 @@ func (t *Tap) identityKegCatalog(ctx context.Context) ([]OrientationKeg, []strin if err != nil { return nil, []string{fmt.Sprintf("KEG listing unavailable: %v", err)}, nil } + return mergeHubKegs(t.orientKegsForHubs(ctx, cfg, "")) +} - var warnings []string - var out []OrientationKeg +// hubKegsResult is one configured hub's identity KEG listing. +type hubKegsResult struct { + name string + rows []OrientationKeg + err error +} + +// orientKegsForHubs lists every configured hub's KEGs concurrently, in hub +// order. The hub named skip is left as a placeholder for a caller that +// already has its rows. +func (t *Tap) orientKegsForHubs(ctx context.Context, cfg *Config, skip string) []hubKegsResult { + var out []hubKegsResult + var entries []HubEntry for _, hubName := range t.allHubNames(cfg) { entry, ok := cfg.Hub(hubName) if !ok { continue } - rows, err := t.orientKegsForHub(ctx, cfg, hubName, entry) - if err != nil { - if apicontract.IsCompatibility(err) { - return nil, warnings, err - } - warnings = append(warnings, fmt.Sprintf("skipped hub %q: %v", hubName, err)) + out = append(out, hubKegsResult{name: hubName}) + entries = append(entries, entry) + } + var wg sync.WaitGroup + for i := range out { + if out[i].name == skip { continue } - for _, row := range rows { - out = append(out, row) + wg.Add(1) + go func(i int) { + defer wg.Done() + out[i].rows, out[i].err = t.orientKegsForHub(ctx, cfg, out[i].name, entries[i]) + }(i) + } + wg.Wait() + return out +} + +// mergeHubKegs folds per-hub listings into the identity catalog in hub order. +// A compatibility error from any hub fails the catalog; any other hub error +// becomes a skipped-hub warning. Rows are deduplicated by Ref and sorted. +func mergeHubKegs(results []hubKegsResult) ([]OrientationKeg, []string, error) { + var warnings []string + var out []OrientationKeg + for _, res := range results { + if res.err != nil { + if apicontract.IsCompatibility(res.err) { + return nil, warnings, res.err + } + warnings = append(warnings, fmt.Sprintf("skipped hub %q: %v", res.name, res.err)) + continue } + out = append(out, res.rows...) } seen := map[string]struct{}{} identity := make([]OrientationKeg, 0, len(out)) @@ -415,7 +488,9 @@ func BuildOrientationPayload(flight *Flight, flightNote, launch string, kegs []O b.WriteString(flightNote + "\n\n") } if flight == nil { - if authority != nil && authority.FullAccess { + if authority != nil && authority.FullAccess && authority.Agent != "" { + fmt.Fprintf(&b, "No flight is active. This session runs as agent `%s`: the agent's tool allowlist governs which tools it may call, and the flight it runs under governs which KEGs it may reach. `keg_list` and `flight_list` list readable resources; `explore_kegs` and `explore_flights` search the public directory and confer no access. Give the agent a flight and start a new connection to narrow its KEG access.\n", orientationTableCell(authority.Agent)) + } else if authority != nil && authority.FullAccess { b.WriteString("No flight is active. This session runs under your full identity authority; nothing here is scoped. `keg_list` and `flight_list` list every readable resource; `explore_kegs` and `explore_flights` search the public directory and confer no access. An explicit flight selects only that flight for one call. Pin a least-privilege flight outside MCP and start a new connection to narrow the session.\n") } else { b.WriteString("The configured flight is unavailable: fail-closed recovery; KEG tools are locked. Repair the selection, then call `session_refresh` and `orient`. Use `flight_list`, `flight_read`, or `session_info` for recovery.\n") @@ -427,7 +502,9 @@ func BuildOrientationPayload(flight *Flight, flightNote, launch string, kegs []O b.WriteString(orientationTableCell(DescriptionPreview(flight.Description)) + "\n\n") } if authority != nil { - if authority.FullAccess { + if authority.Agent != "" { + fmt.Fprintf(&b, "Agent: `%s`. The agent's tool allowlist and this flight govern what this session may do.\n\n", orientationTableCell(authority.Agent)) + } else if authority.FullAccess { b.WriteString("Launch root: (none; identity-authorized full access)\n\n") } if authority.Root != nil { @@ -513,7 +590,7 @@ func OrientationGuide(topic string) (string, error) { case "troubleshooting": names = []string{"troubleshooting.md"} default: - return "", fmt.Errorf("unknown guide topic %q; use operating, authoring, linking, snapshots, tools, or troubleshooting", topic) + return "", fmt.Errorf("unknown guide topic %q; use operating, authoring, linking, snapshots, tools, or troubleshooting: %w", topic, keg.ErrInvalid) } var b strings.Builder if topic == "operating" { diff --git a/pkg/tapper/tap_orient_test.go b/pkg/tapper/tap_orient_test.go index d6278a1c..58369a81 100644 --- a/pkg/tapper/tap_orient_test.go +++ b/pkg/tapper/tap_orient_test.go @@ -333,3 +333,49 @@ func TestBuildOrientationPayload_SingleFlightProjectionRendersOneTable(t *testin require.Contains(t, payload, "@ada/notes") require.NotContains(t, payload, "## Reachable via subflight") } + +// An agent session (TAP_AGENT, set for `tap mcp`) holds no credentials of its +// own, so its orientation must not claim the identity's full authority. +func TestTap_Orient_AgentSessionNamesAllowlistNotIdentityAuthority(t *testing.T) { + t.Parallel() + sb := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + require.NoError(t, sb.Runtime().Env().Set("TAP_AGENT", "@acme/researcher")) + tap, err := tapper.NewTap(tapper.TapOptions{Runtime: sb.Runtime()}) + require.NoError(t, err) + + payload, err := tap.Orient(context.Background(), tapper.OrientOptions{}) + require.NoError(t, err) + + require.NotContains(t, payload, "full identity authority") + require.NotContains(t, payload, "full access") + require.Contains(t, payload, "agent `@acme/researcher`") + require.Contains(t, payload, "tool allowlist") +} + +func TestTap_Orient_PersonWithoutAgentKeepsFullIdentityAuthority(t *testing.T) { + t.Parallel() + payload, err := newOrientTap(t).Orient(context.Background(), tapper.OrientOptions{}) + require.NoError(t, err) + + require.Contains(t, payload, "full identity authority") + require.NotContains(t, payload, "tool allowlist") +} + +func TestBuildOrientationPayload_AgentFlightNamesAllowlistNotFullAccess(t *testing.T) { + t.Parallel() + flight := orientFlight("@ada/+memory", "ada", "memory", + tapper.FlightCover{Namespace: "ada", Keg: "notes", Role: tapper.FlightRoleEditor}) + + payload, err := tapper.BuildOrientationPayload(flight, "", "", nil, nil, + &tapper.OrientationAuthority{Active: flight, FullAccess: true, Agent: "@ada/researcher"}) + require.NoError(t, err) + require.NotContains(t, payload, "full access") + require.Contains(t, payload, "Agent: `@ada/researcher`") + require.Contains(t, payload, "tool allowlist and this flight govern") + + person, err := tapper.BuildOrientationPayload(flight, "", "", nil, nil, + &tapper.OrientationAuthority{Active: flight, FullAccess: true}) + require.NoError(t, err) + require.Contains(t, person, "Launch root: (none; identity-authorized full access)") + require.NotContains(t, person, "tool allowlist") +} diff --git a/pkg/tapper/tap_relay.go b/pkg/tapper/tap_relay.go index bb17178c..bcb16e65 100644 --- a/pkg/tapper/tap_relay.go +++ b/pkg/tapper/tap_relay.go @@ -71,8 +71,9 @@ func (t *Tap) Relay(ctx context.Context, opts RelayOptions) error { hubs := make([]relay.Hub, 0, len(hubURLs)) for _, hubURL := range hubURLs { hubs = append(hubs, relay.Hub{ - URL: hubURL, - Token: func(context.Context) (string, error) { return t.relayToken(cfg, hubURL), nil }, + URL: hubURL, + Token: func(context.Context) (string, error) { return t.relayToken(cfg, hubURL) }, + Refresh: func(ctx context.Context) (string, error) { return t.relayRefresh(ctx, cfg, hubURL) }, }) } @@ -337,9 +338,6 @@ func relayVariants(in map[string]RelayVariant) map[string]relay.Variant { return out } -// relayToken resolves the bearer token for hubURL the same way keg access -// does: a configured hub entry's tokenEnv or token, then the `tap auth login` -// store, refreshing an expiring OAuth token first. // relayHubURLs picks the hubs the relay serves: the explicit hub alone when // one is given, else every hub named in relay.hubs, else the one hub login // resolution picks. URLs are canonical and listed once each. @@ -371,14 +369,62 @@ func relayHubURLs(cfg *Config, rc *RelayConfig, explicit string) ([]string, erro return out, nil } -func (t *Tap) relayToken(cfg *Config, hubURL string) string { +// relayToken resolves the bearer token for hubURL the same way keg access +// does: a configured hub entry's tokenEnv or token, then the `tap auth login` +// store, refreshing an expiring OAuth token first. A refresh that failed for +// a reason that may pass (the hub unreachable, a 5xx) is an error, so the +// relay retries with backoff rather than dial with the expired token and be +// rejected for good. A refresh token the hub refused returns the cached token: +// the sign-in is gone, and the hub's 401 stops the relay. +func (t *Tap) relayToken(cfg *Config, hubURL string) (string, error) { + target := relayTarget(cfg, hubURL) + token, err := t.resolveHubToken(&target) + if err != nil && !errors.Is(err, ErrRefreshRejected) { + return "", fmt.Errorf("relay: refreshing the hub token: %w", err) + } + return token, nil +} + +// relayRefresh is relayToken with a forced refresh: the stored OAuth token is +// renewed whatever its recorded expiry says, because the hub has said it is +// spent or is about to be, and the local clock may disagree. A configured +// tokenEnv or token has nothing to refresh and is returned as relayToken +// would. A refresh token the hub refused means the sign-in is gone, which the +// relay treats as final. +func (t *Tap) relayRefresh(ctx context.Context, cfg *Config, hubURL string) (string, error) { + target := relayTarget(cfg, hubURL) + if target.TokenEnv != "" || target.Token != "" || t.KegService == nil { + return t.relayToken(cfg, hubURL) + } + resolver, _ := t.KegService.tokenResolver().(*authStoreTokenResolver) + hubRoot := hubRootFromTarget(&target) + if resolver == nil || resolver.store == nil || hubRoot == "" { + return t.relayToken(cfg, hubURL) + } + resolver.mu.Lock() + defer resolver.mu.Unlock() + next, err := forceRefreshAuthStoreEntry(ctx, resolver.rt, resolver.store, resolver.storePath, hubRoot, CanonicalHubURL(hubRoot)) + if next != nil { + return next.AccessToken, nil + } + if errors.Is(err, ErrRefreshRejected) { + return "", fmt.Errorf("%w: %v", relay.ErrUnauthorized, err) + } + return "", fmt.Errorf("relay: refreshing the hub token: %w", err) +} + +// relayTarget is the keg target whose credential the relay presents to +// hubURL: a configured hub entry's, when one names that hub. +func relayTarget(cfg *Config, hubURL string) keg.Target { + target := keg.Target{Url: hubURL, HubURL: hubURL} canonical := CanonicalHubURL(hubURL) for _, entry := range cfg.Hubs() { if entry.URL != "" && CanonicalHubURL(hubURLWithScheme(entry.URL)) == canonical { - return t.hubToken(entry) + target, _ = hubEntryTarget(entry) + break } } - return t.hubTokenForTarget(&keg.Target{Url: hubURL, HubURL: hubURL}) + return target } // runtimeHostname reports the host name captured by the runtime, or "" when diff --git a/pkg/tapper/tap_relay_hubs_test.go b/pkg/tapper/tap_relay_hubs_test.go index 7b67ac2f..a2f944a1 100644 --- a/pkg/tapper/tap_relay_hubs_test.go +++ b/pkg/tapper/tap_relay_hubs_test.go @@ -2,6 +2,8 @@ package tapper import ( "context" + "fmt" + "net/http" "os" "path/filepath" "testing" @@ -11,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/jlrickert/tapper/internal/relay" + "github.com/jlrickert/tapper/internal/testapi" ) func TestRelayHubURLs(t *testing.T) { @@ -278,3 +281,71 @@ func TestRelayRunsWithOnlyBuiltins(t *testing.T) { require.ErrorIs(t, tap.Relay(context.Background(), RelayOptions{}), ErrRelayNotConfigured, "no providers, no servers and no runners: nothing to relay") } + +// relayTokenWith seeds an expired, refreshable login for a hub whose token +// endpoint answers with status and body (and a rotated pair on 200), then +// resolves the relay's token for that hub. +func relayTokenWith(t *testing.T, status int, body string) (string, error) { + t.Helper() + srv := testapi.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if status != http.StatusOK { + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"thub_fresh000000","token_type":"Bearer","expires_in":900,"refresh_token":"rt-next"}`)) + })) + t.Cleanup(srv.Close) + sb := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}) + rt := sb.Runtime() + tap, err := NewTap(TapOptions{Runtime: rt}) + require.NoError(t, err) + require.NoError(t, rt.AtomicWriteFile(tap.PathService.UserConfig(), + []byte(fmt.Sprintf("hub: work\nhubs:\n work: {url: %s}\n", srv.URL)), 0o644)) + store, err := LoadAuthStore(context.Background(), rt, tap.PathService.AuthStorePath()) + require.NoError(t, err) + store.Set(CanonicalHubURL(srv.URL), AuthEntry{ + AccessToken: "thub_expired0000", ExpiresAt: rt.Clock().Now().Add(-time.Minute), + RefreshToken: "rt-old", ClientID: "tapper-cli", TokenEndpoint: srv.URL, + }) + require.NoError(t, store.Save(context.Background(), rt, tap.PathService.AuthStorePath())) + cfg, err := tap.ConfigService.Config() + require.NoError(t, err) + return tap.relayToken(cfg, srv.URL) +} + +// A refresh that fails for a passing reason must not hand the relay the +// expired token: the hub would reject the dial, and the relay treats that +// rejection as final. +func TestRelayTokenRetriesWhenRefreshFails(t *testing.T) { + cases := map[string]struct { + status int + body string + }{ + "503 server_error": {http.StatusServiceUnavailable, `{"error":"server_error"}`}, + "400 invalid_request": {http.StatusBadRequest, `{"error":"invalid_request"}`}, + "400 not json": {http.StatusBadRequest, `bad request`}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + token, err := relayTokenWith(t, tc.status, tc.body) + require.Error(t, err, "a transient refresh failure is an error the relay retries") + require.Empty(t, token) + }) + } +} + +func TestRelayTokenRefreshesExpiredLogin(t *testing.T) { + token, err := relayTokenWith(t, http.StatusOK, "") + require.NoError(t, err) + require.Equal(t, "thub_fresh000000", token) +} + +// A refresh token the hub refused is a lost sign-in, not something to retry: +// the relay dials with what it has and the hub's 401 stops it. +func TestRelayTokenStopsRetryingWhenRefreshIsRejected(t *testing.T) { + token, err := relayTokenWith(t, http.StatusBadRequest, `{"error":"invalid_grant"}`) + require.NoError(t, err) + require.Equal(t, "thub_expired0000", token) +} diff --git a/pkg/tapper/tap_relay_refresh_test.go b/pkg/tapper/tap_relay_refresh_test.go new file mode 100644 index 00000000..da819333 --- /dev/null +++ b/pkg/tapper/tap_relay_refresh_test.go @@ -0,0 +1,270 @@ +package tapper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" + "github.com/jlrickert/cli-toolkit/sandbox" + "github.com/stretchr/testify/require" + + "github.com/jlrickert/tapper/internal/relay" + "github.com/jlrickert/tapper/pkg/relaycontract" +) + +// relayAuthHub is a hub that issues OAuth tokens and accepts relays, judging +// every access token by its own clock and records alone. +type relayAuthHub struct { + srv *httptest.Server + + mu sync.Mutex + valid map[string]bool // access tokens the hub accepts + refresh string // the one live refresh token + issued int + revoked bool // the token endpoint refuses every refresh + + tokenPosts atomic.Int32 + dials chan string // the bearer each accepted dial presented + reauths chan string // the token each reauth presented + // onRefused runs when a dial is refused, before the 401 goes out. + onRefused func() +} + +func newRelayAuthHub(t *testing.T, refresh string) *relayAuthHub { + t.Helper() + h := &relayAuthHub{ + valid: map[string]bool{}, + refresh: refresh, + dials: make(chan string, 8), + reauths: make(chan string, 8), + } + h.srv = httptest.NewServer(http.HandlerFunc(h.serve)) + t.Cleanup(h.srv.Close) + return h +} + +func (h *relayAuthHub) serve(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + h.tokenPosts.Add(1) + _ = r.ParseForm() + h.mu.Lock() + defer h.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if h.revoked || r.PostForm.Get("refresh_token") != h.refresh { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"invalid_grant"}`)) + return + } + h.issued++ + access := fmt.Sprintf("thub_fresh_%d", h.issued) + h.refresh = fmt.Sprintf("rt-%d", h.issued) + h.valid[access] = true + _ = json.NewEncoder(w).Encode(map[string]any{ + "access_token": access, "token_type": "Bearer", "expires_in": 3600, "refresh_token": h.refresh, + }) + case relaycontract.ConnectPath: + bearer := r.Header.Get("Authorization") + token := bearer[len("Bearer "):] + h.mu.Lock() + ok := h.valid[token] + onRefused := h.onRefused + h.mu.Unlock() + if !ok { + if onRefused != nil { + onRefused() + } + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + h.dials <- token + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer conn.CloseNow() + h.session(r.Context(), conn) + default: + http.NotFound(w, r) + } +} + +// session registers the relay with a two-second credential and answers +// every reauth that presents a token the hub issued. +func (h *relayAuthHub) session(ctx context.Context, conn *websocket.Conn) { + var env relaycontract.Envelope + if wsjson.Read(ctx, conn, &env) != nil { + return + } + reg, _ := relaycontract.NewEnvelope(relaycontract.TypeRegistered, "", relaycontract.Registered{ + Protocol: relaycontract.ProtocolVersion, Models: []relaycontract.CatalogBinding{}, CredentialExpiresIn: 2, + }) + if wsjson.Write(ctx, conn, reg) != nil { + return + } + for { + if wsjson.Read(ctx, conn, &env) != nil { + return + } + if env.Type != relaycontract.TypeReauth { + continue + } + var re relaycontract.Reauth + if env.Decode(&re) != nil { + return + } + h.reauths <- re.Token + h.mu.Lock() + ok := h.valid[re.Token] + h.mu.Unlock() + var answer relaycontract.Envelope + if ok { + answer, _ = relaycontract.NewEnvelope(relaycontract.TypeReauthed, env.ID, relaycontract.Reauthed{CredentialExpiresIn: 2}) + } else { + answer, _ = relaycontract.NewEnvelope(relaycontract.TypeError, env.ID, relaycontract.Error{Code: relaycontract.CodeReauthRejected}) + } + if wsjson.Write(ctx, conn, answer) != nil { + return + } + } +} + +// skewedRelayTap is a Tap whose clock runs 60 seconds behind the hub, with +// a stored login whose access token the hub already considers expired but +// whose recorded expiry, read on the skewed clock, is 55 seconds away: past +// refreshSkew, so an expiry-based refresh would not renew it. +func skewedRelayTap(t *testing.T, hub *relayAuthHub) (*Tap, *sandbox.Sandbox, string) { + t.Helper() + trueNow := time.Now() + sb := sandbox.NewSandbox(t, &sandbox.Options{Home: "/home/testuser", User: "testuser"}, + sandbox.WithClock(trueNow.Add(-60*time.Second))) + cfg := fmt.Sprintf("hub: test\nhubs:\n test: {url: %q}\n"+ + "relay:\n name: laptop\n providers:\n ollama: {kind: ollama, baseUrl: %q}\n", hub.srv.URL, hub.srv.URL+"/v1") + require.NoError(t, sb.Runtime().AtomicWriteFile("/home/testuser/.config/tapper/config.yaml", []byte(cfg), 0o644)) + tap, err := NewTap(TapOptions{Runtime: sb.Runtime()}) + require.NoError(t, err) + + path := tap.ConfigService.PathService.AuthStorePath() + store := &AuthStore{} + store.Set(CanonicalHubURL(hub.srv.URL), AuthEntry{ + AccessToken: "thub_stale", + TokenType: "Bearer", + ExpiresAt: trueNow.Add(-5 * time.Second), + RefreshToken: "rt-0", + ClientID: "tapper-cli", + TokenEndpoint: hub.srv.URL + "/token", + }) + require.NoError(t, store.Save(context.Background(), sb.Runtime(), path)) + require.False(t, authEntryNeedsRefresh(sb.Runtime(), &AuthEntry{RefreshToken: "rt-0", ExpiresAt: trueNow.Add(-5 * time.Second)}), + "precondition: the skewed clock thinks the stale token is still good") + return tap, sb, path +} + +func runRelay(t *testing.T, tap *Tap) (context.CancelFunc, chan error) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- tap.Relay(ctx, RelayOptions{Version: "test"}) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("relay did not stop") + } + }) + return cancel, done +} + +// Issue 335: with the client clock 60 seconds behind the hub, the relay +// still connects and keeps its session: the refused dial is retried with a +// forced refresh, and the in-session reauth refreshes on the hub's announced +// lifetime, not on the stored expiry. +func TestRelayRefreshesDespiteClockSkew(t *testing.T) { + hub := newRelayAuthHub(t, "rt-0") + tap, sb, path := skewedRelayTap(t, hub) + _, done := runRelay(t, tap) + + select { + case got := <-hub.dials: + require.Equal(t, "thub_fresh_1", got, "the relay connects with a token refreshed after the refusal") + case err := <-done: + done <- err + t.Fatalf("relay stopped instead of refreshing: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("relay never connected") + } + select { + case got := <-hub.reauths: + require.Equal(t, "thub_fresh_2", got, + "the reauth presents a new token although the local clock gives the current one an hour") + case <-time.After(3 * time.Second): + t.Fatal("relay did not reauth before the hub's two-second expiry") + } + + stored, err := LoadAuthStore(context.Background(), sb.Runtime(), path) + require.NoError(t, err) + entry, ok := stored.Get(CanonicalHubURL(hub.srv.URL)) + require.True(t, ok) + require.Equal(t, "thub_fresh_2", entry.AccessToken, "the forced refresh persists the rotated pair") + require.Equal(t, "rt-2", entry.RefreshToken) +} + +// A forced refresh still never spends a refresh token another process has +// already rotated: when the stored login moved on, it adopts that one. +func TestRelayForcedRefreshAdoptsSiblingRotation(t *testing.T) { + hub := newRelayAuthHub(t, "rt-0") + tap, sb, path := skewedRelayTap(t, hub) + var once sync.Once + hub.onRefused = func() { + once.Do(func() { + // Another tap process refreshed while this one held the stale + // token. + hub.mu.Lock() + hub.valid["thub_sibling"] = true + hub.refresh = "rt-sibling" + hub.mu.Unlock() + store, err := LoadAuthStore(context.Background(), sb.Runtime(), path) + if err != nil { + return + } + e, _ := store.Get(CanonicalHubURL(hub.srv.URL)) + e.AccessToken, e.RefreshToken = "thub_sibling", "rt-sibling" + store.Set(CanonicalHubURL(hub.srv.URL), *e) + _ = store.Save(context.Background(), sb.Runtime(), path) + }) + } + runRelay(t, tap) + select { + case got := <-hub.dials: + require.Equal(t, "thub_sibling", got) + case <-time.After(5 * time.Second): + t.Fatal("relay never connected") + } + require.Equal(t, int32(0), hub.tokenPosts.Load(), "no refresh token spent before the reauth") +} + +// Revocation still stops the relay: a refresh the hub refuses is final. +func TestRelayStopsWhenRefreshIsRevoked(t *testing.T) { + hub := newRelayAuthHub(t, "rt-0") + hub.revoked = true + tap, _, _ := skewedRelayTap(t, hub) + _, done := runRelay(t, tap) + select { + case err := <-done: + require.True(t, errors.Is(err, relay.ErrUnauthorized), "err: %v", err) + done <- err + case <-time.After(5 * time.Second): + t.Fatal("relay kept running on a revoked sign-in") + } + require.Equal(t, int32(1), hub.tokenPosts.Load(), "one forced refresh, then final") +} diff --git a/pkg/tapper/tap_schema.go b/pkg/tapper/tap_schema.go index 680d9e24..e1c97eb0 100644 --- a/pkg/tapper/tap_schema.go +++ b/pkg/tapper/tap_schema.go @@ -64,7 +64,7 @@ func (t *Tap) SchemaHash(ctx context.Context, opts SchemaOptions) (string, error // DeleteSchema. Reading schemas stays viewer. func (t *Tap) EditSchema(ctx context.Context, opts EditSchemaOptions) error { typeName := strings.TrimSpace(opts.Type) - if err := keg.ValidSchemaTypeName(typeName); err != nil { + if err := keg.ValidSchemaStorageName(typeName); err != nil { return err } k, err := t.resolveKegForRole(ctx, opts.KegTargetOptions, FlightRoleAdmin) diff --git a/pkg/tapper/tap_search.go b/pkg/tapper/tap_search.go index 22863bfc..75fe9405 100644 --- a/pkg/tapper/tap_search.go +++ b/pkg/tapper/tap_search.go @@ -112,7 +112,10 @@ func (t *Tap) Search(ctx context.Context, opts SearchOptions) (*tapapi.SearchRes return t.searchOneKeg(ctx, opts.KegTargetOptions, flight, req) } - scope := searchScope(kegs, flight) + scope, err := searchScope(kegs, flight) + if err != nil { + return nil, err + } if scope.restricted && len(scope.kegs) == 0 { // An empty KEG list asks the Hub for every readable KEG, so a scope // that narrowed to nothing must never reach it. @@ -216,39 +219,63 @@ type searchScopeResult struct { warnings []string } -// searchScope narrows explicit KEGs to the flight's cover, or without explicit -// KEGs, names the flight's cover. Both read the cover through searchCover, so -// a KEG is searchable on exactly the same terms either way. -func searchScope(explicit []string, flight *Flight) searchScopeResult { +// searchScope names the flight's cover when no KEGs are explicit, and +// otherwise checks that every explicit KEG is inside it. Both read the cover +// through searchCover, so a KEG is searchable on exactly the same terms either +// way. +func searchScope(explicit []string, flight *Flight) (searchScopeResult, error) { if flight == nil { - return searchScopeResult{kegs: explicit, restricted: len(explicit) > 0} + return searchScopeResult{kegs: explicit, restricted: len(explicit) > 0}, nil } cover, warnings := searchCover(flight) - out := searchScopeResult{kegs: []string{}, restricted: true, warnings: warnings} if len(explicit) == 0 { - out.kegs = cover - return out + return searchScopeResult{kegs: cover, restricted: true, warnings: warnings}, nil + } + if err := checkSearchCover(flight, cover, explicit); err != nil { + return searchScopeResult{}, err } + return searchScopeResult{kegs: explicit, restricted: true, warnings: warnings}, nil +} + +// CheckSearchCover refuses a search that names KEGs outside the flight's +// cover, naming every one of them. Naming KEGs only narrows a flight's search; +// it never reaches past the cover. kegs are canonical "@namespace/alias" +// references, as SearchOptions.Validate returns them. A nil flight (no-flight +// full access) restricts nothing. +func CheckSearchCover(flight *Flight, kegs []string) error { + if flight == nil || len(kegs) == 0 { + return nil + } + cover, _ := searchCover(flight) + return checkSearchCover(flight, cover, kegs) +} + +func checkSearchCover(flight *Flight, cover, kegs []string) error { covered := make(map[string]bool, len(cover)) for _, ref := range cover { covered[ref] = true } - for _, ref := range explicit { - if covered[ref] { - out.kegs = append(out.kegs, ref) - continue + var outside []string + for _, ref := range kegs { + if !covered[ref] { + outside = append(outside, ref) } - out.warnings = append(out.warnings, fmt.Sprintf("%s is not in flight %s's cover and was not searched", ref, flight.Name)) } - return out + if len(outside) == 0 { + return nil + } + return fmt.Errorf("flight %s does not cover %s; name only KEGs its cover lists (see orient), omit kegs to search the whole cover, or pass a flight that covers them: %w", + flight.Name, strings.Join(outside, ", "), keg.ErrInvalid) } // searchCover lists, in cover order and once each, the "@namespace/alias" -// references of the flight's cover a search may read: the entries RoleFor -// evaluates, legacy AllowedKegs included, at viewer or above. Ranked search -// names KEGs canonically, so a cover entry without a namespace cannot be -// searched; it is reported as a warning rather than matched against any -// namespace. +// references of the flight's cover a search may read. Each namespaced entry +// names a candidate, and the candidate is kept only when RoleFor grants it +// viewer or above, so search and the role check agree on every KEG: an +// earlier entry that blocks it, bare or namespaced, takes it out of search +// just as it takes it out of reach. Ranked search names KEGs canonically, so +// a cover entry without a namespace cannot itself be searched; it is reported +// as a warning rather than matched against any namespace. func searchCover(flight *Flight) (refs, warnings []string) { refs = []string{} seen := map[string]bool{} @@ -257,12 +284,12 @@ func searchCover(flight *Flight) (refs, warnings []string) { warnings = append(warnings, fmt.Sprintf("cover entry %q names no namespace and was not searched", c.Keg)) continue } - if !normalizeFlightRole(c.Role).AtLeast(FlightRoleViewer) { + ref := "@" + c.Namespace + "/" + c.Keg + if seen[ref] { continue } - ref := "@" + c.Namespace + "/" + c.Keg - if !seen[ref] { - seen[ref] = true + seen[ref] = true + if role, ok := flight.RoleFor("", c.Namespace, c.Keg); ok && role.AtLeast(FlightRoleViewer) { refs = append(refs, ref) } } diff --git a/pkg/tapper/tap_search_test.go b/pkg/tapper/tap_search_test.go index 3cb2f205..ab07e7f9 100644 --- a/pkg/tapper/tap_search_test.go +++ b/pkg/tapper/tap_search_test.go @@ -196,12 +196,19 @@ func TestSearch_FlightCoverScopesTheSearch(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"@acme/notes", "@acme/eng"}, hub.last(t).Kegs) - // A named KEG outside the cover is dropped with a warning. - resp, err := tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@acme/eng", "@other/private"}, Query: "release"}) + // Named KEGs narrow the cover. + _, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@acme/eng"}, Query: "release"}) require.NoError(t, err) require.Equal(t, []string{"@acme/eng"}, hub.last(t).Kegs) - require.Len(t, resp.Warnings, 1) - require.Contains(t, resp.Warnings[0], "@other/private") + + // Naming KEGs outside the cover is refused, naming each of them, and + // nothing is searched. + before := hub.calls.Load() + _, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@acme/eng", "@other/private", "@other/more"}, Query: "release"}) + require.ErrorIs(t, err, keg.ErrInvalid) + require.ErrorContains(t, err, "@other/private, @other/more") + require.NotContains(t, err.Error(), "@acme/eng") + require.Equal(t, before, hub.calls.Load()) // The direct CLI bypasses the flight and searches what it names. _, err = tap.Search(t.Context(), tapper.SearchOptions{ @@ -241,15 +248,16 @@ func TestSearch_GovernedUnscopedSearchLetsTheHubScopeToTheCover(t *testing.T) { require.NotEmpty(t, hub.lastProof(t), "the cover is only implied by the proof, so the proof must travel") // Explicit kegs still narrow, and still only within the cover. - resp, err := tap.Search(ctx, tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{refs[3], "@other/private"}, Query: "release"}) + _, err = tap.Search(ctx, tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{refs[3]}, Query: "release"}) require.NoError(t, err) require.Equal(t, []string{refs[3]}, hub.last(t).Kegs) - require.Contains(t, strings.Join(resp.Warnings, "\n"), "@other/private") + _, err = tap.Search(ctx, tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{refs[3], "@other/private"}, Query: "release"}) + require.ErrorContains(t, err, "@other/private") // A governed flight that covers nothing still never reaches the Hub. before := hub.calls.Load() empty := tapper.KegTargetOptions{FlightContext: &tapper.Flight{Name: "@acme/+empty", EffectiveCover: []tapper.HubEffectiveKeg{}}} - resp, err = tap.Search(ctx, tapper.SearchOptions{KegTargetOptions: empty, Query: "release"}) + resp, err := tap.Search(ctx, tapper.SearchOptions{KegTargetOptions: empty, Query: "release"}) require.NoError(t, err) require.Empty(t, resp.Hits) require.Equal(t, before, hub.calls.Load()) @@ -295,15 +303,71 @@ func TestSearch_ExplicitKegsNeedANamespacedCoverEntry(t *testing.T) { inFlight := tapper.KegTargetOptions{FlightContext: flight} // A namespace-less cover entry matches no namespace, explicit or implicit. - resp, err := tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@other/notes", "@acme/eng"}, Query: "release"}) + _, err := tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@other/notes", "@acme/eng"}, Query: "release"}) + require.ErrorIs(t, err, keg.ErrInvalid) + require.ErrorContains(t, err, "does not cover @other/notes") + + resp, err := tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Query: "release"}) require.NoError(t, err) require.Equal(t, []string{"@acme/eng"}, hub.last(t).Kegs) - require.Contains(t, strings.Join(resp.Warnings, "\n"), "@other/notes is not in flight") + require.Contains(t, strings.Join(resp.Warnings, "\n"), `cover entry "notes" names no namespace`) +} + +// Search reads the same cover the role check does: a KEG the flight blocks +// for a reader is not searched for that reader, whichever cover entry blocks +// it and whatever later entry names it again. +func TestSearch_CoverAgreesWithRoleFor(t *testing.T) { + t.Parallel() + + hub, url := newFakeSearchHub(t, http.StatusOK, `{"hits":[]}`) + tap := newSearchTap(t, url) + cases := map[string][]tapper.FlightCover{ + "blocked first by the same reference": { + {Namespace: "acme", Keg: "eng", Role: tapper.FlightRoleNone}, + {Namespace: "acme", Keg: "eng", Role: tapper.FlightRoleViewer}, + {Namespace: "acme", Keg: "notes", Role: tapper.FlightRoleViewer}, + }, + "blocked first by a bare entry": { + {Keg: "eng", Role: tapper.FlightRoleNone}, + {Namespace: "acme", Keg: "eng", Role: tapper.FlightRoleEditor}, + {Namespace: "acme", Keg: "notes", Role: tapper.FlightRoleViewer}, + }, + } + for name, cover := range cases { + flight := &tapper.Flight{Name: "@acme/+work", FlightManifest: tapper.FlightManifest{Cover: cover}} + _, covered := flight.RoleFor("", "acme", "eng") + require.False(t, covered, "%s: the role check denies @acme/eng", name) + inFlight := tapper.KegTargetOptions{FlightContext: flight} + + _, err := tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Query: "release"}) + require.NoError(t, err, name) + require.Equal(t, []string{"@acme/notes"}, hub.last(t).Kegs, "%s: an unscoped search leaves out what the role check denies", name) + + _, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@acme/eng"}, Query: "release"}) + require.ErrorContains(t, err, "does not cover @acme/eng", "%s: naming a denied KEG is refused", name) + } +} + +// The first matching entry decides, so a block that comes after a grant does +// not take the KEG out of search, just as it does not take it out of reach. +func TestSearch_CoverKeepsWhatRoleForGrants(t *testing.T) { + t.Parallel() - resp, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Query: "release"}) + hub, url := newFakeSearchHub(t, http.StatusOK, `{"hits":[]}`) + tap := newSearchTap(t, url) + flight := &tapper.Flight{Name: "@acme/+work", FlightManifest: tapper.FlightManifest{Cover: []tapper.FlightCover{ + {Namespace: "acme", Keg: "eng", Role: tapper.FlightRoleViewer}, + {Namespace: "acme", Keg: "eng", Role: tapper.FlightRoleNone}, + {Keg: "notes", Role: tapper.FlightRoleViewer}, + }}} + role, covered := flight.RoleFor("", "acme", "eng") + require.True(t, covered) + require.Equal(t, tapper.FlightRoleViewer, role) + + resp, err := tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: tapper.KegTargetOptions{FlightContext: flight}, Query: "release"}) require.NoError(t, err) require.Equal(t, []string{"@acme/eng"}, hub.last(t).Kegs) - require.Contains(t, strings.Join(resp.Warnings, "\n"), `cover entry "notes" names no namespace`) + require.Contains(t, strings.Join(resp.Warnings, "\n"), `cover entry "notes" names no namespace`, "a bare entry is still reported") } func TestSearch_LegacyAllowedKegsScopeBothPaths(t *testing.T) { @@ -320,9 +384,11 @@ func TestSearch_LegacyAllowedKegsScopeBothPaths(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"@acme/notes", "@acme/eng"}, hub.last(t).Kegs, "an unscoped search covers the legacy allowlist") - _, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@acme/eng", "@other/bare"}, Query: "release"}) + _, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@acme/eng"}, Query: "release"}) require.NoError(t, err) require.Equal(t, []string{"@acme/eng"}, hub.last(t).Kegs, "an explicit search honors the same allowlist") + _, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: inFlight, Kegs: []string{"@acme/eng", "@other/bare"}, Query: "release"}) + require.ErrorContains(t, err, "does not cover @other/bare", "an explicit search outside the allowlist is refused") } func TestSearch_EmptyRestrictedScopeNeverReachesTheHub(t *testing.T) { @@ -338,10 +404,8 @@ func TestSearch_EmptyRestrictedScopeNeverReachesTheHub(t *testing.T) { require.Empty(t, resp.Hits) narrow := tapper.KegTargetOptions{FlightContext: &tapper.Flight{Name: "@acme/+work", EffectiveCover: []tapper.HubEffectiveKeg{{Ref: "@acme/notes", Role: tapper.FlightRoleViewer}}}} - resp, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: narrow, Kegs: []string{"@other/private"}, Query: "release"}) - require.NoError(t, err) - require.Empty(t, resp.Hits) - require.NotEmpty(t, resp.Warnings) + _, err = tap.Search(t.Context(), tapper.SearchOptions{KegTargetOptions: narrow, Kegs: []string{"@other/private"}, Query: "release"}) + require.ErrorIs(t, err, keg.ErrInvalid, "a search naming only KEGs outside the cover is refused") require.Zero(t, hub.calls.Load(), "an empty KEG list means everything to the Hub, so a scope narrowed to nothing must not be sent") } diff --git a/pkg/tapper/tap_snapshots.go b/pkg/tapper/tap_snapshots.go index 454ef884..8fe43092 100644 --- a/pkg/tapper/tap_snapshots.go +++ b/pkg/tapper/tap_snapshots.go @@ -103,7 +103,7 @@ func (t *Tap) resolveSnapshotNode(ctx context.Context, targetOpts KegTargetOptio func parseNodeID(raw string) (keg.NodeId, error) { node, err := keg.ParseNode(raw) if err != nil { - return keg.NodeId{}, fmt.Errorf("invalid node ID %q: %w", raw, err) + return keg.NodeId{}, fmt.Errorf("invalid node ID %q: %w: %w", raw, err, keg.ErrInvalid) } if node == nil { return keg.NodeId{}, fmt.Errorf("invalid node ID %q: %w", raw, keg.ErrInvalid) diff --git a/pkg/tapper/tap_stats.go b/pkg/tapper/tap_stats.go index b3cea811..aa0440e2 100644 --- a/pkg/tapper/tap_stats.go +++ b/pkg/tapper/tap_stats.go @@ -3,6 +3,8 @@ package tapper import ( "context" "fmt" + + "github.com/jlrickert/tapper/pkg/keg" ) type StatsOptions struct { @@ -28,7 +30,7 @@ func (t *Tap) Stats(ctx context.Context, opts StatsOptions) (string, error) { return "", fmt.Errorf("unable to inspect node: %w", err) } if !exists { - return "", fmt.Errorf("node %s not found in %s", node.Path(), describeKeg(k)) + return "", fmt.Errorf("node %s not found in %s: %w", node.Path(), describeKeg(k), keg.ErrNotExist) } stats, err := k.GetStats(ctx, node) diff --git a/pkg/tapper/tap_trash.go b/pkg/tapper/tap_trash.go index 94e0751c..a1b2d3f3 100644 --- a/pkg/tapper/tap_trash.go +++ b/pkg/tapper/tap_trash.go @@ -31,7 +31,7 @@ func (t *Tap) TrashRestore(ctx context.Context, opts TrashRestoreOptions) error for i, raw := range opts.NodeIDs { id, err := keg.ParseNode(raw) if err != nil { - return fmt.Errorf("restore %d node %q: %w", i, raw, err) + return fmt.Errorf("restore %d node %q: %w: %w", i, raw, err, keg.ErrInvalid) } ids[i] = *id } diff --git a/schemas/flight-manifest.json b/schemas/flight-manifest.json index 5c0b5f7b..6c9caf21 100644 --- a/schemas/flight-manifest.json +++ b/schemas/flight-manifest.json @@ -75,6 +75,16 @@ "type": "string" } }, + "manageKegs": { + "type": "boolean", + "default": false, + "description": "Let a request rooted on this flight create and delete the KEGs of the flight's own namespace and administer any of them (settings, rename, visibility, grants), even outside its cover; their notes stay out of reach. A subflight in another namespace that sets it adds that namespace for its roots. Never more than the user's own role." + }, + "manageFlights": { + "type": "boolean", + "default": false, + "description": "Let a request rooted on this flight create, edit, and delete the flights of its own namespace, widening them up to the user's own role. A subflight in another namespace that sets it adds that namespace for its roots." + }, "instructions": { "type": "string", "description": "Markdown instructions injected into orientation output while the flight is active."