diff --git a/.changeset/human-actor-and-ignored-trace.md b/.changeset/human-actor-and-ignored-trace.md new file mode 100644 index 00000000..2c297ea4 --- /dev/null +++ b/.changeset/human-actor-and-ignored-trace.md @@ -0,0 +1,9 @@ +--- +"@statelyai/agent": patch +--- + +Events a person sends, and ignored events in traces. + +- **`meta: { actor: "human" }` on a transition** means a person sends its event. `agent.decide` and `decideNext` never offer it: a closed choice lists its options in `withheld`, and a choice of it anyway is retried. `getAcceptedEvents(snapshot, { actor: "agent" })` returns the same model-facing list. `describeMachine` marks the transition `(human)`. +- **New lint check `decide-allows-human-event`** (error): a decision's literal `allowedEvents` names a tagged event. `verifyAgentConfig` fails its `lint` gate on it. +- **`machine.transition` trace events carry `ignored: true`** for the host event the state had no transition for, the one `result.ignored` reports. `foldTraces` counts them per state as `ignored`. diff --git a/docs/debugging.md b/docs/debugging.md index aa934ef7..749b4648 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -24,6 +24,6 @@ Common errors: | `AgentMaxModelCallsExceededError` | The run exceeded its configured model-call budget. | | XState version error | The persisted snapshot version needs the machine's native `migrate` function. | -An event the active state has no transition for is not an error: the run settles normally and the event comes back as `result.ignored`. +An event the active state has no transition for is not an error: the run settles normally and the event comes back as `result.ignored`. Its `machine.transition` trace event carries `ignored: true`. -`lintAgentMachine` reports only Agent-specific mistakes: decisions with no candidate events, direct object request sources that a host cannot bind, and invokes with no error path. General state-machine lint belongs in XState tooling. +`lintAgentMachine` reports only Agent-specific mistakes: decisions with no candidate events, decisions that list an event only a person sends, direct object request sources that a host cannot bind, and invokes with no error path. General state-machine lint belongs in XState tooling. diff --git a/docs/decisions.md b/docs/decisions.md index 5dd12c7c..aca3da87 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -121,6 +121,32 @@ const request = createDecisionRequest({ }); ``` +## Events a person sends + +Tag a transition with `meta: { actor: "human" }` when only a person may send its event, such as an approval. No model-facing decision offers that event in a state that handles it this way. + +```ts no-check +reviewing: { + invoke: { + src: "agent.decide", + input: { model: "quick", allowedEvents: ["revise"] }, + }, + on: { + approve: { target: "approved", meta: { actor: "human" } }, + revise: { target: "revising" }, + }, +}, +``` + +- `agent.decide` and `decideNext` drop the event from the candidates. A closed choice lists its options in `withheld`, and a choice of it anyway is retried. +- `getAcceptedEvents(snapshot, { actor: "agent" })` returns the same model-facing list, for a host that offers events itself. +- `describeMachine` marks the transition `(human)`. +- `lintAgentMachine` reports `decide-allows-human-event` when a decision's literal `allowedEvents` names a tagged event. `verifyAgentConfig` fails its `lint` gate on it. +- Only object transitions can be tagged. A transition written as a function has no static `meta`. +- A plain `createMachine` declares `schemas: { meta: interactionMetaSchema }` to type the tag; `setupAgent` machines have it by default. + +The tag does not change what the machine accepts. A host still sends the event when a person answers, and `parseAgentEvent` still parses it. + ## Validation and retries diff --git a/docs/environment-and-policy.md b/docs/environment-and-policy.md index 2b85dee7..862941a4 100644 --- a/docs/environment-and-policy.md +++ b/docs/environment-and-policy.md @@ -152,6 +152,7 @@ The `vocabulary` gate keeps a candidate inside the environment's contract: it li People own everything that defines "better" and everything the policy cannot touch. - The environment: its states, guards, invariants, and the `canReach` proofs behind them. +- The events a person sends. Tag their transitions `meta: { actor: "human" }` so no decision offers them and `verifyAgentConfig` fails a policy that lists them. See [Events a person sends](decisions.md#events-a-person-sends). - The goal, its success criteria, and the objective function. A model can draft an objective from the goal; a person confirms it before it is used. - The vocabulary a policy may reference. - The scenario set, the held-out split, and the regression tolerance. diff --git a/docs/evals.md b/docs/evals.md index 5e135c0a..4146536e 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -199,7 +199,7 @@ A queue that runs dry throws an `AgentError` with code `seam-script-exhausted`, ## Per-state stats -`foldTraces` (from `@statelyai/agent/testing`) folds many runs' traces into one row per state: how many runs visited it, how many of those failed, and how many options the guards withheld at decisions made there. +`foldTraces` (from `@statelyai/agent/testing`) folds many runs' traces into one row per state: how many runs visited it, how many of those failed, how many options the guards withheld at decisions made there, and how many events it ignored. ```ts no-check import { foldTraces } from "@statelyai/agent/testing"; @@ -211,12 +211,13 @@ const stats = foldTraces(traces, { return end?.type !== "run.end" || !end.snapshot.matches("refunded"); }, }); -// [{ state: "deciding", visits: 2, failures: 1, withheld: 2 }, ...] +// [{ state: "deciding", visits: 2, failures: 1, withheld: 2, ignored: 0 }, ...] ``` - A trace is one run's events, live or from `serializeTraceEvent`. `failed` judges the whole trace. - A run visits each state in a `machine.transition`'s `from.value` and `snapshot.value`. Nested states are dot-joined paths; a parallel state counts once per active leaf. - `withheld` comes from a decision's first `request.start`: the length of `request.withheld`, counted toward the state the run was in, which is the `from.value` of the transition the choice causes. +- `ignored` counts `machine.transition` events with `ignored: true`: a host event the state had no transition for, counted toward that state. - `decideNext` called by the host emits no trace. Push its request onto the trace before sending the event: `trace.push({ type: "request.start", request: result.request })`. - Rows come in the order states were first seen. Sort them yourself, for example by `failures / visits`. diff --git a/docs/observability.md b/docs/observability.md index 130f4cc6..054f75fa 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -17,7 +17,7 @@ await runToQuiescence( Agent trace kinds are `run.start`, `request.start`, `request.end`, `request.error`, `stream.chunk`, `machine.transition`, `emit`, `usage.dropped`, and `run.end`. -A `machine.transition` event carries `from.value`, the state value the event arrived in, next to `snapshot`, the state after it. The first transition of a stream (the init event, or a restore on resume) has no earlier state, so its `from.value` equals its `snapshot.value`. `traceTransitions` sets it the same way for a plain actor. +A `machine.transition` event carries `from.value`, the state value the event arrived in, next to `snapshot`, the state after it. The first transition of a stream (the init event, or a restore on resume) has no earlier state, so its `from.value` equals its `snapshot.value`. `traceTransitions` sets it the same way for a plain actor. When the host's own event had no transition in the active state, the runtime's `machine.transition` also carries `ignored: true` (the event `result.ignored` reports), so evals can count those events from the trace alone. A `request.end` event carries the call's `output` and `raw`, plus what the executor reported about the call itself: `usage` when it reported tokens, and `finishReason` when it reported why the call stopped, normalized to `'stop'`, `'length'`, `'tool-calls'`, `'content-filter'`, or `'other'`. See [Finish reason and truncation](text-requests.md#finish-reason-and-truncation). diff --git a/docs/verify.md b/docs/verify.md index 284c7621..7d293675 100644 --- a/docs/verify.md +++ b/docs/verify.md @@ -40,11 +40,12 @@ skip checks by code. assertAgentMachine(machine, { warnings: true }); ``` -| Code | Severity | Fires when | -| ------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `decide-without-events` | error | A state invokes `agent.decide` but neither it nor any ancestor handles any event, so the chosen event can never be delivered. | -| `invoke-without-on-error` | warning | An invoke declares no `onError`, and neither its state nor any ancestor handles an actor error, so a rejected request or actor lands the machine in an error state with no modeled recovery. Add `onError` targeting a `failed` final state or a bounded retry. | -| `direct-object-src` | warning | An invoke `src` is a direct object or machine value that `runToQuiescence` cannot rebind, so it inherits no host executors. | +| Code | Severity | Fires when | +| --------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `decide-without-events` | error | A state invokes `agent.decide` but neither it nor any ancestor handles any event, so the chosen event can never be delivered. | +| `decide-allows-human-event` | error | A decision's literal `allowedEvents` names an event the state or an ancestor handles through a transition tagged `meta: { actor: "human" }`, so the model is never offered it. See [Events a person sends](decisions.md#events-a-person-sends). | +| `invoke-without-on-error` | warning | An invoke declares no `onError`, and neither its state nor any ancestor handles an actor error, so a rejected request or actor lands the machine in an error state with no modeled recovery. Add `onError` targeting a `failed` final state or a bounded retry. | +| `direct-object-src` | warning | An invoke `src` is a direct object or machine value that `runToQuiescence` cannot rebind, so it inherits no host executors. | ## Test assertions diff --git a/examples/README.md b/examples/README.md index db76b70a..c162597e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -113,6 +113,7 @@ key or a specific runtime, so they set `manual: true` and are not exported from - [cloudflare-agent-host](cloudflare-agent-host): a Durable Object whose SQLite event log is the source of truth - [cloudflare-workers-ai-host](cloudflare-workers-ai-host): a provider with no native tool calling, so the legal events are serialized into the prompt - [next-host](next-host): controlled mode across a stateless Next route handler +- [mcp-workflow](mcp-workflow): a coding agent drives a bug-fix machine through MCP tools; review events are tagged `actor: "human"`, so only a person sends them ## Evals and verification diff --git a/examples/mcp-workflow/index.test.ts b/examples/mcp-workflow/index.test.ts new file mode 100644 index 00000000..ed8b6a2e --- /dev/null +++ b/examples/mcp-workflow/index.test.ts @@ -0,0 +1,106 @@ +import { expect, test } from "vitest"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { answer, createBugfixServer, type RunStatus } from "./index.js"; + +async function connect() { + const [clientSide, serverSide] = InMemoryTransport.createLinkedPair(); + await createBugfixServer().connect(serverSide); + const client = new Client({ name: "test", version: "1.0.0" }); + await client.connect(clientSide); + const call = async (name: string, args: Record) => { + const result = (await client.callTool({ name, arguments: args })) as { + content: { text: string }[]; + isError?: boolean; + }; + const text = result.content[0]!.text; + return result.isError + ? { error: text } + : (JSON.parse(text) as RunStatus & { workflow?: string }); + }; + return { client, call }; +} + +test("lists the three tools", async () => { + const { client } = await connect(); + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).toEqual(["start", "status", "send"]); +}); + +test("the agent works the run; a person approves it", async () => { + const { call } = await connect(); + + const started = await call("start", { issue: "Totals are off by one" }); + if ("error" in started) throw new Error(started.error); + expect(started.workflow).toContain("on approve (human) → approved"); + expect(started.state).toBe("reproducing"); + expect(started.instructions).toBe("Write a test that fails because of the bug."); + expect(started.events.map((event) => event.type)).toEqual(["reproduced", "cannotReproduce"]); + expect(started.events[0]!.payload).toMatchObject({ required: ["test"] }); + + const { runId } = started; + await call("send", { runId, event: { type: "reproduced", test: "totals.test.ts" } }); + await call("send", { runId, event: { type: "fixed", summary: "Use < instead of <=" } }); + const inReview = await call("send", { runId, event: { type: "testsPassed" } }); + expect(inReview).toMatchObject({ + state: "inReview", + waitingFor: "Review the fix: Use < instead of <=", + events: [], + }); + + // Review is a person's call. + expect(await call("send", { runId, event: { type: "approve" } })).toEqual({ + error: "A person sends 'approve'. Call status later to see their answer.", + }); + + await answer(runId, { type: "approve" }); + expect(await call("status", { runId })).toMatchObject({ + state: "approved", + output: { summary: "Use < instead of <=" }, + }); +}); + +test("a bad payload and an event the state does not handle come back as tool errors", async () => { + const { call } = await connect(); + const started = await call("start", { issue: "Crash on empty input" }); + if ("error" in started) throw new Error(started.error); + const { runId } = started; + + const badPayload = await call("send", { runId, event: { type: "reproduced" } }); + expect(badPayload).toHaveProperty("error", expect.stringContaining("reproduced")); + + expect(await call("send", { runId, event: { type: "testsPassed" } })).toEqual({ + error: "'testsPassed' does not apply in state 'reproducing'.", + }); + expect(await call("status", { runId })).toMatchObject({ state: "reproducing" }); +}); + +test("requested changes send the agent back to fixing", async () => { + const { call } = await connect(); + const started = await call("start", { issue: "Slow search" }); + if ("error" in started) throw new Error(started.error); + const { runId } = started; + await call("send", { runId, event: { type: "reproduced", test: "search.test.ts" } }); + await call("send", { runId, event: { type: "fixed", summary: "Add an index" } }); + await call("send", { runId, event: { type: "testsPassed" } }); + + await answer(runId, { type: "requestChanges", notes: "Cover the empty query too" }); + expect(await call("status", { runId })).toMatchObject({ + state: "fixing", + events: [{ type: "fixed" }], + }); +}); + +test("concurrent sends to one run apply one at a time", async () => { + const { call } = await connect(); + const started = await call("start", { issue: "Flaky retry" }); + if ("error" in started) throw new Error(started.error); + const { runId } = started; + + const [first, second] = await Promise.all([ + call("send", { runId, event: { type: "reproduced", test: "retry.test.ts" } }), + call("send", { runId, event: { type: "cannotReproduce", notes: "Passes locally" } }), + ]); + expect(first).toMatchObject({ state: "fixing" }); + expect(second).toEqual({ error: "'cannotReproduce' does not apply in state 'fixing'." }); +}); diff --git a/examples/mcp-workflow/index.ts b/examples/mcp-workflow/index.ts new file mode 100644 index 00000000..510732f9 --- /dev/null +++ b/examples/mcp-workflow/index.ts @@ -0,0 +1,301 @@ +/** + * MCP workflow: a coding agent (Claude Code, Codex, any MCP client) works + * through a machine step by step, over MCP tools. + * + * The agent does the work in its own session; the machine holds the process. + * Three tools bridge them: + * + * - `start` opens a run and returns the workflow, described for a model + * (`describeMachine`), plus the run's status. + * - `status` returns where a run is: its state, the events the agent may + * send there (with payload JSON Schemas), or what it is waiting on a + * person for. + * - `send` delivers one event. The payload is parsed at the boundary + * (`parseAgentEvent`); an event a person sends is not delivered; an event + * the state has no transition for is ignored, and the tool says so. + * + * Review is a person's call: `approve` and `requestChanges` are tagged + * `meta: { actor: "human" }`, so the agent is never offered them and `send` + * will not deliver them. The host's own UI answers with `answer(...)`, and the + * agent's next `status` call picks the run up from there. + * + * Runs live in a `Map` of persisted snapshots, and deliveries to one run go + * one at a time. A real host keeps runs in its own store, keyed the same way, + * and guards each write with a version check. Over stdio the only caller is + * the agent that started the server; a server reachable by several clients + * scopes each run to the caller that started it. + * + * Run: npx tsx examples/mcp-workflow/index.ts + * Add to Claude Code: claude mcp add bugfix -- npx tsx examples/mcp-workflow/index.ts + */ +import { randomUUID } from "node:crypto"; +import { z } from "zod"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { + createAgentRuntime, + describeMachine, + getAcceptedEvents, + getAgentSchemas, + getInteraction, + getJsonSchemaSync, + interactionMetaSchema, + parseAgentEvent, + runToQuiescence, + setupAgent, + type AgentRunResult, + type EventOf, +} from "@statelyai/agent"; + +const agentSetup = setupAgent({ + context: z.object({ + issue: z.string(), + test: z.string().nullable(), + summary: z.string().nullable(), + notes: z.array(z.string()), + }), + input: z.object({ issue: z.string() }), + output: z.object({ summary: z.string() }), + meta: interactionMetaSchema, + events: { + reproduced: z.object({ test: z.string().describe("Path of the failing test") }), + cannotReproduce: z.object({ notes: z.string() }), + fixed: z.object({ summary: z.string().describe("What changed and why") }), + testsPassed: z.object({}), + testsFailed: z.object({ output: z.string() }), + approve: z.object({}), + requestChanges: z.object({ notes: z.string() }), + infoProvided: z.object({ notes: z.string() }), + }, +}); + +export const bugfixMachine = agentSetup.createMachine({ + id: "bugfix", + description: "Fix the reported bug with a regression test, and get the fix approved.", + context: ({ input }) => ({ issue: input.issue, test: null, summary: null, notes: [] }), + output: ({ context }) => ({ summary: context.summary ?? "" }), + initial: "reproducing", + states: { + reproducing: { + description: "Write a test that fails because of the bug.", + on: { + reproduced: ({ event }) => ({ target: "fixing", context: { test: event.test } }), + cannotReproduce: ({ context, event }) => ({ + target: "needsInfo", + context: { notes: [...context.notes, event.notes] }, + }), + }, + }, + fixing: { + description: "Make the failing test pass. Do not touch unrelated code.", + on: { + fixed: ({ event }) => ({ target: "verifying", context: { summary: event.summary } }), + }, + }, + verifying: { + description: "Run the full test suite.", + on: { + testsPassed: { target: "inReview" }, + testsFailed: ({ context, event }) => ({ + target: "fixing", + context: { notes: [...context.notes, event.output] }, + }), + }, + }, + inReview: { + description: "A person reviews the fix.", + meta: { + interaction: { + label: "Review the fix: {summary}", + events: { + approve: { label: "Approve", style: "primary" }, + requestChanges: { label: "Request changes" }, + }, + textEvent: { type: "requestChanges", field: "notes" }, + }, + }, + on: { + approve: { target: "approved", meta: { actor: "human" } }, + requestChanges: { + target: "fixing", + meta: { actor: "human" }, + context: ({ context, event }) => ({ notes: [...context.notes, event.notes] }), + }, + }, + }, + needsInfo: { + description: "A person adds what is needed to reproduce the bug.", + meta: { + interaction: { + label: "The bug did not reproduce. What else is needed?", + textEvent: { type: "infoProvided", field: "notes" }, + }, + }, + on: { + infoProvided: { + target: "reproducing", + meta: { actor: "human" }, + context: ({ context, event }) => ({ notes: [...context.notes, event.notes] }), + }, + }, + }, + approved: { type: "final" }, + }, +}); + +type BugfixEvent = EventOf; + +/** What a run looks like to the agent. */ +export interface RunStatus { + runId: string; + state: string; + /** The current state's description: what to do now. */ + instructions?: string; + /** What the run is waiting on a person for; call `status` again later. */ + waitingFor?: string; + /** The events the agent may send now, each with its payload's JSON Schema. */ + events: { type: string; payload?: Record }[]; + output?: { summary: string }; +} + +interface StoredRun { + /** What a resume restores from. */ + persisted: unknown; + status: RunStatus; + /** Every event the state accepts, a person's included. */ + accepted: string[]; +} + +/** Every run, by id. A real host keeps these in its own store. */ +const runs = new Map(); + +/** The event payload schemas, for the JSON Schemas `status` lists. */ +const schemas = getAgentSchemas(bugfixMachine); + +function save(runId: string, result: AgentRunResult): RunStatus { + if (result.status === "error") throw result.error; + const snapshot = result.snapshot; + const state = String(snapshot.value); + const instructions = bugfixMachine.root.states[state]?.description; + const events = getAcceptedEvents(snapshot, { actor: "agent", schemas }).map((event) => ({ + type: event.type, + ...(event.inputSchema ? { payload: getJsonSchemaSync(event.inputSchema) } : {}), + })); + const interaction = events.length === 0 ? getInteraction(snapshot) : undefined; + const status: RunStatus = { + runId, + state, + ...(instructions ? { instructions } : {}), + ...(interaction ? { waitingFor: interaction.label } : {}), + events, + ...(result.status === "done" ? { output: result.output } : {}), + }; + runs.set(runId, { + persisted: result.persist(), + status, + accepted: getAcceptedEvents(snapshot).map((event) => event.type), + }); + return status; +} + +function load(runId: string): StoredRun { + const run = runs.get(runId); + if (!run) throw new Error(`No run '${runId}'.`); + return run; +} + +/** Each run's last delivery, so deliveries to one run never restore the same snapshot. */ +const deliveries = new Map>(); + +function deliver(runId: string, event: BugfixEvent): Promise { + const delivery = (deliveries.get(runId) ?? Promise.resolve()) + .catch(() => {}) + .then(async () => { + const result = await runToQuiescence(createAgentRuntime(bugfixMachine), { + snapshot: load(runId).persisted as never, + event, + }); + if (result.ignored) { + throw new Error( + `'${event.type}' does not apply in state '${String(result.snapshot.value)}'.`, + ); + } + return save(runId, result); + }); + deliveries.set(runId, delivery); + return delivery; +} + +/** Opens a run for an issue. */ +export async function start(issue: string): Promise { + const result = await runToQuiescence(createAgentRuntime(bugfixMachine), { input: { issue } }); + return save(randomUUID(), result); +} + +/** Where a run is now. */ +export function status(runId: string): RunStatus { + return load(runId).status; +} + +/** The agent's move: parsed at the boundary, never an event a person sends. */ +export async function send(runId: string, payload: unknown): Promise { + const run = load(runId); + const event = parseAgentEvent(bugfixMachine, payload); + const forAgent = run.status.events.some((offered) => offered.type === event.type); + if (!forAgent && run.accepted.includes(event.type)) { + throw new Error(`A person sends '${event.type}'. Call status later to see their answer.`); + } + return deliver(runId, event); +} + +/** A person's answer, from the host's own UI. */ +export function answer(runId: string, event: BugfixEvent): Promise { + return deliver(runId, event); +} + +/** The MCP server: the three tools over the functions above. */ +export function createBugfixServer(): McpServer { + const server = new McpServer({ name: "bugfix", version: "1.0.0" }); + const reply = async (work: () => unknown) => { + try { + return { content: [{ type: "text" as const, text: JSON.stringify(await work(), null, 2) }] }; + } catch (error) { + const text = error instanceof Error ? error.message : String(error); + return { content: [{ type: "text" as const, text }], isError: true }; + } + }; + + server.registerTool( + "start", + { + description: "Open a bug-fix run. Returns the workflow and the run's status.", + inputSchema: { issue: z.string().describe("The bug report") }, + }, + ({ issue }) => + reply(async () => ({ workflow: describeMachine(bugfixMachine), ...(await start(issue)) })), + ); + server.registerTool( + "status", + { + description: "Where a run is: its state and the events you may send, or what it waits on.", + inputSchema: { runId: z.string() }, + }, + ({ runId }) => reply(() => status(runId)), + ); + server.registerTool( + "send", + { + description: "Report what happened: one of the events status lists, with its payload.", + inputSchema: { + runId: z.string(), + event: z.looseObject({ type: z.string() }).describe("{ type, ...payload }"), + }, + }, + ({ runId, event }) => reply(() => send(runId, event)), + ); + return server; +} + +if (process.argv[1] && import.meta.url === new URL(process.argv[1], "file:").href) { + await createBugfixServer().connect(new StdioServerTransport()); +} diff --git a/examples/mcp-workflow/metadata.json b/examples/mcp-workflow/metadata.json new file mode 100644 index 00000000..75706889 --- /dev/null +++ b/examples/mcp-workflow/metadata.json @@ -0,0 +1,16 @@ +{ + "name": "mcp-workflow", + "title": "MCP Workflow Server", + "kind": "host-adapter", + "manual": true, + "origin": { + "type": "platform-adapter", + "source": "Real @modelcontextprotocol/sdk: an McpServer over stdio with start, status, and send tools", + "url": "https://modelcontextprotocol.io" + }, + "comparison": { + "targets": ["Claude Code", "Codex", "MCP clients"], + "purpose": "A coding agent works through a bug-fix machine over MCP. start returns the workflow from describeMachine; status lists the events the agent may send, from getAcceptedEvents with actor: \"agent\"; send parses the payload at the boundary and reports an ignored event. Review events are tagged meta: { actor: \"human\" }, so only the host's own UI sends them." + }, + "starters": [] +} diff --git a/package.json b/package.json index 1d425395..7be06dc7 100644 --- a/package.json +++ b/package.json @@ -156,6 +156,7 @@ "@langchain/core": "^1.2.17", "@langchain/openai": "^1.6.2", "@mastra/core": "^1.75.0", + "@modelcontextprotocol/sdk": "^1.29.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/context-async-hooks": "^2.12.0", "@opentelemetry/sdk-trace-base": "^2.12.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a5beb3e4..da93cad7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: '@mastra/core': specifier: ^1.75.0 version: 1.75.0(@valibot/to-json-schema@1.8.0(valibot@1.5.0(typescript@7.0.2)))(ai@7.0.133(zod@4.6.5))(express@5.2.1)(typebox@1.3.27)(valibot@1.5.0(typescript@7.0.2))(zod-to-json-schema@3.25.2(zod@4.6.5))(zod@4.6.5) + '@modelcontextprotocol/sdk': + specifier: ^1.29.0 + version: 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5) '@opentelemetry/api': specifier: ^1.9.1 version: 1.9.1 diff --git a/src/decide-next.ts b/src/decide-next.ts index 97964150..27e4d5c3 100644 --- a/src/decide-next.ts +++ b/src/decide-next.ts @@ -8,7 +8,12 @@ import { type DecisionAttempt, } from "./decision.js"; import { canonicalEventJson } from "./event-log.js"; -import { getAcceptedEvents, isEventPattern, type EventFromSnapshot } from "./events.js"; +import { + getAcceptedEvents, + getHumanEventTypes, + isEventPattern, + type EventFromSnapshot, +} from "./events.js"; import { isRecord } from "./internal/is-record.js"; import type { ChosenEvent, StandardSchemaV1 } from "./types.js"; @@ -54,7 +59,9 @@ export interface DecideNextResult { /** * The candidates the snapshot rejected: every `allowedEvents` type × payload, * built the same way as the offered options, where `snapshot.can(event)` was - * false. Never offered to the model. Filled when nothing is offered too. + * false or the state handles the type through a transition tagged + * `meta: { actor: "human" }`. Never offered to the model. Filled when + * nothing is offered too. */ withheld: ChosenEvent[]; } @@ -63,7 +70,8 @@ export interface DecideNextResult { * Asks a model to choose the next event for ANY XState snapshot, including a * machine built with plain `createMachine`/`setup` that knows nothing about AI. * The candidates are the `allowedEvents` an active state handles, one option per - * payload, kept only where `snapshot.can(event)` is true; the rest come back as + * payload, kept only where `snapshot.can(event)` is true and no transition + * tagged `meta: { actor: "human" }` handles the type; the rest come back as * `withheld`. Each option is * described from the transition's `description`, else the event schema's, * plus (with `lookahead`) what it changes. The chosen event must equal one @@ -102,17 +110,19 @@ export async function decideNext( const patterns = typeof options.allowedEvents === "string" ? [options.allowedEvents] : options.allowedEvents; const schemas = snapshot.machine.schemas?.events as Record | undefined; - const events = getAcceptedEvents(snapshot, { events: schemas, eventTypes: patterns }).filter( + const candidates = getAcceptedEvents(snapshot, { events: schemas, eventTypes: patterns }).filter( (event) => !isEventPattern(event.type), ); + const humanEventTypes = getHumanEventTypes(snapshot); + const events = candidates.filter((event) => !humanEventTypes.has(event.type)); const payloads = options.payloads?.(snapshot) ?? {}; const accepted: ChosenEvent[] = []; const withheld: ChosenEvent[] = []; - for (const { type } of events) { + for (const { type } of candidates) { for (const payload of payloads[type] ?? [{}]) { const event = { ...payload, type }; - (safeCan(snapshot, event) ? accepted : withheld).push(event); + (!humanEventTypes.has(type) && safeCan(snapshot, event) ? accepted : withheld).push(event); } } diff --git a/src/describe-machine.ts b/src/describe-machine.ts index e310ff95..f8967969 100644 --- a/src/describe-machine.ts +++ b/src/describe-machine.ts @@ -202,7 +202,7 @@ function targetsInSource(source: AnyStateNode, code: string, choice: boolean): s return /\btarget\s*:/.test(code) ? "?" : "(stays)"; } -/** One transition, as a line: `on GO if context.ready → review; sets draft — description`. */ +/** One transition, as a line: `on GO (human) if context.ready → review; sets draft — description`. */ function transitionLine( source: AnyStateNode, head: string, @@ -213,6 +213,7 @@ function transitionLine( let to: string; let guard: string | undefined; let description: string | undefined; + let human = false; let sets: string[]; if (typeof transition === "string") { const node = resolveTarget(source, transition); @@ -233,12 +234,13 @@ function transitionLine( : "(stays)"; guard = guardText(transition[options.guardKey ?? "guard"]); description = typeof transition.description === "string" ? transition.description : undefined; + human = isObject(transition.meta) && transition.meta.actor === "human"; sets = contextKeys(transition, undefined); } else { to = "?"; sets = []; } - return `${head}${guard ? ` if ${guard}` : ""} → ${to}${sets.length ? `; sets ${sets.join(", ")}` : ""}${ + return `${head}${human ? " (human)" : ""}${guard ? ` if ${guard}` : ""} → ${to}${sets.length ? `; sets ${sets.join(", ")}` : ""}${ description ? ` — ${oneLine(description)}` : "" }`; } @@ -365,9 +367,10 @@ function eventLines(machine: AnyStateMachine): string[] { * label and description (`*` marks the active ones when `snapshot` is given), * and under it every transition leaving it, one per line: `on EVENT`, * `always`, `after `, invoke `done`/`error`, and `choice` branches, - * each with its guard (`if …`), where it goes (`→ review`), and what it sets - * in context (`sets draft, attempts`). An `Events` section lists the - * machine's event schemas with their descriptions and fields. + * each with `(human)` when its `meta.actor` is `"human"`, its guard (`if …`), + * where it goes (`→ review`), and what it sets in context (`sets draft, + * attempts`). An `Events` section lists the machine's event schemas with their + * descriptions and fields. * * A transition written as a function shows the targets its source names * (`→ review (conditional)` when it may decline or pick among several), diff --git a/src/events.ts b/src/events.ts index 68506605..6ed87e08 100644 --- a/src/events.ts +++ b/src/events.ts @@ -230,14 +230,20 @@ export function parseAgentEvent & { eventTypes?: readonly string[]; + actor?: "agent"; } = {}, ): AgentEventDescriptor[] { const eventTypes = options.eventTypes; + const humanEventTypes: ReadonlySet = + options.actor === "agent" ? getHumanEventTypes(snapshot) : new Set(); const seen = new Set(); const usedToolNames = new Set(); @@ -249,6 +255,7 @@ export function getAcceptedEvents( eventType === "*" || eventType.startsWith("xstate.") || isReservedAgentEvent(eventType) || + humanEventTypes.has(eventType) || (eventTypes && !eventTypes.some((pattern) => matchesEventPattern(eventType, pattern))) || seen.has(eventType) ) { @@ -270,3 +277,25 @@ export function getAcceptedEvents( ]; }); } + +/** + * The event types a snapshot handles through a transition tagged + * `meta: { actor: "human" }`: a person sends them, so no model is offered them. + * @internal + */ +export function getHumanEventTypes(snapshot: AnyMachineSnapshot): ReadonlySet { + let types = humanEventTypesBySnapshot.get(snapshot); + if (!types) { + types = new Set(); + for (const transition of getNextTransitions(snapshot)) { + if ((transition.meta as { actor?: unknown } | undefined)?.actor === "human") { + types.add(transition.eventType); + } + } + humanEventTypesBySnapshot.set(snapshot, types); + } + return types; +} + +// Snapshots are immutable, so each one's human event types are read once. +const humanEventTypesBySnapshot = new WeakMap>(); diff --git a/src/fold-traces.test.ts b/src/fold-traces.test.ts index 96907efa..a4b94823 100644 --- a/src/fold-traces.test.ts +++ b/src/fold-traces.test.ts @@ -76,9 +76,9 @@ describe("foldTraces", () => { const stats = foldTraces(traces, { failed: (trace) => endedIn(trace) !== "refunded" }); expect(stats).toEqual([ - { state: "deciding", visits: 2, failures: 1, withheld: 2 }, - { state: "refunded", visits: 1, failures: 0, withheld: 0 }, - { state: "escalated", visits: 1, failures: 1, withheld: 0 }, + { state: "deciding", visits: 2, failures: 1, withheld: 2, ignored: 0 }, + { state: "refunded", visits: 1, failures: 0, withheld: 0, ignored: 0 }, + { state: "escalated", visits: 1, failures: 1, withheld: 0, ignored: 0 }, ]); }); @@ -138,8 +138,25 @@ describe("foldTraces", () => { actor.send(result.event!); expect(foldTraces([trace], { failed: () => false })).toEqual([ - { state: "reviewing", visits: 1, failures: 0, withheld: 1 }, - { state: "refunded", visits: 1, failures: 0, withheld: 0 }, + { state: "reviewing", visits: 1, failures: 0, withheld: 1, ignored: 0 }, + { state: "refunded", visits: 1, failures: 0, withheld: 0, ignored: 0 }, + ]); + }); + + test("counts ignored events toward the state that ignored them", () => { + const trace = [ + { type: "machine.transition", from: { value: "idle" }, snapshot: { value: "waiting" } }, + { + type: "machine.transition", + from: { value: "waiting" }, + snapshot: { value: "waiting" }, + ignored: true, + }, + ]; + + expect(foldTraces([trace, trace], { failed: () => false })).toEqual([ + { state: "idle", visits: 2, failures: 0, withheld: 0, ignored: 0 }, + { state: "waiting", visits: 2, failures: 0, withheld: 0, ignored: 2 }, ]); }); diff --git a/src/fold-traces.ts b/src/fold-traces.ts index e8e8d06c..8db47aac 100644 --- a/src/fold-traces.ts +++ b/src/fold-traces.ts @@ -10,6 +10,8 @@ export interface StateStats { failures: number; /** Options withheld by the guards at decisions made in this state, summed over runs. */ withheld: number; + /** Events this state ignored (`machine.transition` with `ignored`), summed over runs. */ + ignored: number; } /** Options for {@link foldTraces}. */ @@ -20,10 +22,11 @@ export interface FoldTracesOptions { /** * Folds many runs' traces into per-state stats: how many runs visited each - * state, how many of those failed, and how many options the guards withheld - * at decisions made there. Pure; reads only: + * state, how many of those failed, how many options the guards withheld + * at decisions made there, and how many events it ignored. Pure; reads only: * - * - `machine.transition`: the run visited `from.value` and `snapshot.value`. + * - `machine.transition`: the run visited `from.value` and `snapshot.value`; + * with `ignored`, each of those states ignored one event. * - `request.start` of a decision, first attempt only: its `request.withheld` * counts toward the state the run was in, the `from.value` of the * transition its choice causes. @@ -48,7 +51,7 @@ export function foldTraces( const statsFor = (state: string): StateStats => { let entry = stats.get(state); if (!entry) { - entry = { state, visits: 0, failures: 0, withheld: 0 }; + entry = { state, visits: 0, failures: 0, withheld: 0, ignored: 0 }; stats.set(state, entry); } return entry; @@ -66,6 +69,9 @@ export function foldTraces( for (const state of statePaths(valueOf(event.from))) visit(state); current = statePaths(valueOf(event.snapshot)); for (const state of current) visit(state); + if (event.ignored === true) { + for (const state of current) statsFor(state).ignored++; + } } else if (event.type === "request.start" && isFirstDecision(event.request)) { const withheld = event.request.withheld; if (!Array.isArray(withheld)) continue; diff --git a/src/human-actor.test.ts b/src/human-actor.test.ts new file mode 100644 index 00000000..27030f98 --- /dev/null +++ b/src/human-actor.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, test } from "vitest"; +import { z } from "zod"; +import { createActor, createMachine } from "xstate"; +import { decideNext } from "./decide-next.js"; +import type { AgentDecisionExecutor, AgentDecisionRequest } from "./decision.js"; +import { describeMachine } from "./describe-machine.js"; +import { getAcceptedEvents } from "./events.js"; +import { interactionMetaSchema } from "./interaction.js"; +import { createAgentRuntime, runToQuiescence } from "./run-agent.js"; +import { createAgentSchemas, setupAgent } from "./setup-agent.js"; +import { explorePaths, lintAgentMachine } from "./verify.js"; + +const schemas = createAgentSchemas({ + context: z.object({}), + input: z.object({}), + events: { approve: z.object({}), revise: z.object({}), cancel: z.object({}) }, +}); + +// A person approves; the model may only ask for a revision. +const review = setupAgent({ schemas }).createMachine({ + context: {}, + initial: "reviewing", + on: { cancel: { target: ".cancelled", meta: { actor: "human" } } }, + states: { + reviewing: { + invoke: { + src: "agent.decide", + input: { model: "quick", allowedEvents: ["revise", "approve"], maxRetries: 1 }, + onError: { target: "failed" }, + }, + on: { + approve: { target: "approved", meta: { actor: "human" } }, + revise: { target: "revised" }, + }, + }, + approved: { type: "final" }, + revised: { type: "final" }, + cancelled: { type: "final" }, + failed: { type: "final" }, + }, +}); + +function recording(choose: (request: AgentDecisionRequest) => { type: string }) { + const requests: AgentDecisionRequest[] = []; + const decide: AgentDecisionExecutor = async (request) => { + requests.push(request); + return { event: choose(request) }; + }; + return { decide, requests }; +} + +describe('transition meta `actor: "human"`', () => { + test("getAcceptedEvents with `actor: 'agent'` drops human-tagged events", () => { + const snapshot = createActor(review, { input: {} }).getSnapshot(); + const types = (actor?: "agent") => + getAcceptedEvents(snapshot, actor ? { actor } : {}).map((event) => event.type); + + expect(types().sort()).toEqual(["approve", "cancel", "revise"]); + expect(types("agent")).toEqual(["revise"]); + }); + + test("agent.decide never offers a human-tagged event", async () => { + const { decide, requests } = recording(() => ({ type: "approve" })); + const result = await runToQuiescence(createAgentRuntime(review, { executors: { decide } }), { + input: {}, + }); + + expect(requests[0]!.events.map((event) => event.type)).toEqual(["revise"]); + // Choosing it anyway is retried, then the decision fails to onError. + expect(requests).toHaveLength(2); + expect(result.snapshot.value).toBe("failed"); + }); + + test("a closed choice withholds the human-tagged options", async () => { + const closed = setupAgent({ schemas }).createMachine({ + context: {}, + initial: "reviewing", + states: { + reviewing: { + invoke: { + src: "agent.decide", + input: { + model: "quick", + allowedEvents: ["revise", "approve"], + payloads: { revise: [{}], approve: [{}] }, + }, + }, + on: { + approve: { target: "approved", meta: { actor: "human" } }, + revise: { target: "revised" }, + }, + }, + approved: { type: "final" }, + revised: { type: "final" }, + }, + }); + const { decide, requests } = recording((request) => request.options![0]!.event); + const result = await runToQuiescence(createAgentRuntime(closed, { executors: { decide } }), { + input: {}, + }); + + expect(requests[0]!.options!.map((option) => option.event)).toEqual([{ type: "revise" }]); + expect(requests[0]!.withheld).toEqual([{ type: "approve" }]); + expect(result.snapshot.value).toBe("revised"); + }); + + test("decideNext withholds a human-tagged event on a plain machine", async () => { + const plain = createMachine({ + schemas: { meta: interactionMetaSchema }, + context: {}, + initial: "reviewing", + states: { + reviewing: { + on: { + approve: { target: "approved", meta: { actor: "human" } }, + revise: { target: "revised" }, + }, + }, + approved: {}, + revised: {}, + }, + }); + const { decide, requests } = recording((request) => request.options![0]!.event); + + const result = await decideNext(createActor(plain).getSnapshot(), { + decide, + model: "quick", + allowedEvents: "*", + }); + + expect(requests[0]!.options!.map((option) => option.event)).toEqual([{ type: "revise" }]); + expect(result.withheld).toEqual([{ type: "approve" }]); + expect(result.event).toEqual({ type: "revise" }); + }); + + test("describeMachine marks human-tagged transitions", () => { + const text = describeMachine(review); + expect(text).toContain("on cancel (human) → cancelled"); + expect(text).toContain("on approve (human) → approved"); + expect(text).toContain("on revise → revised"); + }); + + test("lint: a decision that lists a human-tagged event is an error", () => { + expect(lintAgentMachine(review)).toContainEqual( + expect.objectContaining({ + code: "decide-allows-human-event", + severity: "error", + path: "reviewing", + }), + ); + expect( + lintAgentMachine(review).filter((d) => d.code === "decide-allows-human-event"), + ).toHaveLength(1); + }); + + test("lint reads a single-string `allowedEvents` too", () => { + const single = setupAgent({ schemas }).createMachine({ + context: {}, + initial: "reviewing", + states: { + reviewing: { + invoke: { src: "agent.decide", input: { model: "quick", allowedEvents: "approve" } }, + on: { approve: { target: "approved", meta: { actor: "human" } } }, + }, + approved: { type: "final" }, + }, + }); + expect(lintAgentMachine(single).map((d) => d.code)).toContain("decide-allows-human-event"); + }); + + test("explorePaths branches a decision only on events a model may choose", async () => { + const report = await explorePaths(review, { input: {} }); + expect(report.terminals.map((terminal) => terminal.state)).toEqual(["revised"]); + }); + + test("the meta schema accepts only `actor: 'human'`", () => { + const validate = (value: unknown) => interactionMetaSchema["~standard"].validate(value); + expect(validate({ actor: "human" })).toEqual({ value: { actor: "human" } }); + expect(validate({ actor: "model" })).toHaveProperty("issues"); + }); +}); diff --git a/src/interaction.ts b/src/interaction.ts index 024742fc..9f9eeadc 100644 --- a/src/interaction.ts +++ b/src/interaction.ts @@ -90,6 +90,13 @@ export interface AgentInteractionDescriptor { */ export interface AgentInteractionMeta { interaction?: AgentInteractionDescriptor; + /** + * On a transition: `"human"` means a person sends its event, never a model. + * Model-facing decisions (`agent.decide`, `decideNext`) withhold an event a + * state handles through a transition tagged this way. Read from object + * transitions only; a transition written as a function cannot be tagged. + */ + actor?: "human"; } function issue(message: string, path: (string | number)[]) { @@ -147,7 +154,8 @@ function validateChoice(value: unknown, path: (string | number)[]) { * reads: an object whose optional `interaction` declares a `label` (a string * with `{context.path}` interpolation, or a function of the context), an * `events` map of choices, payload `choices` (an array, or a context path to - * one), and a `textEvent` (a type, or `{ type, field }`) for free-text answers. This is + * one), and a `textEvent` (a type, or `{ type, field }`) for free-text answers; and + * an optional `actor: "human"`, which tags a transition as sent by a person. This is * the default `meta` type `setupAgent` gives a machine; pass it to * `createAgentSchemas({ meta: interactionMetaSchema })` only when building a * schema pack by hand. @@ -160,6 +168,10 @@ export const interactionMetaSchema: StandardSchemaV1 = { if (!value || typeof value !== "object" || Array.isArray(value)) { return { issues: [{ message: "Expected a state meta object" }] }; } + const actor = (value as Record)["actor"]; + if (actor !== undefined && actor !== "human") { + return { issues: [issue('Expected "human"', ["actor"])] }; + } const interaction = (value as Record)["interaction"]; if (interaction === undefined) { return { value: value as AgentInteractionMeta }; diff --git a/src/run-agent.test.ts b/src/run-agent.test.ts index 017a442b..f1274f5c 100644 --- a/src/run-agent.test.ts +++ b/src/run-agent.test.ts @@ -2678,6 +2678,39 @@ describe("Feature B: an event the state does not handle is ignored", () => { expect(second.status === "idle" ? second.snapshot.value : undefined).toBe("reviewing"); }); + test("the trace marks the ignored event's transition", async () => { + const first = await runToQuiescence( + createAgentRuntime(machine, { executors: { generateText } }), + { + input: {}, + }, + ); + if (first.status !== "idle") throw new Error("expected idle"); + + const trace: AgentTraceEvent[] = []; + const runAgain = (event: { type: string }) => + runToQuiescence( + createAgentRuntime(machine, { + executors: { generateText }, + onTrace: (traceEvent) => trace.push(traceEvent), + }), + { snapshot: first.persist(), event: event as never }, + ); + await runAgain({ type: "NOPE" }); + await runAgain({ type: "APPROVE" }); + + const transitions = trace.filter( + (event): event is Extract => + event.type === "machine.transition", + ); + expect(transitions.map((event) => [event.event.type, event.ignored])).toEqual([ + ["@xstate.init", undefined], + ["NOPE", true], + ["@xstate.init", undefined], + ["APPROVE", undefined], + ]); + }); + test("an ignored event is still journaled, and replay ignores it again", async () => { const first = await runToQuiescence( createAgentRuntime(machine, { diff --git a/src/run-agent.ts b/src/run-agent.ts index 398c61f0..da72c0e3 100644 --- a/src/run-agent.ts +++ b/src/run-agent.ts @@ -35,7 +35,7 @@ import { resolveMachineVersion, validateSchemaSync, } from "./utils.js"; -import { getAcceptedEvents, type AgentSchemas } from "./events.js"; +import { getAcceptedEvents, getHumanEventTypes, type AgentSchemas } from "./events.js"; import { GENERATE_TEXT_ACTOR, getCallFinishReason, @@ -222,6 +222,13 @@ export type AgentTraceEvent from: { value: StateValue }; snapshot: SnapshotFrom; event: EventFromLogic; + /** + * `true` when the host's own event (the run's opening `event`, or one + * sent with `runtime.transition`) had no transition in this state, so + * nothing changed: the same event {@link AgentRunResult.ignored} + * reports. Absent otherwise. + */ + ignored?: true; } | { type: "emit"; event: EmittedFrom } | { @@ -1388,6 +1395,14 @@ function bindDecisionLogic(logic: DecisionLogic, runCtx: RunAgentBindContext): D }) : []; + // The model may take an event the snapshot accepts, unless a person sends + // it (a transition tagged `meta: { actor: "human" }`). + const agentCan = (event: ChosenEvent): boolean => { + if (!actorRef) return true; + const snapshot = actorRef.getSnapshot() as AnyMachineSnapshot; + return !getHumanEventTypes(snapshot).has(event.type) && snapshot.can(event); + }; + const lowered = logic.request(input as never); const request: AgentDecisionRequest = { ...lowered, @@ -1396,14 +1411,14 @@ function bindDecisionLogic(logic: DecisionLogic, runCtx: RunAgentBindContext): D input, events, }; - // A closed choice offers only the options the snapshot accepts right + // A closed choice offers only the options the model may take right // now; resolveDecision rejects a choice matching none of them. The rest // ride along as `withheld`, for traces. const withheld: ChosenEvent[] = []; const options = acceptedDecisionOptions(request, (event) => { let accepted: boolean; try { - accepted = actorRef ? (actorRef.getSnapshot() as AnyMachineSnapshot).can(event) : true; + accepted = agentCan(event); } catch { accepted = false; } @@ -1414,6 +1429,12 @@ function bindDecisionLogic(logic: DecisionLogic, runCtx: RunAgentBindContext): D request.options = options; request.withheld = withheld; } + // After the options: a closed choice builds them from `events`, so a + // person's event still lands in `withheld`. + if (actorRef) { + const humanEventTypes = getHumanEventTypes(actorRef.getSnapshot() as AnyMachineSnapshot); + request.events = events.filter((event) => !humanEventTypes.has(event.type)); + } // Nothing to choose from: fail without a model call, the same way // exhausted retries fail, so the invoke's onError handles it. if (options?.length === 0) { @@ -1444,8 +1465,7 @@ function bindDecisionLogic(logic: DecisionLogic, runCtx: RunAgentBindContext): D // the logic's configured budget. maxRetries: lowered.maxRetries ?? logic.maxRetries, signal, - canTake: (event) => - actorRef ? (actorRef.getSnapshot() as AnyMachineSnapshot).can(event) : true, + canTake: agentCan, }, ); @@ -2742,7 +2762,7 @@ export function createAgentRuntime( ? "decision-exhausted" : "machine"; - const commit = (snapshot: AnyMachineSnapshot, event: EventObject): void => { + const commit = (snapshot: AnyMachineSnapshot, event: EventObject, ignored = false): void => { const from = { value: (current ?? snapshot).value }; current = snapshot; onTrace({ @@ -2750,6 +2770,7 @@ export function createAgentRuntime( from, snapshot: snapshot as SnapshotFrom, event: event as EventFromLogic, + ...(ignored ? { ignored: true as const } : {}), }); }; @@ -3114,15 +3135,15 @@ export function createAgentRuntime( // The host's own event (not one the mailbox handed out) was ignored when // the state has no transition for it: nothing changed and nothing is to // be done. Reported as `result.ignored`; still journaled. - if ( + const ignored = !fromMailbox.has(event as object) && (next as unknown) === (state as unknown) && - effects.length === 0 - ) { + effects.length === 0; + if (ignored) { ignoredEvent = event as EventObject; } appendLogEntry(event as EventObject, attempts); - commit(next, event as EventObject); + commit(next, event as EventObject, ignored); return [next as SnapshotFrom, effects]; }, diff --git a/src/steps.ts b/src/steps.ts index 3dbe336e..0a60bf7d 100644 --- a/src/steps.ts +++ b/src/steps.ts @@ -30,6 +30,7 @@ import { import { isDecisionLogic, type AgentDecisionRequest } from "./decision.js"; import { getAcceptedEvents, + getHumanEventTypes, type AgentEventDescriptor, type AgentRequestOptions, type AgentRequestSource, @@ -200,19 +201,31 @@ export function getAgentRequests( allowedEventTypes?: (input: unknown) => readonly string[] | undefined; } ).allowedEventTypes?.(params.input); + // The same model-facing set the runtime offers: no event a person sends. const events = options.snapshot ? getAcceptedEvents(options.snapshot, { events: options.events, schemas: options.schemas, eventTypes: allowedEventTypes, + actor: "agent", }) : []; + const humanEventTypes: ReadonlySet = options.snapshot + ? getHumanEventTypes(options.snapshot) + : new Set(); return [ { ...decisionRequest, id: params.id, events, + ...(decisionRequest.options + ? { + options: decisionRequest.options.filter( + (option) => !humanEventTypes.has(option.event.type), + ), + } + : {}), }, ]; } diff --git a/src/validate/verify-config.test.ts b/src/validate/verify-config.test.ts index f3e5c15b..c336b3ee 100644 --- a/src/validate/verify-config.test.ts +++ b/src/validate/verify-config.test.ts @@ -202,6 +202,14 @@ describe("verifyAgentConfig", () => { expect(result.machine).toBeDefined(); }); + test('lint: a decision that lists an `actor: "human"` event fails', async () => { + const config = refundConfig(); + config.states.triage.on.APPROVE.meta = { actor: "human" }; + const result = await verifyAgentConfig(config, options()); + expect(result.gates.at(-1)).toMatchObject({ name: "lint", ok: false }); + expect(result.gates.at(-1)!.detail).toContain("decide-allows-human-event at 'triage'"); + }); + test("lint: warnings are recorded by default and fail with `warnings: true`", async () => { const config = refundConfig(); delete config.states.triage.invoke.onError; diff --git a/src/verify.ts b/src/verify.ts index c578f946..2b8a2208 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -16,13 +16,14 @@ */ import { getNextTransitions, + machineToJSON, type AnyActorLogic, type AnyMachineSnapshot, type AnyStateMachine, } from "xstate"; import type { ChosenEvent, StandardSchemaV1 } from "./types.js"; import { AgentError } from "./errors.js"; -import { getAcceptedEvents, type AgentSchemas } from "./events.js"; +import { getAcceptedEvents, isEventPattern, type AgentSchemas } from "./events.js"; import { acceptedDecisionOptions, AgentDecisionExhaustedError, @@ -59,7 +60,11 @@ export type AgentLintSeverity = "error" | "warning"; * or config location, and `message` explains the problem and its remedy. */ export interface AgentLintDiagnostic { - code: "decide-without-events" | "direct-object-src" | "invoke-without-on-error"; + code: + | "decide-without-events" + | "decide-allows-human-event" + | "direct-object-src" + | "invoke-without-on-error"; severity: AgentLintSeverity; /** State path (`parent.child`) or config pointer (e.g. `(root)`, `context`) the finding is about. */ path: string; @@ -100,6 +105,7 @@ interface AnyConfig { interface InvokeConfig { id?: unknown; src?: unknown; + input?: unknown; onDone?: unknown; onError?: unknown; } @@ -150,6 +156,8 @@ function buildStateIndex(rootConfig: AnyConfig): Map { interface LintContext { config: AnyConfig; + /** The machine as `machineToJSON` writes it, where invoke inputs stay data. */ + json: AnyConfig | undefined; index: Map; schemas: { context?: unknown; output?: unknown; events?: Record } | undefined; actors: Record; @@ -230,6 +238,81 @@ function checkDecideWithoutEvents(ctx: LintContext): AgentLintDiagnostic[] { return out; } +// The `allowedEvents` of a state's invoke as written: from a literal input, +// or the JSON of a fromConfig `{ "@expr" }` input. A function input has none. +function literalAllowedEvents( + json: AnyConfig | undefined, + path: string, + index: number, +): unknown[] | undefined { + let node: AnyConfig | undefined = json; + for (const key of path.split(".")) node = node?.states?.[key]; + let input = normalizeInvokes(node?.invoke)[index]?.input; + if (isRecord(input) && typeof input["@expr"] === "string") { + try { + input = JSON.parse(input["@expr"]); + } catch { + return undefined; + } + } + const allowedEvents = isRecord(input) ? input.allowedEvents : undefined; + return typeof allowedEvents === "string" + ? [allowedEvents] + : Array.isArray(allowedEvents) + ? allowedEvents + : undefined; +} + +// True when the state handles `eventType` through a transition tagged `meta: { actor: "human" }`. +function handlesAsHuman(config: AnyConfig, eventType: string): boolean { + const transitions = config.on?.[eventType]; + return (Array.isArray(transitions) ? transitions : [transitions]).some( + (transition) => + !!transition && + typeof transition === "object" && + (transition as { meta?: { actor?: unknown } }).meta?.actor === "human", + ); +} + +function checkDecideAllowsHumanEvent(ctx: LintContext): AgentLintDiagnostic[] { + const out: AgentLintDiagnostic[] = []; + for (const node of ctx.index.values()) { + for (const [index, invoke] of node.invokes.entries()) { + if (!isDecisionInvoke(invoke.src, ctx.actors)) { + continue; + } + const allowedEvents = literalAllowedEvents(ctx.json, node.path, index); + if (!allowedEvents) { + continue; + } + const configs = [ + node.config, + ...ancestorChain(node, ctx.index).map((ancestor) => ancestor.config), + ctx.config, + ]; + for (const type of allowedEvents) { + if ( + typeof type !== "string" || + isEventPattern(type) || + !configs.some((config) => handlesAsHuman(config, type)) + ) { + continue; + } + out.push({ + code: "decide-allows-human-event", + severity: "error", + path: node.path, + message: + `State '${node.path}' lets a decision choose '${type}', but a transition tagged ` + + `'meta: { actor: "human" }' handles it, so the model is never offered it. Remove ` + + `'${type}' from 'allowedEvents', or drop the tag if a model may send it.`, + }); + } + } + } + return out; +} + function checkDirectObjectSrc(ctx: LintContext): AgentLintDiagnostic[] { const out: AgentLintDiagnostic[] = []; for (const node of ctx.index.values()) { @@ -296,6 +379,7 @@ function checkInvokeWithoutOnError(ctx: LintContext): AgentLintDiagnostic[] { const LINT_CHECKS: Array<(ctx: LintContext) => AgentLintDiagnostic[]> = [ checkDecideWithoutEvents, + checkDecideAllowsHumanEvent, checkDirectObjectSrc, checkInvokeWithoutOnError, ]; @@ -337,8 +421,15 @@ export function lintAgentMachine( const config = (machine as { config?: AnyConfig }).config ?? {}; const index = buildStateIndex(config); const registered = getRegisteredAgentExecutionOptions(machine); + let json: AnyConfig | undefined; + try { + json = machineToJSON(machine) as AnyConfig; + } catch { + json = undefined; + } const ctx: LintContext = { config, + json, index, schemas: registered.schemas as LintContext["schemas"], actors: