diff --git a/.changeset/studio-dryrun-fixes.md b/.changeset/studio-dryrun-fixes.md new file mode 100644 index 00000000..c5f075ec --- /dev/null +++ b/.changeset/studio-dryrun-fixes.md @@ -0,0 +1,20 @@ +--- +"@statelyai/agent": patch +--- + +Fixes from installing the package in an existing app for the first time. + +- **Plain objects from another realm are JSON.** A jest test's vm context gives objects another `Object.prototype`, so `metadata`, store reads, and errors from there were rejected and a run could silently lose its log. The log's checks now accept them. +- **A run with a `store` fails rather than run unlogged** when its first entry cannot be written: `{ status: "error", cause: "journal" }`. +- **Journaled errors keep their `code`** (a string or a number), and `replay` turns them back into `Error` objects. They carry an `"@error": true` marker, so a thrown plain `{ name, message }` replays as the plain object it was. +- **A throwing `onTrace`, `onTransition`, `onChunk`, `onResult`, `onEvent`, or `on` handler no longer fails the run**; it is reported with `console.error`. +- **Reusing a thread id with different input throws `thread-input-mismatch`** instead of returning the other run's result. +- **`setupAgent` throws `invalid-schema` for a schema without `~standard`** (zod 3.23 and earlier) instead of skipping its validation. +- **`describeMachine` prints `?`** for a transition whose target comes from a helper call, instead of `(stays)`. +- **Subpath types resolve under `moduleResolution: "node"`** (`typesVersions`), and `engines` allows Node 22. +- **`ai` peer floor is 7.0.128,** the first release with `experimental_decide`, which `@statelyai/agent/ai-sdk` imports. +- **Undefined fields are dropped from input and events** before they are journaled, as `JSON.stringify` drops them. Any other value the log cannot hold fails a run that has a `store`, and stops the log with a `console.warn` when it does not, instead of failing one way for input and another for events. +- **`setupAgent` throws `invalid-request`** for a request with neither `prompt` nor `messages`, instead of every call failing inside the invoke. +- **`ExploreSnapshot` and `CanReachTarget` types** from `@statelyai/agent/testing`, for exploration callbacks in an app whose own `xstate` is another version. +- **`CHANGELOG.md` ships in the package,** and a new "Upgrading between alphas" page lists the breaking changes since earlier alphas. +- **`runSeam`'s `respond` may return a promise,** so a seam eval can answer idle pauses with a real async client. diff --git a/docs/environment-and-policy.md b/docs/environment-and-policy.md index a7f69fb2..b720d9e2 100644 --- a/docs/environment-and-policy.md +++ b/docs/environment-and-policy.md @@ -175,7 +175,7 @@ The split pays off when there is a policy worth learning. Skip it in these cases - [Machines as data](machines-as-data.md): the config format and `describeMachine`. - [Testing and verification](verify.md): `canReach`, `explorePaths`, and payload variants. - [Event log](event-log.md) and [Persistence](persistence.md): stores, threads, and replay. -- The [`graph-engineering`](https://github.com/statelyai/agent/blob/main/skills/graph-engineering/SKILL.md) skill routes existing code to [`loop-to-machine`](https://github.com/statelyai/agent/blob/main/skills/loop-to-machine/SKILL.md), which converts the loop and checks parity with `compareRuns`. It routes new policies to [`generate-machine`](https://github.com/statelyai/agent/blob/main/skills/generate-machine/SKILL.md). Then [`add-traces`](https://github.com/statelyai/agent/blob/main/skills/add-traces/SKILL.md) persists runs, [`add-evals`](https://github.com/statelyai/agent/blob/main/skills/add-evals/SKILL.md) writes evaluations for each state, and [`improve-machine`](https://github.com/statelyai/agent/blob/main/skills/improve-machine/SKILL.md) proposes a pull request from traces, or reports that no candidate beat the current policy. +- The [`graph-engineering`](https://github.com/statelyai/agent/blob/next/skills/graph-engineering/SKILL.md) skill routes existing code to [`loop-to-machine`](https://github.com/statelyai/agent/blob/next/skills/loop-to-machine/SKILL.md), which converts the loop and checks parity with `compareRuns`. It routes new policies to [`generate-machine`](https://github.com/statelyai/agent/blob/next/skills/generate-machine/SKILL.md). Then [`add-traces`](https://github.com/statelyai/agent/blob/next/skills/add-traces/SKILL.md) persists runs, [`add-evals`](https://github.com/statelyai/agent/blob/next/skills/add-evals/SKILL.md) writes evaluations for each state, and [`improve-machine`](https://github.com/statelyai/agent/blob/next/skills/improve-machine/SKILL.md) proposes a pull request from traces, or reports that no candidate beat the current policy. - [examples/self-improving-refunds](../examples/self-improving-refunds): a refund desk as the environment, JSON playbooks as the policy, an improvement loop, and a benchmark against a raw tool loop. - [examples/code-review-loop](../examples/code-review-loop): review gates as the environment, a triage policy improved from traces against hidden tests, run with Claude Code in headless mode. - [examples/tic-tac-toe](../examples/tic-tac-toe): a game as the environment, a policy over board features, and a `canReach` proof that it never loses as the bar for promotion. diff --git a/docs/event-log.md b/docs/event-log.md index d0e47b9d..d80fb36c 100644 --- a/docs/event-log.md +++ b/docs/event-log.md @@ -38,12 +38,12 @@ interface AgentLogEntry { - `recordedAt` is acceptance metadata, not machine time. A transition that needs time takes it from the event. - `machineVersion` is the machine's declared `version`, or its structural hash when none is declared. -- `verification.stateHash` is the hash of the persisted snapshot `replay` folds to after the entry is applied. A run records replay's fold, not its live snapshot: replay never starts children, so a live child that already settled while its completion waits to be delivered is not part of the hash. +- `verification.stateHash` is the hash of the persisted snapshot `replay` folds to after the entry is applied. A run records replay's fold, not its live snapshot: replay never starts children, so a live child that already settled while its completion waits to be delivered is not part of the hash. So the hash checks each entry against the fold of the entries before it, not that the log is complete or that the fold matches the live run. A dropped last entry leaves the remaining hashes intact, an altered event that folds to the same state still matches, and a machine that reads something the log does not keep can replay to a different state. Compare the replayed context and output with the live result in a test. - `metadata` is host-owned and stored verbatim. The init entry's `metadata.executionId` names the lineage; read it with `getLogExecutionId(entries)`. - To stamp a run with what produced it (a policy version, model ids, a scenario), pass `metadata` to `createAgentRuntime` or `runAgent`. It lands on the init entry next to `executionId`, which is reserved. A resume keeps the stamp of the run that started the log; a version bridge opens a new segment stamped with the bridging run's `metadata`. A value that is not JSON throws before the run starts. - `attempts` holds the decision attempts the machine did not accept before this entry's event landed. Replay ignores it. See [Decision attempts](#decision-attempts). -Entries are strict JSON. `createReplayEntry`, `initEntry`, and every store append reject values JSON would drop or coerce (`undefined`, functions, symbols, bigint, non-finite numbers, `Date`, `Map`, `Set`, class instances, cycles) with `NonSerializableAgentEventError`, which carries the offending `path`. `Error` payloads on `xstate.error.actor` are normalized to `{ name, message, cause? }`; an `AgentDecisionExhaustedError` also keeps its `attempts`. Use `assertJsonSerializable` and `assertAgentLogEntry` at custom transport boundaries. +Entries are strict JSON, and plain objects and arrays from another realm (a jest test's vm context) count as JSON. Object fields set to `undefined` in an input or event are dropped, as `JSON.stringify` drops them. `createReplayEntry`, `initEntry`, and every store append reject the other values JSON would drop or coerce (functions, symbols, bigint, non-finite numbers, `Date`, `Map`, `Set`, class instances, cycles, `undefined` array items) with `NonSerializableAgentEventError`, which carries the offending `path`. `Error` payloads on `xstate.error.actor` are normalized to `{ name, message, code?, cause?, "@error": true }`, keeping `code` when it is a string or a number; an `AgentDecisionExhaustedError` also keeps its `attempts`. `replay` turns objects marked `"@error"` back into `Error` objects, so an `onError` that checks `instanceof Error` takes the same branch on replay. A thrown plain object stays a plain object. Other properties of the thrown value are not kept. Use `assertJsonSerializable` and `assertAgentLogEntry` at custom transport boundaries. ## Record a log @@ -72,6 +72,8 @@ const result = await runToQuiescence( - A rejected write (an `AgentEventLogConflictError`, or any storage failure) aborts in-flight work and settles the run `{ status: "error", cause: "journal" }`. No further calls run. - A `@agent.usage` entry that arrives after the run settled is still written, but the result does not wait for it. Await `result.drain()` before terminating the process if you care about those straggler entries; it resolves once every write issued so far has landed, and never rejects (a failed write already settled the run with `cause: "journal"`). - `threadId` is required with a `store`; without it `runToQuiescence` throws `AgentError` with code `missing-thread-id`. +- A thread that already has entries is resumed. `input` that differs from the input the thread started with throws `AgentError` with code `thread-input-mismatch`, since two runs sharing a thread id would otherwise get one result. +- If an entry cannot be written (an `input` or event that is not JSON), the run settles `{ status: "error", cause: "journal" }` instead of running without a log. Without a store, the log stops at that entry with a `console.warn`, and the run goes on. - Passing `events` as well makes that log the resume, and it must BE the thread's log: a different length throws `AgentEventLogConflictError`, and an entry-for-entry mismatch at the same length throws `AgentError` with code `event-log-conflict`. ### `onEvent`: observer diff --git a/docs/from-a-loop.md b/docs/from-a-loop.md index fb6ce4b7..4850cba6 100644 --- a/docs/from-a-loop.md +++ b/docs/from-a-loop.md @@ -492,7 +492,7 @@ pnpm exec tsx examples/code-review-loop/run.ts --improve --policy v0 4. Replay the train and held-out tasks with the candidate. 5. Promote only with zero violations on both splits, a held-out objective no lower than before, and a combined objective strictly higher. `run.ts` writes a promoted policy to `policies/current.json`. -Inside Claude Code, the [`improve-machine` skill](https://github.com/statelyai/agent/blob/main/skills/improve-machine/SKILL.md) runs the same round with Claude Code as the improver: +Inside Claude Code, the [`improve-machine` skill](https://github.com/statelyai/agent/blob/next/skills/improve-machine/SKILL.md) runs the same round with Claude Code as the improver: ```bash mkdir -p .claude/skills && cp -R skills/improve-machine .claude/skills/ @@ -512,4 +512,4 @@ The full prompt and its allowed tools are in the [example's README](../examples/ - [Human in the loop](human-in-the-loop.md): idle states, `meta.interaction`, and resuming. - [Environment and policy](environment-and-policy.md): the fixed environment, the generated policy, the objective, and the improvement loop. - [code-review-loop](../examples/code-review-loop): Part 2's factory: `before.ts`, the machine, the policies, and the improvement round on Claude Code. -- [`skills/improve-machine`](https://github.com/statelyai/agent/blob/main/skills/improve-machine/SKILL.md): one improvement round from traces, run by an agent. +- [`skills/improve-machine`](https://github.com/statelyai/agent/blob/next/skills/improve-machine/SKILL.md): one improvement round from traces, run by an agent. diff --git a/docs/generate-machines.md b/docs/generate-machines.md index ca70b4b7..19391b42 100644 --- a/docs/generate-machines.md +++ b/docs/generate-machines.md @@ -88,11 +88,11 @@ A failed gate's `detail` is the repair prompt. Send it back to the model with th The full authoring procedure ships as an agent skill. It covers the environment and policy split, the schema location, the rules a config must follow, a few-shot config, the Ajv adapter, `verifyAgentConfig`, the repair loop, and persisting traces. -**[`skills/generate-machine/SKILL.md`](https://github.com/statelyai/agent/blob/main/skills/generate-machine/SKILL.md)** +**[`skills/generate-machine/SKILL.md`](https://github.com/statelyai/agent/blob/next/skills/generate-machine/SKILL.md)** Copy that directory into your coding agent's skills directory. For Claude Code this is `.claude/skills/`. Other harnesses use their own location. The agent then loads the skill on requests such as "author an agent machine for X" and runs the gates itself. -To improve a generated policy from its run traces, use [`skills/improve-machine`](https://github.com/statelyai/agent/blob/main/skills/improve-machine/SKILL.md). See [Environment and policy](environment-and-policy.md). +To improve a generated policy from its run traces, use [`skills/improve-machine`](https://github.com/statelyai/agent/blob/next/skills/improve-machine/SKILL.md). See [Environment and policy](environment-and-policy.md). ## Limits diff --git a/docs/meta.json b/docs/meta.json index de437672..d2d2f48d 100644 --- a/docs/meta.json +++ b/docs/meta.json @@ -21,6 +21,7 @@ "hosts", "any-stack", "xstate-4-and-5-apps", + "upgrading", "steps", "advanced", "---State & durability---", diff --git a/docs/models-and-providers.md b/docs/models-and-providers.md index c6164301..979b6784 100644 --- a/docs/models-and-providers.md +++ b/docs/models-and-providers.md @@ -148,7 +148,7 @@ export const hostAgent = new Agent({ }); ``` -Any framework that exposes a `LanguageModel` works the same way. The machine and the Mastra agent share one model definition. See [`examples/mastra-host`](https://github.com/statelyai/agent/tree/main/examples/mastra-host) for the full bridge, including pause and resume across tool calls. +Any framework that exposes a `LanguageModel` works the same way. The machine and the Mastra agent share one model definition. See [`examples/mastra-host`](https://github.com/statelyai/agent/tree/next/examples/mastra-host) for the full bridge, including pause and resume across tool calls. ## LangChain models diff --git a/docs/observability.md b/docs/observability.md index 054f75fa..cf74dfff 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -21,6 +21,8 @@ A `machine.transition` event carries `from.value`, the state value the event arr A `request.end` event carries the call's `output` and `raw`, plus what the executor reported about the call itself: `usage` when it reported tokens, and `finishReason` when it reported why the call stopped, normalized to `'stop'`, `'length'`, `'tool-calls'`, `'content-filter'`, or `'other'`. See [Finish reason and truncation](text-requests.md#finish-reason-and-truncation). +Trace handlers are observers: an `onTrace`, `onTransition`, `onChunk`, `onResult`, `onEvent`, or `on` handler that throws is reported with `console.error`, and the run continues. + `serializeTraceEvent` creates a JSON-safe projection. Traces are observations, not a persistence protocol; the [event log](event-log.md) is. `usage.dropped` means the `@agent.usage` event was not delivered to the machine, because no active state accepted it or the leg had already settled. The spend is still journaled and still counted: the entry is in the event log, and `getUsageFromEvents` folds it into the totals. Only the machine event is dropped. diff --git a/docs/scope.md b/docs/scope.md index d8a51ed6..9f2b6392 100644 --- a/docs/scope.md +++ b/docs/scope.md @@ -43,12 +43,12 @@ The host decides how it happens, binding `searchWeb` to the search client, cache The repository examples show the orchestration shape. They are not replacement implementations of the systems listed above. -- [Deep research](https://github.com/statelyai/agent/blob/main/examples/deep-research/index.ts) models planning, concurrent research, reflection, and synthesis. The host still supplies the search implementation. -- [Hierarchical teams](https://github.com/statelyai/agent/blob/main/examples/hierarchical-teams/index.ts) models specialized child agents. External data and side effects remain host-owned. +- [Deep research](https://github.com/statelyai/agent/blob/next/examples/deep-research/index.ts) models planning, concurrent research, reflection, and synthesis. The host still supplies the search implementation. +- [Hierarchical teams](https://github.com/statelyai/agent/blob/next/examples/hierarchical-teams/index.ts) models specialized child agents. External data and side effects remain host-owned. ## Core criteria -The rules for what belongs in core live in [CONTRIBUTING.md](https://github.com/statelyai/agent/blob/main/CONTRIBUTING.md). +The rules for what belongs in core live in [CONTRIBUTING.md](https://github.com/statelyai/agent/blob/next/CONTRIBUTING.md). ## Non-goals diff --git a/docs/upgrading.md b/docs/upgrading.md new file mode 100644 index 00000000..9149a0c0 --- /dev/null +++ b/docs/upgrading.md @@ -0,0 +1,36 @@ +--- +title: Upgrading between alphas +description: The breaking changes you are most likely to hit when you move an app from an earlier 2.0 alpha to the current one, and where to find the rest. +--- + +> **Alpha:** `@statelyai/agent` 2.0 is in alpha. APIs can change between releases; pin an exact version. Feedback: [github.com/statelyai/agent](https://github.com/statelyai/agent/issues). + +This page lists the breaking changes an app on an earlier 2.0 alpha is most likely to hit. Every release's full notes are in `CHANGELOG.md`, which ships in the package: `node_modules/@statelyai/agent/CHANGELOG.md`. + +## Peer versions + +- **XState:** exactly the version in the package's `peerDependencies` (`6.0.0-alpha.69` today). Upgrade it in the same change as the agent package. +- **AI SDK:** `ai` 7.0.128 or newer for `@statelyai/agent/ai-sdk`, which imports `experimental_decide`. An older `ai` fails at import time with `does not provide an export named 'experimental_decide'`. + +## Removed and moved APIs + +| Was | Now | +| ------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `defineModels({ ... })` | A plain `models` map, passed to `setupAgent({ models })` and to `createAiSdkExecutors({ models })`. Pass `executors` to `runAgent` explicitly. | +| `createScriptedExecutors` | Removed; the package ships no model mock. Write plain executors that return `{ result }`, or use the AI SDK's `MockLanguageModelV3` behind `createAiSdkExecutors`. | +| `assertAgentMachine`, `lintAgentMachine`, `simulateAgent`, `explorePaths`, `canReach` from `@statelyai/agent` | The same names from `@statelyai/agent/testing`. `assertAgentMachine(machine)` is also `lintAgentMachine(machine, { throw: true })`. | +| `replay`, `initEntry`, `createReplayEntry`, `getUsageFromEvents`, the stores from `@statelyai/agent` | The same names from `@statelyai/agent/log`. | +| `setupAgent({ isIdle })`, `idleTags` | Removed. A run settles `idle` when the machine waits on an event; `isAgentIdle(snapshot)` tells a human wait from a stuck machine. | + +## Changed shapes + +- **Text request output.** A request invoke's `onDone` output is `{ result, messages }`: read `output.result.field`, not `output.field`. Executors return `{ result, messages?, usage? }`, not `{ output }`. +- **Schemas.** Every schema must be a [Standard Schema](https://standardschema.dev). `setupAgent` throws `invalid-schema` for one that is not, such as a zod 3.23 schema. +- **Undefined fields.** Object fields set to `undefined` in input and events are dropped before they are logged, as `JSON.stringify` drops them, so JSON-sanitizing written only for them can go. Other non-JSON values still fail a run that has a `store`. +- **Exploration types.** Type `payloads`, `dedupeBy`, and `canReach` predicate callbacks with `ExploreSnapshot` (and the target with `CanReachTarget`) from `@statelyai/agent/testing`, not with your own `xstate`'s `AnyMachineSnapshot`. +- **Requests.** A `requests` entry needs a `prompt` or `messages`. `setupAgent` throws `invalid-request` for one with neither, even when the executor builds its own model call. + +## Checking an upgrade + +- Run the type checker first: most of the changes above are type errors. +- Search `CHANGELOG.md` between your old version's heading and the new one for **Breaking** and **Removed**. diff --git a/docs/xstate-4-and-5-apps.md b/docs/xstate-4-and-5-apps.md index ef6eb8b4..0ed8e1e8 100644 --- a/docs/xstate-4-and-5-apps.md +++ b/docs/xstate-4-and-5-apps.md @@ -33,7 +33,7 @@ Put the agent in a workspace package that owns its own dependencies, and give th "dependencies": { "@statelyai/agent": "2.0.0-alpha.29", "@ai-sdk/openai": "^4.0.0", - "ai": "^7.0.0", + "ai": "^7.0.128", "xstate": "6.0.0-alpha.69", "zod": "^4.4.3" } diff --git a/package.json b/package.json index 7be06dc7..7a6d6e70 100644 --- a/package.json +++ b/package.json @@ -21,12 +21,41 @@ "files": [ "dist", "schemas", - "skills" + "skills", + "CHANGELOG.md" ], "type": "module", "main": "dist/index.cjs", "module": "dist/index.mjs", "types": "dist/index.d.mts", + "typesVersions": { + "*": { + "ai-sdk": [ + "./dist/ai-sdk.d.mts" + ], + "log": [ + "./dist/log.d.mts" + ], + "machines": [ + "./dist/machines.d.mts" + ], + "openai": [ + "./dist/openai.d.mts" + ], + "otel": [ + "./dist/otel.d.mts" + ], + "tanstack-ai": [ + "./dist/tanstack-ai.d.mts" + ], + "testing": [ + "./dist/testing.d.mts" + ], + "validate": [ + "./dist/validate.d.mts" + ] + } + }, "exports": { ".": { "import": { @@ -184,7 +213,7 @@ "peerDependencies": { "@opentelemetry/api": "^1", "@tanstack/ai": ">=0.66.0 <1", - "ai": "^7.0.0", + "ai": "^7.0.128", "ajv": "^8.20.0", "openai": ">=5.0.0 <8", "xstate": "6.0.0-alpha.69" @@ -207,7 +236,7 @@ } }, "engines": { - "node": ">=22.18.0" + "node": ">=22" }, "packageManager": "pnpm@10.28.2" } diff --git a/readme.md b/readme.md index 98a557e4..13426d36 100644 --- a/readme.md +++ b/readme.md @@ -14,18 +14,18 @@ Environment and policy: - The environment is code you own and can verify; the policy is something a model can write inside it. - The environment holds the rules and invariants as TypeScript states and guards, fixed and checked with `canReach`. -- The policy is a JSON machine a model can author and improve from traces, gated before it runs and promoted only on a measured gain. See [Environment and policy](docs/environment-and-policy.md). +- The policy is a JSON machine a model can author and improve from traces, gated before it runs and promoted only on a measured gain. See [Environment and policy](https://github.com/statelyai/agent/blob/next/docs/environment-and-policy.md). Stately Agent 2 is in alpha. APIs may change before the stable release. -[Documentation](https://stately.ai/docs/packages/agent) · [Examples](examples/README.md) · [XState](https://github.com/statelyai/xstate) +[Documentation](https://stately.ai/docs/packages/agent) · [Examples](https://github.com/statelyai/agent/blob/next/examples/README.md) · [XState](https://github.com/statelyai/xstate) ## Four starting points -- **Author a new agent.** Build a machine from states, decisions, and typed requests; run it locally with `runAgent`, test it with no API key, then use it in any framework or runtime with zero machine changes. See the [Quickstart](docs/quickstart.md) and [Use in any stack](docs/any-stack.md). -- **Retrofit an existing agent.** Turn a `while` loop into a machine: your SDK calls, tools, and retry code become the executors; the machine replaces only the control flow. See [Migrating from a hand-rolled loop](docs/from-a-loop.md). -- **Copy a known pattern.** ReAct, reflection, plan-and-execute, RAG, supervisor, and more, each a single runnable file you lift in 60 seconds. See [Agent patterns](docs/patterns.md). -- **Let an agent improve its own logic.** Keep the rules in a fixed TypeScript environment and let a model rewrite the JSON policy inside it, gated by `verifyAgentConfig` and promoted only when held-out scenarios improve. See [Environment and policy](docs/environment-and-policy.md) and the [`improve-machine` skill](skills/improve-machine/SKILL.md). +- **Author a new agent.** Build a machine from states, decisions, and typed requests; run it locally with `runAgent`, test it with no API key, then use it in any framework or runtime with zero machine changes. See the [Quickstart](https://github.com/statelyai/agent/blob/next/docs/quickstart.md) and [Use in any stack](https://github.com/statelyai/agent/blob/next/docs/any-stack.md). +- **Retrofit an existing agent.** Turn a `while` loop into a machine: your SDK calls, tools, and retry code become the executors; the machine replaces only the control flow. See [Migrating from a hand-rolled loop](https://github.com/statelyai/agent/blob/next/docs/from-a-loop.md). +- **Copy a known pattern.** ReAct, reflection, plan-and-execute, RAG, supervisor, and more, each a single runnable file you lift in 60 seconds. See [Agent patterns](https://github.com/statelyai/agent/blob/next/docs/patterns.md). +- **Let an agent improve its own logic.** Keep the rules in a fixed TypeScript environment and let a model rewrite the JSON policy inside it, gated by `verifyAgentConfig` and promoted only when held-out scenarios improve. See [Environment and policy](https://github.com/statelyai/agent/blob/next/docs/environment-and-policy.md) and the [`improve-machine` skill](https://github.com/statelyai/agent/blob/next/skills/improve-machine/SKILL.md). ## Install @@ -40,7 +40,7 @@ XState is pinned while its experimental durable restoration API evolves. For the optional Vercel AI SDK executor: ```sh -pnpm add ai@^7 @ai-sdk/openai@^4 +pnpm add ai@^7.0.128 @ai-sdk/openai@^4 ``` For the optional raw OpenAI SDK executor (`@statelyai/agent/openai`): @@ -55,13 +55,43 @@ For the optional TanStack AI executor (`@statelyai/agent/tanstack-ai`): pnpm add @tanstack/ai @tanstack/ai-openai ``` -Classification, grading, relevance, and yes/no checks are judgments: call the AI SDK's `experimental_decide` with `@ai-sdk/typesafe-ai` (`pnpm add ai @ai-sdk/typesafe-ai`) from an ordinary actor; see [Judgments with the AI SDK](docs/judgments.md). +Classification, grading, relevance, and yes/no checks are judgments: call the AI SDK's `experimental_decide` with `@ai-sdk/typesafe-ai` (`pnpm add ai @ai-sdk/typesafe-ai`) from an ordinary actor; see [Judgments with the AI SDK](https://github.com/statelyai/agent/blob/next/docs/judgments.md). + +Upgrading from an earlier alpha: see [Upgrading between alphas](https://github.com/statelyai/agent/blob/next/docs/upgrading.md). Every release's notes are in `CHANGELOG.md`, which ships in the package. Requirements: -- Node 22.18 or newer, and XState `6.0.0-alpha.69`. -- The package is ESM-first. CommonJS builds are published too, so `require()` works. -- Provider packages must match your `ai` major: `@ai-sdk/openai@^4` pairs with `ai@^7`. A bare `@ai-sdk/openai` resolves to `@latest`, which can mismatch the `ai` peer. +- Node 22 or newer, and XState exactly `6.0.0-alpha.69`. +- TypeScript 5.4 or newer. Older versions accept the code, but callback parameters in `createMachine` silently become `any`. +- A [Standard Schema](https://standardschema.dev) library: zod 4, zod 3.24 or newer, Valibot, or ArkType. `setupAgent` throws `invalid-schema` for zod 3.23 and earlier. +- The package is ESM-first. CommonJS builds are published too, so `require()` works. Subpath imports (`@statelyai/agent/testing`) resolve under every `moduleResolution`, including `node`. +- `@statelyai/agent/ai-sdk` needs `ai` 7.0.128 or newer. Provider packages must match your `ai` major: `@ai-sdk/openai@^4` pairs with `ai@^7`. A bare `@ai-sdk/openai` resolves to `@latest`, which can mismatch the `ai` peer. `@statelyai/agent/openai` needs `openai` 5 or newer. + +An app that already uses XState 4 or 5 keeps it: put the agent in its own workspace package. See [Use in an XState 4 or 5 app](https://github.com/statelyai/agent/blob/next/docs/xstate-4-and-5-apps.md). If both versions land in one `node_modules`, the package fails with errors that do not mention a version: + +| Error | Cause | +| --- | --- | +| `createAsyncLogic is not a function` | `xstate` resolved to 4 or 5 from CommonJS | +| `Cannot find module '…/node_modules/xstate/durable'` | `xstate` resolved to 4 or 5 from ESM | +| `unmet peer dependency "xstate@6.0.0-alpha.69"` (pnpm, npm) or `has incorrect peer dependency "xstate@6.0.0-alpha.69"` (yarn 1) | The install's only warning of the two above | + +## Agent skills + +The package ships agent skills in `node_modules/@statelyai/agent/skills/`. Copy them into your coding agent's skills directory (for Claude Code, `.claude/skills/`): + +```sh +mkdir -p .claude/skills +cp -R node_modules/@statelyai/agent/skills/* .claude/skills/ +``` + +| Skill | What it does | +| --- | --- | +| [`graph-engineering`](https://github.com/statelyai/agent/blob/next/skills/graph-engineering/SKILL.md) | Finds which step a project is at and routes to the next skill | +| [`loop-to-machine`](https://github.com/statelyai/agent/blob/next/skills/loop-to-machine/SKILL.md) | Converts existing agent code into a machine and checks parity with `compareRuns` | +| [`generate-machine`](https://github.com/statelyai/agent/blob/next/skills/generate-machine/SKILL.md) | Authors a JSON policy machine from scratch and runs `verifyAgentConfig` | +| [`add-traces`](https://github.com/statelyai/agent/blob/next/skills/add-traces/SKILL.md) | Stamps and persists every run so it replays | +| [`add-evals`](https://github.com/statelyai/agent/blob/next/skills/add-evals/SKILL.md) | Writes evals per state, at the cheapest level that catches the failure | +| [`improve-machine`](https://github.com/statelyai/agent/blob/next/skills/improve-machine/SKILL.md) | Improves a policy from traces and hands the change to a person as a pull request | ## Quick start @@ -159,11 +189,11 @@ const liveResult = await runAgent(refundMachine, { }); ``` -`createAiSdkExecutors(refundMachine)` reads the `models` declared in `setupAgent({ models })`, which also type the machine's model refs. `createAiSdkExecutors({ models })` still works when the host owns the map. Executors are always explicit, and core does not import the AI SDK. See [Models and providers](docs/models-and-providers.md) and [Hosts and executors](docs/hosts.md). +`createAiSdkExecutors(refundMachine)` reads the `models` declared in `setupAgent({ models })`, which also type the machine's model refs. `createAiSdkExecutors({ models })` still works when the host owns the map. Executors are always explicit, and core does not import the AI SDK. See [Models and providers](https://github.com/statelyai/agent/blob/next/docs/models-and-providers.md) and [Hosts and executors](https://github.com/statelyai/agent/blob/next/docs/hosts.md). -For a machine with several requests, the executor routes on `request.name`, or `MockLanguageModelV3` from `ai/test` stands in for the provider behind `createAiSdkExecutors`. See [Evals](docs/evals.md#testing-without-a-provider). +For a machine with several requests, the executor routes on `request.name`, or `MockLanguageModelV3` from `ai/test` stands in for the provider behind `createAiSdkExecutors`. See [Evals](https://github.com/statelyai/agent/blob/next/docs/evals.md#testing-without-a-provider). -A host that needs the transition loop itself (a durable object, events pushed mid-run, a custom checkpoint policy) uses `createAgentRuntime` directly. See [Write your own host loop](docs/hosts.md#write-your-own-host-loop). +A host that needs the transition loop itself (a durable object, events pushed mid-run, a custom checkpoint policy) uses `createAgentRuntime` directly. See [Write your own host loop](https://github.com/statelyai/agent/blob/next/docs/hosts.md#write-your-own-host-loop). ## Architecture @@ -181,38 +211,38 @@ The machine never talks to a model directly, so swapping `createAiSdkExecutors` The example above has one model decision and two final outcomes. Real machines add approval states, retries, parallel work, child agents, and long-running waits without changing how control flow is represented. -The core concepts (machines owning control flow, bounded model decisions, typed requests, host-run executors, and native XState snapshots) are in the [documentation overview](docs/index.md). +The core concepts (machines owning control flow, bounded model decisions, typed requests, host-run executors, and native XState snapshots) are in the [documentation overview](https://github.com/statelyai/agent/blob/next/docs/index.md). ## Examples -- [Twenty Questions](examples/twenty-questions) shows a model choosing legal events in a loop. -- [Chameleon](examples/chameleon) combines hidden information, sequential turns, and a human vote. -- [Snapshot migration](examples/snapshot-migration) uses XState's native `version` and `migrate` contract. -- [Portable XState loop](examples/portable-xstate-loop) runs the artifact through native transitions and effects. -- [Application-owned storage](examples/file-snapshot-store) persists native XState snapshots in application code. -- [Tool calling](examples/review-tool-calls) keeps tools and message history in the AI SDK's native format. -- [Ticket triage](examples/triage) returns structured data from a model request. -- [JSON agent](examples/json-agent) runs a machine defined as data. +- [Twenty Questions](https://github.com/statelyai/agent/tree/next/examples/twenty-questions) shows a model choosing legal events in a loop. +- [Chameleon](https://github.com/statelyai/agent/tree/next/examples/chameleon) combines hidden information, sequential turns, and a human vote. +- [Snapshot migration](https://github.com/statelyai/agent/tree/next/examples/snapshot-migration) uses XState's native `version` and `migrate` contract. +- [Portable XState loop](https://github.com/statelyai/agent/tree/next/examples/portable-xstate-loop) runs the artifact through native transitions and effects. +- [Application-owned storage](https://github.com/statelyai/agent/tree/next/examples/file-snapshot-store) persists native XState snapshots in application code. +- [Tool calling](https://github.com/statelyai/agent/tree/next/examples/review-tool-calls) keeps tools and message history in the AI SDK's native format. +- [Ticket triage](https://github.com/statelyai/agent/tree/next/examples/triage) returns structured data from a model request. +- [JSON agent](https://github.com/statelyai/agent/tree/next/examples/json-agent) runs a machine defined as data. -See [all examples](examples/README.md). +See [all examples](https://github.com/statelyai/agent/blob/next/examples/README.md). ## Related -- [Machines](docs/machines.md) -- [Text requests](docs/text-requests.md) -- [Decisions](docs/decisions.md) -- [Human in the loop](docs/human-in-the-loop.md) -- [Testing and verification](docs/verify.md) -- [Evals](docs/evals.md) -- [Generating machines with an LLM](docs/generate-machines.md) -- [Hosts and executors](docs/hosts.md) -- [Models and providers](docs/models-and-providers.md) -- [Use in any stack](docs/any-stack.md) -- [Persistence](docs/persistence.md) -- [Observability](docs/observability.md) -- [Usage and budgets](docs/usage-and-budgets.md) -- [Agent patterns](docs/patterns.md) -- [Migrating from a hand-rolled loop](docs/from-a-loop.md) -- [Coming from LangGraph](docs/langgraph-comparison.md) +- [Machines](https://github.com/statelyai/agent/blob/next/docs/machines.md) +- [Text requests](https://github.com/statelyai/agent/blob/next/docs/text-requests.md) +- [Decisions](https://github.com/statelyai/agent/blob/next/docs/decisions.md) +- [Human in the loop](https://github.com/statelyai/agent/blob/next/docs/human-in-the-loop.md) +- [Testing and verification](https://github.com/statelyai/agent/blob/next/docs/verify.md) +- [Evals](https://github.com/statelyai/agent/blob/next/docs/evals.md) +- [Generating machines with an LLM](https://github.com/statelyai/agent/blob/next/docs/generate-machines.md) +- [Hosts and executors](https://github.com/statelyai/agent/blob/next/docs/hosts.md) +- [Models and providers](https://github.com/statelyai/agent/blob/next/docs/models-and-providers.md) +- [Use in any stack](https://github.com/statelyai/agent/blob/next/docs/any-stack.md) +- [Persistence](https://github.com/statelyai/agent/blob/next/docs/persistence.md) +- [Observability](https://github.com/statelyai/agent/blob/next/docs/observability.md) +- [Usage and budgets](https://github.com/statelyai/agent/blob/next/docs/usage-and-budgets.md) +- [Agent patterns](https://github.com/statelyai/agent/blob/next/docs/patterns.md) +- [Migrating from a hand-rolled loop](https://github.com/statelyai/agent/blob/next/docs/from-a-loop.md) +- [Coming from LangGraph](https://github.com/statelyai/agent/blob/next/docs/langgraph-comparison.md) diff --git a/skills/add-evals/SKILL.md b/skills/add-evals/SKILL.md index 815c40c7..c78447ef 100644 --- a/skills/add-evals/SKILL.md +++ b/skills/add-evals/SKILL.md @@ -30,7 +30,7 @@ console.log(describeMachine(machine)); // goal, states, transitions, events List: -- the decision states: an `agent.decide` invoke, a `decision` tag, or a state a host answers with `decideNext`; +- the decision states: an `agent.decide` invoke, a `decision` tag, or a state a host answers with `decideNext`. A machine whose model choice is the output of a text request (a tool call it then checks) has no decision states: eval that choice at the **request** level with `runSeam`; - the person's steps: idle states a person answers (`meta.interaction`, the runner's `onIdle`). What the person decides is ground truth, not something to eval; the routing after their answer is a rule; - the guards. `describeMachine` shows where a transition function can go and marks it `(conditional)`, but not the guard's name or logic. Read the source for those; - the sentinel states and the final states. @@ -57,6 +57,8 @@ If the request names nothing ("write evals"), cover every decision state, every A guard or ordering is always a rule eval. Use a decision eval when the guards leave more than one legal option. Use an outcome eval when a person can label the right end result. +Draft labels gate nothing, but rule evals need no labels, so they gate CI from day one. The label-based evals start gating as the user confirms labels (`draft: false`). Say so in the hand-back, so nobody reads a green CI as "the outcome evals pass". + ## 4. Write them Name every eval `: `, so the report reads as a map of the machine. An outcome eval over a whole run takes the final state it expects (`issued: partial-refund refunds 50 dollars`). In `test.each` titles, `$name` is a placeholder: write amounts as `100 dollars`, not `$100`. diff --git a/skills/add-traces/SKILL.md b/skills/add-traces/SKILL.md index 7109bab1..69a1c24e 100644 --- a/skills/add-traces/SKILL.md +++ b/skills/add-traces/SKILL.md @@ -20,7 +20,7 @@ This is step 2 of 4. Before: `loop-to-machine`. Next: `add-evals`, then `improve ## Hard rules -- **Traces never change behavior.** Recording is an observer. No branch reads a trace. +- **Traces never change behavior.** Recording is an observer. No branch reads a trace. An `onTrace` or `onTransition` handler that throws is reported with `console.error` and skipped. The store is different: it is write-ahead, so a failed write stops the run with `cause: "journal"`. Ask the user whether a run should fail when its log cannot be written (the store) or carry on unrecorded (record from `onEvent` instead). - **A trace must name what produced it.** A run with no machine version or policy version cannot be used to compare versions, so it is useless to the improver. - **Replay is the test.** A log that does not replay to the run's final state with zero model calls is not a trace yet. - **No secrets or personal data in metadata or thread ids.** Use ids, not names or emails. The log itself holds the machine's input and every event verbatim (a ticket's text, a customer's reply), and trace snapshots hold the context: in production, redact or encrypt them per your data rules, and ask the user which applies. @@ -33,10 +33,11 @@ Declare `version` on every machine whose runs you record: the environment and, i export const deskMachine = deskSetup.createMachine({ id: "refund-desk", version: "1", ... }); ``` -Also name the **decider version**: what answers the decisions. It is the policy machine's `version` when there is one; otherwise a label for the model and prompt that decide (`gpt-6-luna.p1`), or for a stand-in model in tests (`careful`). Runs by different deciders must never share a version label. +Also name the **decider version**: whatever produces the model's answers. It is the policy machine's `version` when there is one. Otherwise it is a label for the model, its settings, and its prompts (`gpt-6-luna.xhigh.p`); this holds for a machine with only text requests too. When the prompts are private to another module (a route file), export a prompt version from that module rather than guessing; a release or commit id is the fallback. In tests it is the stand-in model's name (`careful`). Runs by different deciders must never share a label. - Every log entry carries `machineVersion`. Without a declared `version` it falls back to a structural hash that skips functions, and in XState v6 the guards and targets live in transition functions. Changing a guard would not change the hash. - Bump `version` on every behavior change. Adding recording is not one: it only observes. `improve-machine` bumps the policy's on promotion. +- A test-only variant built from the same factory (a loosened guard) gets its own `version` (`"1-loose"`), so its runs are never mistaken for the real machine's. ## 2. Pick a store @@ -50,7 +51,9 @@ const result = await runAgent(deskMachine, { input, executors, store, threadId, await result.drain(); // late usage entries, before you read the store or exit ``` -**A thread that already has entries is resumed, not overwritten.** Give every new run a new thread id; reuse one only to continue that run. +**A thread that already has entries is resumed, not overwritten.** Give every new run a new thread id; reuse one only to continue that run. Passing different `input` to a thread that already has entries throws `thread-input-mismatch` instead of handing back another run's result. + +For a production store, use a table of its own (`threadId`, `index`, the entry as JSON, a unique key on both), not a table that already means something else, such as a usage counter. ## 3. Name threads @@ -77,7 +80,7 @@ const metadata = { }; ``` -Read it back with `entries[0].metadata`. It also holds `executionId`, which the library adds and reserves, so compare with `toMatchObject`, not `toEqual`. A resumed run keeps the stamp of the run that started the log. The value type is `JsonValue` from `@statelyai/agent/log`. +Read it back with `entries[0].metadata`. It also holds `executionId` (and `migratedFrom` after a version bridge), which the library adds and reserves, so compare with `toMatchObject`, not `toEqual`. A resumed run keeps the stamp of the run that started the log. Type the stamp as `Record` (`JsonValue` from `@statelyai/agent/log`); an interface without an index signature does not fit. ## 5. Collect trace events @@ -99,8 +102,8 @@ await writeFile(`runs/${file}.trace.json`, JSON.stringify(trace)); Decide with the user whether `runs/` is committed or gitignored. -- In production, send them to OpenTelemetry instead: `createOtelTraceHandler({ tracer })` from `@statelyai/agent/otel` is an `onTrace` handler. -- **A policy run inside a decider** is a separate run. Pass it the same `onTrace` (and a `store`) if you want per-state stats for the policy too. `createPolicyDecider` gives each decision's policy run its own thread, `policy:`; never give it one fixed `threadId`, or the second decision resumes the first. +- In production, send them to OpenTelemetry: `createOtelTraceHandler({ tracer })` from `@statelyai/agent/otel` is an `onTrace` handler. It needs the optional peer `@opentelemetry/api`; without OpenTelemetry in the app, write the serialized events next to the log instead. +- **A policy run inside a decider** is a separate run. Pass it the same `onTrace` (and a `store`) if you want per-state stats for the policy too. `createPolicyDecider` gives each decision's policy run its own thread, `policy::`; never give it one fixed `threadId`, or the second decision resumes the first. - **A plain XState machine** (built with `createMachine` or `setup`, decided with `decideNext`): drive it with `createAgentRuntime` and `runToQuiescence`, which accept plain machines, and pass each `decideNext` result as `{ event, attempts }` so refused choices reach the log. Push `{ type: "request.start", request: result.request }` onto the trace before sending the event, since a host-driven decision emits no trace itself. - **A plain `createActor` run you cannot change:** attach `recordLog(actor)` from `@statelyai/agent/log` before `actor.start()`, and pass `inspect: traceTransitions((event) => trace.push(serializeTraceEvent(event)))` to `createActor`. `recordLog` cannot record decision attempts. @@ -116,15 +119,24 @@ test("a recorded run replays to its final state with no model calls", async () = const result = await runAgent(deskMachine, { input, executors, store, threadId: "test:careful:small-clean:r1", metadata }); await result.drain(); const entries = await store.read("test:careful:small-clean:r1"); - const { snapshot } = replay(deskMachine, entries); // a pure fold: nothing runs - expect(snapshot.value).toEqual(result.snapshot.value); + expect(entries[0]?.event.type).toBe("@agent.init"); // a log was written at all; a run with no outside input has only this entry + const { snapshot } = replay(deskMachine, entries, { verify: "strict" }); // a pure fold: nothing runs + // Compare more than the state value: context and output catch a field the log does not keep. + expect({ status: snapshot.status, value: snapshot.value, context: snapshot.context, output: snapshot.output }).toEqual({ + status: result.snapshot.status, + value: result.snapshot.value, + context: result.snapshot.context, + output: result.snapshot.output, + }); expect(entries[0]!.metadata).toMatchObject({ environmentVersion: "1" }); }); ``` -- `foldTraces(traces, { failed })` over a batch returns a row for every state the runs visited, including each decision state. `failed` judges one whole trace; start with "the last event is not a `run.end` with status `done`, or the run ended in a failure or sentinel state", and ask the user whether a wrong outcome counts too. `withheld` is above 0 only for a closed choice (`payloads`), where the guards refused an option. -- `getDecisionAttempts(entries)` lists the refused choices of a run. A stand-in model that picks from the offered options can never propose an illegal one, so write a raw decide executor for this test that proposes a refused event first: `decide: async (request) => ({ event: request.attempts.length ? request.options![0]!.event : { type: "REFUND", amount: 1000 } })`. -- Running a scenario with and without recording leaves the same world and the same final state. +`verify: "strict"` checks that every entry carries a hash and that each matches the fold. It does not show the log is complete (a dropped last entry leaves the rest intact), nor that the fold matches the live run, which is why the test compares the snapshots too. A journaled `Error` replays as an `Error` with its `name`, `message`, and a string or number `code`; any other property of the thrown value is not kept. A thrown plain object replays as a plain object. + +- `foldTraces(traces, { failed })` over a batch returns a row for every state a run rested in. A `choice` state the run passes through in the same step gets no row. `failed` judges one whole trace; start with "the last event is not a `run.end` with status `done`, or the run ended in a failure or sentinel state", and ask the user whether a wrong outcome counts too. A serialized `run.end` is `{ type: "run.end", status, snapshot: { value, context, … } }`: `failed: (t) => { const end = t.at(-1); return end?.type !== "run.end" || end.status !== "done" || ["failed", "invariantBroken"].includes(String(end.snapshot?.value)); }`. `withheld` is above 0 only for a closed choice (`payloads`), where the guards refused an option. +- `getDecisionAttempts(entries)` lists the refused choices of a run; for a machine with no decisions (`agent.decide` or `decideNext`) it is always empty, so skip this test. A stand-in model that picks from the offered options can never propose an illegal one, so write a raw decide executor for this test that proposes a refused event first: `decide: async (request) => ({ event: request.attempts.length ? request.options![0]!.event : { type: "REFUND", amount: 1000 } })`. +- Running a scenario with and without recording leaves the same world, the same model calls, and the same final state, and the recorded run's log is not empty. ## 7. Hand back diff --git a/skills/generate-machine/SKILL.md b/skills/generate-machine/SKILL.md index 14eeef32..9ce58e44 100644 --- a/skills/generate-machine/SKILL.md +++ b/skills/generate-machine/SKILL.md @@ -31,13 +31,13 @@ import { describeMachine } from "@statelyai/agent"; const text = describeMachine(machine); // "Machine ", then "Goal: …", states, transitions, events ``` -Pass `{ snapshot }` to mark the active states with `*`. Concept page: [Environment and policy](https://github.com/statelyai/agent/blob/main/docs/environment-and-policy.md). +Pass `{ snapshot }` to mark the active states with `*`. Concept page: [Environment and policy](https://github.com/statelyai/agent/blob/next/docs/environment-and-policy.md). ## 1. Read the schema The config shape is fixed by a shipped JSON Schema. Read it first; it is the contract. -- Consuming the package: `node_modules/@statelyai/agent/agent-workflow.json` (importable as `@statelyai/agent/agent-workflow.json`) +- Consuming the package: `node_modules/@statelyai/agent/schemas/agent-workflow.json` (importable as `@statelyai/agent/agent-workflow.json`) - In the `statelyai/agent` repo: `schemas/agent-workflow.json` ## 2. Author the config diff --git a/skills/graph-engineering/SKILL.md b/skills/graph-engineering/SKILL.md index 17d26643..680a2340 100644 --- a/skills/graph-engineering/SKILL.md +++ b/skills/graph-engineering/SKILL.md @@ -18,7 +18,7 @@ Four steps, in order. Each one needs the one before it. Check in this order and start at the first step that is not done: -1. Is the agent's control flow a machine (`setupAgent`, `createMachine`, an `AgentWorkflowConfig`)? If it is a loop or ad-hoc code, step 1. +1. Does the code path production actually runs go through a machine (`setupAgent`, `createMachine`, an `AgentWorkflowConfig`)? Follow the route or entry point to the model call; a machine that nothing calls does not count. If the path is a loop or ad-hoc code, step 1. 2. Do runs persist to an event log store with a declared machine `version` and a `metadata` stamp? If not, step 2. 3. Is there an objective over ground truth, and evals that name the states they cover? If not, step 3. 4. Otherwise, step 4. diff --git a/skills/improve-machine/SKILL.md b/skills/improve-machine/SKILL.md index 9f0cd629..987d5195 100644 --- a/skills/improve-machine/SKILL.md +++ b/skills/improve-machine/SKILL.md @@ -11,7 +11,7 @@ You improve a **policy**: the machine that decides how the agent works toward it locate inputs → read traces → diagnose → propose one diff → gate (≤ 3 repairs) → replay train → replay held-out → recommend or discard → record → pull request ``` -This is step 4 of 4, after `loop-to-machine`, `add-traces`, and `add-evals`. To author a JSON policy from scratch, use the `generate-machine` skill. Concept page: [Environment and policy](https://github.com/statelyai/agent/blob/main/docs/environment-and-policy.md). +This is step 4 of 4, after `loop-to-machine`, `add-traces`, and `add-evals`. To author a JSON policy from scratch, use the `generate-machine` skill. Concept page: [Environment and policy](https://github.com/statelyai/agent/blob/next/docs/environment-and-policy.md). ## Hard rules @@ -232,7 +232,7 @@ Report to the user: rounds run, each candidate with its hypothesis and decision, ## Worked example -[`examples/self-improving-refunds`](https://github.com/statelyai/agent/blob/main/examples/self-improving-refunds) in the `statelyai/agent` repo (not in the installed package) runs this loop end to end for a JSON policy: +[`examples/self-improving-refunds`](https://github.com/statelyai/agent/blob/next/examples/self-improving-refunds) in the `statelyai/agent` repo (not in the installed package) runs this loop end to end for a JSON policy: - the refund desk is the environment: business rules as guards, and an `illegallyIssued` sentinel proven unreachable, - `playbooks/*.json` are policy versions, with the vocabulary listed in `playbooks/README.md`, diff --git a/skills/loop-to-machine/SKILL.md b/skills/loop-to-machine/SKILL.md index 56b214a7..ea40ba04 100644 --- a/skills/loop-to-machine/SKILL.md +++ b/skills/loop-to-machine/SKILL.md @@ -30,7 +30,11 @@ Read the loop and fill this table before writing anything. Each row maps to a ma | A tool with a side effect someone must gate (refund, send) | An actor invoked by one state, entered only through a guarded transition | | A read-only tool (search, lookup) | An actor the machine invokes itself, or a tool on a text request | | A rule: an `if`, a prompt line, a check after the fact | A guard on the transition it protects | +| A model call that produces content (a draft, generated code, a summary) | A text request in `setupAgent({ requests })`, invoked by one state | +| One SDK tool call that both picks the action and carries its content (`edit_graph({ … })`, `send_email({ … })`) | A text request with those tools, then guards on the returned call: exactly one, a tool that was offered, input that passes the tool's schema. Keep it a text request: `agent.decide` would rename the tools to `send_event_` with the description "Choose the '' move.", so the model would no longer see your tool names and descriptions | +| A tool the browser runs (no `execute`; the client sends results back) | The request ends with the tool calls; an idle state waits for a `TOOL_RESULTS` event the route sends when the client posts results | | A point where the model picks what to do | A decision state: `agent.decide`, usually a closed choice | +| A check on the model's output, then a retry or a repair call | An actor or guard that checks it, and a `choice` state that routes to the repair request, the next candidate, or a failure final | | A value the model pulled out of the input (an order id, an amount) | Code when the format is fixed (a regex); a text request when it is not | | A step every correct run takes in the same order (verify, then look up) | A state the machine enters itself; the model no longer chooses it | | A person in the loop: `await approve()`, a returned `pending`, or a tool that asks a person (`askCustomer`, `requestApproval`), even a synchronous one | An idle state with `meta.interaction`, answered by `onIdle` in the runner | @@ -41,14 +45,17 @@ Read the loop and fill this table before writing anything. Each row maps to a ma List the rules as a numbered list in your notes. Every one must end as a guard, a state ordering, or a written reason it was dropped. +A loop with no decision point (generate, validate, repair) is still worth converting: the machine makes the retry cap, the validation rules, and every failure path explicit and provable. It has no `agent.decide`, so step 4 and the decision parts of step 5 do not apply. + ## 2. Pin the baseline The parity test exists before the machine does. 1. **Scenarios.** Reuse the code's tests, fixtures, or benchmark cases. Each has an `id` and, where people know it, the `expected` outcome. If there are none, draft 6–10 that cover every terminal and every rule, and ask the user to confirm them. -2. **Deterministic stand-in models.** Parity with a live model drifts between runs. Use the code's scripted or mock models if it has them; otherwise write one keyed on the input (see [`examples/mock-model.ts`](https://github.com/statelyai/agent/blob/main/examples/mock-model.ts) in the `statelyai/agent` repo). You want two: one that follows the rules, and one that breaks them. A seeded random model needs its seed derived from the scenario id and a fresh instance per run, or every scenario replays the same draws. +2. **Deterministic stand-in models.** Parity with a live model drifts between runs. Use the code's scripted or mock models if it has them; otherwise write one: a function from the request (its prompt, or its `name`) to a fixed answer. ([`examples/mock-model.ts`](https://github.com/statelyai/agent/blob/next/examples/mock-model.ts) in the `statelyai/agent` repo is one, built on the AI SDK's `MockLanguageModelV3`; it is not in the package.) You want two: one that follows the rules, and one that breaks them. A seeded random model needs its seed derived from the scenario id and a fresh instance per run, or every scenario replays the same draws. 3. **Ground truth.** Read outcomes and effects from the world, not from the agent's own report. `effects` holds **every write the world records**: the refunds, and also the approvals, the verifications, the questions asked, the escalations. A refund issued without its approval looks correct if `effects` holds only refunds. Text a model wrote (an escalation reason) differs on every run: compare its count or presence, not the text. If the world can check rules after the fact, return those violations. If it cannot, write the check as a plain function over the effects. -4. **The harness check.** Run the old code on both sides with each stand-in model. Each row's baseline must equal its candidate (the runs are deterministic), and with the rule-following model every row must be `same`. With a rule-breaking model, rows are `regression` by design: the candidate breaks rules. +4. **A baseline with no function to call.** When the loop lives inline in a framework handler (a tRPC procedure, a route), call the handler itself in the test, with its model client and storage replaced by the stand-ins (a module mock, or the framework's test caller). Do not extract the loop into a function first: that changes the code you are trying to pin. +5. **The harness check.** Run the old code on both sides with each stand-in model. Each row's baseline must equal its candidate (the runs are deterministic), and with the rule-following model every row must be `same`. With a rule-breaking model, rows are `regression` by design: the candidate breaks rules. ```ts import { compareRuns, type ParityResult, type ParityRun } from "@statelyai/agent/testing"; @@ -131,9 +138,14 @@ If the code does have that logic, port it as a policy machine that answers `choo Write `runMachine` with the old function's signature and result type, so callers and the parity test change one line. +- **The existing SDK call becomes the executor.** Write `generateText` (or `streamText`) in `executors` as the old call, with the same model, system prompt, tools, and options, and return `{ result }`. The request still needs `prompt` or `messages` in `setupAgent({ requests })` (`setupAgent` throws `invalid-request` otherwise): pass the conversation as `messages: ({ input }) => input.messages` and read `request.messages` in the executor. +- **A streaming route keeps its response, not its function signature.** `createAiSdkExecutors().streamText` forwards text chunks only, so a UI stream with tool calls cannot come out of it. Write the `streamText` executor around the old call, pipe its UI stream to the route's response yourself, and return the final messages as `{ result }`. + - `runAgent(machine, { input, executors, actors, onIdle })`. - `onIdle` answers each idle state with the old code's call (`tools.askCustomer(...)`, `tools.requestApproval(...)`). Write the returned events with `as const` on `type`, or the event union does not narrow. - `executors.decide` adapts the code's model to the decision: the closed choice arrives as `request.options` (each `{ event }`), and the facts as `request.metadata?.facts`, typed `unknown`: parse it with the exported facts schema. Every executor slot is optional in the type; check or assert the ones you call. +- In an `onError` function, the error is `event.error`: `onError: ({ event }) => ({ target: "failed", context: { error: String(event.error) } })`. There is no `error` argument. +- Make input and events JSON-safe at the boundary: no `Date`, `Map`, class instances, or functions (fields set to `undefined` are dropped). With a store, a value the log cannot hold fails the run; without one, the log stops with a warning. - Rebuild the old result from `result.output` and the world. A field that has no meaning anymore (a turn count with no loop) keeps its type and gets a comment saying what it now holds. ## 6. Proofs that need no model @@ -156,10 +168,12 @@ const proofOptions: ExplorePathsOptions = { }; ``` - `options.outputs` lists every output each actor can return (found and not found, owned and not, refunded and not); `options.errors` makes each actor fail once. +- **A machine with a loop** (a repair cycle, a resubmit) needs `options.dedupeBy`, or the search runs to `maxDepth` and reports `complete: false`. Return every context field a guard or choice reads: `dedupeBy: (s) => JSON.stringify([s.value, s.context.attempts, s.context.lastError])`. `outputs` are canned per actor src, not per call: to explore "fails twice, then succeeds", let the guard read a counter from context instead of the output. +- `explorePaths(...).reachedStates` lists only states a run settles in; a `choice` state it passes through is not listed. - A loosened guard makes `canReach` true. Write this test too, or the proof proves nothing. - Each normal final state is reachable, so the search is not vacuous. -Type the options with `ExplorePathsOptions` from `@statelyai/agent/testing`. `errors` takes one error per actor src, and `"agent.decide"` is a valid key. +Type the options with `ExplorePathsOptions`, and a `payloads`, `dedupeBy`, or predicate callback's snapshot with `ExploreSnapshot`, both from `@statelyai/agent/testing`. Do not use `AnyMachineSnapshot` from your app's own `xstate`, which may be another version. `errors` takes one error per actor src, and `"agent.decide"` is a valid key. ## 7. Parity diff --git a/src/describe-machine.test.ts b/src/describe-machine.test.ts index 1f26e826..ecf3a21b 100644 --- a/src/describe-machine.test.ts +++ b/src/describe-machine.test.ts @@ -131,6 +131,47 @@ describe("describeMachine", () => { expect(describeMachine(machine)).toContain("always → stopped; sets prompt"); }); + test("a transition that returns a helper's result is unknown, not (stays)", () => { + const next = (ready: boolean) => (ready ? { target: "done" as const } : undefined); + const machine = setup({}).createMachine({ + context: { ready: false }, + initial: "working", + states: { + working: { + on: { + CHECK: ({ context }) => next(context.ready), + WRAPPED: ({ context }) => next(context.ready), + PICK: ({ context }) => { + const ready = !context.ready; + return next(ready); + }, + NOTE: () => ({ context: { ready: true } }), + MAP: ({ context }) => ({ + context: { ready: [context.ready].map((value) => Boolean(value))[0]! }, + }), + }, + }, + done: {}, + }, + }); + + // As authored: the transpiler drops these grouping parentheses. + const { on } = ( + machine.config as unknown as { states: { working: { on: Record } } } + ).states.working; + Object.defineProperty(on.WRAPPED, "toString", { + value: () => "({ context }) => (next(context.ready))", + }); + + const text = describeMachine(machine); + expect(text).toContain("on CHECK → ?"); + expect(text).toContain("on WRAPPED → ?"); + expect(text).toContain("on PICK → ?"); + expect(text).toContain("on NOTE → (stays)"); + // A call inside a nested callback is not the transition's own result. + expect(text).toContain("on MAP → (stays)"); + }); + test("describes a TS-authored machine", () => { expect(describeMachine(refundMachine)).toMatchInlineSnapshot(` "Machine refund diff --git a/src/describe-machine.ts b/src/describe-machine.ts index f8967969..a4364dd6 100644 --- a/src/describe-machine.ts +++ b/src/describe-machine.ts @@ -180,11 +180,51 @@ const mayDecline = (code: string) => code.replace(/(["'`])(?:\\.|(?!\1)[^\\])*\1/g, '""'), ); +/** + * True when the function returns a call's result (`=> next(ctx)`, `return + * pick(x)`), so its target is not in its own source. Reads only the + * function's own returned expressions, not those of callbacks nested in it. + */ +function returnsCall(code: string): boolean { + const text = code.replace(/(["'`])(?:\\.|(?!\1)[^\\])*\1/g, '""'); + // Grouping parentheses first: `=> (next(x))` returns a call; `=> ({ … })` does not. + const startsWithCall = (at: number) => /^[\s(]*[A-Za-z_$][\w$.]*\s*\(/.test(text.slice(at)); + // The outer function's body: after its arrow, or its first brace, outside the parameters. + let parens = 0; + let body = -1; + for (let i = 0; i < text.length && body < 0; i++) { + const char = text[i]; + if (char === "(") parens++; + else if (char === ")") parens--; + else if (parens === 0 && char === "=" && text[i + 1] === ">") body = i + 2; + else if (parens === 0 && char === "{") body = i; + } + if (body < 0) return false; + while (/\s/.test(text[body] ?? "")) body++; + if (text[body] !== "{") return startsWithCall(body); + // A block body: its own `return` statements, at depth 1. + let depth = 0; + for (let i = body; i < text.length; i++) { + if (text[i] === "{") depth++; + else if (text[i] === "}" && --depth === 0) return false; + else if ( + depth === 1 && + text.startsWith("return", i) && + !/[\w$]/.test(text[i - 1] ?? "") && + !/[\w$]/.test(text[i + 6] ?? "") && + startsWithCall(i + 6) + ) { + return true; + } + } + return false; +} + /** * Where a transition written as a function goes: every string literal after a * `target:` that names a real state node, `(conditional)` when it may decline * or pick among several, `(stays)` when its source has no `target`, `?` when - * it names none that resolves. + * it names none that resolves or returns a helper call's result. */ function targetsInSource(source: AnyStateNode, code: string, choice: boolean): string { const found = new Set(); @@ -199,7 +239,8 @@ function targetsInSource(source: AnyStateNode, code: string, choice: boolean): s const conditional = names.length > 1 || (!choice && mayDecline(code)); return `${names.join(", ")}${conditional ? " (conditional)" : ""}`; } - return /\btarget\s*:/.test(code) ? "?" : "(stays)"; + // A target the source does not spell out, or one a helper call returns, is unknown. + return /\btarget\s*:/.test(code) || returnsCall(code) ? "?" : "(stays)"; } /** One transition, as a line: `on GO (human) if context.ready → review; sets draft — description`. */ diff --git a/src/event-log.test.ts b/src/event-log.test.ts index 562550db..6cda4f1e 100644 --- a/src/event-log.test.ts +++ b/src/event-log.test.ts @@ -1,3 +1,4 @@ +import { runInNewContext } from "node:vm"; import { describe, expect, test } from "vitest"; import { createAsyncLogic, @@ -173,6 +174,7 @@ describe("entry creation", () => { expect((entry.event as unknown as { error: unknown }).error).toEqual({ name: "TypeError", message: "boom", + "@error": true, }); }); @@ -272,6 +274,80 @@ describe("replay", () => { expect(snapshot.value).toBe("failed"); }); + test("a journaled error keeps its code and replays as an Error", () => { + const job = createAsyncLogic({ + run: () => { + throw new Error("never runs"); + }, + }); + const errorMachine = setup({ actors: { job } }).createMachine({ + id: "jobber", + initial: "working", + context: { seen: "" }, + states: { + working: { + invoke: { + id: "job", + src: "job", + input: { attempt: 1 }, + onError: ({ event }) => { + const error = (event as { error?: unknown }).error; + return { + target: "failed", + context: { + seen: + error instanceof Error + ? `${error.name}:${error.message}:${String((error as { code?: unknown }).code)}` + : "plain", + }, + }; + }, + }, + }, + failed: {}, + }, + }); + const entries = [initEntry(errorMachine, {})]; + const sessionId = ( + Object.values((replay(errorMachine, entries).snapshot as AnyMachineSnapshot).children)[0] as { + sessionId: string; + } + ).sessionId; + const error = Object.assign(new Error("down"), { name: "TRPCError", code: "TIMEOUT" }); + entries.push( + createReplayEntry( + errorMachine, + entries, + journaled({ type: "xstate.error.actor", actorId: "job", sessionId, error }), + ), + ); + + expect((entries[1]!.event as { error?: unknown }).error).toEqual({ + name: "TRPCError", + message: "down", + code: "TIMEOUT", + "@error": true, + }); + const roundTripped = JSON.parse(JSON.stringify(entries)) as AgentLogEntry[]; + expect(replay(errorMachine, roundTripped).snapshot.context.seen).toBe("TRPCError:down:TIMEOUT"); + + // A thrown plain object with the same shape replays as the plain object it was. + const plain = [entries[0]!]; + plain.push( + createReplayEntry( + errorMachine, + plain, + journaled({ + type: "xstate.error.actor", + actorId: "job", + sessionId, + error: { name: "RemoteFailure", message: "down" }, + }), + ), + ); + expect(replay(errorMachine, plain).snapshot.context.seen).toBe("plain"); + }); + test("verify: true accepts an untampered log and rejects a tampered hash", () => { const entries = buildLog(); @@ -361,6 +437,36 @@ describe("replay", () => { }); }); +// Jest runs each test file in its own vm context, so values built there (and +// `structuredClone` results, which come from the main realm) have another +// realm's `Object.prototype` and `Error`. +describe("values from another realm", () => { + test("a log read back through another realm's JSON replays", () => { + const entries = runInNewContext("JSON.parse(text)", { + text: JSON.stringify(buildLog({ output: "shipped" })), + }) as AgentLogEntry[]; + + expect(Object.getPrototypeOf(entries[0])).not.toBe(Object.prototype); + expect(replay(machine, entries).snapshot.value).toBe("done"); + }); + + test("another realm's plain object is accepted as init metadata", () => { + const metadata = runInNewContext('({ policyVersion: "v3", models: { decider: "x" } })'); + + const entry = initEntry(machine, { input: { attempt: 1 } }, { metadata }); + + expect(entry.metadata).toMatchObject({ policyVersion: "v3", models: { decider: "x" } }); + }); + + test("another realm's Error is normalized like a local one", () => { + const error = runInNewContext('Object.assign(new Error("nope"), { code: "E_DOWN" })'); + + const { snapshot } = replay(machine, buildLog({ error })); + + expect(snapshot.value).toBe("failed"); + }); +}); + describe("getSnapshotStateHash", () => { test("hashes a reserved key that lives in context as application data", () => { const base = { status: "active", value: "idle", children: {} }; diff --git a/src/event-log.ts b/src/event-log.ts index b717e93f..26cadee1 100644 --- a/src/event-log.ts +++ b/src/event-log.ts @@ -15,6 +15,7 @@ * so {@link replay} needs no side channel. * @module */ +import { isError, isPlainObject } from "./internal/is-record.js"; import { initialTransition, transition, @@ -239,8 +240,7 @@ export function assertJsonSerializable( assertNoHiddenKeys(value, path, (key) => key === "length" || arrayIndexPattern.test(key)); return; } - const prototype = Object.getPrototypeOf(value); - if (prototype !== Object.prototype && prototype !== null) { + if (!isPlainObject(value)) { const type = (value as { constructor?: { name?: string } }).constructor?.name ?? "object"; throw new NonSerializableAgentEventError(path, type); } @@ -374,9 +374,18 @@ function exhaustedAttemptsOf(error: Error): unknown[] | undefined { } /** - * Replaces `Error` instances with plain `{ name, message, cause? }` objects so - * an `xstate.error.actor` payload survives the log's JSON round-trip. A - * decision-exhausted error also keeps its `code` and `attempts`. Cycles + * Marks an object {@link normalizeEventErrors} made from an `Error`, so replay + * restores that object, and only that one, as an `Error`: a thrown plain + * `{ name, message }` stays a plain object. + */ +const NORMALIZED_ERROR_KEY = "@error"; + +/** + * Replaces `Error` instances with plain `{ name, message, code?, cause?, + * "@error": true }` objects so an `xstate.error.actor` payload survives the + * log's JSON round-trip. `code` is kept when it is a string or a number; a + * decision-exhausted error also keeps its `attempts`. Replay turns the + * object back into an `Error` (see `restoreJournaledEvent`). Cycles * are preserved by identity; non-plain objects are left alone (and are then * rejected by {@link assertJsonSerializable}). */ @@ -384,16 +393,20 @@ function normalizeEventErrors( value: unknown, seen: WeakMap = new WeakMap(), ): unknown { - if (value instanceof Error) { + if (isError(value)) { const normalized: Record = { name: value.name, message: value.message, + [NORMALIZED_ERROR_KEY]: true, }; seen.set(value, normalized); if (value.cause !== undefined) normalized.cause = normalizeEventErrors(value.cause, seen); - if ((value as { code?: unknown }).code === DECISION_EXHAUSTED_CODE) { - // The discriminator getDecisionAttempts requires before it reads `attempts`. - normalized.code = DECISION_EXHAUSTED_CODE; + const code = (value as { code?: unknown }).code; + if (typeof code === "string" || (typeof code === "number" && Number.isFinite(code))) { + normalized.code = code; + } + if (code === DECISION_EXHAUSTED_CODE) { + // `code` is the discriminator getDecisionAttempts requires before it reads `attempts`. const attempts = exhaustedAttemptsOf(value); if (attempts !== undefined) normalized.attempts = structuredClone(attempts); } @@ -408,8 +421,7 @@ function normalizeEventErrors( for (const item of value) result.push(normalizeEventErrors(item, seen)); return result; } - const prototype = Object.getPrototypeOf(value); - if (prototype !== Object.prototype && prototype !== null) return value; + if (!isPlainObject(value)) return value; const result: Record = {}; seen.set(value, result); for (const [key, item] of Object.entries(value as Record)) { @@ -460,7 +472,7 @@ function canonicalizeForHash( if (typeof value === "symbol") return `[symbol:${String(value)}]`; if (value === null || typeof value !== "object") return value; if (value instanceof Date) return `[Date:${value.toISOString()}]`; - if (value instanceof Error) { + if (isError(value)) { if (seen.has(value)) return "[circular]"; seen.add(value); try { @@ -607,7 +619,7 @@ function toPersisted( // coalesce, when it resolves `null` as well. Both reach the root machine as an // `xstate.done.actor` event carrying the key explicitly, which // `assertJsonSerializable` rejects. The key is dropped instead; `replay` puts -// it back (`restoreDoneActorOutput`), and the hash already ignores it. +// it back (`restoreJournaledEvent`), and the hash already ignores it. function normalizeDoneActorEvent(event: EventObject): EventObject { if (event.type !== DONE_ACTOR_EVENT_TYPE) return event; const { output, ...rest } = event as EventObject & { output?: unknown }; @@ -634,7 +646,8 @@ export function createReplayEntry( recordedAt: options.recordedAt ?? new Date().toISOString(), machineId: machineIdOf(machine), machineVersion, - event: normalizeDoneActorEvent(normalizeEventErrors(event) as EventObject), + // Object fields set to `undefined` are dropped, as `JSON.stringify` drops them. + event: normalizeDoneActorEvent(pruneUndefined(normalizeEventErrors(event)) as EventObject), ...(options.causationId !== undefined ? { causationId: options.causationId } : {}), ...(options.metadata !== undefined ? { metadata: options.metadata } : {}), ...(options.attempts !== undefined && options.attempts.length > 0 @@ -667,17 +680,24 @@ export type AgentLogInit = // A persisted snapshot carries `output`/`error` as explicit `undefined` before // serialization; JSON drops those keys, and so does this, so the entry passes // `assertJsonSerializable` and hashes identically before and after a round-trip. -function pruneUndefined(value: unknown): unknown { - if (Array.isArray(value)) return value.map(pruneUndefined); +function pruneUndefined(value: unknown, ancestors: WeakSet = new WeakSet()): unknown { if (value === null || typeof value !== "object") return value; - if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) { - return value; + // A cycle is left as is, for `assertJsonSerializable` to reject with its path. + if (ancestors.has(value)) return value; + if (Array.isArray(value)) { + ancestors.add(value); + const result = value.map((item) => pruneUndefined(item, ancestors)); + ancestors.delete(value); + return result; } + if (!isPlainObject(value)) return value; + ancestors.add(value); const result: Record = {}; for (const [key, child] of Object.entries(value as Record)) { if (child === undefined) continue; - result[key] = pruneUndefined(child); + result[key] = pruneUndefined(child, ancestors); } + ancestors.delete(value); return result; } @@ -1006,9 +1026,50 @@ export function rebindActorSession( // completion with an own `output` key (value `undefined`), so the fold gets // the same shape — a guard testing key presence takes the same branch both // times. -function restoreDoneActorOutput(event: EventObject): EventObject { - if (event.type !== DONE_ACTOR_EVENT_TYPE || Object.hasOwn(event, "output")) return event; - return { ...event, output: undefined } as EventObject; +/** + * The event as the machine received it live: a done event's dropped + * `undefined` output restored, and an error event's normalized error turned + * back into an `Error`, so `error instanceof Error` holds on replay too. + */ +function restoreJournaledEvent(event: EventObject): EventObject { + if (event.type === DONE_ACTOR_EVENT_TYPE && !Object.hasOwn(event, "output")) { + return { ...event, output: undefined } as EventObject; + } + if (event.type.startsWith(ERROR_ACTOR_EVENT_TYPE)) { + const error = (event as { error?: unknown }).error; + if (isNormalizedError(error)) return { ...event, error: toError(error) } as EventObject; + } + return event; +} + +function isNormalizedError(value: unknown): value is { name: string; message: string } { + return ( + isPlainObject(value) && + value[NORMALIZED_ERROR_KEY] === true && + typeof value.name === "string" && + typeof value.message === "string" + ); +} + +function toError(normalized: { name: string; message: string; cause?: unknown }): Error { + const { + name, + message, + cause, + [NORMALIZED_ERROR_KEY]: _marker, + ...rest + } = normalized as { + name: string; + message: string; + cause?: unknown; + [NORMALIZED_ERROR_KEY]?: unknown; + }; + const error = new Error( + message, + cause === undefined ? undefined : { cause: isNormalizedError(cause) ? toError(cause) : cause }, + ); + error.name = name; + return Object.assign(error, rest); } /** Options for {@link replay}. */ @@ -1125,7 +1186,7 @@ export function foldLog( const sessions = new Map(); const apply = (entry: AgentLogEntry): AgentPersistedSnapshot => { - const event = restoreDoneActorOutput(rebindActorSession(entry.event, snapshot, sessions)); + const event = restoreJournaledEvent(rebindActorSession(entry.event, snapshot, sessions)); [snapshot] = transition(machine, snapshot as never, event as never) as [ AnyMachineSnapshot, unknown, diff --git a/src/internal/is-record.ts b/src/internal/is-record.ts index 01957cfb..5586a947 100644 --- a/src/internal/is-record.ts +++ b/src/internal/is-record.ts @@ -2,3 +2,35 @@ export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } + +/** + * True for an object whose prototype is `null` or some realm's + * `Object.prototype`. Comparing against this realm's `Object.prototype` alone + * rejects plain objects built in another realm: a jest test file's vm + * context, or `structuredClone`'s output inside one. @internal + */ +export function isPlainObject(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === null || Object.getPrototypeOf(prototype) === null; +} + +/** True for an `Error` from any realm, where `instanceof Error` sees only this one. @internal */ +export function isError(value: unknown): value is Error { + return value instanceof Error || Object.prototype.toString.call(value) === "[object Error]"; +} + +/** JSON text with object keys sorted and `undefined` fields dropped, for comparing JSON-like values. @internal */ +export function canonicalJson(value: unknown): string { + return ( + JSON.stringify(value, (_key, item: unknown) => + isPlainObject(item) + ? Object.fromEntries( + Object.keys(item) + .sort() + .map((key) => [key, item[key]]), + ) + : item, + ) ?? "undefined" + ); +} diff --git a/src/run-agent.test.ts b/src/run-agent.test.ts index f1274f5c..3625f6c5 100644 --- a/src/run-agent.test.ts +++ b/src/run-agent.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { z } from "zod"; import { createActor, @@ -1679,6 +1679,28 @@ describe("emitted events (runAgent `on`)", () => { }, }); + test("a throwing `on` handler is reported, and the run goes on", async () => { + const report = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + const result = await runToQuiescence( + createAgentRuntime(machine, { + on: { + DRAFTING_STARTED: () => { + throw new Error("sink down"); + }, + }, + executors: { generateText: async () => ({ result: "a draft" }) }, + }), + { input: { topic: "rivers" } }, + ); + + expect(result.status).toBe("done"); + expect(String(report.mock.calls[0]![0])).toContain("on['DRAFTING_STARTED']"); + } finally { + report.mockRestore(); + } + }); + test("`on` handlers fire per type, including events emitted during the initial transition", async () => { const started: string[] = []; const drafted: number[] = []; @@ -4498,6 +4520,194 @@ describe("runAgent write-ahead store", () => { expect(stored[0]!.event.type).toBe(AGENT_INIT_EVENT_TYPE); }); + test("an observer that throws is reported and does not change the run", async () => { + const report = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + const result = await runToQuiescence( + createAgentRuntime(makeMachine(), { + executors: executors(), + onTrace: () => { + throw new Error("sink down"); + }, + onTransition: () => { + throw new Error("sink down"); + }, + }), + { input: undefined }, + ); + + expect(result.status).toBe("idle"); + expect(report).toHaveBeenCalled(); + expect(String(report.mock.calls[0]![0])).toContain("onTrace"); + } finally { + report.mockRestore(); + } + }); + + test("a thread started with other input is refused, and the same input resumes it", async () => { + const promptMachine = setup({}).createMachine({ + id: "prompted", + context: ({ input }: { input: { prompt: string } }) => ({ prompt: input.prompt }), + initial: "waiting", + states: { waiting: { on: { GO: { target: "done" } } }, done: { type: "final" } }, + }); + const store = createInMemoryEventLogStore(); + const first = await runToQuiescence( + createAgentRuntime(promptMachine, { store, threadId: "t1" }), + { input: { prompt: "a checkout flow" } }, + ); + expect(first.status).toBe("idle"); + + await expect( + runToQuiescence(createAgentRuntime(promptMachine, { store, threadId: "t1" }), { + input: { prompt: "a login flow" }, + }), + ).rejects.toMatchObject({ code: "thread-input-mismatch" }); + + // The thread's own entries passed as `events` are checked the same way. + await expect( + runToQuiescence(createAgentRuntime(promptMachine, { store, threadId: "t1" }), { + events: first.events, + input: { prompt: "a login flow" }, + }), + ).rejects.toMatchObject({ code: "thread-input-mismatch" }); + + const resumed = await runToQuiescence( + createAgentRuntime(promptMachine, { store, threadId: "t1" }), + { input: { prompt: "a checkout flow" }, event: { type: "GO" } as never }, + ); + expect(resumed.status).toBe("done"); + }); + + test("a thread that began from a snapshot resumes with input", async () => { + const promptMachine = setup({}).createMachine({ + id: "prompted", + context: ({ input }: { input: { prompt: string } }) => ({ prompt: input.prompt }), + initial: "waiting", + states: { waiting: { on: { GO: { target: "done" } } }, done: { type: "final" } }, + }); + const first = await runToQuiescence(createAgentRuntime(promptMachine), { + input: { prompt: "a checkout flow" }, + }); + if (first.status !== "idle") throw new Error("expected idle"); + + const store = createInMemoryEventLogStore(); + await runToQuiescence(createAgentRuntime(promptMachine, { store, threadId: "s1" }), { + snapshot: first.persist(), + }); + const resumed = await runToQuiescence( + createAgentRuntime(promptMachine, { store, threadId: "s1" }), + { input: { prompt: "a checkout flow" }, event: { type: "GO" } as never }, + ); + expect(resumed.status).toBe("done"); + }); + + describe("values JSON cannot hold", () => { + const noteMachine = setup({}).createMachine({ + id: "noted", + context: ({ input }: { input: { note?: string; at?: unknown } }) => ({ note: input.note }), + initial: "waiting", + states: { waiting: { on: { GO: { target: "done" } } }, done: { type: "final" } }, + }); + + test("undefined fields are dropped from input and events, as JSON.stringify drops them", async () => { + const store = createInMemoryEventLogStore(); + const first = await runToQuiescence( + createAgentRuntime(noteMachine, { store, threadId: "u" }), + { input: { note: undefined } }, + ); + const done = await runToQuiescence( + createAgentRuntime(noteMachine, { store, threadId: "u" }), + { + event: { type: "GO", detail: undefined } as never, + }, + ); + + expect(first.status).toBe("idle"); + expect(done.status).toBe("done"); + const entries = await store.read("u"); + expect(entries[0]!.event).not.toHaveProperty("input.note"); + expect(entries.at(-1)!.event).toEqual({ type: "GO" }); + expect(replay(noteMachine, entries).snapshot.status).toBe("done"); + }); + + test("without a store, a non-JSON value warns and stops the log instead of failing the run", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const idle = await runToQuiescence(createAgentRuntime(noteMachine, {}), { + input: { note: "x" }, + }); + const done = await runToQuiescence(createAgentRuntime(noteMachine, {}), { + events: idle.events, + snapshot: idle.persist(), + event: { type: "GO", at: new Date(0) } as never, + }); + + expect(done.status).toBe("done"); + expect(warn).toHaveBeenCalledTimes(1); + expect(String(warn.mock.calls[0]![0])).toMatch(/log stopped/); + } finally { + warn.mockRestore(); + } + }); + + test("with a store, a non-JSON event fails the run", async () => { + const store = createInMemoryEventLogStore(); + await runToQuiescence(createAgentRuntime(noteMachine, { store, threadId: "d" }), { + input: { note: "x" }, + }); + const result = await runToQuiescence( + createAgentRuntime(noteMachine, { store, threadId: "d" }), + { + event: { type: "GO", at: new Date(0) } as never, + }, + ); + + expect(result.status).toBe("error"); + expect(result.status === "error" && result.cause).toBe("journal"); + // The refused event never moved the run: it still rests where the log ends. + expect(result.snapshot.value).toBe("waiting"); + expect((result.persist() as { value?: unknown }).value).toBe("waiting"); + }); + }); + + test("a throwing `onEvent` is reported, and the log keeps one init entry", async () => { + const report = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + const store = createInMemoryEventLogStore(); + const result = await runToQuiescence( + createAgentRuntime(makeMachine(), { + store, + threadId: "main", + executors: executors(), + onEvent: () => { + throw new Error("sink down"); + }, + }), + { input: undefined }, + ); + + expect(result.status).not.toBe("error"); + const stored = await store.read("main"); + expect(stored.filter((entry) => entry.event.type === AGENT_INIT_EVENT_TYPE)).toHaveLength(1); + expect(String(report.mock.calls[0]![0])).toContain("onEvent"); + } finally { + report.mockRestore(); + } + }); + + test("a run whose init entry cannot be built fails instead of running unlogged", async () => { + const store = createInMemoryEventLogStore(); + const result = await runToQuiescence( + createAgentRuntime(makeMachine(), { store, threadId: "main", executors: executors() }), + { input: { at: new Date(0) } as never }, + ); + + expect(result.status).toBe("error"); + expect(result.status === "error" && result.cause).toBe("journal"); + expect(await store.read("main")).toEqual([]); + }); + test("the result resolves only after the last write lands", async () => { const inner = createInMemoryEventLogStore(); let inFlight = 0; diff --git a/src/run-agent.ts b/src/run-agent.ts index da72c0e3..7ea49368 100644 --- a/src/run-agent.ts +++ b/src/run-agent.ts @@ -19,6 +19,7 @@ import { type StateValue, type TimerKey, } from "xstate"; +import { canonicalJson } from "./internal/is-record.js"; import { createDurable, type DurableExecution } from "xstate/durable"; import type { AgentTools, @@ -485,9 +486,13 @@ export interface AgentRuntimeOptions { onEvent?: (entry: AgentLogEntry) => void; /** * Record a `verification.stateHash` on every appended entry (default `true`). - * The hash is taken O(1) from the live actor's persisted snapshot right after - * that entry's transition, so `replay(machine, events)` can prove it lands on - * the same state. Set `false` for a smaller, unverifiable log. + * The hash is of the state the log folds to after that entry, the same fold + * `replay` runs, so `replay(machine, events)` can check each entry against + * the fold before it. It does not show the log is complete (a dropped last + * entry leaves the rest intact), nor compare against the live snapshot: a + * machine that reads something the log does not keep (an error's class, a + * non-JSON field) can replay to a different state. Set `false` for a + * smaller, unverifiable log. */ verification?: boolean; /** @@ -1102,6 +1107,15 @@ interface TraceSinks { ) => void; } +// Observers never control the run: one that throws is reported and skipped. +function observe(name: string, call: () => void): void { + try { + call(); + } catch (error) { + console.error(`@statelyai/agent: ${name} threw; the run continues.`, error); + } +} + /** * Builds the ONE place a trace payload is emitted: it hands the payload to the * envelope-stamping trace sink and, from that same payload, invokes the sugar @@ -1118,19 +1132,21 @@ function createTraceDispatch(sinks: TraceSinks): TraceDispatch { return (payload, self) => { switch (payload.type) { case "stream.chunk": - sinks.onTrace?.(payload, self); - sinks.onChunk?.(payload.chunk, { request: payload.request }); + observe("onTrace", () => sinks.onTrace?.(payload, self)); + observe("onChunk", () => sinks.onChunk?.(payload.chunk, { request: payload.request })); return; case "request.end": - sinks.onResult?.(payload.request, { result: payload.output, raw: payload.raw }); - sinks.onTrace?.(payload, self); + observe("onResult", () => + sinks.onResult?.(payload.request, { result: payload.output, raw: payload.raw }), + ); + observe("onTrace", () => sinks.onTrace?.(payload, self)); return; case "machine.transition": - sinks.onTrace?.(payload, self); - sinks.onTransition?.(payload.snapshot, payload.event); + observe("onTrace", () => sinks.onTrace?.(payload, self)); + observe("onTransition", () => sinks.onTransition?.(payload.snapshot, payload.event)); return; default: - sinks.onTrace?.(payload, self); + observe("onTrace", () => sinks.onTrace?.(payload, self)); } }; } @@ -2346,6 +2362,23 @@ export function createAgentRuntime( // and later entries go unverified rather than failing the run. let logFold: LogFold | null | undefined; + // A value the log cannot hold. With a store the caller asked for a durable + // log, so the run stops; without one, the log stops and the run goes on. + // Either way it is loud: a silently short `result.events` replays wrong. + const stopLogging = (error: unknown): void => { + if (!loggingEnabled) return; + loggingEnabled = false; + if (store !== undefined) { + journalError ??= error; + stopRun({ cause: "journal", error }); + return; + } + console.warn( + `@statelyai/agent: this run's log stopped at entry ${logEntries.length}, so result.events is incomplete: ` + + `${error instanceof Error ? error.message : String(error)}. Pass input and events that JSON can hold.`, + ); + }; + const appendLogEntry = ( event: EventObject, attempts?: DecisionAttempt[], @@ -2353,11 +2386,17 @@ export function createAgentRuntime( if (!loggingEnabled) { return undefined; } - const entry = createReplayEntry(logMachine, logEntries, event, { - machineVersion, - verification: false, - ...(attempts !== undefined ? { attempts } : {}), - }); + let entry: AgentLogEntry; + try { + entry = createReplayEntry(logMachine, logEntries, event, { + machineVersion, + verification: false, + ...(attempts !== undefined ? { attempts } : {}), + }); + } catch (error) { + stopLogging(error); + return undefined; + } if (verificationEnabled && logFold !== null) { try { logFold ??= foldLog(logMachine, logEntries, { machineVersion, verify: false }); @@ -2368,7 +2407,7 @@ export function createAgentRuntime( } logEntries.push(entry); journalEntry(entry); - options.onEvent?.(entry); + observe("onEvent", () => options.onEvent?.(entry)); return entry; }; @@ -2671,9 +2710,9 @@ export function createAgentRuntime( const emitFromRoot = (event: EventObject): void => { onTrace({ type: "emit", event: event as EmittedFrom }); const on = (options.on ?? {}) as Record void) | undefined>; - on[event.type]?.(event); + observe(`on['${event.type}']`, () => on[event.type]?.(event)); if (event.type !== "*") { - on["*"]?.(event); + observe("on['*']", () => on["*"]?.(event)); } }; @@ -2802,6 +2841,7 @@ export function createAgentRuntime( // ─── Resolve the log to resume from ─── let resumeEvents = init.events; + let resumedFromStore = false; if (store !== undefined) { const storedThread = await store.read(threadId!); if (resumeEvents !== undefined) { @@ -2817,8 +2857,10 @@ export function createAgentRuntime( ); } assertThreadMatchesEvents(threadId!, storedThread, resumeEvents); + resumedFromStore = storedThread.length > 0; } else if (storedThread.length > 0) { resumeEvents = storedThread; + resumedFromStore = true; } } @@ -2826,6 +2868,24 @@ export function createAgentRuntime( // start, the replayable init entry, and the `run.start` trace all see // the same post-defaults value, so a replay reproduces this run exactly. const resolvedInput = resolveMachineInput(machine, init.input); + // A thread that has entries is resumed, which ignores `input`. Input + // that differs from the thread's own means two runs share a thread id: + // resuming would hand this caller another run's result. A thread that + // began from a snapshot recorded no input to compare against. + const threadInit = resumeEvents?.[0]?.event as Record | undefined; + if ( + resumedFromStore && + init.input !== undefined && + threadInit !== undefined && + Object.hasOwn(threadInit, "input") + ) { + if (canonicalJson(threadInit.input) !== canonicalJson(resolvedInput)) { + throw new AgentError( + "thread-input-mismatch", + `createAgentRuntime: thread '${threadId}' was started with different input. A thread that already has entries is resumed, not restarted; give a new run a new threadId.`, + ); + } + } let effectiveSnapshot: Snapshot | undefined = init.snapshot; let seedInitEntry: (() => void) | undefined; @@ -2855,24 +2915,24 @@ export function createAgentRuntime( candidates: readonly AgentLogInit[], metadata: Record, ): void => { + let lastError: unknown; for (const candidate of candidates) { for (const verification of verificationEnabled ? [true, false] : [false]) { + let entry: AgentLogEntry; try { - const entry = initEntry(logMachine, candidate, { - machineVersion, - verification, - metadata, - }); - logEntries.push(entry); - journalEntry(entry); - options.onEvent?.(entry); - return; - } catch { + entry = initEntry(logMachine, candidate, { machineVersion, verification, metadata }); + } catch (error) { // Try the next (less exact) form. + lastError = error; + continue; } + logEntries.push(entry); + journalEntry(entry); + observe("onEvent", () => options.onEvent?.(entry)); + return; } } - loggingEnabled = false; + stopLogging(lastError); }; /** Init forms for a snapshot resume, most faithful first. */ @@ -3143,6 +3203,11 @@ export function createAgentRuntime( ignoredEvent = event as EventObject; } appendLogEntry(event as EventObject, attempts); + // The store refused the entry and the run stopped: the transition never + // happened, so the result stays at the last journaled state. + if (stopReason !== undefined) { + return [state, []]; + } commit(next, event as EventObject, ignored); return [next as SnapshotFrom, effects]; }, diff --git a/src/seam.test.ts b/src/seam.test.ts index e5624b91..8b1bae89 100644 --- a/src/seam.test.ts +++ b/src/seam.test.ts @@ -224,6 +224,21 @@ describe("runSeam", () => { expect(run.after.statePath).toEqual(["reviewing", "done"]); }); + test("respond may answer asynchronously", async () => { + const answer = user({ prompt: "Tell the team.", changes: "Add the ship date." }); + const run = await runSeam(machine, { + scripts: { judge: [COMPLETE], writer: [DRAFT, REVISED] }, + respond: async (turn) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + return answer(turn); + }, + seam: { request: "write", occurrence: 1 }, + candidate: async () => ({ result: REVISED }), + }); + + expect(run.after.statePath).toEqual(["reviewing", "done"]); + }); + test("repeatLast replays the last scripted answer down a longer branch", async () => { // One scripted draft, but the user asks for a revision: the `write` queue // is called twice and its last entry covers the second call. diff --git a/src/seam.ts b/src/seam.ts index 491d43c6..0ad52171 100644 --- a/src/seam.ts +++ b/src/seam.ts @@ -212,10 +212,16 @@ export interface RunSeamOptions { candidate?: AgentRequestExecutor; /** * The simulated user. Called at every idle pause; return the event to send, - * or `null`/`undefined` to stop the run there. Omitted, the run stops at the - * first idle pause. + * or `null`/`undefined` to stop the run there, directly or as a promise (a + * real async client). Omitted, the run stops at the first idle pause. */ - respond?: (turn: SeamTurn) => EventFromLogic | null | undefined; + respond?: ( + turn: SeamTurn, + ) => + | EventFromLogic + | null + | undefined + | PromiseLike | null | undefined>; /** Maximum idle pauses to answer before stopping. Default `12`. */ maxTurns?: number; /** @@ -448,7 +454,7 @@ export async function runSeam( break; } - const next = options.respond?.({ + const next = await options.respond?.({ snapshot: result.snapshot, state: (result.snapshot as AnyMachineSnapshot).value, meta: getStateMeta(result.snapshot), diff --git a/src/setup-agent.test.ts b/src/setup-agent.test.ts index 935b6c35..c7f21616 100644 --- a/src/setup-agent.test.ts +++ b/src/setup-agent.test.ts @@ -3237,6 +3237,46 @@ describe("per-state context schemas (setupAgent({ states }))", () => { }); }); +describe("setupAgent schemas that are not Standard Schema", () => { + // The shape of a zod 3.23 schema: a parser, but no `~standard` property. + const legacy = { _def: { typeName: "ZodObject" }, parse: (value: unknown) => value }; + + test.each([ + ["context", { context: legacy }], + ["input", { context: z.object({}), input: legacy }], + ["an event", { context: z.object({}), events: { GO: legacy } }], + [ + "a request", + { context: z.object({}), requests: { ask: { schemas: { output: legacy }, prompt: "hi" } } }, + ], + [ + "a nested state's context", + { + context: z.object({}), + states: { work: { states: { review: { schemas: { context: legacy } } } } }, + }, + ], + ])("%s is rejected with a clear error", (_where, config) => { + expect(() => setupAgent(config as never)).toThrow(/not a Standard Schema.*zod/is); + }); + + test("a request with neither prompt nor messages is rejected at setup", () => { + expect(() => + setupAgent({ context: z.object({}), requests: { chatStep: { model: "m" } } } as never), + ).toThrow(/'chatStep' has neither `prompt` nor `messages`/); + expect(() => + setupAgent({ + context: z.object({}), + requests: { chatStep: { model: "m", prompt: undefined } }, + } as never), + ).toThrow(/'chatStep' has neither `prompt` nor `messages`/); + }); + + test("an empty event payload is still accepted", () => { + expect(() => setupAgent({ context: z.object({}), events: { GO: {} } })).not.toThrow(); + }); +}); + describe("setupAgent reserved agent.* actor keys", () => { const schemas = createAgentSchemas({ context: z.object({}), diff --git a/src/setup-agent.ts b/src/setup-agent.ts index 76850ce8..cbcd8cee 100644 --- a/src/setup-agent.ts +++ b/src/setup-agent.ts @@ -31,6 +31,8 @@ import { type TextLogicConfig, } from "./text-logic.js"; import { createDecideActor } from "./decision.js"; +import { AgentError } from "./errors.js"; +import { isPlainObject } from "./internal/is-record.js"; import { AGENT_USAGE_EVENT_TYPE, type AgentUsageEvent } from "./usage.js"; import type { AgentInteractionMeta } from "./interaction.js"; import { getAgentExecutionOptions, setAgentExecutionOptions } from "./internal/registry.js"; @@ -276,13 +278,51 @@ function normalizeEventSchemas( events: T, ): NormalizedEventSchemas { return Object.fromEntries( - Object.entries(events).map(([type, schema]) => [ - type, - schema && typeof schema === "object" && "~standard" in schema ? schema : emptyEventSchema, - ]), + Object.entries(events).map(([type, schema]) => { + if (schema && typeof schema === "object" && "~standard" in schema) return [type, schema]; + // `{}` is the shorthand for an event with no payload; anything else + // without `~standard` is a schema this package cannot read. + if (!(isPlainObject(schema) && Object.keys(schema).length === 0)) { + assertStandardSchema(schema, `events.${type}`); + } + return [type, emptyEventSchema]; + }), ) as NormalizedEventSchemas; } +/** + * Throws for a declared schema without the Standard Schema `~standard` + * property. Such a schema would otherwise be skipped, so values it should + * reject would pass unchecked. Zod 3.23 and earlier are the usual cause. + */ +function assertStandardSchema(schema: unknown, where: string): void { + if ( + schema === undefined || + (typeof schema === "object" && schema !== null && "~standard" in schema) + ) { + return; + } + throw new AgentError( + "invalid-schema", + `setupAgent: '${where}' is not a Standard Schema (it has no '~standard' property), so it cannot be used to validate. ` + + `Use zod 4 or zod 3.24 and later, Valibot, ArkType, or any other Standard Schema library; zod 3.23 and earlier do not implement it.`, + ); +} + +// State-level `schemas` (`states..schemas.context`, …), at any depth. +function assertStateSchemasAreStandard( + states: Record | undefined, + path: readonly string[] = [], +): void { + for (const [key, state] of Object.entries(states ?? {})) { + const statePath = [...path, key]; + for (const [name, schema] of Object.entries(state?.schemas ?? {})) { + assertStandardSchema(schema, `states.${statePath.join(".states.")}.schemas.${name}`); + } + assertStateSchemasAreStandard(state?.states, statePath); + } +} + /** * Builds a machine's {@link AgentSchemaPack} from a partial schema * declaration — only `context` is required; `events`/`input`/`output`/`meta` @@ -733,6 +773,14 @@ export function setupAgent< TMetaSchema, TEmittedSchemas >(config); + for (const key of ["context", "input", "output", "meta"] as const) { + assertStandardSchema(schemas[key], key); + } + for (const [type, schema] of Object.entries(schemas.emitted ?? {})) { + assertStandardSchema(schema, `emitted.${type}`); + } + assertStateSchemasAreStandard(config.states); + // `requests` is optional; an omitted map is simply empty. const requests = (config.requests ?? {}) as AgentRequestInput< TRequestSchemas, @@ -743,6 +791,20 @@ export function setupAgent< config.actors, requestActors, ); + for (const [name, request] of Object.entries(config.requests ?? {})) { + const source = request as { prompt?: unknown; messages?: unknown }; + if (source.prompt === undefined && source.messages === undefined) { + throw new AgentError( + "invalid-request", + `setupAgent: request '${name}' has neither \`prompt\` nor \`messages\`, so every call to it fails. ` + + `A request whose executor builds its own model call still needs one: pass the conversation as ` + + `\`messages: ({ input }) => input.messages\` (or a \`prompt\`) and read it in the executor.`, + ); + } + const requestSchemas = (request as { schemas?: { input?: unknown; output?: unknown } }).schemas; + assertStandardSchema(requestSchemas?.input, `requests.${name}.schemas.input`); + assertStandardSchema(requestSchemas?.output, `requests.${name}.schemas.output`); + } // The plain-object config passed to xstate's setup(...) (see // AgentSetupXStateConfig's note on why it's not SetupConfig<...>). const setupConfig: AgentSetupXStateConfig< diff --git a/src/testing/index.ts b/src/testing/index.ts index 9cdcc142..738549ac 100644 --- a/src/testing/index.ts +++ b/src/testing/index.ts @@ -22,6 +22,8 @@ export type { AgentPathTerminal, AssertAgentMachineOptions, CanReachResult, + CanReachTarget, + ExploreSnapshot, ExplorePathsOptions, LintAgentMachineOptions, SimulateAgentOptions, diff --git a/src/text-logic.ts b/src/text-logic.ts index 684a267a..bcf6939a 100644 --- a/src/text-logic.ts +++ b/src/text-logic.ts @@ -1,4 +1,5 @@ import { createAsyncLogic, type AsyncActorLogic, type EventObject } from "xstate"; +import { isPlainObject } from "./internal/is-record.js"; import type { AgentMessage, AgentToolChoice, @@ -256,7 +257,7 @@ export function responseMessagesOf(raw: unknown): AgentMessage[] { // every other value (strings, typed arrays, URLs, class instances) is kept as-is. function dropUndefined(value: unknown): unknown { if (Array.isArray(value)) return value.map(dropUndefined); - if (value && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) { + if (isPlainObject(value)) { const out: Record = {}; for (const [key, entry] of Object.entries(value)) { if (entry !== undefined) out[key] = dropUndefined(entry); diff --git a/src/utils.ts b/src/utils.ts index 2efbd31c..2b1871fe 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,4 +1,5 @@ import type { AnyMachineSnapshot, AnyStateMachine, StateValue } from "xstate"; +import { isPlainObject } from "./internal/is-record.js"; import type { AgentMessage, AssistantMessage, @@ -82,8 +83,7 @@ export function findNonSerializableContextPaths(context: unknown, limit = 5): st return; } - const proto = Object.getPrototypeOf(value); - if (proto === Object.prototype || proto === null) { + if (isPlainObject(value)) { // Plain object — walk its entries. seen.add(obj); for (const [key, item] of Object.entries(value as Record)) { diff --git a/src/verify.ts b/src/verify.ts index 2b8a2208..9147f76b 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -1003,6 +1003,16 @@ function scriptDryError( // ─── Path exploration ─── /** Options for {@link explorePaths}. */ +/** + * The snapshot type the exploration callbacks (`payloads`, `dedupeBy`, a + * `canReach` predicate) receive. Import it from here rather than from your + * own `xstate`, which can be another version than the one this package uses. + */ +export type ExploreSnapshot = AnyMachineSnapshot; + +/** What {@link canReach} looks for: a state id or path, or a snapshot predicate. */ +export type CanReachTarget = string | ((snapshot: ExploreSnapshot) => boolean); + export interface ExplorePathsOptions { input?: unknown; /** Max branch points (decisions + external-event forks) along any path. Default 8. */ @@ -1065,7 +1075,7 @@ export interface ExplorePathsOptions { * were not all tried, so it is listed in `unexpandedEvents`. A type with no * schema at all is applied as `{ type }`. */ - payloads?: (snapshot: AnyMachineSnapshot) => Partial[]>>; + payloads?: (snapshot: ExploreSnapshot) => Partial[]>>; /** * A projection of the snapshot that determines its future. Settled * snapshots with an equal key are explored once (again only when reached @@ -1074,7 +1084,7 @@ export interface ExplorePathsOptions { * a key that drops a field the machine reads makes the search unsound. * Without it, every path is explored in full. */ - dedupeBy?: (snapshot: AnyMachineSnapshot) => string; + dedupeBy?: (snapshot: ExploreSnapshot) => string; } /** A single explored path's terminal outcome. */ @@ -1794,7 +1804,7 @@ function resolveStateTarget(machine: AnyStateMachine, target: string): string { */ export async function canReach( machine: AnyStateMachine, - target: string | ((snapshot: AnyMachineSnapshot) => boolean), + target: CanReachTarget, options: ExplorePathsOptions = {}, ): Promise { const resolvedTarget =