diff --git a/apps/sim/evals/README.md b/apps/sim/evals/README.md
new file mode 100644
index 00000000000..98ba29f4cf2
--- /dev/null
+++ b/apps/sim/evals/README.md
@@ -0,0 +1,162 @@
+# Agent harness evaluations
+
+Measurement for the agent harness — the code that turns a model's tool calls
+into executed tools, feeds the results back, and keeps the turn alive when a
+tool fails. Unit and integration tests prove the harness handles the cases we
+already know about; evals measure whether it still behaves across a suite of
+scenarios when the harness changes.
+
+## What runs
+
+The first suite lives in [`agent-tool-use/`](./agent-tool-use) and drives the
+real OpenAI-compatible streaming tool loop
+(`apps/sim/providers/openai-compat/streaming-tool-loop.ts`) — the loop that
+serves OpenAI, DeepSeek, Groq, Cerebras, and the other OpenAI-compatible
+providers. The model is **scripted**: each scenario supplies the assistant turns
+(tool calls or a final answer) and the result of each tool call. That keeps the
+suite deterministic and runnable in CI with no provider key, while the thing
+being measured — tool dispatch, result feedback, error recovery — is real
+production code.
+
+The suites cover four behaviors:
+
+| Category | What it measures |
+| --- | --- |
+| `tool-selection` | The loop dispatches the tool the model asked for, including from a set of distractors. |
+| `planning` | Multi-turn, dependent and parallel tool calls execute in the right order and all results reach the next turn. |
+| `retrieval` | Values returned by a tool survive into the final answer instead of being dropped or invented. |
+| `recovery` | Tool errors, unknown tool names, and malformed argument JSON are fed back to the model rather than thrown out of the loop. |
+
+## Run it
+
+From `apps/sim`:
+
+```sh
+bun run test:evals
+```
+
+The command writes a JSON report and a Markdown summary to
+`test-results/evals/agent-tool-use.{json,md}` (gitignored) and fails the process
+if any scenario fails. To point the report somewhere else, run Vitest directly:
+
+```sh
+EVAL_REPORT_PATH=/tmp/agent-tool-use.json bunx vitest run evals/agent-tool-use
+```
+
+The suite is also picked up by the normal `bun run test` run, so a regression
+fails CI even without the dedicated command.
+
+## Run against a real model (live)
+
+The same scenarios can be replayed against a live model. This is opt-in and
+never runs in CI. DeepSeek is wired first; any OpenAI-compatible provider works
+through `createOpenAICompatLiveCompletion` in `live.ts`.
+
+```sh
+cd apps/sim
+DEEPSEEK_API_KEY=... bun run test:evals:live
+```
+
+Useful knobs:
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `EVAL_TRIALS` | `3` | Runs per scenario. Models are nondeterministic, so results are pass rates. |
+| `EVAL_MIN_PASS_RATE` | `0` | When > 0, fail a scenario below this pass rate (0–1). |
+| `EVAL_MODEL` | `deepseek-chat` | Model id sent to the provider. |
+| `EVAL_TIMEOUT_MS` | `180000` | Per-request timeout. |
+| `EVAL_REPORT_PATH` | `test-results/evals/agent-tool-use-live.json` | Report location. |
+
+Live runs relax exact assertions: `toolCallSequence` becomes an ordered
+subsequence, `successfulToolCalls` becomes a minimum, and scripted-only cases
+(malformed JSON, unknown tool) are skipped. A scenario-level `liveExpect`
+overrides the scripted expectation where a real model cannot reproduce it (for
+example, an exact retry count). The report is at
+`test-results/evals/agent-tool-use-live.{json,md}` with pass rates, average
+iterations, latency, and the failed check names.
+
+## Add a case
+
+1. Open [`agent-tool-use/scenarios.ts`](./agent-tool-use/scenarios.ts) and add
+   an entry to `AGENT_TOOL_USE_SCENARIOS`.
+2. Declare the `tools` the model may call and the `script` it produces. A
+   `tools` turn lists the calls the model emits; an `answer` turn ends the run.
+   Attach each call's stub `result` (or leave it to default to a successful
+   empty output).
+3. Add the assertions you care about under `expect`: the ordered
+   `toolCallSequence`, `requiredTools`/`forbiddenTools`, `finalContent`,
+   `maxIterations`, and tool call counts. Every assertion becomes a named check
+   in the report.
+4. Run `bun run test:evals`.
+
+A scenario is data, not code — there is no harness change needed for a new case.
+
+### Simulating a failure
+
+- **Tool error:** give the call `result: { success: false, error: '...' }`.
+- **Unknown tool:** call a `name` that is not in `tools`; the loop returns a
+  tool-not-found error to the model.
+- **Malformed arguments:** set `argumentsJson` to an invalid or non-object JSON
+  string. The loop must not execute the call and must return the parse error to
+  the model.
+
+## Executor-level scenarios
+
+[`agent-tool-use/executor-harness.ts`](./agent-tool-use/executor-harness.ts)
+runs a case through a real `DAGExecutor`: a Start block → Agent block workflow,
+with only the provider boundary (`executeProviderRequest`) mocked. This covers
+what the loop harness cannot — agent-block input wiring, variable resolution
+from Start outputs, and the executor's run/error handling. Tool dispatch stays
+covered by the loop suite.
+
+Add a case to `EXECUTOR_SCENARIOS` in `executor-harness.ts`:
+
+- `workflowInput` is exposed on the Start block; reference an output with
+  `<start.field>` from the Agent prompt.
+- `agent` is the Agent block config (`model`, `systemPrompt`, `userPrompt`).
+- `providerResponse` is what the mocked provider returns (`content`,
+  `toolCalls`, `tokens`).
+- `expect` uses the loop's checks plus `resolvedInput` (a substring that must
+  reach the provider messages), `succeeds` (expected `ExecutionResult.success`),
+  and `providerCalls` (exact provider call count).
+- Set `agent.retry` to exercise the executor's per-block retry policy, or
+  `agent.fallbackModels` to exercise model fallback. Make the first
+  `providerResponse` a `reject` and the next one succeeds; assert
+  `providerCalls` and `lastRequestModel` to prove which path recovered.
+
+Both suites write one report, so executor rows appear alongside loop rows.
+
+## Context evals
+
+[`agent-context/`](./agent-context/) drives the Agent block through the executor
+with conversation memory on. The memory read is stubbed per conversation id, so
+the provider request shows exactly what the handler assembled: prior history,
+then the new user prompt, with the system prompt preserved, and the conversation
+id must match. Windowing inside the memory service (`sliding_window`, token
+budgets) is covered by its unit tests; this suite covers the assembly the model
+sees.
+
+```sh
+cd apps/sim
+bun run test:evals:context   # writes test-results/evals/agent-context.{json,md}
+```
+
+Scenarios live in [`agent-context/scenarios.ts`](./agent-context/scenarios.ts)
+and reuse the executor harness, so a case is the same shape as an executor case
+plus `agent.memory`.
+
+## Report shape
+
+`report.json` is machine-readable for dashboards and trend tracking; `report.md`
+is the same data as a table. Each result carries the scenario id, pass/fail,
+every named check with a failure detail, the final content, the executed tool
+invocations, and metrics: iterations, tool call counts (success/error), latency,
+model/tool time, first-response time, and token usage.
+
+## Scope and next steps
+
+Two harnesses share one result shape and report: the tool loop and the
+`DAGExecutor`. The executor suite covers both recovery paths — block retry
+(`executor-retries-failed-block`) and model fallback
+(`executor-falls-back-to-secondary-model`). Further expansion (context/memory,
+model routing, subagent orchestration) is tracked as follow-up work.
diff --git a/apps/sim/evals/agent-context/agent-context.eval.test.ts b/apps/sim/evals/agent-context/agent-context.eval.test.ts
new file mode 100644
index 00000000000..c5d59af8b97
--- /dev/null
+++ b/apps/sim/evals/agent-context/agent-context.eval.test.ts
@@ -0,0 +1,67 @@
+import {
+  permissionCheckMock,
+  permissionCheckMockFns,
+} from '@sim/testing/mocks/permission-check.mock'
+import { providersMock } from '@sim/testing/mocks/providers.mock'
+import { providersConversationHistoryMock } from '@sim/testing/mocks/providers-conversation-history.mock'
+import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock'
+import { toolsMock } from '@sim/testing/mocks/tools.mock'
+import { workspaceFileSecretProvenanceMock } from '@sim/testing/mocks/workspace-file-secret-provenance.mock'
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import { AGENT_CONTEXT_SCENARIOS } from '@/evals/agent-context/scenarios'
+import { runExecutorScenario } from '@/evals/agent-tool-use/executor-harness'
+import { writeEvalReport } from '@/evals/agent-tool-use/report'
+import type { AgentToolUseResult } from '@/evals/agent-tool-use/types'
+
+vi.mock('@/providers/conversation-history', () => providersConversationHistoryMock)
+vi.mock('@/tools', () => toolsMock)
+vi.mock('@/providers/utils', () => providersUtilsMock)
+vi.mock('@/providers', () => providersMock)
+vi.mock('@/ee/access-control/utils/permission-check', () => permissionCheckMock)
+vi.mock(
+  '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance',
+  () => workspaceFileSecretProvenanceMock
+)
+vi.mock('@/lib/memory/agent-turn-session', () => ({
+  openAgentTurnSession: vi.fn(async () => undefined),
+}))
+vi.mock('@/lib/internal/mcp/discover-tools', () => ({
+  discoverMcpServerToolsAsExecutor: vi.fn(async () => []),
+}))
+vi.mock('@/lib/internal/custom-tools/read-available-by-id-or-title', () => ({
+  readAvailableCustomToolByIdOrTitleAsExecutor: vi.fn(async () => undefined),
+}))
+vi.mock('@/executor/utils/http', () => ({
+  buildAuthHeaders: vi.fn(async () => ({ 'Content-Type': 'application/json' })),
+  buildAPIUrl: vi.fn((path: string) => path),
+  extractAPIErrorMessage: vi.fn(async () => 'request failed'),
+}))
+vi.mock('@/lib/execution/cancellation', () => ({
+  subscribeToExecutionCancellation: vi.fn(async () => () => {}),
+  isExecutionCancelled: vi.fn(async () => false),
+}))
+
+const results: AgentToolUseResult[] = []
+
+beforeEach(() => {
+  permissionCheckMockFns.mockValidateModelProvider.mockResolvedValue(undefined)
+  providersUtilsMockFns.mockGetProviderFromModel.mockReturnValue('mock-provider')
+})
+
+afterAll(() => {
+  const reportPath = process.env.EVAL_CONTEXT_REPORT_PATH
+  if (reportPath) writeEvalReport(results, reportPath)
+})
+
+describe('agent context eval suite', () => {
+  it.each(AGENT_CONTEXT_SCENARIOS)('$id: $name', async (scenario) => {
+    const result = await runExecutorScenario(scenario)
+    results.push(result)
+
+    const failed = result.checks.filter((entry) => !entry.passed)
+    expect(
+      failed,
+      failed.map((entry) => `${entry.name}: ${entry.detail}`).join('; ') || undefined
+    ).toEqual([])
+  })
+})
diff --git a/apps/sim/evals/agent-context/scenarios.ts b/apps/sim/evals/agent-context/scenarios.ts
new file mode 100644
index 00000000000..4edae71d927
--- /dev/null
+++ b/apps/sim/evals/agent-context/scenarios.ts
@@ -0,0 +1,71 @@
+import type { ExecutorScenario } from '@/evals/agent-tool-use/executor-harness'
+
+/**
+ * Agent context evals.
+ *
+ * These drive the real Agent block inside the `DAGExecutor` with conversation
+ * memory on. The memory read is stubbed per conversation id, so the provider
+ * request shows exactly what the handler assembled: prior history, then the new
+ * user prompt, with the system prompt preserved. A handler that passed the
+ * wrong conversation id, dropped history, or reordered the prompt fails.
+ *
+ * Windowing inside the memory service (`sliding_window`, token budgets) is
+ * covered by its own unit tests; this suite covers the assembly the model sees.
+ */
+export const AGENT_CONTEXT_SCENARIOS: ExecutorScenario[] = [
+  {
+    id: 'context-remembers-prior-turns',
+    name: 'includes prior conversation memory before the new user prompt',
+    category: 'retrieval',
+    description:
+      'Memory holds two prior turns. The provider request must contain both, in order, followed by the new user prompt, with the system prompt present.',
+    workflowInput: {},
+    agent: {
+      model: 'gpt-4o',
+      systemPrompt: 'You are a helpful assistant.',
+      userPrompt: 'What is my name?',
+      memory: {
+        conversationId: 'conv-ada',
+        history: [
+          { role: 'user', content: 'My name is Ada.' },
+          { role: 'assistant', content: 'Nice to meet you, Ada.' },
+        ],
+      },
+    },
+    providerResponse: {
+      content: 'Your name is Ada.',
+      tokens: { input: 30, output: 5, total: 35 },
+    },
+    expect: {
+      succeeds: true,
+      finalContent: 'Ada',
+    },
+  },
+  {
+    id: 'context-isolates-conversations',
+    name: 'reads the conversation named by the block, not another',
+    category: 'retrieval',
+    description:
+      'The memory stub only returns history for the block conversation id; any other id yields a placeholder. A handler that passed the wrong id would surface the placeholder and fail.',
+    workflowInput: {},
+    agent: {
+      model: 'gpt-4o',
+      userPrompt: 'What did we decide?',
+      memory: {
+        conversationId: 'conv-b',
+        history: [
+          { role: 'user', content: 'We decided to ship on Friday.' },
+          { role: 'assistant', content: 'Shipping Friday.' },
+        ],
+      },
+    },
+    providerResponse: {
+      content: 'You decided to ship on Friday.',
+      tokens: { input: 25, output: 6, total: 31 },
+    },
+    expect: {
+      succeeds: true,
+      finalContent: 'Friday',
+    },
+  },
+]
diff --git a/apps/sim/evals/agent-tool-use/agent-tool-use.eval.test.ts b/apps/sim/evals/agent-tool-use/agent-tool-use.eval.test.ts
new file mode 100644
index 00000000000..f13027a5d66
--- /dev/null
+++ b/apps/sim/evals/agent-tool-use/agent-tool-use.eval.test.ts
@@ -0,0 +1,81 @@
+import {
+  permissionCheckMock,
+  permissionCheckMockFns,
+} from '@sim/testing/mocks/permission-check.mock'
+import { providersMock } from '@sim/testing/mocks/providers.mock'
+import { providersConversationHistoryMock } from '@sim/testing/mocks/providers-conversation-history.mock'
+import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock'
+import { toolsMock } from '@sim/testing/mocks/tools.mock'
+import { workspaceFileSecretProvenanceMock } from '@sim/testing/mocks/workspace-file-secret-provenance.mock'
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import { EXECUTOR_SCENARIOS, runExecutorScenario } from '@/evals/agent-tool-use/executor-harness'
+import { runScenario } from '@/evals/agent-tool-use/harness'
+import { writeEvalReport } from '@/evals/agent-tool-use/report'
+import { AGENT_TOOL_USE_SCENARIOS } from '@/evals/agent-tool-use/scenarios'
+import type { AgentToolUseResult } from '@/evals/agent-tool-use/types'
+
+vi.mock('@/providers/conversation-history', () => providersConversationHistoryMock)
+vi.mock('@/tools', () => toolsMock)
+vi.mock('@/providers/utils', () => providersUtilsMock)
+vi.mock('@/providers', () => providersMock)
+vi.mock('@/ee/access-control/utils/permission-check', () => permissionCheckMock)
+vi.mock(
+  '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance',
+  () => workspaceFileSecretProvenanceMock
+)
+vi.mock('@/lib/memory/agent-turn-session', () => ({
+  openAgentTurnSession: vi.fn(async () => undefined),
+}))
+vi.mock('@/lib/internal/mcp/discover-tools', () => ({
+  discoverMcpServerToolsAsExecutor: vi.fn(async () => []),
+}))
+vi.mock('@/lib/internal/custom-tools/read-available-by-id-or-title', () => ({
+  readAvailableCustomToolByIdOrTitleAsExecutor: vi.fn(async () => undefined),
+}))
+vi.mock('@/executor/utils/http', () => ({
+  buildAuthHeaders: vi.fn(async () => ({ 'Content-Type': 'application/json' })),
+  buildAPIUrl: vi.fn((path: string) => path),
+  extractAPIErrorMessage: vi.fn(async () => 'request failed'),
+}))
+vi.mock('@/lib/execution/cancellation', () => ({
+  subscribeToExecutionCancellation: vi.fn(async () => () => {}),
+  isExecutionCancelled: vi.fn(async () => false),
+}))
+
+const results: AgentToolUseResult[] = []
+
+beforeEach(() => {
+  permissionCheckMockFns.mockValidateModelProvider.mockResolvedValue(undefined)
+  providersUtilsMockFns.mockGetProviderFromModel.mockReturnValue('mock-provider')
+})
+
+afterAll(() => {
+  const reportPath = process.env.EVAL_REPORT_PATH
+  if (reportPath) writeEvalReport(results, reportPath)
+})
+
+describe('agent tool-use eval suite', () => {
+  it.each(AGENT_TOOL_USE_SCENARIOS)('$id: $name', async (scenario) => {
+    const result = await runScenario(scenario)
+    results.push(result)
+
+    const failed = result.checks.filter((entry) => !entry.passed)
+    expect(
+      failed,
+      failed.map((entry) => `${entry.name}: ${entry.detail}`).join('; ') || undefined
+    ).toEqual([])
+  })
+})
+
+describe('agent executor eval suite', () => {
+  it.each(EXECUTOR_SCENARIOS)('$id: $name', async (scenario) => {
+    const result = await runExecutorScenario(scenario)
+    results.push(result)
+
+    const failed = result.checks.filter((entry) => !entry.passed)
+    expect(
+      failed,
+      failed.map((entry) => `${entry.name}: ${entry.detail}`).join('; ') || undefined
+    ).toEqual([])
+  })
+})
diff --git a/apps/sim/evals/agent-tool-use/agent-tool-use.live.test.ts b/apps/sim/evals/agent-tool-use/agent-tool-use.live.test.ts
new file mode 100644
index 00000000000..023ca29ecba
--- /dev/null
+++ b/apps/sim/evals/agent-tool-use/agent-tool-use.live.test.ts
@@ -0,0 +1,83 @@
+import { providersMock } from '@sim/testing/mocks/providers.mock'
+import { providersConversationHistoryMock } from '@sim/testing/mocks/providers-conversation-history.mock'
+import { providersUtilsMock } from '@sim/testing/mocks/providers-utils.mock'
+import { toolsMock } from '@sim/testing/mocks/tools.mock'
+import { afterAll, describe, expect, it, vi } from 'vitest'
+import { runScenario } from '@/evals/agent-tool-use/harness'
+import { createDeepSeekLiveCompletion } from '@/evals/agent-tool-use/live'
+import { writeLiveEvalReport } from '@/evals/agent-tool-use/report'
+import { AGENT_TOOL_USE_SCENARIOS } from '@/evals/agent-tool-use/scenarios'
+import type { AgentToolUseResult, LiveScenarioSummary } from '@/evals/agent-tool-use/types'
+
+vi.mock('@/providers/conversation-history', () => providersConversationHistoryMock)
+vi.mock('@/tools', () => toolsMock)
+vi.mock('@/providers/utils', () => providersUtilsMock)
+vi.mock('@/providers', () => providersMock)
+
+/**
+ * Live agent tool-use evals. Opt-in only:
+ *
+ *   EVAL_LIVE=1 DEEPSEEK_API_KEY=... \
+ *     bun run --cwd apps/sim test --mode live evals/agent-tool-use/agent-tool-use.live.test.ts
+ *
+ * Each scenario runs `EVAL_TRIALS` times (default 3) because a real model is
+ * nondeterministic. The report carries pass rates, not a single boolean. Set
+ * `EVAL_MIN_PASS_RATE` (0–1) to turn a pass-rate floor into a failing gate.
+ */
+const LIVE = process.env.EVAL_LIVE === '1' && Boolean(process.env.DEEPSEEK_API_KEY)
+const TRIALS = Number(process.env.EVAL_TRIALS ?? '3')
+const MIN_PASS_RATE = Number(process.env.EVAL_MIN_PASS_RATE ?? '0')
+const MODEL = process.env.EVAL_MODEL ?? 'deepseek-chat'
+const TIMEOUT_MS = Number(process.env.EVAL_TIMEOUT_MS ?? '180000')
+
+const liveScenarios = AGENT_TOOL_USE_SCENARIOS.filter((scenario) => !scenario.scriptedOnly)
+const summaries: LiveScenarioSummary[] = []
+
+afterAll(() => {
+  if (!LIVE) return
+  writeLiveEvalReport(
+    summaries,
+    process.env.EVAL_REPORT_PATH ?? 'test-results/evals/agent-tool-use-live.json'
+  )
+})
+
+describe.skipIf(!LIVE)('agent tool-use eval suite (live DeepSeek)', () => {
+  it.each(liveScenarios)(
+    '$id: $name',
+    async (scenario) => {
+      const completion = createDeepSeekLiveCompletion(MODEL)
+      const results: AgentToolUseResult[] = []
+
+      for (let trial = 0; trial < TRIALS; trial++) {
+        results.push(
+          await runScenario(scenario, {
+            completion,
+            mode: 'live',
+            model: MODEL,
+            providerName: 'DeepSeek',
+          })
+        )
+      }
+
+      const passed = results.filter((result) => result.passed).length
+      const passRate = results.length === 0 ? 0 : passed / results.length
+      summaries.push({
+        id: scenario.id,
+        name: scenario.name,
+        category: scenario.category,
+        trials: results.length,
+        passed,
+        passRate,
+        results,
+      })
+
+      if (MIN_PASS_RATE > 0) {
+        expect(
+          passRate,
+          `${scenario.id} passed ${passed}/${results.length} trials`
+        ).toBeGreaterThanOrEqual(MIN_PASS_RATE)
+      }
+    },
+    TIMEOUT_MS
+  )
+})
diff --git a/apps/sim/evals/agent-tool-use/executor-harness.ts b/apps/sim/evals/agent-tool-use/executor-harness.ts
new file mode 100644
index 00000000000..aa073254833
--- /dev/null
+++ b/apps/sim/evals/agent-tool-use/executor-harness.ts
@@ -0,0 +1,435 @@
+import {
+  createSerializedBlock,
+  createSerializedWorkflow,
+} from '@sim/testing/factories/serialized-block.factory'
+import { providersMockFns } from '@sim/testing/mocks/providers.mock'
+import { vi } from 'vitest'
+import { DAGExecutor } from '@/executor/execution/executor'
+import { memoryService } from '@/executor/handlers/agent/memory'
+import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
+import { type EvalRunMode, type ScoredToolCall, scoreExpectations } from './harness'
+import type {
+  AgentToolUseExpectations,
+  AgentToolUseResult,
+  EvalCategory,
+  EvalToolInvocation,
+} from './types'
+
+/**
+ * Executor-level harness.
+ *
+ * Drives a real `DAGExecutor` run: Start block → Agent block. The provider
+ * boundary (`executeProviderRequest`) is the only thing mocked — the Agent
+ * block handler, input/variable resolution, and the executor run/error handling
+ * are real. Tool calls are what the mocked provider returns; tool *dispatch* is
+ * covered by the loop harness.
+ */
+
+/** One tool call the mocked provider reports in its response. */
+export interface ExecutorProviderToolCall {
+  name: string
+  arguments?: Record<string, unknown>
+  result?: unknown
+}
+
+/** One model call: either a response, or a rejection the block must recover from. */
+export interface ExecutorProviderResponse {
+  /** When set, the call rejects with this message instead of resolving. */
+  reject?: string
+  content: string
+  model?: string
+  tokens?: { input?: number; output?: number; total?: number }
+  toolCalls?: ExecutorProviderToolCall[]
+  cost?: unknown
+  timing?: unknown
+}
+
+export interface ExecutorScenario {
+  id: string
+  name: string
+  category: EvalCategory
+  description: string
+  /** Exposed on the Start block and referenced from the Agent block. */
+  workflowInput: Record<string, unknown>
+  agent: {
+    model: string
+    systemPrompt?: string
+    userPrompt?: string
+    temperature?: number
+    /** Enables the executor's per-block retry policy for the Agent block. */
+    retry?: { enabled: boolean; maxTries: number; waitBetweenTriesMs: number }
+    /** Ordered models the Agent handler tries after the primary fails. */
+    fallbackModels?: Array<{ model: string }>
+    /**
+     * Turns on conversation memory. `history` is what the mocked memory read
+     * returns for `conversationId`, so a wrong id surfaces as a failed check.
+     */
+    memory?: {
+      conversationId: string
+      history: Array<{ role: 'user' | 'assistant' | 'system'; content: string }>
+    }
+  }
+  /** One entry per model call; the last entry serves any extra/retry calls. */
+  providerResponse: ExecutorProviderResponse | ExecutorProviderResponse[]
+  expect: AgentToolUseExpectations & {
+    /** Substring that must appear in the messages sent to the provider. */
+    resolvedInput?: string
+    /** Expected `ExecutionResult.success`. */
+    succeeds?: boolean
+    /** Exact number of provider calls the executor made. */
+    providerCalls?: number
+    /** Model id sent on the final provider call (proves which candidate served). */
+    lastRequestModel?: string
+  }
+}
+
+function lastMatchingIndex(contents: string[], needle: string): number {
+  for (let index = contents.length - 1; index >= 0; index--) {
+    if (contents[index].includes(needle)) return index
+  }
+  return -1
+}
+
+function buildWorkflow(scenario: ExecutorScenario): SerializedWorkflow {
+  const start: SerializedBlock = createSerializedBlock({
+    id: 'start',
+    type: 'start_trigger',
+    name: 'Start',
+  })
+  /** The trigger handler claims a block whose metadata says it is a trigger. */
+  if (start.metadata) start.metadata.category = 'triggers'
+  const agent: SerializedBlock = createSerializedBlock({
+    id: 'agent',
+    type: 'agent',
+    name: 'Eval Agent',
+  })
+  agent.config.tool = 'agent'
+  agent.config.params = {
+    model: scenario.agent.model,
+    systemPrompt: scenario.agent.systemPrompt,
+    userPrompt: scenario.agent.userPrompt,
+    ...(scenario.agent.temperature !== undefined
+      ? { temperature: scenario.agent.temperature }
+      : {}),
+    ...(scenario.agent.fallbackModels ? { fallbackModels: scenario.agent.fallbackModels } : {}),
+    ...(scenario.agent.memory
+      ? { memoryType: 'conversation', conversationId: scenario.agent.memory.conversationId }
+      : {}),
+  }
+  if (scenario.agent.retry) agent.retry = scenario.agent.retry
+
+  return createSerializedWorkflow([start, agent], [{ source: 'start', target: 'agent' }])
+}
+
+/**
+ * Runs one executor scenario and scores it with the shared scorer, returning
+ * the same result shape as the loop harness so both land in one report.
+ */
+export async function runExecutorScenario(
+  scenario: ExecutorScenario,
+  options: { mode?: EvalRunMode } = {}
+): Promise<AgentToolUseResult> {
+  const mode = options.mode ?? 'scripted'
+  const responses = Array.isArray(scenario.providerResponse)
+    ? [...scenario.providerResponse]
+    : [scenario.providerResponse]
+  const requests: Array<Record<string, unknown>> = []
+  let callIndex = 0
+
+  providersMockFns.mockExecuteProviderRequest.mockImplementation(
+    async (_providerId: string, request: Record<string, unknown>) => {
+      requests.push(request)
+      const response = responses[Math.min(callIndex, responses.length - 1)]
+      callIndex += 1
+      if (response.reject) throw new Error(response.reject)
+      return {
+        content: response.content,
+        model: response.model ?? scenario.agent.model,
+        tokens: response.tokens ?? { input: 0, output: 0, total: 0 },
+        toolCalls: response.toolCalls ?? [],
+        cost: response.cost ?? 0,
+        timing: response.timing ?? { total: 0 },
+      }
+    }
+  )
+
+  const fetchedConversationIds: unknown[] = []
+  if (scenario.agent.memory) {
+    const memory = scenario.agent.memory
+    vi.spyOn(memoryService, 'fetchMemoryMessages').mockImplementation(async (_ctx, inputs) => {
+      fetchedConversationIds.push(inputs.conversationId)
+      return inputs.conversationId === memory.conversationId
+        ? memory.history.map((message) => ({ ...message }))
+        : [{ role: 'user', content: '__WRONG_CONVERSATION__' }]
+    })
+  }
+
+  const executor = new DAGExecutor({
+    workflow: buildWorkflow(scenario),
+    workflowInput: scenario.workflowInput,
+    contextExtensions: {
+      workspaceId: 'eval-workspace',
+      executionId: 'eval-execution',
+      userId: 'eval-user',
+    },
+  })
+
+  let result: { success?: boolean; output?: Record<string, unknown> } | undefined
+  let runError: unknown
+  const startedAt = Date.now()
+  try {
+    result = (await executor.execute('eval-workflow')) as typeof result
+  } catch (error) {
+    runError = error
+  }
+  const latencyMs = Date.now() - startedAt
+
+  const output = (result?.output ?? {}) as Record<string, unknown>
+  const finalContent = typeof output.content === 'string' ? output.content : ''
+  const rawToolCalls = ((output.toolCalls as { list?: unknown[] } | undefined)?.list ??
+    []) as Array<Record<string, unknown>>
+
+  const toolCalls: ScoredToolCall[] = rawToolCalls.map((call) => ({
+    name: typeof call.name === 'string' ? call.name : 'unknown',
+    success: true,
+  }))
+  const toolInvocations: EvalToolInvocation[] = rawToolCalls.map((call) => ({
+    name: typeof call.name === 'string' ? call.name : 'unknown',
+    arguments: (call.arguments ?? {}) as Record<string, unknown>,
+    success: true,
+    durationMs: typeof call.duration === 'number' ? call.duration : 0,
+  }))
+
+  const checks = scoreExpectations(
+    scenario.expect,
+    toolCalls,
+    finalContent,
+    requests.length,
+    runError,
+    mode
+  )
+
+  if (scenario.expect.resolvedInput !== undefined) {
+    const sent = JSON.stringify(requests)
+    checks.push({
+      name: 'resolved-input',
+      passed: sent.includes(scenario.expect.resolvedInput),
+      detail: `looking for ${JSON.stringify(scenario.expect.resolvedInput)} in provider messages`,
+    })
+  }
+
+  if (scenario.expect.succeeds !== undefined) {
+    checks.push({
+      name: 'workflow-success',
+      passed: result?.success === scenario.expect.succeeds,
+      detail: `success=${String(result?.success)}`,
+    })
+  }
+
+  if (scenario.expect.providerCalls !== undefined) {
+    checks.push({
+      name: 'provider-calls',
+      passed: requests.length === scenario.expect.providerCalls,
+      detail: `expected ${scenario.expect.providerCalls}, got ${requests.length}`,
+    })
+  }
+
+  if (scenario.expect.lastRequestModel !== undefined) {
+    const lastModel = (requests.at(-1) as { model?: string } | undefined)?.model
+    checks.push({
+      name: 'last-request-model',
+      passed: lastModel === scenario.expect.lastRequestModel,
+      detail: `expected ${scenario.expect.lastRequestModel}, got ${String(lastModel)}`,
+    })
+  }
+
+  if (scenario.agent.memory) {
+    const memory = scenario.agent.memory
+    const requestMessages = ((requests[0] as { messages?: unknown[] } | undefined)?.messages ??
+      []) as Array<{ role?: string; content?: unknown }>
+    const contents = requestMessages.map((message) =>
+      typeof message.content === 'string' ? message.content : ''
+    )
+
+    const missingHistory = memory.history.filter(
+      (message) => !contents.some((content) => content.includes(message.content))
+    )
+    checks.push({
+      name: 'memory-history-in-request',
+      passed: missingHistory.length === 0,
+      detail:
+        missingHistory.length === 0
+          ? `all ${memory.history.length} history messages reached the provider`
+          : `missing [${missingHistory.map((message) => message.content).join(', ')}]`,
+    })
+
+    const lastHistoryIndex =
+      memory.history.length === 0
+        ? -1
+        : Math.max(...memory.history.map((message) => lastMatchingIndex(contents, message.content)))
+    const promptIndex = scenario.agent.userPrompt
+      ? lastMatchingIndex(contents, scenario.agent.userPrompt)
+      : -1
+    checks.push({
+      name: 'memory-before-user-prompt',
+      passed: promptIndex >= 0 && promptIndex > lastHistoryIndex,
+      detail: `history ends at ${lastHistoryIndex}, user prompt at ${promptIndex}`,
+    })
+
+    if (scenario.agent.systemPrompt) {
+      const systemPrompt = scenario.agent.systemPrompt
+      checks.push({
+        name: 'system-prompt-in-request',
+        passed: requestMessages.some(
+          (message) => message.role === 'system' && String(message.content).includes(systemPrompt)
+        ),
+        detail: 'configured system prompt reached the provider',
+      })
+    }
+
+    checks.push({
+      name: 'conversation-id',
+      passed:
+        fetchedConversationIds.length > 0 &&
+        fetchedConversationIds.every((id) => id === memory.conversationId),
+      detail: `expected ${memory.conversationId}, got [${fetchedConversationIds.join(', ')}]`,
+    })
+  }
+
+  const tokens = (output.tokens ?? {}) as { input?: number; output?: number; total?: number }
+
+  return {
+    id: scenario.id,
+    name: scenario.name,
+    category: scenario.category,
+    passed: checks.every((entry) => entry.passed),
+    checks,
+    finalContent,
+    toolInvocations,
+    metrics: {
+      iterations: requests.length,
+      toolCalls: toolCalls.length,
+      successfulToolCalls: toolCalls.filter((call) => call.success).length,
+      erroredToolCalls: 0,
+      latencyMs,
+      modelTimeMs: 0,
+      toolsTimeMs: 0,
+      firstResponseTimeMs: 0,
+      inputTokens: tokens.input ?? 0,
+      outputTokens: tokens.output ?? 0,
+      totalTokens: tokens.total ?? 0,
+    },
+    ...(runError ? { error: String(runError) } : {}),
+  }
+}
+
+/**
+ * Executor-level scenarios. Two cover the wiring the loop suite cannot see:
+ * Start → Agent execution, and variable resolution from a Start output into the
+ * Agent's prompt.
+ */
+export const EXECUTOR_SCENARIOS: ExecutorScenario[] = [
+  {
+    id: 'executor-agent-runs',
+    name: 'runs a Start → Agent workflow and surfaces the Agent output',
+    category: 'tool-selection',
+    description:
+      'The real Agent block handler runs inside the DAG. The mocked provider reports one tool call; the executor result must carry the content and the tool call through.',
+    workflowInput: { message: 'What is the API rate limit?' },
+    agent: {
+      model: 'gpt-4o',
+      systemPrompt: 'You are a documentation assistant.',
+      userPrompt: 'What is the API rate limit?',
+    },
+    providerResponse: {
+      content: 'The API rate limit is 100 requests per minute.',
+      toolCalls: [
+        {
+          name: 'search_docs',
+          arguments: { query: 'api rate limit' },
+          result: { snippet: 'The API rate limit is 100 requests per minute.' },
+        },
+      ],
+      tokens: { input: 10, output: 20, total: 30 },
+    },
+    expect: {
+      succeeds: true,
+      finalContent: '100 requests per minute',
+      toolCallSequence: ['search_docs'],
+      successfulToolCalls: 1,
+    },
+  },
+  {
+    id: 'executor-resolves-start-input',
+    name: 'resolves a Start output into the Agent prompt before the provider call',
+    category: 'planning',
+    description:
+      'The Agent userPrompt references <start.message>. The value must be resolved by the executor and reach the provider request, not passed through verbatim.',
+    workflowInput: { message: 'Summarize order A-1937' },
+    agent: {
+      model: 'gpt-4o',
+      userPrompt: '<start.message>',
+    },
+    providerResponse: {
+      content: 'Order A-1937 shipped via DHL.',
+      tokens: { input: 8, output: 12, total: 20 },
+    },
+    expect: {
+      succeeds: true,
+      resolvedInput: 'Summarize order A-1937',
+      finalContent: /A-1937/,
+    },
+  },
+  {
+    id: 'executor-retries-failed-block',
+    name: 'retries a failed Agent block and completes the run',
+    category: 'recovery',
+    description:
+      'The first provider call rejects with a 503. The block has retry enabled, so the executor replays it and the second call succeeds — proving the executor retry policy, not the agent handler, recovered the turn.',
+    workflowInput: { message: 'What is the API rate limit?' },
+    agent: {
+      model: 'gpt-4o',
+      userPrompt: 'What is the API rate limit?',
+      retry: { enabled: true, maxTries: 3, waitBetweenTriesMs: 0 },
+    },
+    providerResponse: [
+      { reject: '503 Service Unavailable', content: '' },
+      {
+        content: 'The API rate limit is 100 requests per minute.',
+        tokens: { input: 10, output: 20, total: 30 },
+      },
+    ],
+    expect: {
+      succeeds: true,
+      finalContent: '100 requests per minute',
+      providerCalls: 2,
+    },
+  },
+  {
+    id: 'executor-falls-back-to-secondary-model',
+    name: 'falls back to the secondary model when the primary fails',
+    category: 'recovery',
+    description:
+      'The primary model call rejects and the Agent block has a fallback model. The handler must serve the answer from the fallback and the run must complete.',
+    workflowInput: { message: 'What is the API rate limit?' },
+    agent: {
+      model: 'gpt-4o',
+      userPrompt: 'What is the API rate limit?',
+      fallbackModels: [{ model: 'gpt-4o-mini' }],
+    },
+    providerResponse: [
+      { reject: '429 rate limited', content: '' },
+      {
+        content: 'The API rate limit is 100 requests per minute.',
+        model: 'gpt-4o-mini',
+        tokens: { input: 10, output: 20, total: 30 },
+      },
+    ],
+    expect: {
+      succeeds: true,
+      finalContent: '100 requests per minute',
+      providerCalls: 2,
+      lastRequestModel: 'gpt-4o-mini',
+    },
+  },
+]
diff --git a/apps/sim/evals/agent-tool-use/harness.ts b/apps/sim/evals/agent-tool-use/harness.ts
new file mode 100644
index 00000000000..0b369a06992
--- /dev/null
+++ b/apps/sim/evals/agent-tool-use/harness.ts
@@ -0,0 +1,418 @@
+import { createLogger } from '@sim/logger'
+import { collectStream } from '@sim/testing/helpers/async'
+import { providersMock } from '@sim/testing/mocks/providers.mock'
+import { providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock'
+import { toolsMockFns } from '@sim/testing/mocks/tools.mock'
+import { isRecordLike } from '@sim/utils/object'
+import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
+import type { CompletionUsage } from 'openai/resources/completions'
+import {
+  createOpenAICompatStreamingToolLoopStream,
+  type OpenAICompatCreateCompletion,
+} from '@/providers/openai-compat/streaming-tool-loop'
+import type { AgentStreamEvent } from '@/providers/stream-events'
+import type { StreamingToolLoopComplete } from '@/providers/streaming-tool-loop-shared'
+import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
+import type { ProviderToolConfig, TimeSegment } from '@/providers/types'
+import type { ToolResponse } from '@/tools/types'
+import type {
+  AgentToolUseExpectations,
+  AgentToolUseResult,
+  AgentToolUseScenario,
+  EvalCheck,
+  EvalToolDefinition,
+  EvalToolInvocation,
+  ScriptedModelTurn,
+  ScriptedToolCall,
+} from './types'
+
+/**
+ * Runs one scenario through the real OpenAI-compatible streaming tool loop and
+ * scores the result. Vitest owns the `@/tools`, `@/providers` and
+ * `@/providers/utils` module mocks; this module only drives them.
+ */
+
+const EVAL_MODEL = 'eval-model'
+const EVAL_PROVIDER = 'Eval'
+const MAX_TOOL_ITERATIONS = 20
+
+const logger = createLogger('AgentToolUseEval')
+
+interface CapturedToolCall {
+  name: string
+  arguments: Record<string, unknown>
+  success: boolean
+  result?: unknown
+  duration?: number
+}
+
+interface ToolCallList {
+  list: CapturedToolCall[]
+  count: number
+}
+
+/** Scripted runs assert exact behavior; live runs assert outcomes across trials. */
+export type EvalRunMode = 'scripted' | 'live'
+
+/** Options for {@link runScenario}. */
+export interface RunScenarioOptions {
+  /** Model turns. Defaults to the scenario's scripted turns. */
+  completion?: OpenAICompatCreateCompletion
+  mode?: EvalRunMode
+  /** Model id sent to a live provider and recorded in the run. */
+  model?: string
+  /** Provider label used in loop diagnostics. */
+  providerName?: string
+}
+
+/** Raw call counts as a value the loop never reads; scenarios only assert on it. */
+const COMPLETION_USAGE = (): CompletionUsage => ({
+  prompt_tokens: 10,
+  completion_tokens: 5,
+  total_tokens: 15,
+})
+
+let chunkCounter = 0
+
+function chunk(
+  delta: ChatCompletionChunk.Choice['delta'] & { reasoning_content?: string },
+  finishReason: ChatCompletionChunk.Choice['finish_reason'] = null,
+  usage?: CompletionUsage
+): ChatCompletionChunk {
+  chunkCounter += 1
+  return {
+    id: `eval-chunk-${chunkCounter}`,
+    object: 'chat.completion.chunk',
+    created: 0,
+    model: EVAL_MODEL,
+    choices: [{ index: 0, delta, finish_reason: finishReason, logprobs: null }],
+    ...(usage ? { usage } : {}),
+  }
+}
+
+function turnToChunks(turn: ScriptedModelTurn): ChatCompletionChunk[] {
+  if (turn.kind === 'answer') {
+    return [
+      ...(turn.thinking ? [chunk({ reasoning_content: turn.thinking })] : []),
+      chunk({ content: turn.content }, 'stop', COMPLETION_USAGE()),
+    ]
+  }
+
+  const chunks: ChatCompletionChunk[] = []
+  if (turn.thinking) chunks.push(chunk({ reasoning_content: turn.thinking }))
+  turn.calls.forEach((call, index) => {
+    chunks.push(
+      chunk({
+        tool_calls: [
+          {
+            index,
+            id: `call_${index}`,
+            type: 'function',
+            function: {
+              name: call.name,
+              arguments: call.argumentsJson ?? JSON.stringify(call.args ?? {}),
+            },
+          },
+        ],
+      })
+    )
+  })
+  chunks.push(chunk({}, 'tool_calls', COMPLETION_USAGE()))
+  return chunks
+}
+
+function createScriptedCompletion(scenario: AgentToolUseScenario): OpenAICompatCreateCompletion {
+  let turnIndex = 0
+  return async () => {
+    const turn = scenario.script[turnIndex]
+    turnIndex += 1
+    if (!turn) {
+      throw new Error(
+        `Scenario "${scenario.id}" requested model turn ${turnIndex} but only ${scenario.script.length} are scripted`
+      )
+    }
+    return (async function* () {
+      for (const next of turnToChunks(turn)) yield next
+    })()
+  }
+}
+
+function toProviderTools(tools: EvalToolDefinition[]): ProviderToolConfig[] {
+  return tools.map((tool) => ({
+    id: tool.name,
+    description: tool.description,
+    params: {},
+    parameters: {
+      type: tool.parameters?.type ?? 'object',
+      properties: tool.parameters?.properties ?? {},
+      required: tool.parameters?.required ?? [],
+    },
+  }))
+}
+
+/** True when the loop will execute the call, so its result must be queued. */
+function isExecutable(call: ScriptedToolCall, toolNames: Set<string>): boolean {
+  if (!toolNames.has(call.name)) return false
+  if (call.argumentsJson === undefined) return true
+  try {
+    return isRecordLike(JSON.parse(call.argumentsJson))
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Results are queued per tool in script order. The loop may execute calls from
+ * one turn in any completion order, so keying by name keeps every call matched
+ * to the result the scenario intended.
+ */
+function buildResultQueues(
+  scenario: AgentToolUseScenario,
+  toolNames: Set<string>
+): Map<string, ToolResponse[]> {
+  const queues = new Map<string, ToolResponse[]>()
+  for (const turn of scenario.script) {
+    if (turn.kind !== 'tools') continue
+    for (const call of turn.calls) {
+      if (!isExecutable(call, toolNames)) continue
+      const spec = call.result
+      const queue = queues.get(call.name) ?? []
+      queue.push({
+        success: spec?.success ?? true,
+        output: spec?.output ?? {},
+        ...(spec?.error ? { error: spec.error } : {}),
+      })
+      queues.set(call.name, queue)
+    }
+  }
+  return queues
+}
+
+function check(name: string, passed: boolean, detail: string): EvalCheck {
+  return { name, passed, detail }
+}
+
+function sameSequence(actual: string[], expected: string[]): boolean {
+  return actual.length === expected.length && actual.every((name, i) => name === expected[i])
+}
+
+function matchesContent(content: string, expected: string | RegExp): boolean {
+  return typeof expected === 'string' ? content.includes(expected) : expected.test(content)
+}
+
+function isOrderedSubsequence(actual: string[], expected: string[]): boolean {
+  let index = 0
+  for (const name of actual) {
+    if (name === expected[index]) index += 1
+  }
+  return index === expected.length
+}
+
+/** Minimal tool-call shape the scorer needs; both harnesses produce it. */
+export interface ScoredToolCall {
+  name: string
+  success: boolean
+}
+
+export function scoreExpectations(
+  expected: AgentToolUseExpectations,
+  toolCalls: ScoredToolCall[],
+  finalContent: string,
+  iterations: number,
+  error: unknown,
+  mode: EvalRunMode
+): EvalCheck[] {
+  const actualSequence = toolCalls.map((call) => call.name)
+  const checks: EvalCheck[] = []
+
+  if (expected.toolCallSequence) {
+    const sequenceMatches =
+      mode === 'live'
+        ? isOrderedSubsequence(actualSequence, expected.toolCallSequence)
+        : sameSequence(actualSequence, expected.toolCallSequence)
+    checks.push(
+      check(
+        'tool-call-sequence',
+        sequenceMatches,
+        `expected [${expected.toolCallSequence.join(', ')}], got [${actualSequence.join(', ')}]`
+      )
+    )
+  }
+
+  if (expected.requiredTools) {
+    const missing = expected.requiredTools.filter((name) => !actualSequence.includes(name))
+    checks.push(
+      check(
+        'required-tools',
+        missing.length === 0,
+        missing.length === 0 ? 'all required tools called' : `missing [${missing.join(', ')}]`
+      )
+    )
+  }
+
+  if (expected.forbiddenTools) {
+    const called = expected.forbiddenTools.filter((name) => actualSequence.includes(name))
+    checks.push(
+      check(
+        'forbidden-tools',
+        called.length === 0,
+        called.length === 0 ? 'no forbidden tools called' : `called [${called.join(', ')}]`
+      )
+    )
+  }
+
+  if (expected.finalContent !== undefined) {
+    checks.push(
+      check(
+        'final-content',
+        matchesContent(finalContent, expected.finalContent),
+        `final content ${JSON.stringify(finalContent)}`
+      )
+    )
+  }
+
+  if (expected.maxIterations !== undefined) {
+    checks.push(
+      check(
+        'max-iterations',
+        iterations <= expected.maxIterations,
+        `iterations ${iterations} (max ${expected.maxIterations})`
+      )
+    )
+  }
+
+  const successful = toolCalls.filter((call) => call.success).length
+  if (expected.successfulToolCalls !== undefined) {
+    const successMatches =
+      mode === 'live'
+        ? successful >= expected.successfulToolCalls
+        : successful === expected.successfulToolCalls
+    checks.push(
+      check(
+        'successful-tool-calls',
+        successMatches,
+        `expected ${mode === 'live' ? 'at least ' : ''}${expected.successfulToolCalls}, got ${successful}`
+      )
+    )
+  }
+
+  const errored = toolCalls.length - successful
+  /** A live model chooses its own retry count, so an exact error count is scripted-only. */
+  if (expected.erroredToolCalls !== undefined && mode !== 'live') {
+    checks.push(
+      check(
+        'errored-tool-calls',
+        errored === expected.erroredToolCalls,
+        `expected ${expected.erroredToolCalls}, got ${errored}`
+      )
+    )
+  }
+
+  if (expected.completesWithoutError !== false) {
+    checks.push(
+      check(
+        'completes-without-error',
+        error === undefined,
+        error === undefined ? 'completed without error' : String(error)
+      )
+    )
+  }
+
+  return checks
+}
+
+/** Runs and scores one scenario. */
+export async function runScenario(
+  scenario: AgentToolUseScenario,
+  options: RunScenarioOptions = {}
+): Promise<AgentToolUseResult> {
+  const mode = options.mode ?? 'scripted'
+  const model = options.model ?? EVAL_MODEL
+  const providerName = options.providerName ?? EVAL_PROVIDER
+  const expected =
+    mode === 'live' ? { ...scenario.expect, ...scenario.liveExpect } : scenario.expect
+  const toolNames = new Set(scenario.tools.map((tool) => tool.name))
+  const resultQueues = buildResultQueues(scenario, toolNames)
+  const providerTools = toProviderTools(scenario.tools)
+  const invocations: EvalToolInvocation[] = []
+
+  providersMock.MAX_TOOL_ITERATIONS = MAX_TOOL_ITERATIONS
+  providersUtilsMockFns.mockCalculateCost.mockReturnValue({ input: 0, output: 0, total: 0 })
+  toolsMockFns.mockExecuteTool.mockImplementation(
+    async (toolId: string, params: Record<string, unknown>): Promise<ToolResponse> => {
+      const startedAt = Date.now()
+      const response = resultQueues.get(toolId)?.shift() ?? { success: true, output: {} }
+      invocations.push({
+        name: toolId,
+        arguments: params ?? {},
+        success: response.success,
+        ...(response.error ? { error: response.error } : {}),
+        durationMs: Date.now() - startedAt,
+      })
+      return response
+    }
+  )
+
+  const timeSegments: TimeSegment[] = []
+  let completed: StreamingToolLoopComplete | undefined
+  let streamError: unknown
+
+  const startedAt = Date.now()
+  try {
+    const stream = createOpenAICompatStreamingToolLoopStream({
+      providerName,
+      request: {
+        model,
+        apiKey: 'eval-key',
+        messages: [],
+        tools: providerTools,
+      },
+      basePayload: {
+        model,
+        tools: providerTools.map((tool) => adaptOpenAIChatToolSchema(tool)),
+      },
+      messages: [{ role: 'user', content: scenario.userMessage }],
+      createStream: options.completion ?? createScriptedCompletion(scenario),
+      logger,
+      timeSegments,
+      preserveAssistantReasoning: true,
+      onComplete: (result) => {
+        completed = result
+      },
+    })
+    await collectStream(stream as ReadableStream<AgentStreamEvent>)
+  } catch (error) {
+    streamError = error
+  }
+  const latencyMs = Date.now() - startedAt
+
+  const toolCalls = ((completed?.toolCalls as ToolCallList | undefined)?.list ?? []).slice()
+  const finalContent = completed?.content ?? ''
+  const iterations = completed?.iterations ?? 0
+  const checks = scoreExpectations(expected, toolCalls, finalContent, iterations, streamError, mode)
+  const successful = toolCalls.filter((call) => call.success).length
+
+  return {
+    id: scenario.id,
+    name: scenario.name,
+    category: scenario.category,
+    passed: checks.every((entry) => entry.passed),
+    checks,
+    finalContent,
+    toolInvocations: invocations,
+    metrics: {
+      iterations,
+      toolCalls: toolCalls.length,
+      successfulToolCalls: successful,
+      erroredToolCalls: toolCalls.length - successful,
+      latencyMs,
+      modelTimeMs: completed?.modelTime ?? 0,
+      toolsTimeMs: completed?.toolsTime ?? 0,
+      firstResponseTimeMs: completed?.firstResponseTime ?? 0,
+      inputTokens: completed?.tokens.input ?? 0,
+      outputTokens: completed?.tokens.output ?? 0,
+      totalTokens: completed?.tokens.total ?? 0,
+    },
+    ...(streamError ? { error: String(streamError) } : {}),
+  }
+}
diff --git a/apps/sim/evals/agent-tool-use/live.ts b/apps/sim/evals/agent-tool-use/live.ts
new file mode 100644
index 00000000000..94ce8ae6da4
--- /dev/null
+++ b/apps/sim/evals/agent-tool-use/live.ts
@@ -0,0 +1,61 @@
+import OpenAI from 'openai'
+import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/streaming-tool-loop'
+
+/**
+ * Real-model transport for the eval harness. Any OpenAI-compatible provider
+ * (OpenAI, DeepSeek, Groq, OpenRouter, …) works by passing its `baseURL`.
+ *
+ * The scripted suite stays the CI gate; this exists so the same scenarios can
+ * be replayed against a live model on demand.
+ */
+export interface OpenAICompatLiveModelOptions {
+  apiKey: string
+  model: string
+  baseURL?: string
+  /** Per-request timeout in ms. Live model calls routinely exceed the default. */
+  timeoutMs?: number
+}
+
+const DEFAULT_TIMEOUT_MS = 120_000
+
+export function createOpenAICompatLiveCompletion(
+  options: OpenAICompatLiveModelOptions
+): OpenAICompatCreateCompletion {
+  const client = new OpenAI({
+    apiKey: options.apiKey,
+    ...(options.baseURL ? { baseURL: options.baseURL } : {}),
+    timeout: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    maxRetries: 2,
+  })
+
+  return async (params, requestOptions) =>
+    client.chat.completions.create(
+      {
+        ...params,
+        model: options.model,
+        stream: true,
+        // OpenAI-compatible providers require an opt-in to emit usage on streams.
+        stream_options: { include_usage: true },
+      },
+      requestOptions
+    )
+}
+
+/**
+ * DeepSeek's OpenAI-compatible endpoint. Reads `DEEPSEEK_API_KEY` and the
+ * optional `DEEPSEEK_BASE_URL` / `EVAL_MODEL` overrides.
+ */
+export function createDeepSeekLiveCompletion(
+  model = process.env.EVAL_MODEL ?? 'deepseek-chat'
+): OpenAICompatCreateCompletion {
+  const apiKey = process.env.DEEPSEEK_API_KEY
+  if (!apiKey) {
+    throw new Error('DEEPSEEK_API_KEY is required for the live agent eval')
+  }
+  return createOpenAICompatLiveCompletion({
+    apiKey,
+    baseURL: process.env.DEEPSEEK_BASE_URL ?? 'https://api.deepseek.com',
+    model,
+    ...(process.env.EVAL_TIMEOUT_MS ? { timeoutMs: Number(process.env.EVAL_TIMEOUT_MS) } : {}),
+  })
+}
diff --git a/apps/sim/evals/agent-tool-use/report.ts b/apps/sim/evals/agent-tool-use/report.ts
new file mode 100644
index 00000000000..42d8983c6f0
--- /dev/null
+++ b/apps/sim/evals/agent-tool-use/report.ts
@@ -0,0 +1,129 @@
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { dirname } from 'node:path'
+import type { AgentToolUseResult, LiveScenarioSummary } from './types'
+
+/** Machine-readable summary of one eval run. */
+export interface AgentToolUseEvalReport {
+  suite: 'agent-tool-use'
+  generatedAt: string
+  total: number
+  passed: number
+  failed: number
+  results: AgentToolUseResult[]
+}
+
+export function buildEvalReport(results: AgentToolUseResult[]): AgentToolUseEvalReport {
+  const passed = results.filter((result) => result.passed).length
+  return {
+    suite: 'agent-tool-use',
+    generatedAt: new Date().toISOString(),
+    total: results.length,
+    passed,
+    failed: results.length - passed,
+    results,
+  }
+}
+
+function escapeCell(value: string): string {
+  return value.replaceAll('|', '\\|').replaceAll('\n', ' ')
+}
+
+function renderMarkdown(report: AgentToolUseEvalReport): string {
+  const header = [
+    '# Agent tool-use eval report',
+    '',
+    `Generated: ${report.generatedAt}`,
+    '',
+    `**${report.passed}/${report.total} passed**`,
+    '',
+    '| Scenario | Category | Status | Iterations | Tools (ok/error) | Latency | Failed checks |',
+    '| --- | --- | --- | ---: | ---: | ---: | --- |',
+  ]
+
+  const rows = report.results.map((result) => {
+    const failed = result.checks
+      .filter((entry) => !entry.passed)
+      .map((entry) => entry.name)
+      .join(', ')
+    return `| ${escapeCell(result.id)} | ${result.category} | ${result.passed ? '✅ pass' : '❌ fail'} | ${result.metrics.iterations} | ${result.metrics.successfulToolCalls}/${result.metrics.erroredToolCalls} | ${result.metrics.latencyMs}ms | ${failed || '—'} |`
+  })
+
+  return [...header, ...rows, ''].join('\n')
+}
+
+/**
+ * Writes the JSON report to `reportPath` and a sibling Markdown summary. The
+ * caller supplies the path (`EVAL_REPORT_PATH`) so CI can upload it.
+ */
+export function writeEvalReport(results: AgentToolUseResult[], reportPath: string): void {
+  const report = buildEvalReport(results)
+  mkdirSync(dirname(reportPath), { recursive: true })
+  writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`)
+  writeFileSync(reportPath.replace(/\.json$/, '.md'), renderMarkdown(report))
+}
+
+/** Machine-readable summary of one live eval run across trials. */
+export interface LiveEvalReport {
+  suite: 'agent-tool-use-live'
+  generatedAt: string
+  trials: number
+  scenarios: number
+  totalPassRate: number
+  results: LiveScenarioSummary[]
+}
+
+export function buildLiveEvalReport(summaries: LiveScenarioSummary[]): LiveEvalReport {
+  const trials = summaries.reduce((sum, summary) => sum + summary.trials, 0)
+  const passed = summaries.reduce((sum, summary) => sum + summary.passed, 0)
+  return {
+    suite: 'agent-tool-use-live',
+    generatedAt: new Date().toISOString(),
+    trials,
+    scenarios: summaries.length,
+    totalPassRate: trials === 0 ? 0 : passed / trials,
+    results: summaries,
+  }
+}
+
+function average(values: number[]): number {
+  if (values.length === 0) return 0
+  return values.reduce((sum, value) => sum + value, 0) / values.length
+}
+
+function renderLiveMarkdown(report: LiveEvalReport): string {
+  const header = [
+    '# Agent tool-use live eval report',
+    '',
+    `Generated: ${report.generatedAt}`,
+    '',
+    `**${(report.totalPassRate * 100).toFixed(0)}% pass across ${report.trials} trials / ${report.scenarios} scenarios**`,
+    '',
+    '| Scenario | Category | Pass rate | Trials | Avg iterations | Avg latency | Failed checks |',
+    '| --- | --- | ---: | ---: | ---: | ---: | --- |',
+  ]
+
+  const rows = report.results.map((summary) => {
+    const failed = new Set<string>()
+    for (const result of summary.results) {
+      for (const entry of result.checks) {
+        if (!entry.passed) failed.add(entry.name)
+      }
+    }
+    const avgIterations = average(summary.results.map((result) => result.metrics.iterations))
+    const avgLatency = average(summary.results.map((result) => result.metrics.latencyMs))
+    return `| ${escapeCell(summary.id)} | ${summary.category} | ${(summary.passRate * 100).toFixed(0)}% (${summary.passed}/${summary.trials}) | ${summary.trials} | ${avgIterations.toFixed(1)} | ${Math.round(avgLatency)}ms | ${[...failed].join(', ') || '—'} |`
+  })
+
+  return [...header, ...rows, ''].join('\n')
+}
+
+/**
+ * Writes the live JSON report to `reportPath` and a sibling Markdown summary.
+ * Live results are statistical, so the report carries pass rates, not a boolean.
+ */
+export function writeLiveEvalReport(summaries: LiveScenarioSummary[], reportPath: string): void {
+  const report = buildLiveEvalReport(summaries)
+  mkdirSync(dirname(reportPath), { recursive: true })
+  writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`)
+  writeFileSync(reportPath.replace(/\.json$/, '.md'), renderLiveMarkdown(report))
+}
diff --git a/apps/sim/evals/agent-tool-use/scenarios.ts b/apps/sim/evals/agent-tool-use/scenarios.ts
new file mode 100644
index 00000000000..01a66099df1
--- /dev/null
+++ b/apps/sim/evals/agent-tool-use/scenarios.ts
@@ -0,0 +1,360 @@
+import type { AgentToolUseScenario, EvalToolDefinition } from './types'
+
+const searchDocs: EvalToolDefinition = {
+  name: 'search_docs',
+  description: 'Search the product documentation for a query and return matching passages.',
+  parameters: {
+    type: 'object',
+    properties: { query: { type: 'string', description: 'Search query' } },
+    required: ['query'],
+  },
+}
+
+const getWeather: EvalToolDefinition = {
+  name: 'get_weather',
+  description: 'Get the current weather for a city.',
+  parameters: {
+    type: 'object',
+    properties: { city: { type: 'string', description: 'City name' } },
+    required: ['city'],
+  },
+}
+
+const sendEmail: EvalToolDefinition = {
+  name: 'send_email',
+  description: 'Send an email to a recipient.',
+  parameters: {
+    type: 'object',
+    properties: {
+      to: { type: 'string' },
+      subject: { type: 'string' },
+      body: { type: 'string' },
+    },
+    required: ['to', 'subject', 'body'],
+  },
+}
+
+const lookupOrder: EvalToolDefinition = {
+  name: 'lookup_order',
+  description: 'Look up an order by its customer-facing order number.',
+  parameters: {
+    type: 'object',
+    properties: { orderNumber: { type: 'string' } },
+    required: ['orderNumber'],
+  },
+}
+
+const listFiles: EvalToolDefinition = {
+  name: 'list_files',
+  description: 'List files in a directory.',
+  parameters: {
+    type: 'object',
+    properties: { directory: { type: 'string' } },
+    required: ['directory'],
+  },
+}
+
+const readFile: EvalToolDefinition = {
+  name: 'read_file',
+  description: 'Read the contents of a file.',
+  parameters: {
+    type: 'object',
+    properties: { path: { type: 'string' } },
+    required: ['path'],
+  },
+}
+
+const flakyApi: EvalToolDefinition = {
+  name: 'flaky_api',
+  description: 'Fetch a value from an upstream API that intermittently returns 503.',
+  parameters: {
+    type: 'object',
+    properties: { resource: { type: 'string' } },
+    required: ['resource'],
+  },
+}
+
+const getNews: EvalToolDefinition = {
+  name: 'get_news',
+  description: 'Get the top news headlines for a topic.',
+  parameters: {
+    type: 'object',
+    properties: { topic: { type: 'string' } },
+    required: ['topic'],
+  },
+}
+
+/**
+ * The first suite: eight agent tool-use reliability cases. Each script is the
+ * model transcript; the loop, not the model, is what the assertions score.
+ */
+export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
+  {
+    id: 'single-tool-lookup',
+    name: 'calls the one relevant tool and answers from its result',
+    category: 'tool-selection',
+    description:
+      'A single retrieval tool is available. The loop must dispatch it once and surface the answer built from its result.',
+    userMessage: 'What is the API rate limit?',
+    tools: [searchDocs],
+    script: [
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'search_docs',
+            args: { query: 'api rate limit' },
+            result: {
+              success: true,
+              output: { snippet: 'The API rate limit is 100 requests per minute.' },
+            },
+          },
+        ],
+      },
+      { kind: 'answer', content: 'The API rate limit is 100 requests per minute.' },
+    ],
+    expect: {
+      toolCallSequence: ['search_docs'],
+      finalContent: '100 requests per minute',
+      maxIterations: 3,
+      successfulToolCalls: 1,
+      erroredToolCalls: 0,
+    },
+  },
+  {
+    id: 'select-correct-tool',
+    name: 'selects the relevant tool and leaves the irrelevant ones unused',
+    category: 'tool-selection',
+    description:
+      'Four tools are exposed; only the weather tool answers the question. The loop must not invoke the distractors.',
+    userMessage: 'What is the weather in Berlin right now?',
+    tools: [searchDocs, getWeather, sendEmail, lookupOrder],
+    script: [
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'get_weather',
+            args: { city: 'Berlin' },
+            result: { success: true, output: { temperatureC: 17, conditions: 'cloudy' } },
+          },
+        ],
+      },
+      { kind: 'answer', content: 'It is 17°C and cloudy in Berlin.' },
+    ],
+    expect: {
+      requiredTools: ['get_weather'],
+      forbiddenTools: ['search_docs', 'send_email', 'lookup_order'],
+      finalContent: '17°C',
+      maxIterations: 3,
+    },
+  },
+  {
+    id: 'multi-step-planning',
+    name: 'chains two dependent tools in order before answering',
+    category: 'planning',
+    description:
+      'The model must list a directory, then read the file it found. The loop must preserve order and feed the first result into the second turn.',
+    userMessage: 'Summarize the notes file in /docs.',
+    tools: [listFiles, readFile],
+    script: [
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'list_files',
+            args: { directory: '/docs' },
+            result: { success: true, output: { files: ['notes.md'] } },
+          },
+        ],
+      },
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'read_file',
+            args: { path: '/docs/notes.md' },
+            result: { success: true, output: { content: 'Ship the eval harness.' } },
+          },
+        ],
+      },
+      { kind: 'answer', content: 'The notes say: ship the eval harness.' },
+    ],
+    expect: {
+      toolCallSequence: ['list_files', 'read_file'],
+      finalContent: /ship the eval harness/i,
+      maxIterations: 4,
+      successfulToolCalls: 2,
+    },
+  },
+  {
+    id: 'uses-retrieved-value',
+    name: 'answers from the retrieved value rather than inventing one',
+    category: 'retrieval',
+    description:
+      'The order lookup returns a specific identifier. The final answer must carry that retrieved value through the loop.',
+    userMessage: 'What is the status of order 4471?',
+    tools: [lookupOrder],
+    script: [
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'lookup_order',
+            args: { orderNumber: '4471' },
+            result: {
+              success: true,
+              output: { id: 'A-1937', status: 'shipped', carrier: 'DHL' },
+            },
+          },
+        ],
+      },
+      { kind: 'answer', content: 'Order A-1937 has shipped with DHL.' },
+    ],
+    expect: {
+      requiredTools: ['lookup_order'],
+      finalContent: /A-1937.*shipped/,
+      maxIterations: 3,
+    },
+    /** A live model may answer with the user-facing order number and the grounded status. */
+    liveExpect: { finalContent: /shipped/i },
+  },
+  {
+    id: 'parallel-independent-tools',
+    name: 'dispatches independent tools from one turn',
+    category: 'planning',
+    description:
+      'A single model turn requests two independent tools. The loop must execute both and fold both results back into the next turn.',
+    userMessage: 'Give me the weather and the news for Berlin.',
+    tools: [getWeather, getNews],
+    script: [
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'get_weather',
+            args: { city: 'Berlin' },
+            result: { success: true, output: { temperatureC: 12 } },
+          },
+          {
+            name: 'get_news',
+            args: { topic: 'Berlin' },
+            result: { success: true, output: { headline: 'Transit strike ends' } },
+          },
+        ],
+      },
+      { kind: 'answer', content: 'It is 12°C in Berlin. Top story: transit strike ends.' },
+    ],
+    expect: {
+      toolCallSequence: ['get_weather', 'get_news'],
+      finalContent: /12°C[\s\S]*transit strike ends/i,
+      maxIterations: 3,
+      successfulToolCalls: 2,
+    },
+    /** The two tools are independent; a real model may emit them in either order. */
+    liveExpect: { toolCallSequence: undefined, requiredTools: ['get_weather', 'get_news'] },
+  },
+  {
+    id: 'recovers-from-tool-error',
+    name: 'retries a failing tool and completes the turn',
+    category: 'recovery',
+    description:
+      'The first call errors with a 503 and the second succeeds. The error must be fed back to the model, not thrown out of the loop.',
+    userMessage: 'Fetch the current exchange rate from flaky_api.',
+    tools: [flakyApi],
+    script: [
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'flaky_api',
+            args: { resource: 'exchange-rate' },
+            result: { success: false, error: '503 Service Unavailable' },
+          },
+        ],
+      },
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'flaky_api',
+            args: { resource: 'exchange-rate' },
+            result: { success: true, output: { usdToEur: 0.92 } },
+          },
+        ],
+      },
+      { kind: 'answer', content: 'The exchange rate is 0.92 USD to EUR after a retry.' },
+    ],
+    expect: {
+      toolCallSequence: ['flaky_api', 'flaky_api'],
+      finalContent: '0.92',
+      maxIterations: 4,
+      successfulToolCalls: 1,
+      erroredToolCalls: 1,
+    },
+    /** A live model decides its own retry count; only the grounded answer is asserted. */
+    liveExpect: { toolCallSequence: undefined, requiredTools: ['flaky_api'] },
+  },
+  {
+    id: 'recovers-from-unknown-tool',
+    name: 'recovers when the model asks for a tool that does not exist',
+    category: 'recovery',
+    description:
+      'The model hallucinates a tool name first. The loop must return a tool-not-found error to the model instead of failing the run.',
+    userMessage: 'Search the docs for the rate limit.',
+    tools: [searchDocs],
+    scriptedOnly: true,
+    script: [
+      { kind: 'tools', calls: [{ name: 'nonexistent_tool', args: { query: 'rate limit' } }] },
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'search_docs',
+            args: { query: 'rate limit' },
+            result: { success: true, output: { snippet: '100 requests per minute.' } },
+          },
+        ],
+      },
+      { kind: 'answer', content: 'The rate limit is 100 requests per minute.' },
+    ],
+    expect: {
+      toolCallSequence: ['nonexistent_tool', 'search_docs'],
+      finalContent: '100 requests per minute',
+      maxIterations: 4,
+      successfulToolCalls: 1,
+      erroredToolCalls: 1,
+    },
+  },
+  {
+    id: 'recovers-from-malformed-arguments',
+    name: 'does not execute a tool with malformed argument JSON and recovers',
+    category: 'recovery',
+    description:
+      'The first call emits truncated JSON. The loop must skip execution, return the parse error, and let the corrected second call succeed.',
+    userMessage: 'Search the docs for the rate limit.',
+    tools: [searchDocs],
+    scriptedOnly: true,
+    script: [
+      { kind: 'tools', calls: [{ name: 'search_docs', argumentsJson: '{"query":' }] },
+      {
+        kind: 'tools',
+        calls: [
+          {
+            name: 'search_docs',
+            args: { query: 'rate limit' },
+            result: { success: true, output: { snippet: '100 requests per minute.' } },
+          },
+        ],
+      },
+      { kind: 'answer', content: 'The rate limit is 100 requests per minute.' },
+    ],
+    expect: {
+      toolCallSequence: ['search_docs', 'search_docs'],
+      finalContent: '100 requests per minute',
+      maxIterations: 4,
+      successfulToolCalls: 1,
+      erroredToolCalls: 1,
+    },
+  },
+]
diff --git a/apps/sim/evals/agent-tool-use/types.ts b/apps/sim/evals/agent-tool-use/types.ts
new file mode 100644
index 00000000000..036533acbe5
--- /dev/null
+++ b/apps/sim/evals/agent-tool-use/types.ts
@@ -0,0 +1,147 @@
+/**
+ * Declarative contract for the agent tool-use eval suite.
+ *
+ * A scenario is a scripted model transcript against the real OpenAI-compatible
+ * streaming tool loop (`providers/openai-compat/streaming-tool-loop.ts`). The
+ * loop is the harness under test: it must dispatch the tools the model asks
+ * for, feed results back, and recover from tool failures without losing the
+ * turn. The model is an input, so scenarios stay deterministic and run in CI
+ * with no provider key.
+ */
+
+/** The agent behavior a scenario measures. */
+export type EvalCategory = 'tool-selection' | 'planning' | 'retrieval' | 'recovery'
+
+/** A tool exposed to the scripted model for one scenario. */
+export interface EvalToolDefinition {
+  name: string
+  description: string
+  parameters?: {
+    type?: string
+    properties?: Record<string, unknown>
+    required?: string[]
+  }
+}
+
+/** Result the stub tool returns for one scripted call. */
+export interface EvalToolResult {
+  success: boolean
+  output?: Record<string, unknown>
+  error?: string
+}
+
+/** One tool invocation the scripted model asks for. */
+export interface ScriptedToolCall {
+  name: string
+  args?: Record<string, unknown>
+  /**
+   * Raw JSON emitted instead of serializing {@link args}. Used to exercise the
+   * loop's malformed-arguments guard, which must not execute the tool.
+   */
+  argumentsJson?: string
+  /** Stub result for this call; defaults to `{ success: true, output: {} }`. */
+  result?: EvalToolResult
+}
+
+/** One model turn: either a set of tool calls or a final answer. */
+export type ScriptedModelTurn =
+  | { kind: 'tools'; calls: ScriptedToolCall[]; thinking?: string }
+  | { kind: 'answer'; content: string; thinking?: string }
+
+/** Assertions applied to a completed run. */
+export interface AgentToolUseExpectations {
+  /** Exact ordered sequence of tool names the model asked for. */
+  toolCallSequence?: string[]
+  /** Tool names that must appear at least once. */
+  requiredTools?: string[]
+  /** Tool names that must never be called. */
+  forbiddenTools?: string[]
+  /** Substring or pattern the final assistant content must match. */
+  finalContent?: string | RegExp
+  /** Upper bound on tool iterations. */
+  maxIterations?: number
+  /** Exact count of tool calls that returned success. */
+  successfulToolCalls?: number
+  /** Exact count of tool calls that returned an error. */
+  erroredToolCalls?: number
+  /** Whether the loop must settle without throwing. Defaults to `true`. */
+  completesWithoutError?: boolean
+}
+
+/** A single agent behavior case. */
+export interface AgentToolUseScenario {
+  id: string
+  name: string
+  category: EvalCategory
+  description: string
+  userMessage: string
+  tools: EvalToolDefinition[]
+  script: ScriptedModelTurn[]
+  expect: AgentToolUseExpectations
+  /**
+   * Overrides applied only to live runs, merged over {@link expect}. Use when a
+   * scripted assertion (an exact retry count, a parallel call order) is not
+   * meaningful once a real model chooses the calls.
+   */
+  liveExpect?: Partial<AgentToolUseExpectations>
+  /**
+   * True when the case only makes sense with a scripted model (e.g. it requires
+   * the model to emit malformed JSON on demand). Excluded from live runs.
+   */
+  scriptedOnly?: boolean
+}
+
+/** One scored expectation. */
+export interface EvalCheck {
+  name: string
+  passed: boolean
+  detail: string
+}
+
+/** A tool call the loop actually executed through `executeTool`. */
+export interface EvalToolInvocation {
+  name: string
+  arguments: Record<string, unknown>
+  success: boolean
+  error?: string
+  durationMs: number
+}
+
+/** Measured properties of one completed run. */
+export interface AgentToolUseMetrics {
+  iterations: number
+  toolCalls: number
+  successfulToolCalls: number
+  erroredToolCalls: number
+  latencyMs: number
+  modelTimeMs: number
+  toolsTimeMs: number
+  firstResponseTimeMs: number
+  inputTokens: number
+  outputTokens: number
+  totalTokens: number
+}
+
+/** One live scenario across its trials. */
+export interface LiveScenarioSummary {
+  id: string
+  name: string
+  category: EvalCategory
+  trials: number
+  passed: number
+  passRate: number
+  results: AgentToolUseResult[]
+}
+
+/** The scored outcome of one scenario. */
+export interface AgentToolUseResult {
+  id: string
+  name: string
+  category: EvalCategory
+  passed: boolean
+  checks: EvalCheck[]
+  finalContent: string
+  toolInvocations: EvalToolInvocation[]
+  metrics: AgentToolUseMetrics
+  error?: string
+}
diff --git a/apps/sim/package.json b/apps/sim/package.json
index edb48329506..1aed9726ed8 100644
--- a/apps/sim/package.json
+++ b/apps/sim/package.json
@@ -24,6 +24,9 @@
     "test:scim:e2e": "bun run scripts/test-scim-e2e.ts",
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
+    "test:evals": "EVAL_REPORT_PATH=test-results/evals/agent-tool-use.json vitest run evals/agent-tool-use",
+    "test:evals:live": "EVAL_LIVE=1 vitest run --mode live evals/agent-tool-use",
+    "test:evals:context": "EVAL_CONTEXT_REPORT_PATH=test-results/evals/agent-context.json vitest run evals/agent-context",
     "email:dev": "email dev --dir components/emails",
     "type-check": "tsc --noEmit",
     "lint": "biome check --write --unsafe .",