From 184fe6daf4f7746993965aaf898085b5ee77dade Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Tue, 14 Jul 2026 16:45:38 -0700 Subject: [PATCH 01/15] feat: add built-in Foundry adapter and cortyx eval cases Move the Foundry agent adapter from a standalone custom adapter file into beval as a built-in protocol (protocol: foundry). Add cortyx eCommerce analytics cases (100 cases across 11 files) for evaluating Foundry agents with deterministic graders. - Add python/src/beval/adapters/foundry.py with lazy azure SDK imports - Register "foundry" protocol in adapters/__init__.py - Add "foundry" optional dependency group in pyproject.toml - Update samples/foundry-agent to use protocol: foundry - Remove standalone beval_foundry_adapter.py - Add run.sh convenience script Co-Authored-By: Claude Opus 4.6 (1M context) --- python/pyproject.toml | 5 +- python/src/beval/adapters/__init__.py | 6 +- .../src/beval/adapters/foundry.py | 43 ++- python/uv.lock | 53 ++- samples/foundry-agent/README.md | 111 +++---- samples/foundry-agent/agent.yaml | 23 +- .../cases/cortyx/cart-analytics.yaml | 144 +++++++++ .../cases/cortyx/cross-domain.yaml | 111 +++++++ .../cases/cortyx/customer-analytics.yaml | 224 +++++++++++++ .../cases/cortyx/data-profiling.yaml | 306 ++++++++++++++++++ .../cases/cortyx/data-quality.yaml | 112 +++++++ .../foundry-agent/cases/cortyx/payments.yaml | 113 +++++++ .../cases/cortyx/product-analytics.yaml | 208 ++++++++++++ .../foundry-agent/cases/cortyx/returns.yaml | 80 +++++ .../cases/cortyx/revenue-sales.yaml | 209 ++++++++++++ .../foundry-agent/cases/cortyx/reviews.yaml | 147 +++++++++ .../foundry-agent/cases/cortyx/shipping.yaml | 111 +++++++ samples/foundry-agent/eval.config.yaml | 21 +- samples/foundry-agent/run.sh | 5 + 19 files changed, 1912 insertions(+), 120 deletions(-) rename samples/foundry-agent/beval_foundry_adapter.py => python/src/beval/adapters/foundry.py (68%) create mode 100644 samples/foundry-agent/cases/cortyx/cart-analytics.yaml create mode 100644 samples/foundry-agent/cases/cortyx/cross-domain.yaml create mode 100644 samples/foundry-agent/cases/cortyx/customer-analytics.yaml create mode 100644 samples/foundry-agent/cases/cortyx/data-profiling.yaml create mode 100644 samples/foundry-agent/cases/cortyx/data-quality.yaml create mode 100644 samples/foundry-agent/cases/cortyx/payments.yaml create mode 100644 samples/foundry-agent/cases/cortyx/product-analytics.yaml create mode 100644 samples/foundry-agent/cases/cortyx/returns.yaml create mode 100644 samples/foundry-agent/cases/cortyx/revenue-sales.yaml create mode 100644 samples/foundry-agent/cases/cortyx/reviews.yaml create mode 100644 samples/foundry-agent/cases/cortyx/shipping.yaml create mode 100755 samples/foundry-agent/run.sh diff --git a/python/pyproject.toml b/python/pyproject.toml index ae588c0..63733bb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -40,11 +40,12 @@ tracing = [ "opentelemetry-api>=1.20.0", "opentelemetry-sdk>=1.20.0", ] +foundry = ["azure-ai-projects>=2.0.0", "azure-identity>=1.15.0"] acp = ["agent-client-protocol>=0.1.0"] a2a = ["a2a-sdk>=0.3.0", "httpx>=0.27.0"] judge = ["openai>=1.0.0", "azure-identity>=1.15.0"] a2a-server = ["a2a-sdk[http-server]>=0.3.0", "uvicorn>=0.30.0", "fastapi>=0.115.0", "openai>=1.0.0", "azure-identity>=1.15.0"] -all = ["beval[validate,tracing,acp,a2a,judge]"] +all = ["beval[validate,tracing,acp,a2a,foundry,judge]"] dev = [ "beval[all]", "pytest>=8.0", @@ -100,7 +101,7 @@ module = ["azure.*"] ignore_missing_imports = true [[tool.mypy.overrides]] -module = ["beval.adapters.acp", "beval.adapters.a2a"] +module = ["beval.adapters.acp", "beval.adapters.a2a", "beval.adapters.foundry"] disallow_untyped_defs = false check_untyped_defs = false warn_return_any = false diff --git a/python/src/beval/adapters/__init__.py b/python/src/beval/adapters/__init__.py index a7408b7..185d673 100644 --- a/python/src/beval/adapters/__init__.py +++ b/python/src/beval/adapters/__init__.py @@ -83,7 +83,7 @@ def _replace(m: re.Match[str]) -> str: _REQUIRED_FIELDS = {"name", "protocol", "connection"} -_KNOWN_PROTOCOLS = {"acp", "a2a", "custom"} +_KNOWN_PROTOCOLS = {"acp", "a2a", "custom", "foundry"} def load_agent( @@ -200,6 +200,10 @@ def create_adapter( from beval.adapters.a2a import A2AAdapter return A2AAdapter(agent_def) + elif protocol == "foundry": + from beval.adapters.foundry import FoundryAdapter + + return FoundryAdapter(agent_def) elif protocol == "custom": from beval.adapters.custom import CustomAdapter diff --git a/samples/foundry-agent/beval_foundry_adapter.py b/python/src/beval/adapters/foundry.py similarity index 68% rename from samples/foundry-agent/beval_foundry_adapter.py rename to python/src/beval/adapters/foundry.py index 42f24d8..d24b90e 100644 --- a/samples/foundry-agent/beval_foundry_adapter.py +++ b/python/src/beval/adapters/foundry.py @@ -1,34 +1,56 @@ -"""beval custom adapter for Microsoft Foundry Agent Service.""" +"""Foundry Agent Service adapter. See SPEC §13.4. + +Uses the ``azure-ai-projects`` SDK with Entra ID authentication. +Requires ``azure-ai-projects`` and ``azure-identity`` packages. +""" from __future__ import annotations import time from typing import Any -from azure.ai.projects import AIProjectClient -from azure.core.exceptions import ClientAuthenticationError, HttpResponseError -from azure.identity import DefaultAzureCredential - from beval.adapters import AdapterInput, AdapterInterface from beval.types import Subject class FoundryAdapter(AdapterInterface): - """Connects beval to a Microsoft Foundry prompt agent.""" + """Adapter for Microsoft Foundry prompt agents. + + Uses ``AIProjectClient`` from the ``azure-ai-projects`` SDK to invoke + agents via the OpenAI-compatible Responses API. - def __init__(self, config: dict[str, Any]) -> None: + Configuration (``connection`` keys): + - ``endpoint``: Foundry project endpoint (required) + - ``agent_name``: Name of the Foundry agent to invoke + - ``model``: Fallback model when agent_name is not set (default: gpt-4o) + """ + + def __init__(self, agent_def: dict[str, Any]) -> None: + try: + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + except ImportError as exc: + raise ImportError( + "Foundry adapter requires azure-ai-projects and azure-identity. " + "Install with: pip install beval[foundry]" + ) from exc + + connection = agent_def.get("connection", {}) + config = connection.get("config", connection) endpoint = config.get("endpoint") if not endpoint: import sys print( - "Error: set FOUNDRY_PROJECT_ENDPOINT (adapter config 'endpoint' is required).", + "Error: set FOUNDRY_PROJECT_ENDPOINT " + "(adapter config 'endpoint' is required).", file=sys.stderr, ) raise SystemExit(2) self._agent_name = config.get("agent_name") self._model = config.get("model", "gpt-4o") + self._timeout = agent_def.get("timeout", 60) self._credential = DefaultAzureCredential() self._client = AIProjectClient( endpoint=endpoint, @@ -38,6 +60,8 @@ def __init__(self, config: dict[str, Any]) -> None: self._conversation_id: str | None = None def invoke(self, adapter_input: AdapterInput) -> Subject: + from azure.core.exceptions import ClientAuthenticationError + query = adapter_input.query if isinstance(query, list): query = " ".join( @@ -72,9 +96,6 @@ def invoke(self, adapter_input: AdapterInput) -> Subject: file=sys.stderr, ) raise SystemExit(3) from exc - raise RuntimeError( - f"Foundry API error (status {exc.status_code}): {exc.message}" - ) from exc except TimeoutError as exc: raise RuntimeError(f"Foundry API timed out: {exc}") from exc diff --git a/python/uv.lock b/python/uv.lock index e29113f..bf4cacb 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -82,6 +82,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, ] +[[package]] +name = "azure-ai-projects" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "azure-identity" }, + { name = "azure-storage-blob" }, + { name = "isodate" }, + { name = "openai" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/29/ab1a80ce483fcc36ec2ac0a3db8e043aa954700b34b77db43ed5f4f9c488/azure_ai_projects-2.3.0.tar.gz", hash = "sha256:6e3006b7b8aa51c6ff9db61ef4aac3717f8a712cd1a183d5a1d34e2eb33450bd", size = 27563233, upload-time = "2026-07-01T21:09:00.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/bd/7e64f2d6e43d19fc8cf464769804f947d997833eb4dce1c86c2ce76146d2/azure_ai_projects-2.3.0-py3-none-any.whl", hash = "sha256:1da20aeac9663740a97644efedbb9f59eb2a332d3e01bfde7aa03027936edf44", size = 349581, upload-time = "2026-07-01T21:09:04.32Z" }, +] + [[package]] name = "azure-core" version = "1.38.2" @@ -111,6 +128,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/77/f658c76f9e9a52c784bd836aaca6fd5b9aae176f1f53273e758a2bcda695/azure_identity-1.25.2-py3-none-any.whl", hash = "sha256:1b40060553d01a72ba0d708b9a46d0f61f56312e215d8896d836653ffdc6753d", size = 191423, upload-time = "2026-02-11T01:55:44.245Z" }, ] +[[package]] +name = "azure-storage-blob" +version = "12.30.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "isodate" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/48/84a820d898267f662b5c06f7cd76fdb8a9e272b44aa9376cef3ec0f6a294/azure_storage_blob-12.30.0.tar.gz", hash = "sha256:2cd74d4d5731e5eb6b8d5c5056ee115a5e88f8fdf22517b739836fda685018be", size = 618229, upload-time = "2026-06-08T11:45:35.575Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/0b/e106f0fd7fa785867d9ffcc47dc9e6237c0e58f51058473b777487a98edc/azure_storage_blob-12.30.0-py3-none-any.whl", hash = "sha256:d415ac50b67a8da6b3ae7e9f1014b1b55cd7aafa0b8d4ca9b380568dc7360423", size = 435610, upload-time = "2026-06-08T11:45:37.213Z" }, +] + [[package]] name = "beval" version = "0.1.0" @@ -137,6 +169,7 @@ acp = [ all = [ { name = "a2a-sdk" }, { name = "agent-client-protocol" }, + { name = "azure-ai-projects" }, { name = "azure-identity" }, { name = "httpx" }, { name = "jsonschema" }, @@ -147,6 +180,7 @@ all = [ dev = [ { name = "a2a-sdk" }, { name = "agent-client-protocol" }, + { name = "azure-ai-projects" }, { name = "azure-identity" }, { name = "httpx" }, { name = "jsonschema" }, @@ -159,6 +193,10 @@ dev = [ { name = "types-jsonschema" }, { name = "types-pyyaml" }, ] +foundry = [ + { name = "azure-ai-projects" }, + { name = "azure-identity" }, +] judge = [ { name = "azure-identity" }, { name = "openai" }, @@ -176,10 +214,12 @@ requires-dist = [ { name = "a2a-sdk", marker = "extra == 'a2a'", specifier = ">=0.3.0" }, { name = "a2a-sdk", extras = ["http-server"], marker = "extra == 'a2a-server'", specifier = ">=0.3.0" }, { name = "agent-client-protocol", marker = "extra == 'acp'", specifier = ">=0.1.0" }, + { name = "azure-ai-projects", marker = "extra == 'foundry'", specifier = ">=2.0.0" }, { name = "azure-identity", marker = "extra == 'a2a-server'", specifier = ">=1.15.0" }, + { name = "azure-identity", marker = "extra == 'foundry'", specifier = ">=1.15.0" }, { name = "azure-identity", marker = "extra == 'judge'", specifier = ">=1.15.0" }, { name = "beval", extras = ["all"], marker = "extra == 'dev'" }, - { name = "beval", extras = ["validate", "tracing", "acp", "a2a", "judge"], marker = "extra == 'all'" }, + { name = "beval", extras = ["validate", "tracing", "acp", "a2a", "foundry", "judge"], marker = "extra == 'all'" }, { name = "fastapi", marker = "extra == 'a2a-server'", specifier = ">=0.115.0" }, { name = "httpx", marker = "extra == 'a2a'", specifier = ">=0.27.0" }, { name = "jsonschema", marker = "extra == 'validate'", specifier = ">=4.20.0" }, @@ -195,7 +235,7 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0" }, { name = "uvicorn", marker = "extra == 'a2a-server'", specifier = ">=0.30.0" }, ] -provides-extras = ["validate", "tracing", "acp", "a2a", "judge", "a2a-server", "all", "dev"] +provides-extras = ["validate", "tracing", "foundry", "acp", "a2a", "judge", "a2a-server", "all", "dev"] [[package]] name = "certifi" @@ -613,6 +653,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "isodate" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, +] + [[package]] name = "jiter" version = "0.13.0" diff --git a/samples/foundry-agent/README.md b/samples/foundry-agent/README.md index b411c35..ca4b9d8 100644 --- a/samples/foundry-agent/README.md +++ b/samples/foundry-agent/README.md @@ -1,24 +1,25 @@ --- title: Foundry Agent Sample Evaluation -description: Evaluates a Microsoft Foundry prompt agent using beval's custom adapter protocol with Entra ID authentication. -ms.date: 2026-06-19 +description: Evaluates a Microsoft Foundry prompt agent using beval's built-in Foundry adapter with Entra ID authentication. +ms.date: 2026-07-14 ms.topic: tutorial --- ## Foundry Agent Sample Evaluation -This sample evaluates a Microsoft Foundry prompt agent using beval's custom -adapter protocol. The adapter connects directly to the Foundry Responses API +This sample evaluates a Microsoft Foundry prompt agent using beval's built-in +`foundry` adapter protocol. The adapter connects to the Foundry Responses API via the `azure-ai-projects` SDK with Entra ID authentication. ## Files ```text -├── agent.yaml # Agent definition (custom adapter) -├── beval_foundry_adapter.py # Custom adapter implementation -├── eval.config.yaml # Evaluation configuration (judge, agents) +├── agent.yaml # Agent definition +├── eval.config.yaml # Evaluation configuration +├── run.sh # Run script with defaults ├── cases/ -│ └── foundry-eval.yaml # 2 evaluation cases +│ ├── foundry-eval.yaml # 2 basic evaluation cases +│ └── cortyx/ # eCommerce analytics cases (100 cases) └── README.md ``` @@ -27,7 +28,7 @@ via the `azure-ai-projects` SDK with Entra ID authentication. * Python 3.10+ * Azure CLI (`az login`) for Entra ID authentication * A Microsoft Foundry project with a deployed prompt agent (or model deployment) -* **Foundry User** role assigned at the project level +* **Foundry Project Manager** role assigned at the project scope * (Optional) [Ollama](https://ollama.com) for local LLM judging ## Environment Variables @@ -35,10 +36,8 @@ via the `azure-ai-projects` SDK with Entra ID authentication. | Variable | Required | Description | |----------------------------|----------|------------------------------------------------------| | `FOUNDRY_PROJECT_ENDPOINT` | Yes | Foundry project endpoint URL | -| `FOUNDRY_AGENT_NAME` | No | Agent name (omit for direct model calls) | -| `FOUNDRY_MODEL_NAME` | No | Model name (default: `gpt-4o`) | -| `JUDGE_MODEL_NAME` | No | Ollama judge model (default: `gemma4:e2b`) | -| `OPENAI_API_KEY` | No | Required for LLM judge; set to `ollama` when local | +| `FOUNDRY_AGENT_NAME` | No | Agent name (default: `ecommerce-agent`) | +| `FOUNDRY_MODEL_NAME` | No | Fallback model when agent name is not set (default: `gpt-4o`) | ## Setup @@ -48,75 +47,55 @@ via the `azure-ai-projects` SDK with Entra ID authentication. az login ``` -1. Install the required Python packages: +1. Install beval with the Foundry extra: ```bash - pip install azure-ai-projects azure-identity + pip install "beval[foundry] @ git+https://github.com/microsoft/beval.git#subdirectory=python" ``` -1. Set the required environment variables. `FOUNDRY_PROJECT_ENDPOINT` is - required; the others have defaults or are optional. - - **Bash / Linux / macOS:** +1. Set the required environment variables: ```bash export FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" export FOUNDRY_AGENT_NAME="MyAgent" ``` - **PowerShell / Windows:** - - ```powershell - $env:FOUNDRY_PROJECT_ENDPOINT = "https://.services.ai.azure.com/api/projects/" - $env:FOUNDRY_AGENT_NAME = "MyAgent" - ``` - ## Run -The custom adapter module (`beval_foundry_adapter.py`) must be importable when -beval starts. Run from this directory with `PYTHONPATH=.` so Python can find -the module. - -**Dev mode** runs only deterministic graders (fast, no LLM judge required): +**Dev mode** runs only deterministic graders (completion time, response length — no LLM judge required): ```bash cd samples/foundry-agent -PYTHONPATH=. beval run --cases cases --agent agent.yaml -m dev --verbose +beval -c eval.config.yaml run --cases cases --verbose ``` -**Validation mode** includes AI-judged criteria and requires a judge -configured in `eval.config.yaml` (see [LLM Judge](#llm-judge) below). Pass -`-c` before the `run` subcommand: +Or use the run script with defaults: ```bash -PYTHONPATH=. OPENAI_API_KEY=ollama beval -c eval.config.yaml run \ - --cases cases --agent agent.yaml -m validation --verbose +./run.sh ``` -On PowerShell, set environment variables first: +**Validation mode** includes AI-judged criteria and requires a judge +configured in `eval.config.yaml` (see [LLM Judge](#llm-judge) below): -```powershell -cd samples\foundry-agent -$env:PYTHONPATH = "." -$env:OPENAI_API_KEY = "ollama" -beval -c eval.config.yaml run --cases cases --agent agent.yaml -m validation --verbose +```bash +beval -c eval.config.yaml run --cases cases -m validation --verbose ``` -**From the `python/` directory:** +**From the `python/` directory** (development): ```bash cd python -PYTHONPATH=../samples/foundry-agent uv run beval run \ +uv run beval -c ../samples/foundry-agent/eval.config.yaml run \ --cases ../samples/foundry-agent/cases \ - --agent ../samples/foundry-agent/agent.yaml \ -m dev --verbose ``` ## LLM Judge -The `eval.config.yaml` file configures the LLM judge used for AI-judged -graders in validation mode. Any OpenAI-compatible endpoint works, including -Ollama for fully local evaluation. +To enable AI-judged graders in validation mode, add a `judge` section to +`eval.config.yaml`. Any OpenAI-compatible endpoint works, including Ollama +for fully local evaluation. ### Ollama (local) @@ -126,7 +105,7 @@ Ollama for fully local evaluation. ollama pull gemma4:e2b ``` -1. The judge is already configured in `eval.config.yaml`: +1. Add the judge config to `eval.config.yaml`: ```yaml eval: @@ -136,8 +115,6 @@ Ollama for fully local evaluation. base_url: http://localhost:11434/v1 ``` - Override the model at runtime by setting `JUDGE_MODEL_NAME`. - 1. Set `OPENAI_API_KEY` to any non-empty value (Ollama ignores it, but the `openai` Python package requires it): @@ -145,10 +122,6 @@ Ollama for fully local evaluation. export OPENAI_API_KEY=ollama ``` -> [!NOTE] -> Judge quality depends on model reasoning capability. Larger models like -> `gemma4` or `llama3:70b` produce more reliable scores. - ### OpenAI ```yaml @@ -160,31 +133,21 @@ eval: Set `OPENAI_API_KEY` to your actual API key. -## Inline Agent Definition (Alternative) +## Agent Definition -You can define the agent inline in `eval.config.yaml` instead of using a -separate `agent.yaml`: +The agent is defined inline in `eval.config.yaml` using the built-in `foundry` +protocol: ```yaml eval: - mode: validation agents: default: foundry-agent definitions: - name: foundry-agent - protocol: custom + protocol: foundry connection: - module: beval_foundry_adapter - class: FoundryAdapter - config: - endpoint: ${FOUNDRY_PROJECT_ENDPOINT} - agent_name: ${FOUNDRY_AGENT_NAME:-} - model: ${FOUNDRY_MODEL_NAME:-gpt-4o} - timeout: 60 -``` - -Then run without `--agent`: - -```bash -PYTHONPATH=. beval run --cases cases + endpoint: ${FOUNDRY_PROJECT_ENDPOINT} + agent_name: ${FOUNDRY_AGENT_NAME:-ecommerce-agent} + model: ${FOUNDRY_MODEL_NAME:-gpt-4o} + timeout: 120 ``` diff --git a/samples/foundry-agent/agent.yaml b/samples/foundry-agent/agent.yaml index e4a21d8..f5b6c75 100644 --- a/samples/foundry-agent/agent.yaml +++ b/samples/foundry-agent/agent.yaml @@ -1,26 +1,15 @@ name: foundry-agent description: > - Microsoft Foundry prompt agent evaluated via beval custom adapter. + Microsoft Foundry prompt agent evaluated via beval's built-in Foundry adapter. Uses the azure-ai-projects SDK with Entra ID authentication. -protocol: custom +protocol: foundry connection: - module: beval_foundry_adapter - class: FoundryAdapter - config: - endpoint: ${FOUNDRY_PROJECT_ENDPOINT} - agent_name: ${FOUNDRY_AGENT_NAME:-} - model: ${FOUNDRY_MODEL_NAME:-gpt-4o} + endpoint: ${FOUNDRY_PROJECT_ENDPOINT} + agent_name: ${FOUNDRY_AGENT_NAME:-} + model: ${FOUNDRY_MODEL_NAME:-gpt-4o} -timeout: 60 - -# Note: retry config is defined for schema compliance but has no effect -# for custom adapters — the beval CustomAdapter wrapper does not implement -# retry logic. -retry: - max_attempts: 2 - backoff: 2.0 +timeout: 120 metadata: platform: azure-foundry - sdk_version: "2.2.0" diff --git a/samples/foundry-agent/cases/cortyx/cart-analytics.yaml b/samples/foundry-agent/cases/cortyx/cart-analytics.yaml new file mode 100644 index 0000000..2414a34 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/cart-analytics.yaml @@ -0,0 +1,144 @@ +# eCommerce Cart Analytics — evaluation cases +# +# These cases evaluate the agent's ability to analyze cart behavior, +# abandonment patterns, and conversion metrics. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses accuracy +# of cart metrics and correct conversion calculations + +background: + category: cart-analytics + given: + domain: ecommerce-analytics + +cases: + # ── Cart Value & Items ────────────────────────────────────────── + + - id: avg_cart_value_by_status + name: Average cart value by cart status + tags: [carts, value] + given: + query: "What is the average cart value by cart status?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the average cart value for each cart status + + - id: avg_items_per_cart + name: Average items per cart + tags: [carts, items] + given: + query: "What is the average number of items per cart based on the quantity column in cart_items?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + the average item quantity per cart + + - id: avg_cart_value_by_device + name: Average cart value by device type + tags: [carts, device, value] + given: + query: "How does average cart value vary by device type?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + average cart value for each device type + + - id: avg_cart_value_by_traffic_source + name: Average cart value by traffic source + tags: [carts, traffic, value] + given: + query: "How does average cart value vary by traffic source?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + average cart value for each traffic source + + # ── Conversion & Abandonment ──────────────────────────────────── + + - id: cart_abandonment_conversion_rate + name: Cart abandonment and conversion rate + tags: [carts, abandonment, conversion, core] + given: + query: "What is the cart abandonment rate and conversion rate when filtering the carts data status to be abandoned or converted?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + the percentage of carts that are abandoned and the + percentage that converted + + - id: conversion_rate_by_traffic_source + name: Cart-to-order conversion rate by traffic source + tags: [carts, conversion, traffic] + given: + query: "What is the cart-to-order conversion rate by traffic source?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the conversion rate from cart to order for each traffic source + + # ── Trends & Comparisons ──────────────────────────────────────── + + - id: category_carts_vs_orders + name: Category preferences in carts vs actual orders + tags: [carts, orders, categories] + given: + query: "How do category preferences differ between carts and actual orders?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a comparison of product category distribution in carts vs + in completed orders + + - id: carts_created_monthly_trend + name: Carts created per month trend + tags: [carts, trends] + given: + query: "What is the trend of carts created per month?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + monthly counts of new carts created showing the trend + over time diff --git a/samples/foundry-agent/cases/cortyx/cross-domain.yaml b/samples/foundry-agent/cases/cortyx/cross-domain.yaml new file mode 100644 index 0000000..313f3c4 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/cross-domain.yaml @@ -0,0 +1,111 @@ +# eCommerce Cross-Domain — evaluation cases +# +# These cases evaluate the agent's ability to join and correlate data +# across multiple tables to answer complex analytical questions. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses accuracy +# of cross-table joins and multi-step reasoning + +background: + category: cross-domain + given: + domain: ecommerce-analytics + +cases: + # ── Returns × Other Domains ───────────────────────────────────── + + - id: repeat_returners + name: Repeat returners with 3+ returns + tags: [returns, customers, behavior] + given: + query: "How many users are repeat returners with 3 or more returns?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + the count of users who have 3 or more return records + + - id: return_rate_by_region + name: Return rate by shipping region + tags: [returns, shipping, region] + given: + query: "How does the return rate vary by shipping region?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the return rate for each shipping region + + - id: returns_vs_review_sentiment_trends + name: Monthly return trends vs review sentiment trends + tags: [returns, reviews, trends] + given: + query: "How do monthly return trends compare to review sentiment trends?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + a month-by-month comparison of return volume and review + sentiment metrics + + - id: returns_by_status_type + name: Returns by status type + tags: [returns, status] + given: + query: "How many returns are there by status type in returns?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a count of returns grouped by each status type + + # ── Shipping × Spending ───────────────────────────────────────── + + - id: carrier_shipping_cost_comparison + name: Carriers compared on average shipping cost + tags: [shipping, carriers, cost] + given: + query: "How do carriers compare on average shipping cost when looking at the shipping cost column?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + average shipping cost for each carrier for comparison + + - id: top_5_net_spend_after_returns + name: Top 5 customers by net spend after returns + tags: [customers, returns, revenue, top-n, core] + given: + query: "Which top 5 customers have the highest net spend after returns?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a ranked list of 5 customers with the highest total spend + minus refund amounts diff --git a/samples/foundry-agent/cases/cortyx/customer-analytics.yaml b/samples/foundry-agent/cases/cortyx/customer-analytics.yaml new file mode 100644 index 0000000..c958593 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/customer-analytics.yaml @@ -0,0 +1,224 @@ +# eCommerce Customer Analytics — evaluation cases +# +# These cases evaluate the agent's ability to analyze customer +# segments, lifetime value, cohorts, and behavioral patterns. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses accuracy +# of customer metrics and correct cohort logic + +background: + category: customer-analytics + given: + domain: ecommerce-analytics + +cases: + # ── Customer Distribution ─────────────────────────────────────── + + - id: customers_by_country_status + name: Customers by country and account status + tags: [customers, geography, status] + given: + query: "How are customers distributed by country and account status?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a breakdown of customer counts by country and account status + + - id: customers_by_acquisition_channel + name: Customers by acquisition channel + tags: [customers, acquisition] + given: + query: "How are customers distributed by acquisition channel?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a count of customers for each acquisition channel + + - id: customers_by_order_frequency + name: Customers by order frequency + tags: [customers, frequency] + given: + query: "How are customers distributed by order frequency?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a distribution of customers grouped by how many orders + they have placed + + - id: top_10_cities_by_customers + name: Top 10 cities by customer count + tags: [customers, geography, top-n] + given: + query: "What are the top 10 cities by customer count?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a ranked list of the 10 cities with the most customers + + # ── Lifetime Value & Spend ────────────────────────────────────── + + - id: top_20_customers_ltv + name: Top 20 customers by lifetime value + tags: [customers, ltv, top-n, core] + given: + query: "Who are the top 20 customers by lifetime value?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + a ranked list of 20 customers with the highest total spending + + - id: avg_orders_per_customer + name: Average orders per customer + tags: [customers, orders, average] + given: + query: "What is the average number of orders per customer for customers with an order?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + the average order count per customer, excluding customers + with zero orders + + # ── Revenue Segmentation ──────────────────────────────────────── + + - id: revenue_first_vs_repeat + name: Revenue split between first-time and repeat orders + tags: [customers, revenue, retention] + given: + query: "How does revenue split between first-time and repeat orders?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals or percentages for first-time orders vs + repeat orders + + - id: revenue_by_loyalty_tier + name: Revenue by loyalty tier + tags: [customers, loyalty, revenue] + given: + query: "How does revenue break down by loyalty tier?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals for each loyalty tier + + - id: aov_by_loyalty_tier + name: Average order value per loyalty tier + tags: [customers, loyalty, aov] + given: + query: "What is the average order value per loyalty tier?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the average order value for each loyalty tier + + - id: revenue_by_gender + name: Revenue by customer gender + tags: [customers, demographics, revenue] + given: + query: "How does revenue break down by customer gender?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals grouped by customer gender + + # ── Cohorts & Retention ───────────────────────────────────────── + + - id: cohort_order_volume + name: Monthly registration cohorts vs order volume + tags: [customers, cohorts, orders] + given: + query: "How do monthly registration cohorts compare in order volume?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + order counts grouped by customer registration month cohort + + - id: cohort_retention + name: Customer cohort retention over time + tags: [customers, cohorts, retention] + given: + query: "What does customer cohort retention look like over time?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + a cohort retention analysis showing what percentage of each + monthly registration cohort returns to purchase in + subsequent months + + - id: optin_rate_by_channel + name: Marketing opt-in rate per acquisition channel + tags: [customers, marketing, acquisition] + given: + query: "What is the marketing opt-in rate per acquisition channel?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the percentage of customers opted into marketing for each + acquisition channel diff --git a/samples/foundry-agent/cases/cortyx/data-profiling.yaml b/samples/foundry-agent/cases/cortyx/data-profiling.yaml new file mode 100644 index 0000000..9f4c1c7 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/data-profiling.yaml @@ -0,0 +1,306 @@ +# eCommerce Data Profiling — evaluation cases +# +# These cases evaluate the agent's ability to profile and describe +# the structure, shape, and distributions of the ecommerce dataset. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses completeness +# and accuracy of profiling results + +background: + category: data-profiling + given: + domain: ecommerce-analytics + +cases: + # ── Schema & Shape ────────────────────────────────────────────── + + - id: row_counts + name: Row counts for each table + tags: [schema, counts] + given: + query: "How many rows are in each table?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a complete listing of row counts for each table in the dataset + + - id: distinct_primary_keys + name: Distinct values per primary key column + tags: [schema, keys] + given: + query: "How many distinct values does each primary key column have?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - response should contain: "distinct" + - the answer should be: > + a listing of distinct value counts for each primary key column + + - id: date_ranges + name: Date range for each date column + tags: [schema, dates] + given: + query: "What is the date range for each date column?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a listing of min and max dates for each date column across datasets + + - id: null_values + name: NULL value counts per column + tags: [quality, nulls] + given: + query: "Across each dataset which columns have NULL values and how many?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a breakdown of null counts per column across each dataset + + # ── Distributions ─────────────────────────────────────────────── + + - id: order_status_distribution + name: Distribution of order statuses + tags: [distribution, orders] + given: + query: "What is the distribution of order statuses?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a count or percentage breakdown of each order status value + + - id: user_status_distribution + name: Distribution of user statuses + tags: [distribution, users] + given: + query: "What is the distribution of user statuses?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a count or percentage breakdown of each user status value + + - id: product_category_brand_distribution + name: Products distributed across categories and brands + tags: [distribution, products] + given: + query: "How are products distributed across categories and brands?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a breakdown of product counts by category and brand + + - id: payment_method_distribution + name: Payment method frequency + tags: [distribution, payments] + given: + query: "What payment methods are used and how often?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a frequency count of each payment method + + - id: shipping_carrier_distribution + name: Shipping carrier frequency + tags: [distribution, shipping] + given: + query: "Which shipping carriers are used and how often?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a frequency count of each shipping carrier + + - id: return_reasons + name: Most common reasons for returns + tags: [distribution, returns] + given: + query: "What are the most common reasons for returns?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a ranked list of return reasons by frequency + + - id: cart_status_distribution + name: Distribution of cart statuses + tags: [distribution, carts] + given: + query: "What is the distribution of cart statuses?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a count or percentage breakdown of each cart status + + - id: review_sentiment_distribution + name: Reviews distributed across sentiment buckets + tags: [distribution, reviews] + given: + query: "How are reviews distributed across sentiment buckets?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - response should contain: "sentiment" + - the answer should be: > + a count breakdown of reviews by sentiment category + + - id: review_rating_distribution + name: Distribution of review ratings (1-5 stars) + tags: [distribution, reviews] + given: + query: "What is the distribution of review ratings (1-5 stars)?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a count of reviews for each star rating from 1 to 5 + + - id: shipping_status_distribution + name: Distribution of shipping statuses + tags: [distribution, shipping] + given: + query: "What is the distribution of shipping statuses?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a count or percentage breakdown of each shipping status + + - id: payment_risk_flags + name: Payments with risk flags + tags: [distribution, payments, risk] + given: + query: "How many payments have risk flags?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + a count of payments flagged as high risk vs not flagged + + - id: loyalty_tier_distribution + name: Users distributed across loyalty tiers + tags: [distribution, users, loyalty] + given: + query: "How are users distributed across loyalty tiers?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a count of users in each loyalty tier + + - id: product_active_inactive + name: Active vs inactive products + tags: [distribution, products] + given: + query: "How many products are active vs inactive?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + a count of active products and inactive products + + - id: coupon_usage_count + name: Orders with coupon vs without + tags: [distribution, orders, coupons] + given: + query: "How many orders used a coupon vs did not?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + a count of orders with coupon applied vs orders without + + - id: first_vs_repeat_orders + name: First-time vs repeat purchases + tags: [distribution, orders] + given: + query: "How many orders are first-time vs repeat purchases?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + a count of first-time purchase orders vs repeat purchase orders diff --git a/samples/foundry-agent/cases/cortyx/data-quality.yaml b/samples/foundry-agent/cases/cortyx/data-quality.yaml new file mode 100644 index 0000000..ec6ecef --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/data-quality.yaml @@ -0,0 +1,112 @@ +# eCommerce Data Quality — evaluation cases +# +# These cases evaluate the agent's ability to detect data quality +# issues, anomalies, and cross-table inconsistencies. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses whether +# anomalies are correctly identified and explained + +background: + category: data-quality + given: + domain: ecommerce-analytics + +cases: + # ── Cross-table inconsistencies ──────────────────────────────── + + - id: payment_order_mismatch + name: Orders with mismatched payment and order totals + tags: [anomaly, payments, orders] + given: + query: "Which orders have a mismatch between payment amounts and order totals?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a list of order IDs where the payment amount does not match + the order total + + - id: delivered_with_return + name: Orders marked delivered but also returned + tags: [anomaly, orders, returns] + given: + query: "Which orders are marked as delivered but also have a return?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a list of order IDs that have both delivered status and an + associated return record + + - id: multiple_payments + name: Orders with multiple payment records + tags: [anomaly, payments] + given: + query: "Which orders have multiple payment records?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a list of order IDs with more than one payment record + + - id: carts_no_orders + name: Users with carts but no orders + tags: [anomaly, carts, users] + given: + query: "Which users have carts but have never placed an order?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a list of user IDs who have cart records but no order records + + - id: missing_carrier + name: Shipments missing a carrier + tags: [anomaly, shipping] + given: + query: "Which shipments are missing a carrier?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a list of shipment IDs or order IDs where the carrier field + is null or empty + + - id: return_date_anomalies + name: Anomalies in returns requested_at column + tags: [anomaly, returns, dates] + given: + query: "Are there anomalies in the returns requested_at column?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + an analysis of any date anomalies such as future dates, + dates before order creation, or null values in the returns + requested_at column diff --git a/samples/foundry-agent/cases/cortyx/payments.yaml b/samples/foundry-agent/cases/cortyx/payments.yaml new file mode 100644 index 0000000..be2ecf2 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/payments.yaml @@ -0,0 +1,113 @@ +# eCommerce Payments — evaluation cases +# +# These cases evaluate the agent's ability to analyze payment +# processing metrics, failure rates, and risk indicators. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses accuracy +# of payment analysis and correct status interpretation + +background: + category: payments + given: + domain: ecommerce-analytics + +cases: + # ── Payment Status & Methods ──────────────────────────────────── + + - id: payment_success_failure_rate + name: Payment success vs failure rate + tags: [payments, status, core] + given: + query: "What is the payment success vs failure rate (captured vs failed)?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + a count or percentage of captured payments vs failed payments + + - id: revenue_by_payment_method + name: Revenue by payment method + tags: [payments, revenue] + given: + query: "How does revenue break down by payment method?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals grouped by each payment method + + - id: aov_by_payment_method + name: Average order value per payment method + tags: [payments, aov] + given: + query: "What is the average order value per payment method?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the average order value for each payment method + + - id: failures_by_payment_method + name: Payment methods with most failures + tags: [payments, failures] + given: + query: "Which payment methods have the most failures?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a ranked list of payment methods by number of failed + transactions + + # ── Risk & Cross-reference ────────────────────────────────────── + + - id: high_risk_amount_distribution + name: Amount distribution for high risk payments + tags: [payments, risk] + given: + query: "What is the amount distribution for high risk payments. Divide it into 2 categories <100 and >=100" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + a count of high-risk payments split into two buckets: + amounts less than 100 and amounts greater than or equal + to 100 + + - id: payment_vs_order_status + name: Payment status compared with order status + tags: [payments, orders, cross-reference] + given: + query: "How does payment status from payments compare with order status from orders?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a cross-tabulation or comparison showing how payment + statuses align with corresponding order statuses diff --git a/samples/foundry-agent/cases/cortyx/product-analytics.yaml b/samples/foundry-agent/cases/cortyx/product-analytics.yaml new file mode 100644 index 0000000..b976079 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/product-analytics.yaml @@ -0,0 +1,208 @@ +# eCommerce Product Analytics — evaluation cases +# +# These cases evaluate the agent's ability to analyze product +# performance, category breakdowns, and inventory metrics. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses accuracy +# of product-level analytics and correct join logic + +background: + category: product-analytics + given: + domain: ecommerce-analytics + +cases: + # ── Product Revenue Rankings ──────────────────────────────────── + + - id: top_10_products_by_revenue + name: Top 10 products by revenue + tags: [products, revenue, top-n, core] + given: + query: "What are the top 10 products by revenue when merging the order items and products datasets?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + a ranked list of the 10 highest-revenue products with + product names and revenue amounts + + - id: bottom_10_products_by_revenue + name: Bottom 10 products by revenue + tags: [products, revenue, bottom-n] + given: + query: "What are the bottom 10 products by revenue when merging the order items and products datasets?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + a ranked list of the 10 lowest-revenue products with + product names and revenue amounts + + # ── Category & Brand Performance ──────────────────────────────── + + - id: revenue_units_per_category + name: Revenue and units sold per product category + tags: [products, categories, revenue] + given: + query: "What is the revenue and units sold per product category?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + revenue and unit count for each product category + + - id: revenue_units_per_brand + name: Revenue and units sold per brand + tags: [products, brands, revenue] + given: + query: "What is the revenue and units sold per brand?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + revenue and unit count for each brand + + - id: avg_price_per_category + name: Average product price per category + tags: [products, categories, pricing] + given: + query: "What is the average product price per category?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the average price for each product category + + - id: top_5_per_category + name: Top 5 revenue products in each category + tags: [products, categories, top-n] + given: + query: "What are the top 5 revenue-generating products in each category?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + the top 5 products by revenue within each product category + + # ── Inventory & Velocity ──────────────────────────────────────── + + - id: zero_sales_products + name: Products with zero sales + tags: [products, performance] + given: + query: "Which products have zero sales?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a list of products that have never been ordered + + - id: sales_velocity_vs_inventory + name: Sales velocity relative to inventory + tags: [products, inventory, velocity] + given: + query: "What is the sales velocity relative to inventory for each product (total units sold / current inventory_qty)?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a ratio of total units sold divided by current inventory + quantity for each product + + - id: inactive_products_ordered + name: Inactive products still being ordered + tags: [products, anomaly] + given: + query: "Are there inactive products that are still being ordered?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 10000] + - the answer should be: > + a list of products marked as inactive that still appear + in order items + + # ── Revenue by Product Attributes ────────────────────────────── + + - id: revenue_by_margin_band + name: Revenue by margin band + tags: [products, margins, revenue] + given: + query: "How does revenue break down by margin band?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals grouped by margin band categories + + - id: revenue_by_launch_season + name: Revenue by launch season + tags: [products, seasonal, revenue] + given: + query: "How does revenue break down by launch season?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals grouped by product launch season + + - id: price_distribution_bins + name: Product price distribution in 10 bins + tags: [products, pricing, distribution] + given: + query: "What does the product price distribution look like when divided into 10 bins?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a histogram-style breakdown of product prices divided into + 10 equal-width bins with counts per bin diff --git a/samples/foundry-agent/cases/cortyx/returns.yaml b/samples/foundry-agent/cases/cortyx/returns.yaml new file mode 100644 index 0000000..236be4d --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/returns.yaml @@ -0,0 +1,80 @@ +# eCommerce Returns — evaluation cases +# +# These cases evaluate the agent's ability to analyze return rates, +# refund breakdowns, and product-level return patterns. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses accuracy +# of return metrics and correct filtering logic + +background: + category: returns + given: + domain: ecommerce-analytics + +cases: + # ── Return Metrics ────────────────────────────────────────────── + + - id: overall_return_rate + name: Overall return rate + tags: [returns, rate, core] + given: + query: "What is the overall return rate based on the distinct count of returned orders?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + the percentage of orders that have been returned based on + distinct order count + + - id: refund_by_reason + name: Refund amounts by return reason + tags: [returns, refunds, reasons] + given: + query: "How do refund amounts break down by return reason?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + total refund amounts grouped by each return reason + + - id: return_rate_per_category + name: Return rate per product category + tags: [returns, categories, rate] + given: + query: "What is the breakdown of the overall return rate per product category using the returns data when the status is approved, refunded, or requested?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + return rates for each product category, considering only + returns with status approved, refunded, or requested + + - id: top_10_returned_products + name: Top 10 most-returned products + tags: [returns, products, top-n] + given: + query: "What are the top 10 most-returned products?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + a ranked list of the 10 products with the highest number + of returns diff --git a/samples/foundry-agent/cases/cortyx/revenue-sales.yaml b/samples/foundry-agent/cases/cortyx/revenue-sales.yaml new file mode 100644 index 0000000..4c63427 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/revenue-sales.yaml @@ -0,0 +1,209 @@ +# eCommerce Revenue & Sales — evaluation cases +# +# These cases evaluate the agent's ability to compute revenue metrics, +# identify trends, and break down sales by various dimensions. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length, +# keyword presence +# - AI-judged layer (validation mode): LLM judge assesses numeric +# accuracy and completeness of revenue analysis + +background: + category: revenue-sales + given: + domain: ecommerce-analytics + +cases: + # ── Summary Metrics ───────────────────────────────────────────── + + - id: total_revenue_metrics + name: Total revenue, order count, and AOV + tags: [revenue, summary, core] + given: + query: "What is the total revenue, order count, and average order value?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - response should contain: "revenue" + - the answer should be: > + numeric values for total revenue, total order count, and + average order value + + - id: monthly_revenue_trend + name: Monthly revenue trend + tags: [revenue, trends] + given: + query: "What is the monthly revenue trend?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + a month-by-month breakdown of revenue showing the trend + over time + + # ── Revenue Breakdowns ────────────────────────────────────────── + + - id: revenue_by_channel + name: Revenue by sales channel + tags: [revenue, channels] + given: + query: "How does revenue break down by sales channel?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals grouped by each sales channel + + - id: revenue_by_device + name: Revenue by device type + tags: [revenue, device] + given: + query: "How does revenue break down by device type?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals grouped by device type + + - id: revenue_by_region + name: Revenue by shipping region + tags: [revenue, region] + given: + query: "How does revenue break down by shipping region?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals grouped by shipping region + + - id: avg_discount_pct + name: Average discount percentage per order + tags: [revenue, discounts] + given: + query: "What is the average discount percentage per order for orders with a subtotal greater than 0?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + the average discount as a percentage, computed only for + orders with subtotal greater than zero + + - id: coupon_impact_revenue + name: Coupon usage rate and impact on revenue + tags: [revenue, coupons] + given: + query: "What is the coupon usage rate and its impact on revenue?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - response should contain: "coupon" + - the answer should be: > + the percentage of orders using coupons and the revenue + comparison between coupon and non-coupon orders + + - id: revenue_components + name: Revenue split between subtotal, tax, shipping, and discounts + tags: [revenue, breakdown] + given: + query: "How does revenue split between subtotal, tax, shipping fees, and discounts?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a breakdown of total subtotal, total tax, total shipping + fees, and total discounts across all orders + + - id: top_10_orders + name: Top 10 highest-value orders + tags: [revenue, top-n] + given: + query: "What are the top 10 highest-value orders?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + a list of the 10 orders with the highest total value + including order IDs and amounts + + - id: revenue_by_day_of_week + name: Revenue by day of the week + tags: [revenue, trends] + given: + query: "How does revenue vary by day of the week?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals or averages for each day of the week + + - id: mom_order_growth + name: Month-over-month order count growth + tags: [orders, trends, growth] + given: + query: "What is the month-over-month order count growth?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [100, 10000] + - the answer should be: > + monthly order counts with growth rate or percentage change + month over month + + - id: revenue_by_currency + name: Revenue by currency + tags: [revenue, currency] + given: + query: "How does revenue break down by currency?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + revenue totals grouped by currency diff --git a/samples/foundry-agent/cases/cortyx/reviews.yaml b/samples/foundry-agent/cases/cortyx/reviews.yaml new file mode 100644 index 0000000..4f40ca4 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/reviews.yaml @@ -0,0 +1,147 @@ +# eCommerce Reviews — evaluation cases +# +# These cases evaluate the agent's ability to analyze review ratings, +# sentiment patterns, and review-related customer insights. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses accuracy +# of review analysis and correct aggregation logic + +background: + category: reviews + given: + domain: ecommerce-analytics + +cases: + # ── Ratings & Sentiment ───────────────────────────────────────── + + - id: avg_rating_overall_per_category + name: Average review rating overall and per category + tags: [reviews, ratings, categories, core] + given: + query: "What is the average review rating overall and per category?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the overall average review rating and average rating for + each product category + + - id: verified_vs_unverified_reviews + name: Verified vs unverified purchase reviews + tags: [reviews, verified] + given: + query: "How do verified purchase reviews compare to unverified ones?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a comparison of average rating, count, or other metrics + between verified and unverified purchase reviews + + - id: brand_ratings_high_low + name: Brands with highest and lowest average ratings + tags: [reviews, brands, ratings] + given: + query: "Which brands have the highest and lowest average ratings?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + the brands with the highest average ratings and the brands + with the lowest + + - id: monthly_review_volume_rating + name: Monthly trend in review volume and average rating + tags: [reviews, trends] + given: + query: "What is the monthly trend in review volume and average rating?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + monthly review counts and average ratings showing trends + over time + + - id: review_length_by_sentiment + name: Review length by sentiment + tags: [reviews, sentiment, length] + given: + query: "How does review length vary by sentiment?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - response should contain: "sentiment" + - the answer should be: > + average review text length for each sentiment category + + # ── Review × Customer Cross-reference ─────────────────────────── + + - id: low_rating_high_return_products + name: Low-rating products with high return rates + tags: [reviews, returns, products, cross-reference] + given: + query: "Which products are in the bottom quartile for ratings and top quartile for return rates using the rating from reviews and return rate from returns?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 10000] + - the answer should be: > + a list of products that are both in the bottom 25% for + average rating and the top 25% for return rate + + - id: customers_no_reviews + name: Customers with orders but no reviews + tags: [reviews, customers] + given: + query: "How many customers have placed orders but never left a review?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + the count of customers who have at least one order but + zero reviews + + - id: top_spenders_no_reviews + name: Top 5 highest-spending customers with no reviews + tags: [reviews, customers, top-n] + given: + query: "Who are the top 5 highest-spending customers that have never left a review?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + a list of the 5 customers with the highest total spend who + have never left a review diff --git a/samples/foundry-agent/cases/cortyx/shipping.yaml b/samples/foundry-agent/cases/cortyx/shipping.yaml new file mode 100644 index 0000000..ebc4b45 --- /dev/null +++ b/samples/foundry-agent/cases/cortyx/shipping.yaml @@ -0,0 +1,111 @@ +# eCommerce Shipping & Delivery — evaluation cases +# +# These cases evaluate the agent's ability to analyze shipping +# performance, carrier metrics, and delivery time patterns. +# +# Grading approach: +# - Deterministic layer (dev mode): completion time, response length +# - AI-judged layer (validation mode): LLM judge assesses accuracy +# of shipping analysis and correct metric computation + +background: + category: shipping + given: + domain: ecommerce-analytics + +cases: + # ── Carrier Performance ───────────────────────────────────────── + + - id: avg_delivery_time_by_carrier + name: Average delivery time by carrier + tags: [shipping, carriers, delivery-time, core] + given: + query: "What is the average delivery time by carrier?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the average delivery time for each shipping carrier + + - id: delay_risk_by_carrier + name: Delay risk rate by carrier + tags: [shipping, carriers, risk] + given: + query: "What is the delay risk rate by carrier?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the percentage or rate of delayed shipments for each carrier + + - id: avg_shipping_cost_by_carrier + name: Average shipping cost by carrier + tags: [shipping, carriers, cost] + given: + query: "What is the average shipping cost by carrier including when carrier is missing?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the average shipping cost for each carrier including a + category for missing carriers + + # ── Regional Analysis ─────────────────────────────────────────── + + - id: avg_shipping_cost_by_region + name: Average shipping cost by region + tags: [shipping, region, cost] + given: + query: "What is the average shipping cost by region?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + the average shipping cost for each shipping region + + - id: shipping_cost_pct_of_order + name: Shipping cost as percentage of order total + tags: [shipping, cost, ratio] + given: + query: "What percentage of the order total does shipping cost represent?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [30, 5000] + - the answer should be: > + the average or overall percentage that shipping cost + represents relative to total order value + + - id: delivery_time_by_region + name: Delivery time by shipping region + tags: [shipping, region, delivery-time] + given: + query: "How does delivery time vary by shipping region?" + stages: + - when: the agent processes the request + then: + - completion time should be under: 100 + - when: the agent responds + then: + - response length should be: [50, 5000] + - the answer should be: > + average delivery time for each shipping region diff --git a/samples/foundry-agent/eval.config.yaml b/samples/foundry-agent/eval.config.yaml index ebd0b8f..50528ed 100644 --- a/samples/foundry-agent/eval.config.yaml +++ b/samples/foundry-agent/eval.config.yaml @@ -1,19 +1,14 @@ eval: - mode: validation + mode: dev # deterministic graders only, no LLM judge + thresholds: + case_pass: 0.0 # pass all cases; focus on score, not pass/fail agents: default: foundry-agent definitions: - name: foundry-agent - protocol: custom + protocol: foundry connection: - module: beval_foundry_adapter - class: FoundryAdapter - config: - endpoint: ${FOUNDRY_PROJECT_ENDPOINT} - agent_name: ${FOUNDRY_AGENT_NAME:-} - model: ${FOUNDRY_MODEL_NAME:-gpt-4o} - timeout: 60 - judge: - protocol: openai - model: ${JUDGE_MODEL_NAME:-gemma4:e2b} # whatever model you've pulled in Ollama - base_url: http://localhost:11434/v1 + endpoint: ${FOUNDRY_PROJECT_ENDPOINT} + agent_name: ${FOUNDRY_AGENT_NAME:-ecommerce-agent} + model: ${FOUNDRY_MODEL_NAME:-gpt-4o} + timeout: 120 diff --git a/samples/foundry-agent/run.sh b/samples/foundry-agent/run.sh new file mode 100755 index 0000000..ceca557 --- /dev/null +++ b/samples/foundry-agent/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export FOUNDRY_PROJECT_ENDPOINT="https://cortyxmvefoundry.services.ai.azure.com/api/projects/cortyx-mve-orchestration" +export FOUNDRY_AGENT_NAME="ecommerce-agent" + +uv run --project ../../python beval -c eval.config.yaml run --cases cases/cortyx --verbose -o results.json From c053b0aa556ebdabcb7c2e583c758059d5582c81 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Wed, 15 Jul 2026 14:19:04 -0700 Subject: [PATCH 02/15] chore: update run.sh to accept cases arg and fix install URL - run.sh accepts optional cases path argument (defaults to cases/cortyx) - Update README install URL to point to current branch Co-Authored-By: Claude Opus 4.6 (1M context) --- samples/foundry-agent/README.md | 2 +- samples/foundry-agent/run.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/samples/foundry-agent/README.md b/samples/foundry-agent/README.md index ca4b9d8..cf8b00a 100644 --- a/samples/foundry-agent/README.md +++ b/samples/foundry-agent/README.md @@ -50,7 +50,7 @@ via the `azure-ai-projects` SDK with Entra ID authentication. 1. Install beval with the Foundry extra: ```bash - pip install "beval[foundry] @ git+https://github.com/microsoft/beval.git#subdirectory=python" + pip install "beval[foundry] @ git+https://github.com/vyta/beval.git@eedorenko/update-foundry-adapter#subdirectory=python" ``` 1. Set the required environment variables: diff --git a/samples/foundry-agent/run.sh b/samples/foundry-agent/run.sh index ceca557..796ee28 100755 --- a/samples/foundry-agent/run.sh +++ b/samples/foundry-agent/run.sh @@ -2,4 +2,5 @@ export FOUNDRY_PROJECT_ENDPOINT="https://cortyxmvefoundry.services.ai.azure.com/api/projects/cortyx-mve-orchestration" export FOUNDRY_AGENT_NAME="ecommerce-agent" -uv run --project ../../python beval -c eval.config.yaml run --cases cases/cortyx --verbose -o results.json +CASES="${1:-cases/cortyx}" +uv run --project ../../python beval -c eval.config.yaml run --cases "$CASES" --verbose -o results.json From 12426d99406c0240c341a957b7e0f2c68868b752 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Wed, 15 Jul 2026 14:22:10 -0700 Subject: [PATCH 03/15] chore: remove cortyx cases from beval repo Cases live in the cortyx repo (microsoft/cortyx/cases/). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../cases/cortyx/cart-analytics.yaml | 144 --------- .../cases/cortyx/cross-domain.yaml | 111 ------- .../cases/cortyx/customer-analytics.yaml | 224 ------------- .../cases/cortyx/data-profiling.yaml | 306 ------------------ .../cases/cortyx/data-quality.yaml | 112 ------- .../foundry-agent/cases/cortyx/payments.yaml | 113 ------- .../cases/cortyx/product-analytics.yaml | 208 ------------ .../foundry-agent/cases/cortyx/returns.yaml | 80 ----- .../cases/cortyx/revenue-sales.yaml | 209 ------------ .../foundry-agent/cases/cortyx/reviews.yaml | 147 --------- .../foundry-agent/cases/cortyx/shipping.yaml | 111 ------- 11 files changed, 1765 deletions(-) delete mode 100644 samples/foundry-agent/cases/cortyx/cart-analytics.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/cross-domain.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/customer-analytics.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/data-profiling.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/data-quality.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/payments.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/product-analytics.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/returns.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/revenue-sales.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/reviews.yaml delete mode 100644 samples/foundry-agent/cases/cortyx/shipping.yaml diff --git a/samples/foundry-agent/cases/cortyx/cart-analytics.yaml b/samples/foundry-agent/cases/cortyx/cart-analytics.yaml deleted file mode 100644 index 2414a34..0000000 --- a/samples/foundry-agent/cases/cortyx/cart-analytics.yaml +++ /dev/null @@ -1,144 +0,0 @@ -# eCommerce Cart Analytics — evaluation cases -# -# These cases evaluate the agent's ability to analyze cart behavior, -# abandonment patterns, and conversion metrics. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses accuracy -# of cart metrics and correct conversion calculations - -background: - category: cart-analytics - given: - domain: ecommerce-analytics - -cases: - # ── Cart Value & Items ────────────────────────────────────────── - - - id: avg_cart_value_by_status - name: Average cart value by cart status - tags: [carts, value] - given: - query: "What is the average cart value by cart status?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the average cart value for each cart status - - - id: avg_items_per_cart - name: Average items per cart - tags: [carts, items] - given: - query: "What is the average number of items per cart based on the quantity column in cart_items?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - the average item quantity per cart - - - id: avg_cart_value_by_device - name: Average cart value by device type - tags: [carts, device, value] - given: - query: "How does average cart value vary by device type?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - average cart value for each device type - - - id: avg_cart_value_by_traffic_source - name: Average cart value by traffic source - tags: [carts, traffic, value] - given: - query: "How does average cart value vary by traffic source?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - average cart value for each traffic source - - # ── Conversion & Abandonment ──────────────────────────────────── - - - id: cart_abandonment_conversion_rate - name: Cart abandonment and conversion rate - tags: [carts, abandonment, conversion, core] - given: - query: "What is the cart abandonment rate and conversion rate when filtering the carts data status to be abandoned or converted?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - the percentage of carts that are abandoned and the - percentage that converted - - - id: conversion_rate_by_traffic_source - name: Cart-to-order conversion rate by traffic source - tags: [carts, conversion, traffic] - given: - query: "What is the cart-to-order conversion rate by traffic source?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the conversion rate from cart to order for each traffic source - - # ── Trends & Comparisons ──────────────────────────────────────── - - - id: category_carts_vs_orders - name: Category preferences in carts vs actual orders - tags: [carts, orders, categories] - given: - query: "How do category preferences differ between carts and actual orders?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a comparison of product category distribution in carts vs - in completed orders - - - id: carts_created_monthly_trend - name: Carts created per month trend - tags: [carts, trends] - given: - query: "What is the trend of carts created per month?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - monthly counts of new carts created showing the trend - over time diff --git a/samples/foundry-agent/cases/cortyx/cross-domain.yaml b/samples/foundry-agent/cases/cortyx/cross-domain.yaml deleted file mode 100644 index 313f3c4..0000000 --- a/samples/foundry-agent/cases/cortyx/cross-domain.yaml +++ /dev/null @@ -1,111 +0,0 @@ -# eCommerce Cross-Domain — evaluation cases -# -# These cases evaluate the agent's ability to join and correlate data -# across multiple tables to answer complex analytical questions. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses accuracy -# of cross-table joins and multi-step reasoning - -background: - category: cross-domain - given: - domain: ecommerce-analytics - -cases: - # ── Returns × Other Domains ───────────────────────────────────── - - - id: repeat_returners - name: Repeat returners with 3+ returns - tags: [returns, customers, behavior] - given: - query: "How many users are repeat returners with 3 or more returns?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - the count of users who have 3 or more return records - - - id: return_rate_by_region - name: Return rate by shipping region - tags: [returns, shipping, region] - given: - query: "How does the return rate vary by shipping region?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the return rate for each shipping region - - - id: returns_vs_review_sentiment_trends - name: Monthly return trends vs review sentiment trends - tags: [returns, reviews, trends] - given: - query: "How do monthly return trends compare to review sentiment trends?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - a month-by-month comparison of return volume and review - sentiment metrics - - - id: returns_by_status_type - name: Returns by status type - tags: [returns, status] - given: - query: "How many returns are there by status type in returns?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a count of returns grouped by each status type - - # ── Shipping × Spending ───────────────────────────────────────── - - - id: carrier_shipping_cost_comparison - name: Carriers compared on average shipping cost - tags: [shipping, carriers, cost] - given: - query: "How do carriers compare on average shipping cost when looking at the shipping cost column?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - average shipping cost for each carrier for comparison - - - id: top_5_net_spend_after_returns - name: Top 5 customers by net spend after returns - tags: [customers, returns, revenue, top-n, core] - given: - query: "Which top 5 customers have the highest net spend after returns?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a ranked list of 5 customers with the highest total spend - minus refund amounts diff --git a/samples/foundry-agent/cases/cortyx/customer-analytics.yaml b/samples/foundry-agent/cases/cortyx/customer-analytics.yaml deleted file mode 100644 index c958593..0000000 --- a/samples/foundry-agent/cases/cortyx/customer-analytics.yaml +++ /dev/null @@ -1,224 +0,0 @@ -# eCommerce Customer Analytics — evaluation cases -# -# These cases evaluate the agent's ability to analyze customer -# segments, lifetime value, cohorts, and behavioral patterns. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses accuracy -# of customer metrics and correct cohort logic - -background: - category: customer-analytics - given: - domain: ecommerce-analytics - -cases: - # ── Customer Distribution ─────────────────────────────────────── - - - id: customers_by_country_status - name: Customers by country and account status - tags: [customers, geography, status] - given: - query: "How are customers distributed by country and account status?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a breakdown of customer counts by country and account status - - - id: customers_by_acquisition_channel - name: Customers by acquisition channel - tags: [customers, acquisition] - given: - query: "How are customers distributed by acquisition channel?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a count of customers for each acquisition channel - - - id: customers_by_order_frequency - name: Customers by order frequency - tags: [customers, frequency] - given: - query: "How are customers distributed by order frequency?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a distribution of customers grouped by how many orders - they have placed - - - id: top_10_cities_by_customers - name: Top 10 cities by customer count - tags: [customers, geography, top-n] - given: - query: "What are the top 10 cities by customer count?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a ranked list of the 10 cities with the most customers - - # ── Lifetime Value & Spend ────────────────────────────────────── - - - id: top_20_customers_ltv - name: Top 20 customers by lifetime value - tags: [customers, ltv, top-n, core] - given: - query: "Who are the top 20 customers by lifetime value?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - a ranked list of 20 customers with the highest total spending - - - id: avg_orders_per_customer - name: Average orders per customer - tags: [customers, orders, average] - given: - query: "What is the average number of orders per customer for customers with an order?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - the average order count per customer, excluding customers - with zero orders - - # ── Revenue Segmentation ──────────────────────────────────────── - - - id: revenue_first_vs_repeat - name: Revenue split between first-time and repeat orders - tags: [customers, revenue, retention] - given: - query: "How does revenue split between first-time and repeat orders?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals or percentages for first-time orders vs - repeat orders - - - id: revenue_by_loyalty_tier - name: Revenue by loyalty tier - tags: [customers, loyalty, revenue] - given: - query: "How does revenue break down by loyalty tier?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals for each loyalty tier - - - id: aov_by_loyalty_tier - name: Average order value per loyalty tier - tags: [customers, loyalty, aov] - given: - query: "What is the average order value per loyalty tier?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the average order value for each loyalty tier - - - id: revenue_by_gender - name: Revenue by customer gender - tags: [customers, demographics, revenue] - given: - query: "How does revenue break down by customer gender?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals grouped by customer gender - - # ── Cohorts & Retention ───────────────────────────────────────── - - - id: cohort_order_volume - name: Monthly registration cohorts vs order volume - tags: [customers, cohorts, orders] - given: - query: "How do monthly registration cohorts compare in order volume?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - order counts grouped by customer registration month cohort - - - id: cohort_retention - name: Customer cohort retention over time - tags: [customers, cohorts, retention] - given: - query: "What does customer cohort retention look like over time?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - a cohort retention analysis showing what percentage of each - monthly registration cohort returns to purchase in - subsequent months - - - id: optin_rate_by_channel - name: Marketing opt-in rate per acquisition channel - tags: [customers, marketing, acquisition] - given: - query: "What is the marketing opt-in rate per acquisition channel?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the percentage of customers opted into marketing for each - acquisition channel diff --git a/samples/foundry-agent/cases/cortyx/data-profiling.yaml b/samples/foundry-agent/cases/cortyx/data-profiling.yaml deleted file mode 100644 index 9f4c1c7..0000000 --- a/samples/foundry-agent/cases/cortyx/data-profiling.yaml +++ /dev/null @@ -1,306 +0,0 @@ -# eCommerce Data Profiling — evaluation cases -# -# These cases evaluate the agent's ability to profile and describe -# the structure, shape, and distributions of the ecommerce dataset. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses completeness -# and accuracy of profiling results - -background: - category: data-profiling - given: - domain: ecommerce-analytics - -cases: - # ── Schema & Shape ────────────────────────────────────────────── - - - id: row_counts - name: Row counts for each table - tags: [schema, counts] - given: - query: "How many rows are in each table?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a complete listing of row counts for each table in the dataset - - - id: distinct_primary_keys - name: Distinct values per primary key column - tags: [schema, keys] - given: - query: "How many distinct values does each primary key column have?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - response should contain: "distinct" - - the answer should be: > - a listing of distinct value counts for each primary key column - - - id: date_ranges - name: Date range for each date column - tags: [schema, dates] - given: - query: "What is the date range for each date column?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a listing of min and max dates for each date column across datasets - - - id: null_values - name: NULL value counts per column - tags: [quality, nulls] - given: - query: "Across each dataset which columns have NULL values and how many?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a breakdown of null counts per column across each dataset - - # ── Distributions ─────────────────────────────────────────────── - - - id: order_status_distribution - name: Distribution of order statuses - tags: [distribution, orders] - given: - query: "What is the distribution of order statuses?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a count or percentage breakdown of each order status value - - - id: user_status_distribution - name: Distribution of user statuses - tags: [distribution, users] - given: - query: "What is the distribution of user statuses?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a count or percentage breakdown of each user status value - - - id: product_category_brand_distribution - name: Products distributed across categories and brands - tags: [distribution, products] - given: - query: "How are products distributed across categories and brands?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a breakdown of product counts by category and brand - - - id: payment_method_distribution - name: Payment method frequency - tags: [distribution, payments] - given: - query: "What payment methods are used and how often?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a frequency count of each payment method - - - id: shipping_carrier_distribution - name: Shipping carrier frequency - tags: [distribution, shipping] - given: - query: "Which shipping carriers are used and how often?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a frequency count of each shipping carrier - - - id: return_reasons - name: Most common reasons for returns - tags: [distribution, returns] - given: - query: "What are the most common reasons for returns?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a ranked list of return reasons by frequency - - - id: cart_status_distribution - name: Distribution of cart statuses - tags: [distribution, carts] - given: - query: "What is the distribution of cart statuses?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a count or percentage breakdown of each cart status - - - id: review_sentiment_distribution - name: Reviews distributed across sentiment buckets - tags: [distribution, reviews] - given: - query: "How are reviews distributed across sentiment buckets?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - response should contain: "sentiment" - - the answer should be: > - a count breakdown of reviews by sentiment category - - - id: review_rating_distribution - name: Distribution of review ratings (1-5 stars) - tags: [distribution, reviews] - given: - query: "What is the distribution of review ratings (1-5 stars)?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a count of reviews for each star rating from 1 to 5 - - - id: shipping_status_distribution - name: Distribution of shipping statuses - tags: [distribution, shipping] - given: - query: "What is the distribution of shipping statuses?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a count or percentage breakdown of each shipping status - - - id: payment_risk_flags - name: Payments with risk flags - tags: [distribution, payments, risk] - given: - query: "How many payments have risk flags?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - a count of payments flagged as high risk vs not flagged - - - id: loyalty_tier_distribution - name: Users distributed across loyalty tiers - tags: [distribution, users, loyalty] - given: - query: "How are users distributed across loyalty tiers?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a count of users in each loyalty tier - - - id: product_active_inactive - name: Active vs inactive products - tags: [distribution, products] - given: - query: "How many products are active vs inactive?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - a count of active products and inactive products - - - id: coupon_usage_count - name: Orders with coupon vs without - tags: [distribution, orders, coupons] - given: - query: "How many orders used a coupon vs did not?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - a count of orders with coupon applied vs orders without - - - id: first_vs_repeat_orders - name: First-time vs repeat purchases - tags: [distribution, orders] - given: - query: "How many orders are first-time vs repeat purchases?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - a count of first-time purchase orders vs repeat purchase orders diff --git a/samples/foundry-agent/cases/cortyx/data-quality.yaml b/samples/foundry-agent/cases/cortyx/data-quality.yaml deleted file mode 100644 index ec6ecef..0000000 --- a/samples/foundry-agent/cases/cortyx/data-quality.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# eCommerce Data Quality — evaluation cases -# -# These cases evaluate the agent's ability to detect data quality -# issues, anomalies, and cross-table inconsistencies. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses whether -# anomalies are correctly identified and explained - -background: - category: data-quality - given: - domain: ecommerce-analytics - -cases: - # ── Cross-table inconsistencies ──────────────────────────────── - - - id: payment_order_mismatch - name: Orders with mismatched payment and order totals - tags: [anomaly, payments, orders] - given: - query: "Which orders have a mismatch between payment amounts and order totals?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a list of order IDs where the payment amount does not match - the order total - - - id: delivered_with_return - name: Orders marked delivered but also returned - tags: [anomaly, orders, returns] - given: - query: "Which orders are marked as delivered but also have a return?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a list of order IDs that have both delivered status and an - associated return record - - - id: multiple_payments - name: Orders with multiple payment records - tags: [anomaly, payments] - given: - query: "Which orders have multiple payment records?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a list of order IDs with more than one payment record - - - id: carts_no_orders - name: Users with carts but no orders - tags: [anomaly, carts, users] - given: - query: "Which users have carts but have never placed an order?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a list of user IDs who have cart records but no order records - - - id: missing_carrier - name: Shipments missing a carrier - tags: [anomaly, shipping] - given: - query: "Which shipments are missing a carrier?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a list of shipment IDs or order IDs where the carrier field - is null or empty - - - id: return_date_anomalies - name: Anomalies in returns requested_at column - tags: [anomaly, returns, dates] - given: - query: "Are there anomalies in the returns requested_at column?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - an analysis of any date anomalies such as future dates, - dates before order creation, or null values in the returns - requested_at column diff --git a/samples/foundry-agent/cases/cortyx/payments.yaml b/samples/foundry-agent/cases/cortyx/payments.yaml deleted file mode 100644 index be2ecf2..0000000 --- a/samples/foundry-agent/cases/cortyx/payments.yaml +++ /dev/null @@ -1,113 +0,0 @@ -# eCommerce Payments — evaluation cases -# -# These cases evaluate the agent's ability to analyze payment -# processing metrics, failure rates, and risk indicators. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses accuracy -# of payment analysis and correct status interpretation - -background: - category: payments - given: - domain: ecommerce-analytics - -cases: - # ── Payment Status & Methods ──────────────────────────────────── - - - id: payment_success_failure_rate - name: Payment success vs failure rate - tags: [payments, status, core] - given: - query: "What is the payment success vs failure rate (captured vs failed)?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - a count or percentage of captured payments vs failed payments - - - id: revenue_by_payment_method - name: Revenue by payment method - tags: [payments, revenue] - given: - query: "How does revenue break down by payment method?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals grouped by each payment method - - - id: aov_by_payment_method - name: Average order value per payment method - tags: [payments, aov] - given: - query: "What is the average order value per payment method?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the average order value for each payment method - - - id: failures_by_payment_method - name: Payment methods with most failures - tags: [payments, failures] - given: - query: "Which payment methods have the most failures?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a ranked list of payment methods by number of failed - transactions - - # ── Risk & Cross-reference ────────────────────────────────────── - - - id: high_risk_amount_distribution - name: Amount distribution for high risk payments - tags: [payments, risk] - given: - query: "What is the amount distribution for high risk payments. Divide it into 2 categories <100 and >=100" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - a count of high-risk payments split into two buckets: - amounts less than 100 and amounts greater than or equal - to 100 - - - id: payment_vs_order_status - name: Payment status compared with order status - tags: [payments, orders, cross-reference] - given: - query: "How does payment status from payments compare with order status from orders?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a cross-tabulation or comparison showing how payment - statuses align with corresponding order statuses diff --git a/samples/foundry-agent/cases/cortyx/product-analytics.yaml b/samples/foundry-agent/cases/cortyx/product-analytics.yaml deleted file mode 100644 index b976079..0000000 --- a/samples/foundry-agent/cases/cortyx/product-analytics.yaml +++ /dev/null @@ -1,208 +0,0 @@ -# eCommerce Product Analytics — evaluation cases -# -# These cases evaluate the agent's ability to analyze product -# performance, category breakdowns, and inventory metrics. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses accuracy -# of product-level analytics and correct join logic - -background: - category: product-analytics - given: - domain: ecommerce-analytics - -cases: - # ── Product Revenue Rankings ──────────────────────────────────── - - - id: top_10_products_by_revenue - name: Top 10 products by revenue - tags: [products, revenue, top-n, core] - given: - query: "What are the top 10 products by revenue when merging the order items and products datasets?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - a ranked list of the 10 highest-revenue products with - product names and revenue amounts - - - id: bottom_10_products_by_revenue - name: Bottom 10 products by revenue - tags: [products, revenue, bottom-n] - given: - query: "What are the bottom 10 products by revenue when merging the order items and products datasets?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - a ranked list of the 10 lowest-revenue products with - product names and revenue amounts - - # ── Category & Brand Performance ──────────────────────────────── - - - id: revenue_units_per_category - name: Revenue and units sold per product category - tags: [products, categories, revenue] - given: - query: "What is the revenue and units sold per product category?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - revenue and unit count for each product category - - - id: revenue_units_per_brand - name: Revenue and units sold per brand - tags: [products, brands, revenue] - given: - query: "What is the revenue and units sold per brand?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - revenue and unit count for each brand - - - id: avg_price_per_category - name: Average product price per category - tags: [products, categories, pricing] - given: - query: "What is the average product price per category?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the average price for each product category - - - id: top_5_per_category - name: Top 5 revenue products in each category - tags: [products, categories, top-n] - given: - query: "What are the top 5 revenue-generating products in each category?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - the top 5 products by revenue within each product category - - # ── Inventory & Velocity ──────────────────────────────────────── - - - id: zero_sales_products - name: Products with zero sales - tags: [products, performance] - given: - query: "Which products have zero sales?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a list of products that have never been ordered - - - id: sales_velocity_vs_inventory - name: Sales velocity relative to inventory - tags: [products, inventory, velocity] - given: - query: "What is the sales velocity relative to inventory for each product (total units sold / current inventory_qty)?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a ratio of total units sold divided by current inventory - quantity for each product - - - id: inactive_products_ordered - name: Inactive products still being ordered - tags: [products, anomaly] - given: - query: "Are there inactive products that are still being ordered?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 10000] - - the answer should be: > - a list of products marked as inactive that still appear - in order items - - # ── Revenue by Product Attributes ────────────────────────────── - - - id: revenue_by_margin_band - name: Revenue by margin band - tags: [products, margins, revenue] - given: - query: "How does revenue break down by margin band?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals grouped by margin band categories - - - id: revenue_by_launch_season - name: Revenue by launch season - tags: [products, seasonal, revenue] - given: - query: "How does revenue break down by launch season?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals grouped by product launch season - - - id: price_distribution_bins - name: Product price distribution in 10 bins - tags: [products, pricing, distribution] - given: - query: "What does the product price distribution look like when divided into 10 bins?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a histogram-style breakdown of product prices divided into - 10 equal-width bins with counts per bin diff --git a/samples/foundry-agent/cases/cortyx/returns.yaml b/samples/foundry-agent/cases/cortyx/returns.yaml deleted file mode 100644 index 236be4d..0000000 --- a/samples/foundry-agent/cases/cortyx/returns.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# eCommerce Returns — evaluation cases -# -# These cases evaluate the agent's ability to analyze return rates, -# refund breakdowns, and product-level return patterns. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses accuracy -# of return metrics and correct filtering logic - -background: - category: returns - given: - domain: ecommerce-analytics - -cases: - # ── Return Metrics ────────────────────────────────────────────── - - - id: overall_return_rate - name: Overall return rate - tags: [returns, rate, core] - given: - query: "What is the overall return rate based on the distinct count of returned orders?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - the percentage of orders that have been returned based on - distinct order count - - - id: refund_by_reason - name: Refund amounts by return reason - tags: [returns, refunds, reasons] - given: - query: "How do refund amounts break down by return reason?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - total refund amounts grouped by each return reason - - - id: return_rate_per_category - name: Return rate per product category - tags: [returns, categories, rate] - given: - query: "What is the breakdown of the overall return rate per product category using the returns data when the status is approved, refunded, or requested?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - return rates for each product category, considering only - returns with status approved, refunded, or requested - - - id: top_10_returned_products - name: Top 10 most-returned products - tags: [returns, products, top-n] - given: - query: "What are the top 10 most-returned products?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - a ranked list of the 10 products with the highest number - of returns diff --git a/samples/foundry-agent/cases/cortyx/revenue-sales.yaml b/samples/foundry-agent/cases/cortyx/revenue-sales.yaml deleted file mode 100644 index 4c63427..0000000 --- a/samples/foundry-agent/cases/cortyx/revenue-sales.yaml +++ /dev/null @@ -1,209 +0,0 @@ -# eCommerce Revenue & Sales — evaluation cases -# -# These cases evaluate the agent's ability to compute revenue metrics, -# identify trends, and break down sales by various dimensions. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length, -# keyword presence -# - AI-judged layer (validation mode): LLM judge assesses numeric -# accuracy and completeness of revenue analysis - -background: - category: revenue-sales - given: - domain: ecommerce-analytics - -cases: - # ── Summary Metrics ───────────────────────────────────────────── - - - id: total_revenue_metrics - name: Total revenue, order count, and AOV - tags: [revenue, summary, core] - given: - query: "What is the total revenue, order count, and average order value?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - response should contain: "revenue" - - the answer should be: > - numeric values for total revenue, total order count, and - average order value - - - id: monthly_revenue_trend - name: Monthly revenue trend - tags: [revenue, trends] - given: - query: "What is the monthly revenue trend?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - a month-by-month breakdown of revenue showing the trend - over time - - # ── Revenue Breakdowns ────────────────────────────────────────── - - - id: revenue_by_channel - name: Revenue by sales channel - tags: [revenue, channels] - given: - query: "How does revenue break down by sales channel?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals grouped by each sales channel - - - id: revenue_by_device - name: Revenue by device type - tags: [revenue, device] - given: - query: "How does revenue break down by device type?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals grouped by device type - - - id: revenue_by_region - name: Revenue by shipping region - tags: [revenue, region] - given: - query: "How does revenue break down by shipping region?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals grouped by shipping region - - - id: avg_discount_pct - name: Average discount percentage per order - tags: [revenue, discounts] - given: - query: "What is the average discount percentage per order for orders with a subtotal greater than 0?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - the average discount as a percentage, computed only for - orders with subtotal greater than zero - - - id: coupon_impact_revenue - name: Coupon usage rate and impact on revenue - tags: [revenue, coupons] - given: - query: "What is the coupon usage rate and its impact on revenue?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - response should contain: "coupon" - - the answer should be: > - the percentage of orders using coupons and the revenue - comparison between coupon and non-coupon orders - - - id: revenue_components - name: Revenue split between subtotal, tax, shipping, and discounts - tags: [revenue, breakdown] - given: - query: "How does revenue split between subtotal, tax, shipping fees, and discounts?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a breakdown of total subtotal, total tax, total shipping - fees, and total discounts across all orders - - - id: top_10_orders - name: Top 10 highest-value orders - tags: [revenue, top-n] - given: - query: "What are the top 10 highest-value orders?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - a list of the 10 orders with the highest total value - including order IDs and amounts - - - id: revenue_by_day_of_week - name: Revenue by day of the week - tags: [revenue, trends] - given: - query: "How does revenue vary by day of the week?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals or averages for each day of the week - - - id: mom_order_growth - name: Month-over-month order count growth - tags: [orders, trends, growth] - given: - query: "What is the month-over-month order count growth?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [100, 10000] - - the answer should be: > - monthly order counts with growth rate or percentage change - month over month - - - id: revenue_by_currency - name: Revenue by currency - tags: [revenue, currency] - given: - query: "How does revenue break down by currency?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - revenue totals grouped by currency diff --git a/samples/foundry-agent/cases/cortyx/reviews.yaml b/samples/foundry-agent/cases/cortyx/reviews.yaml deleted file mode 100644 index 4f40ca4..0000000 --- a/samples/foundry-agent/cases/cortyx/reviews.yaml +++ /dev/null @@ -1,147 +0,0 @@ -# eCommerce Reviews — evaluation cases -# -# These cases evaluate the agent's ability to analyze review ratings, -# sentiment patterns, and review-related customer insights. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses accuracy -# of review analysis and correct aggregation logic - -background: - category: reviews - given: - domain: ecommerce-analytics - -cases: - # ── Ratings & Sentiment ───────────────────────────────────────── - - - id: avg_rating_overall_per_category - name: Average review rating overall and per category - tags: [reviews, ratings, categories, core] - given: - query: "What is the average review rating overall and per category?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the overall average review rating and average rating for - each product category - - - id: verified_vs_unverified_reviews - name: Verified vs unverified purchase reviews - tags: [reviews, verified] - given: - query: "How do verified purchase reviews compare to unverified ones?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a comparison of average rating, count, or other metrics - between verified and unverified purchase reviews - - - id: brand_ratings_high_low - name: Brands with highest and lowest average ratings - tags: [reviews, brands, ratings] - given: - query: "Which brands have the highest and lowest average ratings?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - the brands with the highest average ratings and the brands - with the lowest - - - id: monthly_review_volume_rating - name: Monthly trend in review volume and average rating - tags: [reviews, trends] - given: - query: "What is the monthly trend in review volume and average rating?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - monthly review counts and average ratings showing trends - over time - - - id: review_length_by_sentiment - name: Review length by sentiment - tags: [reviews, sentiment, length] - given: - query: "How does review length vary by sentiment?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - response should contain: "sentiment" - - the answer should be: > - average review text length for each sentiment category - - # ── Review × Customer Cross-reference ─────────────────────────── - - - id: low_rating_high_return_products - name: Low-rating products with high return rates - tags: [reviews, returns, products, cross-reference] - given: - query: "Which products are in the bottom quartile for ratings and top quartile for return rates using the rating from reviews and return rate from returns?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 10000] - - the answer should be: > - a list of products that are both in the bottom 25% for - average rating and the top 25% for return rate - - - id: customers_no_reviews - name: Customers with orders but no reviews - tags: [reviews, customers] - given: - query: "How many customers have placed orders but never left a review?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - the count of customers who have at least one order but - zero reviews - - - id: top_spenders_no_reviews - name: Top 5 highest-spending customers with no reviews - tags: [reviews, customers, top-n] - given: - query: "Who are the top 5 highest-spending customers that have never left a review?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - a list of the 5 customers with the highest total spend who - have never left a review diff --git a/samples/foundry-agent/cases/cortyx/shipping.yaml b/samples/foundry-agent/cases/cortyx/shipping.yaml deleted file mode 100644 index ebc4b45..0000000 --- a/samples/foundry-agent/cases/cortyx/shipping.yaml +++ /dev/null @@ -1,111 +0,0 @@ -# eCommerce Shipping & Delivery — evaluation cases -# -# These cases evaluate the agent's ability to analyze shipping -# performance, carrier metrics, and delivery time patterns. -# -# Grading approach: -# - Deterministic layer (dev mode): completion time, response length -# - AI-judged layer (validation mode): LLM judge assesses accuracy -# of shipping analysis and correct metric computation - -background: - category: shipping - given: - domain: ecommerce-analytics - -cases: - # ── Carrier Performance ───────────────────────────────────────── - - - id: avg_delivery_time_by_carrier - name: Average delivery time by carrier - tags: [shipping, carriers, delivery-time, core] - given: - query: "What is the average delivery time by carrier?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the average delivery time for each shipping carrier - - - id: delay_risk_by_carrier - name: Delay risk rate by carrier - tags: [shipping, carriers, risk] - given: - query: "What is the delay risk rate by carrier?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the percentage or rate of delayed shipments for each carrier - - - id: avg_shipping_cost_by_carrier - name: Average shipping cost by carrier - tags: [shipping, carriers, cost] - given: - query: "What is the average shipping cost by carrier including when carrier is missing?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the average shipping cost for each carrier including a - category for missing carriers - - # ── Regional Analysis ─────────────────────────────────────────── - - - id: avg_shipping_cost_by_region - name: Average shipping cost by region - tags: [shipping, region, cost] - given: - query: "What is the average shipping cost by region?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - the average shipping cost for each shipping region - - - id: shipping_cost_pct_of_order - name: Shipping cost as percentage of order total - tags: [shipping, cost, ratio] - given: - query: "What percentage of the order total does shipping cost represent?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [30, 5000] - - the answer should be: > - the average or overall percentage that shipping cost - represents relative to total order value - - - id: delivery_time_by_region - name: Delivery time by shipping region - tags: [shipping, region, delivery-time] - given: - query: "How does delivery time vary by shipping region?" - stages: - - when: the agent processes the request - then: - - completion time should be under: 100 - - when: the agent responds - then: - - response length should be: [50, 5000] - - the answer should be: > - average delivery time for each shipping region From 77df2f80644e09ab798b11f5add2c81014d154c4 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Wed, 15 Jul 2026 14:25:24 -0700 Subject: [PATCH 04/15] chore: clean up PR to focus on built-in Foundry adapter - Revert uv.lock changes - Restore eval.config.yaml to standard sample config - Clean up README to be generic (no branch-specific URLs) Co-Authored-By: Claude Opus 4.6 (1M context) --- python/uv.lock | 53 +------------------------- samples/foundry-agent/README.md | 41 ++++++-------------- samples/foundry-agent/eval.config.yaml | 12 +++--- samples/foundry-agent/run.sh | 6 --- 4 files changed, 21 insertions(+), 91 deletions(-) delete mode 100755 samples/foundry-agent/run.sh diff --git a/python/uv.lock b/python/uv.lock index bf4cacb..e29113f 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -82,23 +82,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, ] -[[package]] -name = "azure-ai-projects" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "azure-identity" }, - { name = "azure-storage-blob" }, - { name = "isodate" }, - { name = "openai" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/29/ab1a80ce483fcc36ec2ac0a3db8e043aa954700b34b77db43ed5f4f9c488/azure_ai_projects-2.3.0.tar.gz", hash = "sha256:6e3006b7b8aa51c6ff9db61ef4aac3717f8a712cd1a183d5a1d34e2eb33450bd", size = 27563233, upload-time = "2026-07-01T21:09:00.018Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/bd/7e64f2d6e43d19fc8cf464769804f947d997833eb4dce1c86c2ce76146d2/azure_ai_projects-2.3.0-py3-none-any.whl", hash = "sha256:1da20aeac9663740a97644efedbb9f59eb2a332d3e01bfde7aa03027936edf44", size = 349581, upload-time = "2026-07-01T21:09:04.32Z" }, -] - [[package]] name = "azure-core" version = "1.38.2" @@ -128,21 +111,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/77/f658c76f9e9a52c784bd836aaca6fd5b9aae176f1f53273e758a2bcda695/azure_identity-1.25.2-py3-none-any.whl", hash = "sha256:1b40060553d01a72ba0d708b9a46d0f61f56312e215d8896d836653ffdc6753d", size = 191423, upload-time = "2026-02-11T01:55:44.245Z" }, ] -[[package]] -name = "azure-storage-blob" -version = "12.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "isodate" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/48/84a820d898267f662b5c06f7cd76fdb8a9e272b44aa9376cef3ec0f6a294/azure_storage_blob-12.30.0.tar.gz", hash = "sha256:2cd74d4d5731e5eb6b8d5c5056ee115a5e88f8fdf22517b739836fda685018be", size = 618229, upload-time = "2026-06-08T11:45:35.575Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/0b/e106f0fd7fa785867d9ffcc47dc9e6237c0e58f51058473b777487a98edc/azure_storage_blob-12.30.0-py3-none-any.whl", hash = "sha256:d415ac50b67a8da6b3ae7e9f1014b1b55cd7aafa0b8d4ca9b380568dc7360423", size = 435610, upload-time = "2026-06-08T11:45:37.213Z" }, -] - [[package]] name = "beval" version = "0.1.0" @@ -169,7 +137,6 @@ acp = [ all = [ { name = "a2a-sdk" }, { name = "agent-client-protocol" }, - { name = "azure-ai-projects" }, { name = "azure-identity" }, { name = "httpx" }, { name = "jsonschema" }, @@ -180,7 +147,6 @@ all = [ dev = [ { name = "a2a-sdk" }, { name = "agent-client-protocol" }, - { name = "azure-ai-projects" }, { name = "azure-identity" }, { name = "httpx" }, { name = "jsonschema" }, @@ -193,10 +159,6 @@ dev = [ { name = "types-jsonschema" }, { name = "types-pyyaml" }, ] -foundry = [ - { name = "azure-ai-projects" }, - { name = "azure-identity" }, -] judge = [ { name = "azure-identity" }, { name = "openai" }, @@ -214,12 +176,10 @@ requires-dist = [ { name = "a2a-sdk", marker = "extra == 'a2a'", specifier = ">=0.3.0" }, { name = "a2a-sdk", extras = ["http-server"], marker = "extra == 'a2a-server'", specifier = ">=0.3.0" }, { name = "agent-client-protocol", marker = "extra == 'acp'", specifier = ">=0.1.0" }, - { name = "azure-ai-projects", marker = "extra == 'foundry'", specifier = ">=2.0.0" }, { name = "azure-identity", marker = "extra == 'a2a-server'", specifier = ">=1.15.0" }, - { name = "azure-identity", marker = "extra == 'foundry'", specifier = ">=1.15.0" }, { name = "azure-identity", marker = "extra == 'judge'", specifier = ">=1.15.0" }, { name = "beval", extras = ["all"], marker = "extra == 'dev'" }, - { name = "beval", extras = ["validate", "tracing", "acp", "a2a", "foundry", "judge"], marker = "extra == 'all'" }, + { name = "beval", extras = ["validate", "tracing", "acp", "a2a", "judge"], marker = "extra == 'all'" }, { name = "fastapi", marker = "extra == 'a2a-server'", specifier = ">=0.115.0" }, { name = "httpx", marker = "extra == 'a2a'", specifier = ">=0.27.0" }, { name = "jsonschema", marker = "extra == 'validate'", specifier = ">=4.20.0" }, @@ -235,7 +195,7 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0" }, { name = "uvicorn", marker = "extra == 'a2a-server'", specifier = ">=0.30.0" }, ] -provides-extras = ["validate", "tracing", "foundry", "acp", "a2a", "judge", "a2a-server", "all", "dev"] +provides-extras = ["validate", "tracing", "acp", "a2a", "judge", "a2a-server", "all", "dev"] [[package]] name = "certifi" @@ -653,15 +613,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "isodate" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, -] - [[package]] name = "jiter" version = "0.13.0" diff --git a/samples/foundry-agent/README.md b/samples/foundry-agent/README.md index cf8b00a..2160753 100644 --- a/samples/foundry-agent/README.md +++ b/samples/foundry-agent/README.md @@ -1,7 +1,7 @@ --- title: Foundry Agent Sample Evaluation description: Evaluates a Microsoft Foundry prompt agent using beval's built-in Foundry adapter with Entra ID authentication. -ms.date: 2026-07-14 +ms.date: 2026-07-15 ms.topic: tutorial --- @@ -16,10 +16,8 @@ via the `azure-ai-projects` SDK with Entra ID authentication. ```text ├── agent.yaml # Agent definition ├── eval.config.yaml # Evaluation configuration -├── run.sh # Run script with defaults ├── cases/ -│ ├── foundry-eval.yaml # 2 basic evaluation cases -│ └── cortyx/ # eCommerce analytics cases (100 cases) +│ └── foundry-eval.yaml # 2 evaluation cases └── README.md ``` @@ -36,7 +34,7 @@ via the `azure-ai-projects` SDK with Entra ID authentication. | Variable | Required | Description | |----------------------------|----------|------------------------------------------------------| | `FOUNDRY_PROJECT_ENDPOINT` | Yes | Foundry project endpoint URL | -| `FOUNDRY_AGENT_NAME` | No | Agent name (default: `ecommerce-agent`) | +| `FOUNDRY_AGENT_NAME` | No | Agent name (omit for direct model calls) | | `FOUNDRY_MODEL_NAME` | No | Fallback model when agent name is not set (default: `gpt-4o`) | ## Setup @@ -50,7 +48,7 @@ via the `azure-ai-projects` SDK with Entra ID authentication. 1. Install beval with the Foundry extra: ```bash - pip install "beval[foundry] @ git+https://github.com/vyta/beval.git@eedorenko/update-foundry-adapter#subdirectory=python" + pip install beval[foundry] ``` 1. Set the required environment variables: @@ -66,20 +64,14 @@ via the `azure-ai-projects` SDK with Entra ID authentication. ```bash cd samples/foundry-agent -beval -c eval.config.yaml run --cases cases --verbose -``` - -Or use the run script with defaults: - -```bash -./run.sh +beval -c eval.config.yaml run --cases cases -m dev --verbose ``` **Validation mode** includes AI-judged criteria and requires a judge configured in `eval.config.yaml` (see [LLM Judge](#llm-judge) below): ```bash -beval -c eval.config.yaml run --cases cases -m validation --verbose +beval -c eval.config.yaml run --cases cases --verbose ``` **From the `python/` directory** (development): @@ -93,9 +85,8 @@ uv run beval -c ../samples/foundry-agent/eval.config.yaml run \ ## LLM Judge -To enable AI-judged graders in validation mode, add a `judge` section to -`eval.config.yaml`. Any OpenAI-compatible endpoint works, including Ollama -for fully local evaluation. +The `eval.config.yaml` configures an Ollama-based LLM judge for validation +mode. Any OpenAI-compatible endpoint works. ### Ollama (local) @@ -105,16 +96,6 @@ for fully local evaluation. ollama pull gemma4:e2b ``` -1. Add the judge config to `eval.config.yaml`: - - ```yaml - eval: - judge: - protocol: openai - model: ${JUDGE_MODEL_NAME:-gemma4:e2b} - base_url: http://localhost:11434/v1 - ``` - 1. Set `OPENAI_API_KEY` to any non-empty value (Ollama ignores it, but the `openai` Python package requires it): @@ -124,6 +105,8 @@ for fully local evaluation. ### OpenAI +Override the judge in `eval.config.yaml`: + ```yaml eval: judge: @@ -147,7 +130,7 @@ eval: protocol: foundry connection: endpoint: ${FOUNDRY_PROJECT_ENDPOINT} - agent_name: ${FOUNDRY_AGENT_NAME:-ecommerce-agent} + agent_name: ${FOUNDRY_AGENT_NAME:-} model: ${FOUNDRY_MODEL_NAME:-gpt-4o} - timeout: 120 + timeout: 60 ``` diff --git a/samples/foundry-agent/eval.config.yaml b/samples/foundry-agent/eval.config.yaml index 50528ed..d5846b7 100644 --- a/samples/foundry-agent/eval.config.yaml +++ b/samples/foundry-agent/eval.config.yaml @@ -1,7 +1,5 @@ eval: - mode: dev # deterministic graders only, no LLM judge - thresholds: - case_pass: 0.0 # pass all cases; focus on score, not pass/fail + mode: validation agents: default: foundry-agent definitions: @@ -9,6 +7,10 @@ eval: protocol: foundry connection: endpoint: ${FOUNDRY_PROJECT_ENDPOINT} - agent_name: ${FOUNDRY_AGENT_NAME:-ecommerce-agent} + agent_name: ${FOUNDRY_AGENT_NAME:-} model: ${FOUNDRY_MODEL_NAME:-gpt-4o} - timeout: 120 + timeout: 60 + judge: + protocol: openai + model: ${JUDGE_MODEL_NAME:-gemma4:e2b} + base_url: http://localhost:11434/v1 diff --git a/samples/foundry-agent/run.sh b/samples/foundry-agent/run.sh deleted file mode 100755 index 796ee28..0000000 --- a/samples/foundry-agent/run.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -export FOUNDRY_PROJECT_ENDPOINT="https://cortyxmvefoundry.services.ai.azure.com/api/projects/cortyx-mve-orchestration" -export FOUNDRY_AGENT_NAME="ecommerce-agent" - -CASES="${1:-cases/cortyx}" -uv run --project ../../python beval -c eval.config.yaml run --cases "$CASES" --verbose -o results.json From cc09769bc09a9e2ee51e6276ad051cb189f17f55 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Wed, 15 Jul 2026 14:28:50 -0700 Subject: [PATCH 05/15] style: fix ruff format in foundry adapter Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/beval/adapters/foundry.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/src/beval/adapters/foundry.py b/python/src/beval/adapters/foundry.py index d24b90e..6db72f3 100644 --- a/python/src/beval/adapters/foundry.py +++ b/python/src/beval/adapters/foundry.py @@ -64,9 +64,7 @@ def invoke(self, adapter_input: AdapterInput) -> Subject: query = adapter_input.query if isinstance(query, list): - query = " ".join( - m.get("content", "") for m in query if isinstance(m, dict) - ) + query = " ".join(m.get("content", "") for m in query if isinstance(m, dict)) start = time.monotonic() From 39efcf487bd165ad507c2cc32d8f5ad1a8702c58 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Wed, 15 Jul 2026 14:42:36 -0700 Subject: [PATCH 06/15] docs: add missing env vars and OPENAI_API_KEY to validation example Co-Authored-By: Claude Opus 4.6 (1M context) --- samples/foundry-agent/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/foundry-agent/README.md b/samples/foundry-agent/README.md index 2160753..6dd4038 100644 --- a/samples/foundry-agent/README.md +++ b/samples/foundry-agent/README.md @@ -36,6 +36,8 @@ via the `azure-ai-projects` SDK with Entra ID authentication. | `FOUNDRY_PROJECT_ENDPOINT` | Yes | Foundry project endpoint URL | | `FOUNDRY_AGENT_NAME` | No | Agent name (omit for direct model calls) | | `FOUNDRY_MODEL_NAME` | No | Fallback model when agent name is not set (default: `gpt-4o`) | +| `JUDGE_MODEL_NAME` | No | Ollama judge model (default: `gemma4:e2b`) | +| `OPENAI_API_KEY` | No | Required for validation mode; set to `ollama` for local Ollama | ## Setup @@ -71,7 +73,7 @@ beval -c eval.config.yaml run --cases cases -m dev --verbose configured in `eval.config.yaml` (see [LLM Judge](#llm-judge) below): ```bash -beval -c eval.config.yaml run --cases cases --verbose +OPENAI_API_KEY=ollama beval -c eval.config.yaml run --cases cases --verbose ``` **From the `python/` directory** (development): From 226d8202fda0af080e5fd264e3e0008ea380697b Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Mon, 20 Jul 2026 13:31:48 -0700 Subject: [PATCH 07/15] fix: append /openai/v1 path for Foundry judge endpoints Azure AI Foundry project endpoints use the OpenAI v1 API path (/openai/v1/chat/completions) rather than the Azure-native path that requires api-version query params. Without this, the judge fails with "Missing required query parameter: api-version" or "API version not supported". Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/beval/judge.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/python/src/beval/judge.py b/python/src/beval/judge.py index 0faf241..8405d48 100644 --- a/python/src/beval/judge.py +++ b/python/src/beval/judge.py @@ -367,13 +367,20 @@ def _create_client_explicit( # Azure AI Foundry: use plain OpenAI with bearer token header. # Preserve the full path but strip trailing endpoint segments # (/responses, /chat/completions) so the SDK can append its own. + # Append /openai/v1 so the SDK constructs the correct URL + # (e.g. .../openai/v1/chat/completions). parsed = urlparse(base_url) path = parsed.path.rstrip("/") - for suffix in ("/responses", "/chat/completions", "/completions"): + for suffix in ( + "/openai/v1", + "/responses", + "/chat/completions", + "/completions", + ): if path.endswith(suffix): path = path[: -len(suffix)] break - clean_url = f"{parsed.scheme}://{parsed.netloc}{path}" + clean_url = f"{parsed.scheme}://{parsed.netloc}{path}/openai/v1" scope = "https://ai.azure.com/.default" token_provider = get_bearer_token_provider( DefaultAzureCredential(), scope @@ -399,14 +406,20 @@ def _create_client_explicit( if _is_foundry and base_url: # Azure AI Foundry with api_key: plain OpenAI client, full path. + # Append /openai/v1 so the SDK constructs the correct URL. parsed = urlparse(base_url) path = parsed.path.rstrip("/") - for suffix in ("/responses", "/chat/completions", "/completions"): + for suffix in ( + "/openai/v1", + "/responses", + "/chat/completions", + "/completions", + ): if path.endswith(suffix): path = path[: -len(suffix)] break - clean_url = f"{parsed.scheme}://{parsed.netloc}{path}" - kwargs = {"base_url": clean_url} + clean_url = f"{parsed.scheme}://{parsed.netloc}{path}/openai/v1" + kwargs: dict[str, Any] = {"base_url": clean_url} if api_key: kwargs["api_key"] = api_key return openai_cls(**kwargs) From b14ca74f125708a92bbed1de5d46e971e563bab8 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Mon, 20 Jul 2026 13:39:31 -0700 Subject: [PATCH 08/15] fix: resolve mypy no-redef error in Foundry judge kwargs Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/beval/judge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/src/beval/judge.py b/python/src/beval/judge.py index 8405d48..22433e3 100644 --- a/python/src/beval/judge.py +++ b/python/src/beval/judge.py @@ -419,10 +419,10 @@ def _create_client_explicit( path = path[: -len(suffix)] break clean_url = f"{parsed.scheme}://{parsed.netloc}{path}/openai/v1" - kwargs: dict[str, Any] = {"base_url": clean_url} + foundry_kwargs: dict[str, Any] = {"base_url": clean_url} if api_key: - kwargs["api_key"] = api_key - return openai_cls(**kwargs) + foundry_kwargs["api_key"] = api_key + return openai_cls(**foundry_kwargs) if _is_azure_oai and base_url: # Classic Azure OpenAI with api_key: AzureOpenAI, strip path. From c59b0ebf0ccac1b619b81a698bebe222a5c1aac2 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Wed, 22 Jul 2026 15:14:02 -0700 Subject: [PATCH 09/15] feat: add Bar 4 MVP quality metrics to run summary Add effective_latency (geometric mean of time+1, capped at 100s), p95_latency, fail_rate, keyword_recall, and accuracy_rate with binomial standard errors to RunSummary. These metrics appear in results.json summary when computed, and are omitted when not applicable (backward compatible). Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/beval/reporter.py | 4 ++ python/src/beval/runner.py | 103 +++++++++++++++++++++++++++++++++++ python/src/beval/types.py | 10 ++++ 3 files changed, 117 insertions(+) diff --git a/python/src/beval/reporter.py b/python/src/beval/reporter.py index c8c00bc..f5ad811 100644 --- a/python/src/beval/reporter.py +++ b/python/src/beval/reporter.py @@ -77,6 +77,10 @@ def _prepare(result: RunResult, *, scrub: bool = False) -> dict[str, Any]: if "config" in raw: raw["config"] = _strip_defaults(raw["config"]) + # Summary: drop null optional fields (Bar 4 metrics) + if "summary" in raw and isinstance(raw["summary"], dict): + raw["summary"] = {k: v for k, v in raw["summary"].items() if v is not None} + # Cases: drop null optional fields and false high_variance for case in raw.get("cases", []): for key in _CASE_OPTIONAL: diff --git a/python/src/beval/runner.py b/python/src/beval/runner.py index e56cc6a..188fa0d 100644 --- a/python/src/beval/runner.py +++ b/python/src/beval/runner.py @@ -6,6 +6,7 @@ from __future__ import annotations +import math import statistics import time from typing import Any @@ -74,6 +75,37 @@ def _filter_cases( return result +def _process_latency( + times: list[float], max_latency: float = 100.0 +) -> tuple[float, float]: + """Compute effective latency via geometric mean of (time+1), capped. + + Returns (effective_latency, stderr) per Bar 4 spec. + """ + capped = [min(t, max_latency) for t in times] + xl = [math.log(1.0 + t) for t in capped] + avg = statistics.mean(xl) + se = avg + if len(xl) > 1: + se = statistics.stdev(xl) / math.sqrt(len(xl) - 1) + eff = math.exp(avg) - 1.0 + stderr = (math.exp(avg + se) - math.exp(avg - se)) / 2.0 + return eff, stderr + + +def _compute_pass_rate(values: list[bool]) -> tuple[float, float]: + """Compute pass rate with binomial stderr. + + Returns (rate, stderr). + """ + n = len(values) + if n == 0: + return 0.0, 0.0 + p = sum(1 for v in values if v) / n + se = math.sqrt(p * (1 - p) / n) if n > 1 else 0.0 + return p, se + + def _aggregate_grades( grades: list[Grade], config: RunConfig, @@ -716,6 +748,68 @@ def _build_summary(self, cases: list[CaseResult]) -> RunSummary: for m in metrics: metrics[m] /= metric_counts[m] + # Bar 4: Effective latency — geometric mean of (time+1), capped at 100s + effective_latency: float | None = None + effective_latency_stderr: float | None = None + p95_latency: float | None = None + times = [ + c.time_seconds + for c in cases + if c.time_seconds is not None and c.time_seconds >= 0 + ] + if times: + effective_latency, effective_latency_stderr = _process_latency(times) + sorted_times = sorted(times) + p95_idx = max(0, int(math.ceil(0.95 * len(sorted_times))) - 1) + p95_latency = sorted_times[p95_idx] + + # Bar 4: Fail rate — binomial proportion with stderr + n_total = len(cases) + fail_rate: float | None = None + fail_rate_stderr: float | None = None + if n_total > 0: + fail_rate = errored / n_total + fail_rate_stderr = ( + math.sqrt(fail_rate * (1 - fail_rate) / n_total) if n_total > 1 else 0.0 + ) + + # Bar 4: Keyword recall — from "response should contain" grades + recall_values: list[float] = [] + for c in non_errored: + keyword_grades = [ + g + for g in c.grades + if g.criterion.lower().startswith("response should contain") + and not g.skipped + ] + if keyword_grades: + case_recall = sum(g.score for g in keyword_grades) / len(keyword_grades) + recall_values.append(case_recall) + keyword_recall: float | None = None + keyword_recall_stderr: float | None = None + if recall_values: + keyword_recall = statistics.mean(recall_values) + n_recall = len(recall_values) + keyword_recall_stderr = ( + math.sqrt(keyword_recall * (1 - keyword_recall) / n_recall) + if n_recall > 1 + else 0.0 + ) + + # Bar 4: Accuracy rate — from ai_judged "the answer should" grades + accuracy_values: list[bool] = [] + for c in non_errored: + for g in c.grades: + if ( + g.criterion.lower().startswith("the answer should") + and not g.skipped + ): + accuracy_values.append(g.passed) + accuracy_rate: float | None = None + accuracy_rate_stderr: float | None = None + if accuracy_values: + accuracy_rate, accuracy_rate_stderr = _compute_pass_rate(accuracy_values) + return RunSummary( overall_score=overall, passed=passed, @@ -723,4 +817,13 @@ def _build_summary(self, cases: list[CaseResult]) -> RunSummary: errored=errored, total=len(cases), metrics=metrics, + effective_latency=effective_latency, + effective_latency_stderr=effective_latency_stderr, + p95_latency=p95_latency, + fail_rate=fail_rate, + fail_rate_stderr=fail_rate_stderr, + keyword_recall=keyword_recall, + keyword_recall_stderr=keyword_recall_stderr, + accuracy_rate=accuracy_rate, + accuracy_rate_stderr=accuracy_rate_stderr, ) diff --git a/python/src/beval/types.py b/python/src/beval/types.py index 5a9bdd5..08a95b4 100644 --- a/python/src/beval/types.py +++ b/python/src/beval/types.py @@ -226,3 +226,13 @@ class RunSummary: errored: int total: int metrics: dict[str, float] + # Bar 4 aggregate metrics + effective_latency: float | None = None + effective_latency_stderr: float | None = None + p95_latency: float | None = None + fail_rate: float | None = None + fail_rate_stderr: float | None = None + keyword_recall: float | None = None + keyword_recall_stderr: float | None = None + accuracy_rate: float | None = None + accuracy_rate_stderr: float | None = None From 6049f4396ef792147e02edfc5e2b474dec6f8a83 Mon Sep 17 00:00:00 2001 From: Eugene Bobukh Date: Thu, 23 Jul 2026 16:23:04 -0700 Subject: [PATCH 10/15] Bug fix for redacted keyword_recall metrics --- python/src/beval/reporter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/src/beval/reporter.py b/python/src/beval/reporter.py index f5ad811..9b57252 100644 --- a/python/src/beval/reporter.py +++ b/python/src/beval/reporter.py @@ -47,8 +47,9 @@ } # Patterns identifying sensitive keys for scrubbing (§10.1). +# Excludes keyword_recall and keyword_recall_stderr via negative lookbehind. _SENSITIVE_KEY_RE = re.compile( - r"(secret|password|token|key|credential|auth|api.?key)", + r"(? Date: Fri, 24 Jul 2026 12:35:53 -0700 Subject: [PATCH 11/15] Regex fix for keyword_recall being redacted --- python/src/beval/reporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/beval/reporter.py b/python/src/beval/reporter.py index 9b57252..fae417c 100644 --- a/python/src/beval/reporter.py +++ b/python/src/beval/reporter.py @@ -47,9 +47,9 @@ } # Patterns identifying sensitive keys for scrubbing (§10.1). -# Excludes keyword_recall and keyword_recall_stderr via negative lookbehind. +# Excludes keyword_recall and keyword_recall_stderr. _SENSITIVE_KEY_RE = re.compile( - r"(? Date: Tue, 28 Jul 2026 12:01:00 -0700 Subject: [PATCH 12/15] fix: normalize thousands-separator commas in keyword graders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strip comma thousands separators (e.g. 1,000 → 1000) from both the keyword and the agent response before matching. This fixes false negatives where a case expects "9786" but the agent responds with "9,786.07". Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/beval/graders/deterministic.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/python/src/beval/graders/deterministic.py b/python/src/beval/graders/deterministic.py index 5e1aee0..82b60fe 100644 --- a/python/src/beval/graders/deterministic.py +++ b/python/src/beval/graders/deterministic.py @@ -12,6 +12,14 @@ from beval.graders import grader from beval.types import EvalContext, Grade, GraderLayer, Subject +# Matches numbers with comma thousands separators (e.g. "1,000" or "32,548.45"). +_COMMA_NUMBER_RE = re.compile(r"\d{1,3}(?:,\d{3})+(?:\.\d+)?") + + +def _strip_thousands_commas(text: str) -> str: + """Remove thousands-separator commas from numbers in *text*.""" + return _COMMA_NUMBER_RE.sub(lambda m: m.group().replace(",", ""), text) + @grader( "completion time should be under", @@ -49,7 +57,11 @@ def _response_contains_grader( ) -> Grade: """Grade based on keyword/phrase presence in output.""" keyword = args[0].lower() if args else "" - found = keyword in subject.answer.lower() + answer = subject.answer.lower() + found = keyword in answer + if not found: + # Retry with thousands-separator commas stripped from both sides. + found = _strip_thousands_commas(keyword) in _strip_thousands_commas(answer) return Grade( criterion=criterion, score=1.0 if found else 0.0, @@ -66,7 +78,11 @@ def _response_not_contains_grader( ) -> Grade: """Grade based on keyword/phrase absence in output.""" keyword = args[0].lower() if args else "" - absent = keyword not in subject.answer.lower() + answer = subject.answer.lower() + absent = keyword not in answer + if absent: + # Also check with thousands-separator commas stripped. + absent = _strip_thousands_commas(keyword) not in _strip_thousands_commas(answer) return Grade( criterion=criterion, score=1.0 if absent else 0.0, From 44740ebfbefe27ceeaa2246eb2cfc30bb851a079 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Tue, 28 Jul 2026 12:09:11 -0700 Subject: [PATCH 13/15] fix: normalize trailing zeros in keyword graders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strip trailing decimal zeros from both keyword and output before matching (e.g. 104.90 → 104.9, 1.00 → 1). Combined with the existing comma normalization, this handles all common number formatting variations without losing meaningful precision. Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/beval/graders/deterministic.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/python/src/beval/graders/deterministic.py b/python/src/beval/graders/deterministic.py index 82b60fe..79bc014 100644 --- a/python/src/beval/graders/deterministic.py +++ b/python/src/beval/graders/deterministic.py @@ -15,12 +15,27 @@ # Matches numbers with comma thousands separators (e.g. "1,000" or "32,548.45"). _COMMA_NUMBER_RE = re.compile(r"\d{1,3}(?:,\d{3})+(?:\.\d+)?") +# Matches any decimal number (e.g. "104.90", "1.00", "3.0"). +_DECIMAL_RE = re.compile(r"\d+\.\d+") + def _strip_thousands_commas(text: str) -> str: """Remove thousands-separator commas from numbers in *text*.""" return _COMMA_NUMBER_RE.sub(lambda m: m.group().replace(",", ""), text) +def _normalize_trailing_zeros(text: str) -> str: + """Strip trailing zeros from decimals: 104.90 → 104.9, 1.00 → 1.""" + return _DECIMAL_RE.sub( + lambda m: m.group().rstrip("0").rstrip("."), text + ) + + +def _normalize_numbers(text: str) -> str: + """Normalize number formatting: strip commas and trailing zeros.""" + return _normalize_trailing_zeros(_strip_thousands_commas(text)) + + @grader( "completion time should be under", layer=GraderLayer.DETERMINISTIC, @@ -60,8 +75,8 @@ def _response_contains_grader( answer = subject.answer.lower() found = keyword in answer if not found: - # Retry with thousands-separator commas stripped from both sides. - found = _strip_thousands_commas(keyword) in _strip_thousands_commas(answer) + # Retry with normalized number formatting (commas, trailing zeros). + found = _normalize_numbers(keyword) in _normalize_numbers(answer) return Grade( criterion=criterion, score=1.0 if found else 0.0, @@ -81,8 +96,8 @@ def _response_not_contains_grader( answer = subject.answer.lower() absent = keyword not in answer if absent: - # Also check with thousands-separator commas stripped. - absent = _strip_thousands_commas(keyword) not in _strip_thousands_commas(answer) + # Also check with normalized number formatting (commas, trailing zeros). + absent = _normalize_numbers(keyword) not in _normalize_numbers(answer) return Grade( criterion=criterion, score=1.0 if absent else 0.0, From d2d9fb8a786bf81aca1ad8c758180b998e332e6c Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Tue, 28 Jul 2026 13:29:33 -0700 Subject: [PATCH 14/15] fix: add bidirectional rounding match to keyword graders When a numeric keyword isn't found literally, check if rounding the more-precise value to the less-precise one's scale produces a match. Works both ways: keyword 1.577 matches output 1.58, and keyword 9786 matches output 9786.07. Limited to at most 2 decimal places difference to avoid overly aggressive matching. Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/beval/graders/deterministic.py | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/python/src/beval/graders/deterministic.py b/python/src/beval/graders/deterministic.py index 79bc014..99b815a 100644 --- a/python/src/beval/graders/deterministic.py +++ b/python/src/beval/graders/deterministic.py @@ -18,6 +18,9 @@ # Matches any decimal number (e.g. "104.90", "1.00", "3.0"). _DECIMAL_RE = re.compile(r"\d+\.\d+") +# Matches any number (integer or decimal, with optional comma separators). +_NUMBER_RE = re.compile(r"\d[\d,]*(?:\.\d+)?") + def _strip_thousands_commas(text: str) -> str: """Remove thousands-separator commas from numbers in *text*.""" @@ -36,6 +39,46 @@ def _normalize_numbers(text: str) -> str: return _normalize_trailing_zeros(_strip_thousands_commas(text)) +def _decimal_places(s: str) -> int: + """Return the number of decimal places in a numeric string.""" + if "." in s: + return len(s.rsplit(".", 1)[1]) + return 0 + + +def _rounding_match(keyword: str, text: str) -> bool: + """Check if *keyword* matches any number in *text* after rounding. + + Rounding is checked in both directions: + - keyword rounded to output precision (e.g. kw ``1.577`` matches ``1.58``) + - output rounded to keyword precision (e.g. kw ``9786`` matches ``9786.07``) + + At most 2 decimal places may be rounded away in either direction. + """ + try: + kw_val = float(keyword.replace(",", "")) + except ValueError: + return False + kw_dp = _decimal_places(keyword.replace(",", "")) + for m in _NUMBER_RE.finditer(text): + token = m.group().replace(",", "") + try: + out_val = float(token) + except ValueError: + continue + out_dp = _decimal_places(token) + dp_diff = abs(kw_dp - out_dp) + # Only allow rounding away at most 2 decimal places. + if dp_diff > 2: + continue + # Round the more-precise value to the less-precise one's scale. + if out_dp <= kw_dp and round(kw_val, out_dp) == out_val: + return True + if kw_dp <= out_dp and round(out_val, kw_dp) == kw_val: + return True + return False + + @grader( "completion time should be under", layer=GraderLayer.DETERMINISTIC, @@ -77,6 +120,9 @@ def _response_contains_grader( if not found: # Retry with normalized number formatting (commas, trailing zeros). found = _normalize_numbers(keyword) in _normalize_numbers(answer) + if not found: + # Retry with rounding: keyword 1.577 matches output 1.58. + found = _rounding_match(keyword, answer) return Grade( criterion=criterion, score=1.0 if found else 0.0, @@ -98,6 +144,9 @@ def _response_not_contains_grader( if absent: # Also check with normalized number formatting (commas, trailing zeros). absent = _normalize_numbers(keyword) not in _normalize_numbers(answer) + if absent: + # Also check with rounding awareness. + absent = not _rounding_match(keyword, answer) return Grade( criterion=criterion, score=1.0 if absent else 0.0, From 75df000d80ac776a587a1d84c282ebc2a3013440 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Fri, 31 Jul 2026 13:58:29 -0700 Subject: [PATCH 15/15] fix: use AzureOpenAI client for Foundry judge to auto-refresh tokens The Foundry entra_id path was using plain OpenAI client with a static bearer token baked into headers. After 60 minutes the token expired, causing 401 errors for all remaining cases. Switch to AzureOpenAI client with azure_ad_token_provider so tokens refresh automatically on every request. Co-Authored-By: Claude Opus 4.6 (1M context) --- python/src/beval/judge.py | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/python/src/beval/judge.py b/python/src/beval/judge.py index 22433e3..c9652af 100644 --- a/python/src/beval/judge.py +++ b/python/src/beval/judge.py @@ -364,36 +364,12 @@ def _create_client_explicit( raise ImportError(msg) from exc if _is_foundry and base_url: - # Azure AI Foundry: use plain OpenAI with bearer token header. - # Preserve the full path but strip trailing endpoint segments - # (/responses, /chat/completions) so the SDK can append its own. - # Append /openai/v1 so the SDK constructs the correct URL - # (e.g. .../openai/v1/chat/completions). - parsed = urlparse(base_url) - path = parsed.path.rstrip("/") - for suffix in ( - "/openai/v1", - "/responses", - "/chat/completions", - "/completions", - ): - if path.endswith(suffix): - path = path[: -len(suffix)] - break - clean_url = f"{parsed.scheme}://{parsed.netloc}{path}/openai/v1" scope = "https://ai.azure.com/.default" - token_provider = get_bearer_token_provider( - DefaultAzureCredential(), scope - ) - return openai_cls( - api_key="placeholder", # required by OpenAI SDK but unused - base_url=clean_url, - default_headers={"Authorization": f"Bearer {token_provider()}"}, - ) - - # Classic Azure OpenAI: use AzureOpenAI with azure_endpoint. - scope = "https://cognitiveservices.azure.com/.default" - token_provider = get_bearer_token_provider(DefaultAzureCredential(), scope) + else: + scope = "https://cognitiveservices.azure.com/.default" + token_provider = get_bearer_token_provider( + DefaultAzureCredential(), scope + ) api_version = os.environ.get("OPENAI_API_VERSION", "2024-12-01-preview") kwargs: dict[str, Any] = { "azure_ad_token_provider": token_provider,