diff --git a/python/pyproject.toml b/python/pyproject.toml index ae588c0..63733bb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -40,11 +40,12 @@ tracing = [ "opentelemetry-api>=1.20.0", "opentelemetry-sdk>=1.20.0", ] +foundry = ["azure-ai-projects>=2.0.0", "azure-identity>=1.15.0"] acp = ["agent-client-protocol>=0.1.0"] a2a = ["a2a-sdk>=0.3.0", "httpx>=0.27.0"] judge = ["openai>=1.0.0", "azure-identity>=1.15.0"] a2a-server = ["a2a-sdk[http-server]>=0.3.0", "uvicorn>=0.30.0", "fastapi>=0.115.0", "openai>=1.0.0", "azure-identity>=1.15.0"] -all = ["beval[validate,tracing,acp,a2a,judge]"] +all = ["beval[validate,tracing,acp,a2a,foundry,judge]"] dev = [ "beval[all]", "pytest>=8.0", @@ -100,7 +101,7 @@ module = ["azure.*"] ignore_missing_imports = true [[tool.mypy.overrides]] -module = ["beval.adapters.acp", "beval.adapters.a2a"] +module = ["beval.adapters.acp", "beval.adapters.a2a", "beval.adapters.foundry"] disallow_untyped_defs = false check_untyped_defs = false warn_return_any = false diff --git a/python/src/beval/adapters/__init__.py b/python/src/beval/adapters/__init__.py index a7408b7..185d673 100644 --- a/python/src/beval/adapters/__init__.py +++ b/python/src/beval/adapters/__init__.py @@ -83,7 +83,7 @@ def _replace(m: re.Match[str]) -> str: _REQUIRED_FIELDS = {"name", "protocol", "connection"} -_KNOWN_PROTOCOLS = {"acp", "a2a", "custom"} +_KNOWN_PROTOCOLS = {"acp", "a2a", "custom", "foundry"} def load_agent( @@ -200,6 +200,10 @@ def create_adapter( from beval.adapters.a2a import A2AAdapter return A2AAdapter(agent_def) + elif protocol == "foundry": + from beval.adapters.foundry import FoundryAdapter + + return FoundryAdapter(agent_def) elif protocol == "custom": from beval.adapters.custom import CustomAdapter diff --git a/samples/foundry-agent/beval_foundry_adapter.py b/python/src/beval/adapters/foundry.py similarity index 65% rename from samples/foundry-agent/beval_foundry_adapter.py rename to python/src/beval/adapters/foundry.py index 42f24d8..6db72f3 100644 --- a/samples/foundry-agent/beval_foundry_adapter.py +++ b/python/src/beval/adapters/foundry.py @@ -1,34 +1,56 @@ -"""beval custom adapter for Microsoft Foundry Agent Service.""" +"""Foundry Agent Service adapter. See SPEC §13.4. + +Uses the ``azure-ai-projects`` SDK with Entra ID authentication. +Requires ``azure-ai-projects`` and ``azure-identity`` packages. +""" from __future__ import annotations import time from typing import Any -from azure.ai.projects import AIProjectClient -from azure.core.exceptions import ClientAuthenticationError, HttpResponseError -from azure.identity import DefaultAzureCredential - from beval.adapters import AdapterInput, AdapterInterface from beval.types import Subject class FoundryAdapter(AdapterInterface): - """Connects beval to a Microsoft Foundry prompt agent.""" + """Adapter for Microsoft Foundry prompt agents. + + Uses ``AIProjectClient`` from the ``azure-ai-projects`` SDK to invoke + agents via the OpenAI-compatible Responses API. + + Configuration (``connection`` keys): + - ``endpoint``: Foundry project endpoint (required) + - ``agent_name``: Name of the Foundry agent to invoke + - ``model``: Fallback model when agent_name is not set (default: gpt-4o) + """ + + def __init__(self, agent_def: dict[str, Any]) -> None: + try: + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + except ImportError as exc: + raise ImportError( + "Foundry adapter requires azure-ai-projects and azure-identity. " + "Install with: pip install beval[foundry]" + ) from exc - def __init__(self, config: dict[str, Any]) -> None: + connection = agent_def.get("connection", {}) + config = connection.get("config", connection) endpoint = config.get("endpoint") if not endpoint: import sys print( - "Error: set FOUNDRY_PROJECT_ENDPOINT (adapter config 'endpoint' is required).", + "Error: set FOUNDRY_PROJECT_ENDPOINT " + "(adapter config 'endpoint' is required).", file=sys.stderr, ) raise SystemExit(2) self._agent_name = config.get("agent_name") self._model = config.get("model", "gpt-4o") + self._timeout = agent_def.get("timeout", 60) self._credential = DefaultAzureCredential() self._client = AIProjectClient( endpoint=endpoint, @@ -38,11 +60,11 @@ def __init__(self, config: dict[str, Any]) -> None: self._conversation_id: str | None = None def invoke(self, adapter_input: AdapterInput) -> Subject: + from azure.core.exceptions import ClientAuthenticationError + query = adapter_input.query if isinstance(query, list): - query = " ".join( - m.get("content", "") for m in query if isinstance(m, dict) - ) + query = " ".join(m.get("content", "") for m in query if isinstance(m, dict)) start = time.monotonic() @@ -72,9 +94,6 @@ def invoke(self, adapter_input: AdapterInput) -> Subject: file=sys.stderr, ) raise SystemExit(3) from exc - raise RuntimeError( - f"Foundry API error (status {exc.status_code}): {exc.message}" - ) from exc except TimeoutError as exc: raise RuntimeError(f"Foundry API timed out: {exc}") from exc diff --git a/python/src/beval/graders/deterministic.py b/python/src/beval/graders/deterministic.py index 5e1aee0..99b815a 100644 --- a/python/src/beval/graders/deterministic.py +++ b/python/src/beval/graders/deterministic.py @@ -12,6 +12,72 @@ from beval.graders import grader from beval.types import EvalContext, Grade, GraderLayer, Subject +# Matches numbers with comma thousands separators (e.g. "1,000" or "32,548.45"). +_COMMA_NUMBER_RE = re.compile(r"\d{1,3}(?:,\d{3})+(?:\.\d+)?") + +# Matches any decimal number (e.g. "104.90", "1.00", "3.0"). +_DECIMAL_RE = re.compile(r"\d+\.\d+") + +# Matches any number (integer or decimal, with optional comma separators). +_NUMBER_RE = re.compile(r"\d[\d,]*(?:\.\d+)?") + + +def _strip_thousands_commas(text: str) -> str: + """Remove thousands-separator commas from numbers in *text*.""" + return _COMMA_NUMBER_RE.sub(lambda m: m.group().replace(",", ""), text) + + +def _normalize_trailing_zeros(text: str) -> str: + """Strip trailing zeros from decimals: 104.90 → 104.9, 1.00 → 1.""" + return _DECIMAL_RE.sub( + lambda m: m.group().rstrip("0").rstrip("."), text + ) + + +def _normalize_numbers(text: str) -> str: + """Normalize number formatting: strip commas and trailing zeros.""" + return _normalize_trailing_zeros(_strip_thousands_commas(text)) + + +def _decimal_places(s: str) -> int: + """Return the number of decimal places in a numeric string.""" + if "." in s: + return len(s.rsplit(".", 1)[1]) + return 0 + + +def _rounding_match(keyword: str, text: str) -> bool: + """Check if *keyword* matches any number in *text* after rounding. + + Rounding is checked in both directions: + - keyword rounded to output precision (e.g. kw ``1.577`` matches ``1.58``) + - output rounded to keyword precision (e.g. kw ``9786`` matches ``9786.07``) + + At most 2 decimal places may be rounded away in either direction. + """ + try: + kw_val = float(keyword.replace(",", "")) + except ValueError: + return False + kw_dp = _decimal_places(keyword.replace(",", "")) + for m in _NUMBER_RE.finditer(text): + token = m.group().replace(",", "") + try: + out_val = float(token) + except ValueError: + continue + out_dp = _decimal_places(token) + dp_diff = abs(kw_dp - out_dp) + # Only allow rounding away at most 2 decimal places. + if dp_diff > 2: + continue + # Round the more-precise value to the less-precise one's scale. + if out_dp <= kw_dp and round(kw_val, out_dp) == out_val: + return True + if kw_dp <= out_dp and round(out_val, kw_dp) == kw_val: + return True + return False + @grader( "completion time should be under", @@ -49,7 +115,14 @@ def _response_contains_grader( ) -> Grade: """Grade based on keyword/phrase presence in output.""" keyword = args[0].lower() if args else "" - found = keyword in subject.answer.lower() + answer = subject.answer.lower() + found = keyword in answer + if not found: + # Retry with normalized number formatting (commas, trailing zeros). + found = _normalize_numbers(keyword) in _normalize_numbers(answer) + if not found: + # Retry with rounding: keyword 1.577 matches output 1.58. + found = _rounding_match(keyword, answer) return Grade( criterion=criterion, score=1.0 if found else 0.0, @@ -66,7 +139,14 @@ def _response_not_contains_grader( ) -> Grade: """Grade based on keyword/phrase absence in output.""" keyword = args[0].lower() if args else "" - absent = keyword not in subject.answer.lower() + answer = subject.answer.lower() + absent = keyword not in answer + if absent: + # Also check with normalized number formatting (commas, trailing zeros). + absent = _normalize_numbers(keyword) not in _normalize_numbers(answer) + if absent: + # Also check with rounding awareness. + absent = not _rounding_match(keyword, answer) return Grade( criterion=criterion, score=1.0 if absent else 0.0, diff --git a/python/src/beval/judge.py b/python/src/beval/judge.py index 0faf241..c9652af 100644 --- a/python/src/beval/judge.py +++ b/python/src/beval/judge.py @@ -364,29 +364,12 @@ def _create_client_explicit( raise ImportError(msg) from exc if _is_foundry and base_url: - # Azure AI Foundry: use plain OpenAI with bearer token header. - # Preserve the full path but strip trailing endpoint segments - # (/responses, /chat/completions) so the SDK can append its own. - parsed = urlparse(base_url) - path = parsed.path.rstrip("/") - for suffix in ("/responses", "/chat/completions", "/completions"): - if path.endswith(suffix): - path = path[: -len(suffix)] - break - clean_url = f"{parsed.scheme}://{parsed.netloc}{path}" scope = "https://ai.azure.com/.default" - token_provider = get_bearer_token_provider( - DefaultAzureCredential(), scope - ) - return openai_cls( - api_key="placeholder", # required by OpenAI SDK but unused - base_url=clean_url, - default_headers={"Authorization": f"Bearer {token_provider()}"}, - ) - - # Classic Azure OpenAI: use AzureOpenAI with azure_endpoint. - scope = "https://cognitiveservices.azure.com/.default" - token_provider = get_bearer_token_provider(DefaultAzureCredential(), scope) + else: + scope = "https://cognitiveservices.azure.com/.default" + token_provider = get_bearer_token_provider( + DefaultAzureCredential(), scope + ) api_version = os.environ.get("OPENAI_API_VERSION", "2024-12-01-preview") kwargs: dict[str, Any] = { "azure_ad_token_provider": token_provider, @@ -399,17 +382,23 @@ def _create_client_explicit( if _is_foundry and base_url: # Azure AI Foundry with api_key: plain OpenAI client, full path. + # Append /openai/v1 so the SDK constructs the correct URL. parsed = urlparse(base_url) path = parsed.path.rstrip("/") - for suffix in ("/responses", "/chat/completions", "/completions"): + for suffix in ( + "/openai/v1", + "/responses", + "/chat/completions", + "/completions", + ): if path.endswith(suffix): path = path[: -len(suffix)] break - clean_url = f"{parsed.scheme}://{parsed.netloc}{path}" - kwargs = {"base_url": clean_url} + clean_url = f"{parsed.scheme}://{parsed.netloc}{path}/openai/v1" + foundry_kwargs: dict[str, Any] = {"base_url": clean_url} if api_key: - kwargs["api_key"] = api_key - return openai_cls(**kwargs) + foundry_kwargs["api_key"] = api_key + return openai_cls(**foundry_kwargs) if _is_azure_oai and base_url: # Classic Azure OpenAI with api_key: AzureOpenAI, strip path. diff --git a/python/src/beval/reporter.py b/python/src/beval/reporter.py index c8c00bc..fae417c 100644 --- a/python/src/beval/reporter.py +++ b/python/src/beval/reporter.py @@ -47,8 +47,9 @@ } # Patterns identifying sensitive keys for scrubbing (§10.1). +# Excludes keyword_recall and keyword_recall_stderr. _SENSITIVE_KEY_RE = re.compile( - r"(secret|password|token|key|credential|auth|api.?key)", + r"^(?!keyword_recall(_stderr)?$).*(secret|password|token|key|credential|auth|api.?key)", re.IGNORECASE, ) _REDACTED = "***REDACTED***" @@ -77,6 +78,10 @@ def _prepare(result: RunResult, *, scrub: bool = False) -> dict[str, Any]: if "config" in raw: raw["config"] = _strip_defaults(raw["config"]) + # Summary: drop null optional fields (Bar 4 metrics) + if "summary" in raw and isinstance(raw["summary"], dict): + raw["summary"] = {k: v for k, v in raw["summary"].items() if v is not None} + # Cases: drop null optional fields and false high_variance for case in raw.get("cases", []): for key in _CASE_OPTIONAL: diff --git a/python/src/beval/runner.py b/python/src/beval/runner.py index e56cc6a..188fa0d 100644 --- a/python/src/beval/runner.py +++ b/python/src/beval/runner.py @@ -6,6 +6,7 @@ from __future__ import annotations +import math import statistics import time from typing import Any @@ -74,6 +75,37 @@ def _filter_cases( return result +def _process_latency( + times: list[float], max_latency: float = 100.0 +) -> tuple[float, float]: + """Compute effective latency via geometric mean of (time+1), capped. + + Returns (effective_latency, stderr) per Bar 4 spec. + """ + capped = [min(t, max_latency) for t in times] + xl = [math.log(1.0 + t) for t in capped] + avg = statistics.mean(xl) + se = avg + if len(xl) > 1: + se = statistics.stdev(xl) / math.sqrt(len(xl) - 1) + eff = math.exp(avg) - 1.0 + stderr = (math.exp(avg + se) - math.exp(avg - se)) / 2.0 + return eff, stderr + + +def _compute_pass_rate(values: list[bool]) -> tuple[float, float]: + """Compute pass rate with binomial stderr. + + Returns (rate, stderr). + """ + n = len(values) + if n == 0: + return 0.0, 0.0 + p = sum(1 for v in values if v) / n + se = math.sqrt(p * (1 - p) / n) if n > 1 else 0.0 + return p, se + + def _aggregate_grades( grades: list[Grade], config: RunConfig, @@ -716,6 +748,68 @@ def _build_summary(self, cases: list[CaseResult]) -> RunSummary: for m in metrics: metrics[m] /= metric_counts[m] + # Bar 4: Effective latency — geometric mean of (time+1), capped at 100s + effective_latency: float | None = None + effective_latency_stderr: float | None = None + p95_latency: float | None = None + times = [ + c.time_seconds + for c in cases + if c.time_seconds is not None and c.time_seconds >= 0 + ] + if times: + effective_latency, effective_latency_stderr = _process_latency(times) + sorted_times = sorted(times) + p95_idx = max(0, int(math.ceil(0.95 * len(sorted_times))) - 1) + p95_latency = sorted_times[p95_idx] + + # Bar 4: Fail rate — binomial proportion with stderr + n_total = len(cases) + fail_rate: float | None = None + fail_rate_stderr: float | None = None + if n_total > 0: + fail_rate = errored / n_total + fail_rate_stderr = ( + math.sqrt(fail_rate * (1 - fail_rate) / n_total) if n_total > 1 else 0.0 + ) + + # Bar 4: Keyword recall — from "response should contain" grades + recall_values: list[float] = [] + for c in non_errored: + keyword_grades = [ + g + for g in c.grades + if g.criterion.lower().startswith("response should contain") + and not g.skipped + ] + if keyword_grades: + case_recall = sum(g.score for g in keyword_grades) / len(keyword_grades) + recall_values.append(case_recall) + keyword_recall: float | None = None + keyword_recall_stderr: float | None = None + if recall_values: + keyword_recall = statistics.mean(recall_values) + n_recall = len(recall_values) + keyword_recall_stderr = ( + math.sqrt(keyword_recall * (1 - keyword_recall) / n_recall) + if n_recall > 1 + else 0.0 + ) + + # Bar 4: Accuracy rate — from ai_judged "the answer should" grades + accuracy_values: list[bool] = [] + for c in non_errored: + for g in c.grades: + if ( + g.criterion.lower().startswith("the answer should") + and not g.skipped + ): + accuracy_values.append(g.passed) + accuracy_rate: float | None = None + accuracy_rate_stderr: float | None = None + if accuracy_values: + accuracy_rate, accuracy_rate_stderr = _compute_pass_rate(accuracy_values) + return RunSummary( overall_score=overall, passed=passed, @@ -723,4 +817,13 @@ def _build_summary(self, cases: list[CaseResult]) -> RunSummary: errored=errored, total=len(cases), metrics=metrics, + effective_latency=effective_latency, + effective_latency_stderr=effective_latency_stderr, + p95_latency=p95_latency, + fail_rate=fail_rate, + fail_rate_stderr=fail_rate_stderr, + keyword_recall=keyword_recall, + keyword_recall_stderr=keyword_recall_stderr, + accuracy_rate=accuracy_rate, + accuracy_rate_stderr=accuracy_rate_stderr, ) diff --git a/python/src/beval/types.py b/python/src/beval/types.py index 5a9bdd5..08a95b4 100644 --- a/python/src/beval/types.py +++ b/python/src/beval/types.py @@ -226,3 +226,13 @@ class RunSummary: errored: int total: int metrics: dict[str, float] + # Bar 4 aggregate metrics + effective_latency: float | None = None + effective_latency_stderr: float | None = None + p95_latency: float | None = None + fail_rate: float | None = None + fail_rate_stderr: float | None = None + keyword_recall: float | None = None + keyword_recall_stderr: float | None = None + accuracy_rate: float | None = None + accuracy_rate_stderr: float | None = None diff --git a/samples/foundry-agent/README.md b/samples/foundry-agent/README.md index b411c35..6dd4038 100644 --- a/samples/foundry-agent/README.md +++ b/samples/foundry-agent/README.md @@ -1,22 +1,21 @@ --- title: Foundry Agent Sample Evaluation -description: Evaluates a Microsoft Foundry prompt agent using beval's custom adapter protocol with Entra ID authentication. -ms.date: 2026-06-19 +description: Evaluates a Microsoft Foundry prompt agent using beval's built-in Foundry adapter with Entra ID authentication. +ms.date: 2026-07-15 ms.topic: tutorial --- ## Foundry Agent Sample Evaluation -This sample evaluates a Microsoft Foundry prompt agent using beval's custom -adapter protocol. The adapter connects directly to the Foundry Responses API +This sample evaluates a Microsoft Foundry prompt agent using beval's built-in +`foundry` adapter protocol. The adapter connects to the Foundry Responses API via the `azure-ai-projects` SDK with Entra ID authentication. ## Files ```text -├── agent.yaml # Agent definition (custom adapter) -├── beval_foundry_adapter.py # Custom adapter implementation -├── eval.config.yaml # Evaluation configuration (judge, agents) +├── agent.yaml # Agent definition +├── eval.config.yaml # Evaluation configuration ├── cases/ │ └── foundry-eval.yaml # 2 evaluation cases └── README.md @@ -27,7 +26,7 @@ via the `azure-ai-projects` SDK with Entra ID authentication. * Python 3.10+ * Azure CLI (`az login`) for Entra ID authentication * A Microsoft Foundry project with a deployed prompt agent (or model deployment) -* **Foundry User** role assigned at the project level +* **Foundry Project Manager** role assigned at the project scope * (Optional) [Ollama](https://ollama.com) for local LLM judging ## Environment Variables @@ -36,9 +35,9 @@ via the `azure-ai-projects` SDK with Entra ID authentication. |----------------------------|----------|------------------------------------------------------| | `FOUNDRY_PROJECT_ENDPOINT` | Yes | Foundry project endpoint URL | | `FOUNDRY_AGENT_NAME` | No | Agent name (omit for direct model calls) | -| `FOUNDRY_MODEL_NAME` | No | Model name (default: `gpt-4o`) | +| `FOUNDRY_MODEL_NAME` | No | Fallback model when agent name is not set (default: `gpt-4o`) | | `JUDGE_MODEL_NAME` | No | Ollama judge model (default: `gemma4:e2b`) | -| `OPENAI_API_KEY` | No | Required for LLM judge; set to `ollama` when local | +| `OPENAI_API_KEY` | No | Required for validation mode; set to `ollama` for local Ollama | ## Setup @@ -48,75 +47,48 @@ via the `azure-ai-projects` SDK with Entra ID authentication. az login ``` -1. Install the required Python packages: +1. Install beval with the Foundry extra: ```bash - pip install azure-ai-projects azure-identity + pip install beval[foundry] ``` -1. Set the required environment variables. `FOUNDRY_PROJECT_ENDPOINT` is - required; the others have defaults or are optional. - - **Bash / Linux / macOS:** +1. Set the required environment variables: ```bash export FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" export FOUNDRY_AGENT_NAME="MyAgent" ``` - **PowerShell / Windows:** - - ```powershell - $env:FOUNDRY_PROJECT_ENDPOINT = "https://.services.ai.azure.com/api/projects/" - $env:FOUNDRY_AGENT_NAME = "MyAgent" - ``` - ## Run -The custom adapter module (`beval_foundry_adapter.py`) must be importable when -beval starts. Run from this directory with `PYTHONPATH=.` so Python can find -the module. - -**Dev mode** runs only deterministic graders (fast, no LLM judge required): +**Dev mode** runs only deterministic graders (completion time, response length — no LLM judge required): ```bash cd samples/foundry-agent -PYTHONPATH=. beval run --cases cases --agent agent.yaml -m dev --verbose +beval -c eval.config.yaml run --cases cases -m dev --verbose ``` **Validation mode** includes AI-judged criteria and requires a judge -configured in `eval.config.yaml` (see [LLM Judge](#llm-judge) below). Pass -`-c` before the `run` subcommand: +configured in `eval.config.yaml` (see [LLM Judge](#llm-judge) below): ```bash -PYTHONPATH=. OPENAI_API_KEY=ollama beval -c eval.config.yaml run \ - --cases cases --agent agent.yaml -m validation --verbose +OPENAI_API_KEY=ollama beval -c eval.config.yaml run --cases cases --verbose ``` -On PowerShell, set environment variables first: - -```powershell -cd samples\foundry-agent -$env:PYTHONPATH = "." -$env:OPENAI_API_KEY = "ollama" -beval -c eval.config.yaml run --cases cases --agent agent.yaml -m validation --verbose -``` - -**From the `python/` directory:** +**From the `python/` directory** (development): ```bash cd python -PYTHONPATH=../samples/foundry-agent uv run beval run \ +uv run beval -c ../samples/foundry-agent/eval.config.yaml run \ --cases ../samples/foundry-agent/cases \ - --agent ../samples/foundry-agent/agent.yaml \ -m dev --verbose ``` ## LLM Judge -The `eval.config.yaml` file configures the LLM judge used for AI-judged -graders in validation mode. Any OpenAI-compatible endpoint works, including -Ollama for fully local evaluation. +The `eval.config.yaml` configures an Ollama-based LLM judge for validation +mode. Any OpenAI-compatible endpoint works. ### Ollama (local) @@ -126,18 +98,6 @@ Ollama for fully local evaluation. ollama pull gemma4:e2b ``` -1. The judge is already configured in `eval.config.yaml`: - - ```yaml - eval: - judge: - protocol: openai - model: ${JUDGE_MODEL_NAME:-gemma4:e2b} - base_url: http://localhost:11434/v1 - ``` - - Override the model at runtime by setting `JUDGE_MODEL_NAME`. - 1. Set `OPENAI_API_KEY` to any non-empty value (Ollama ignores it, but the `openai` Python package requires it): @@ -145,12 +105,10 @@ Ollama for fully local evaluation. export OPENAI_API_KEY=ollama ``` -> [!NOTE] -> Judge quality depends on model reasoning capability. Larger models like -> `gemma4` or `llama3:70b` produce more reliable scores. - ### OpenAI +Override the judge in `eval.config.yaml`: + ```yaml eval: judge: @@ -160,31 +118,21 @@ eval: Set `OPENAI_API_KEY` to your actual API key. -## Inline Agent Definition (Alternative) +## Agent Definition -You can define the agent inline in `eval.config.yaml` instead of using a -separate `agent.yaml`: +The agent is defined inline in `eval.config.yaml` using the built-in `foundry` +protocol: ```yaml eval: - mode: validation agents: default: foundry-agent definitions: - name: foundry-agent - protocol: custom + protocol: foundry connection: - module: beval_foundry_adapter - class: FoundryAdapter - config: - endpoint: ${FOUNDRY_PROJECT_ENDPOINT} - agent_name: ${FOUNDRY_AGENT_NAME:-} - model: ${FOUNDRY_MODEL_NAME:-gpt-4o} + endpoint: ${FOUNDRY_PROJECT_ENDPOINT} + agent_name: ${FOUNDRY_AGENT_NAME:-} + model: ${FOUNDRY_MODEL_NAME:-gpt-4o} timeout: 60 ``` - -Then run without `--agent`: - -```bash -PYTHONPATH=. beval run --cases cases -``` diff --git a/samples/foundry-agent/agent.yaml b/samples/foundry-agent/agent.yaml index e4a21d8..f5b6c75 100644 --- a/samples/foundry-agent/agent.yaml +++ b/samples/foundry-agent/agent.yaml @@ -1,26 +1,15 @@ name: foundry-agent description: > - Microsoft Foundry prompt agent evaluated via beval custom adapter. + Microsoft Foundry prompt agent evaluated via beval's built-in Foundry adapter. Uses the azure-ai-projects SDK with Entra ID authentication. -protocol: custom +protocol: foundry connection: - module: beval_foundry_adapter - class: FoundryAdapter - config: - endpoint: ${FOUNDRY_PROJECT_ENDPOINT} - agent_name: ${FOUNDRY_AGENT_NAME:-} - model: ${FOUNDRY_MODEL_NAME:-gpt-4o} + endpoint: ${FOUNDRY_PROJECT_ENDPOINT} + agent_name: ${FOUNDRY_AGENT_NAME:-} + model: ${FOUNDRY_MODEL_NAME:-gpt-4o} -timeout: 60 - -# Note: retry config is defined for schema compliance but has no effect -# for custom adapters — the beval CustomAdapter wrapper does not implement -# retry logic. -retry: - max_attempts: 2 - backoff: 2.0 +timeout: 120 metadata: platform: azure-foundry - sdk_version: "2.2.0" diff --git a/samples/foundry-agent/eval.config.yaml b/samples/foundry-agent/eval.config.yaml index ebd0b8f..d5846b7 100644 --- a/samples/foundry-agent/eval.config.yaml +++ b/samples/foundry-agent/eval.config.yaml @@ -4,16 +4,13 @@ eval: default: foundry-agent definitions: - name: foundry-agent - protocol: custom + protocol: foundry connection: - module: beval_foundry_adapter - class: FoundryAdapter - config: - endpoint: ${FOUNDRY_PROJECT_ENDPOINT} - agent_name: ${FOUNDRY_AGENT_NAME:-} - model: ${FOUNDRY_MODEL_NAME:-gpt-4o} + endpoint: ${FOUNDRY_PROJECT_ENDPOINT} + agent_name: ${FOUNDRY_AGENT_NAME:-} + model: ${FOUNDRY_MODEL_NAME:-gpt-4o} timeout: 60 judge: protocol: openai - model: ${JUDGE_MODEL_NAME:-gemma4:e2b} # whatever model you've pulled in Ollama + model: ${JUDGE_MODEL_NAME:-gemma4:e2b} base_url: http://localhost:11434/v1