Add built-in Foundry adapter

python/pyproject.toml

@@ -40,11 +40,12 @@ tracing = [
     "opentelemetry-api>=1.20.0",
     "opentelemetry-sdk>=1.20.0",
 ]
+foundry = ["azure-ai-projects>=2.0.0", "azure-identity>=1.15.0"]
 acp = ["agent-client-protocol>=0.1.0"]
 a2a = ["a2a-sdk>=0.3.0", "httpx>=0.27.0"]
 judge = ["openai>=1.0.0", "azure-identity>=1.15.0"]
 a2a-server = ["a2a-sdk[http-server]>=0.3.0", "uvicorn>=0.30.0", "fastapi>=0.115.0", "openai>=1.0.0", "azure-identity>=1.15.0"]
-all = ["beval[validate,tracing,acp,a2a,judge]"]
+all = ["beval[validate,tracing,acp,a2a,foundry,judge]"]
 dev = [
     "beval[all]",
     "pytest>=8.0",
@@ -100,7 +101,7 @@ module = ["azure.*"]
 ignore_missing_imports = true
 
 [[tool.mypy.overrides]]
-module = ["beval.adapters.acp", "beval.adapters.a2a"]
+module = ["beval.adapters.acp", "beval.adapters.a2a", "beval.adapters.foundry"]
 disallow_untyped_defs = false
 check_untyped_defs = false
 warn_return_any = false

python/src/beval/adapters/__init__.py

@@ -83,7 +83,7 @@ def _replace(m: re.Match[str]) -> str:
 
 
 _REQUIRED_FIELDS = {"name", "protocol", "connection"}
-_KNOWN_PROTOCOLS = {"acp", "a2a", "custom"}
+_KNOWN_PROTOCOLS = {"acp", "a2a", "custom", "foundry"}
 
 
 def load_agent(
@@ -200,6 +200,10 @@ def create_adapter(
         from beval.adapters.a2a import A2AAdapter
 
         return A2AAdapter(agent_def)
+    elif protocol == "foundry":
+        from beval.adapters.foundry import FoundryAdapter
+
+        return FoundryAdapter(agent_def)
     elif protocol == "custom":
         from beval.adapters.custom import CustomAdapter
 

samples/foundry-agent/beval_foundry_adapter.py → python/src/beval/adapters/foundry.py

@@ -1,34 +1,56 @@
-"""beval custom adapter for Microsoft Foundry Agent Service."""
+"""Foundry Agent Service adapter. See SPEC §13.4.
+
+Uses the ``azure-ai-projects`` SDK with Entra ID authentication.
+Requires ``azure-ai-projects`` and ``azure-identity`` packages.
+"""
 
 from __future__ import annotations
 
 import time
 from typing import Any
 
-from azure.ai.projects import AIProjectClient
-from azure.core.exceptions import ClientAuthenticationError, HttpResponseError
-from azure.identity import DefaultAzureCredential
-
 from beval.adapters import AdapterInput, AdapterInterface
 from beval.types import Subject
 
 
 class FoundryAdapter(AdapterInterface):
-    """Connects beval to a Microsoft Foundry prompt agent."""
+    """Adapter for Microsoft Foundry prompt agents.
+
+    Uses ``AIProjectClient`` from the ``azure-ai-projects`` SDK to invoke
+    agents via the OpenAI-compatible Responses API.
+
+    Configuration (``connection`` keys):
+      - ``endpoint``: Foundry project endpoint (required)
+      - ``agent_name``: Name of the Foundry agent to invoke
+      - ``model``: Fallback model when agent_name is not set (default: gpt-4o)
+    """
+
+    def __init__(self, agent_def: dict[str, Any]) -> None:
+        try:
+            from azure.ai.projects import AIProjectClient
+            from azure.identity import DefaultAzureCredential
+        except ImportError as exc:
+            raise ImportError(
+                "Foundry adapter requires azure-ai-projects and azure-identity. "
+                "Install with: pip install beval[foundry]"
+            ) from exc
 
-    def __init__(self, config: dict[str, Any]) -> None:
+        connection = agent_def.get("connection", {})
+        config = connection.get("config", connection)
         endpoint = config.get("endpoint")
         if not endpoint:
             import sys
 
             print(
-                "Error: set FOUNDRY_PROJECT_ENDPOINT (adapter config 'endpoint' is required).",
+                "Error: set FOUNDRY_PROJECT_ENDPOINT "
+                "(adapter config 'endpoint' is required).",
                 file=sys.stderr,
             )
             raise SystemExit(2)
 
         self._agent_name = config.get("agent_name")
         self._model = config.get("model", "gpt-4o")
+        self._timeout = agent_def.get("timeout", 60)
         self._credential = DefaultAzureCredential()
         self._client = AIProjectClient(
             endpoint=endpoint,
@@ -38,11 +60,11 @@ def __init__(self, config: dict[str, Any]) -> None:
         self._conversation_id: str | None = None
 
     def invoke(self, adapter_input: AdapterInput) -> Subject:
+        from azure.core.exceptions import ClientAuthenticationError
+
         query = adapter_input.query
         if isinstance(query, list):
-            query = " ".join(
-                m.get("content", "") for m in query if isinstance(m, dict)
-            )
+            query = " ".join(m.get("content", "") for m in query if isinstance(m, dict))
 
         start = time.monotonic()
 
@@ -72,9 +94,6 @@ def invoke(self, adapter_input: AdapterInput) -> Subject:
                 file=sys.stderr,
             )
             raise SystemExit(3) from exc
-            raise RuntimeError(
-                f"Foundry API error (status {exc.status_code}): {exc.message}"
-            ) from exc
         except TimeoutError as exc:
             raise RuntimeError(f"Foundry API timed out: {exc}") from exc
 

python/src/beval/graders/deterministic.py

@@ -12,6 +12,72 @@
 from beval.graders import grader
 from beval.types import EvalContext, Grade, GraderLayer, Subject
 
+# Matches numbers with comma thousands separators (e.g. "1,000" or "32,548.45").
+_COMMA_NUMBER_RE = re.compile(r"\d{1,3}(?:,\d{3})+(?:\.\d+)?")
+
+# Matches any decimal number (e.g. "104.90", "1.00", "3.0").
+_DECIMAL_RE = re.compile(r"\d+\.\d+")
+
+# Matches any number (integer or decimal, with optional comma separators).
+_NUMBER_RE = re.compile(r"\d[\d,]*(?:\.\d+)?")
+
+
+def _strip_thousands_commas(text: str) -> str:
+    """Remove thousands-separator commas from numbers in *text*."""
+    return _COMMA_NUMBER_RE.sub(lambda m: m.group().replace(",", ""), text)
+
+
+def _normalize_trailing_zeros(text: str) -> str:
+    """Strip trailing zeros from decimals: 104.90 → 104.9, 1.00 → 1."""
+    return _DECIMAL_RE.sub(
+        lambda m: m.group().rstrip("0").rstrip("."), text
+    )
+
+
+def _normalize_numbers(text: str) -> str:
+    """Normalize number formatting: strip commas and trailing zeros."""
+    return _normalize_trailing_zeros(_strip_thousands_commas(text))
+
+
+def _decimal_places(s: str) -> int:
+    """Return the number of decimal places in a numeric string."""
+    if "." in s:
+        return len(s.rsplit(".", 1)[1])
+    return 0
+
+
+def _rounding_match(keyword: str, text: str) -> bool:
+    """Check if *keyword* matches any number in *text* after rounding.
+
+    Rounding is checked in both directions:
+    - keyword rounded to output precision (e.g. kw ``1.577`` matches ``1.58``)
+    - output rounded to keyword precision (e.g. kw ``9786`` matches ``9786.07``)
+
+    At most 2 decimal places may be rounded away in either direction.
+    """
+    try:
+        kw_val = float(keyword.replace(",", ""))
+    except ValueError:
+        return False
+    kw_dp = _decimal_places(keyword.replace(",", ""))
+    for m in _NUMBER_RE.finditer(text):
+        token = m.group().replace(",", "")
+        try:
+            out_val = float(token)
+        except ValueError:
+            continue
+        out_dp = _decimal_places(token)
+        dp_diff = abs(kw_dp - out_dp)
+        # Only allow rounding away at most 2 decimal places.
+        if dp_diff > 2:
+            continue
+        # Round the more-precise value to the less-precise one's scale.
+        if out_dp <= kw_dp and round(kw_val, out_dp) == out_val:
+            return True
+        if kw_dp <= out_dp and round(out_val, kw_dp) == kw_val:
+            return True
+    return False
+
 
 @grader(
     "completion time should be under",
@@ -49,7 +115,14 @@ def _response_contains_grader(
 ) -> Grade:
     """Grade based on keyword/phrase presence in output."""
     keyword = args[0].lower() if args else ""
-    found = keyword in subject.answer.lower()
+    answer = subject.answer.lower()
+    found = keyword in answer
+    if not found:
+        # Retry with normalized number formatting (commas, trailing zeros).
+        found = _normalize_numbers(keyword) in _normalize_numbers(answer)
+    if not found:
+        # Retry with rounding: keyword 1.577 matches output 1.58.
+        found = _rounding_match(keyword, answer)
     return Grade(
         criterion=criterion,
         score=1.0 if found else 0.0,
@@ -66,7 +139,14 @@ def _response_not_contains_grader(
 ) -> Grade:
     """Grade based on keyword/phrase absence in output."""
     keyword = args[0].lower() if args else ""
-    absent = keyword not in subject.answer.lower()
+    answer = subject.answer.lower()
+    absent = keyword not in answer
+    if absent:
+        # Also check with normalized number formatting (commas, trailing zeros).
+        absent = _normalize_numbers(keyword) not in _normalize_numbers(answer)
+    if absent:
+        # Also check with rounding awareness.
+        absent = not _rounding_match(keyword, answer)
     return Grade(
         criterion=criterion,
         score=1.0 if absent else 0.0,

python/src/beval/judge.py

@@ -364,29 +364,12 @@ def _create_client_explicit(
                 raise ImportError(msg) from exc
 
             if _is_foundry and base_url:
-                # Azure AI Foundry: use plain OpenAI with bearer token header.
-                # Preserve the full path but strip trailing endpoint segments
-                # (/responses, /chat/completions) so the SDK can append its own.
-                parsed = urlparse(base_url)
-                path = parsed.path.rstrip("/")
-                for suffix in ("/responses", "/chat/completions", "/completions"):
-                    if path.endswith(suffix):
-                        path = path[: -len(suffix)]
-                        break
-                clean_url = f"{parsed.scheme}://{parsed.netloc}{path}"
                 scope = "https://ai.azure.com/.default"
-                token_provider = get_bearer_token_provider(
-                    DefaultAzureCredential(), scope
-                )
-                return openai_cls(
-                    api_key="placeholder",  # required by OpenAI SDK but unused
-                    base_url=clean_url,
-                    default_headers={"Authorization": f"Bearer {token_provider()}"},
-                )
-
-            # Classic Azure OpenAI: use AzureOpenAI with azure_endpoint.
-            scope = "https://cognitiveservices.azure.com/.default"
-            token_provider = get_bearer_token_provider(DefaultAzureCredential(), scope)
+            else:
+                scope = "https://cognitiveservices.azure.com/.default"
+            token_provider = get_bearer_token_provider(
+                DefaultAzureCredential(), scope
+            )
             api_version = os.environ.get("OPENAI_API_VERSION", "2024-12-01-preview")
             kwargs: dict[str, Any] = {
                 "azure_ad_token_provider": token_provider,
@@ -399,17 +382,23 @@ def _create_client_explicit(
 
         if _is_foundry and base_url:
             # Azure AI Foundry with api_key: plain OpenAI client, full path.
+            # Append /openai/v1 so the SDK constructs the correct URL.
             parsed = urlparse(base_url)
             path = parsed.path.rstrip("/")
-            for suffix in ("/responses", "/chat/completions", "/completions"):
+            for suffix in (
+                "/openai/v1",
+                "/responses",
+                "/chat/completions",
+                "/completions",
+            ):
                 if path.endswith(suffix):
                     path = path[: -len(suffix)]
                     break
-            clean_url = f"{parsed.scheme}://{parsed.netloc}{path}"
-            kwargs = {"base_url": clean_url}
+            clean_url = f"{parsed.scheme}://{parsed.netloc}{path}/openai/v1"
+            foundry_kwargs: dict[str, Any] = {"base_url": clean_url}
             if api_key:
-                kwargs["api_key"] = api_key
-            return openai_cls(**kwargs)
+                foundry_kwargs["api_key"] = api_key
+            return openai_cls(**foundry_kwargs)
 
         if _is_azure_oai and base_url:
             # Classic Azure OpenAI with api_key: AzureOpenAI, strip path.

python/src/beval/reporter.py

@@ -47,8 +47,9 @@
 }
 
 # Patterns identifying sensitive keys for scrubbing (§10.1).
+# Excludes keyword_recall and keyword_recall_stderr.
 _SENSITIVE_KEY_RE = re.compile(
-    r"(secret|password|token|key|credential|auth|api.?key)",
+    r"^(?!keyword_recall(_stderr)?$).*(secret|password|token|key|credential|auth|api.?key)",
     re.IGNORECASE,
 )
 _REDACTED = "***REDACTED***"
@@ -77,6 +78,10 @@ def _prepare(result: RunResult, *, scrub: bool = False) -> dict[str, Any]:
     if "config" in raw:
         raw["config"] = _strip_defaults(raw["config"])
 
+    # Summary: drop null optional fields (Bar 4 metrics)
+    if "summary" in raw and isinstance(raw["summary"], dict):
+        raw["summary"] = {k: v for k, v in raw["summary"].items() if v is not None}
+
     # Cases: drop null optional fields and false high_variance
     for case in raw.get("cases", []):
         for key in _CASE_OPTIONAL: