diff --git a/authbridge/sparc-service/Dockerfile b/authbridge/sparc-service/Dockerfile
index 5d348bfae..35edde3c0 100644
--- a/authbridge/sparc-service/Dockerfile
+++ b/authbridge/sparc-service/Dockerfile
@@ -15,7 +15,7 @@ COPY sparc_service ./sparc_service
RUN pip install --upgrade pip && pip install .
# Drop privileges.
-RUN useradd --create-home --uid 10001 sparc
+RUN useradd --create-home --uid 10001 sparc && chown -R sparc:sparc /app
USER sparc
EXPOSE 8090
diff --git a/authbridge/sparc-service/deploy/Makefile b/authbridge/sparc-service/deploy/Makefile
index a494c760b..1d1588ba2 100644
--- a/authbridge/sparc-service/deploy/Makefile
+++ b/authbridge/sparc-service/deploy/Makefile
@@ -51,7 +51,12 @@ image: ## Build the sparc-service image locally and load it into kind
@echo "[*] building $(IMAGE)"
$(CONTAINER_RUNTIME) build -t $(IMAGE) ..
@echo "[*] kind load $(IMAGE) into $(KIND_CLUSTER_NAME)"
- kind load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME)
+ @# kind load docker-image fails on Linux with rootful Podman — use image-archive instead.
+ if kind load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME) 2>/dev/null; then \
+ true; \
+ else \
+ $(CONTAINER_RUNTIME) save $(IMAGE) | kind load image-archive /dev/stdin --name $(KIND_CLUSTER_NAME); \
+ fi
@# Let containerd resolve the bare docker.io/library/
ref kubelet uses.
-$(CONTAINER_RUNTIME) exec $(KIND_NODE) ctr -n k8s.io images tag localhost/$(IMAGE) docker.io/library/$(IMAGE) >/dev/null 2>&1 || true
diff --git a/authbridge/sparc-service/sparc_service/__main__.py b/authbridge/sparc-service/sparc_service/__main__.py
index f545a7f91..ca7868b92 100644
--- a/authbridge/sparc-service/sparc_service/__main__.py
+++ b/authbridge/sparc-service/sparc_service/__main__.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import logging
+import os
import uvicorn
@@ -10,9 +11,14 @@
def main() -> None:
- logging.basicConfig(level=logging.INFO)
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s:%(name)s:%(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ")
+ # Demote noisy third-party loggers — their INFO adds no operational value
+ logging.getLogger("LiteLLM").setLevel(logging.WARNING)
+ if os.getenv("SPARC_DEBUG_LLM", "").strip().lower() in ("1", "true", "yes"):
+ logging.getLogger("sparc_service.llm_debug").setLevel(logging.DEBUG)
+ logging.getLogger("altk").setLevel(logging.DEBUG)
settings = Settings.from_env()
- uvicorn.run("sparc_service.api:app", host=settings.host, port=settings.port, log_level="info")
+ uvicorn.run("sparc_service.api:app", host=settings.host, port=settings.port, log_level="info", access_log=False)
if __name__ == "__main__":
diff --git a/authbridge/sparc-service/sparc_service/api.py b/authbridge/sparc-service/sparc_service/api.py
index fd0487ca5..940953c0e 100644
--- a/authbridge/sparc-service/sparc_service/api.py
+++ b/authbridge/sparc-service/sparc_service/api.py
@@ -4,12 +4,18 @@
POST /reflect — run SPARC on a proposed tool call, return the verdict.
GET /healthz — liveness (always ok if the process is up).
GET /readyz — readiness (config valid and component buildable).
+
+Log levels:
+ INFO — clean operational log: startup skip list + evaluated verdicts only.
+ DEBUG — adds per-call skip entries and full request payloads
+ (payloads only when SPARC_LOG_REQUESTS=true).
"""
from __future__ import annotations
import json
import logging
+import os
from fastapi import FastAPI, HTTPException
from fastapi.concurrency import run_in_threadpool
@@ -20,8 +26,16 @@
log = logging.getLogger(__name__)
-# _LOG_REQUESTS and _STRIP_KEYS are now read from Settings (via Settings.from_env)
-# so all config comes from a single place. See settings.py.
+# SPARC_LOG_REQUESTS and SPARC_STRIP_TOOL_ARG_KEYS are read via Settings.from_env
+# so all config flows through a single place. See settings.py.
+
+# SPARC_SKIP_TOOLS — comma-separated tool names to auto-approve without SPARC.
+# Use for infrastructure tools (e.g. message, calculate) that have no policy
+# risk and would cause false-positive rejects.
+# Example: SPARC_SKIP_TOOLS=message,calculate
+_SKIP_TOOLS: frozenset[str] = frozenset(
+ t.strip() for t in os.getenv("SPARC_SKIP_TOOLS", "").split(",") if t.strip()
+)
def _strip_tool_arg_keys(tool_calls: list[dict], keys: frozenset[str]) -> list[dict]:
@@ -57,6 +71,10 @@ def create_app(engine: ReflectionEngine | None = None) -> FastAPI:
app.state.engine = engine
app.state.settings = settings
+ # INFO: announce skip list once at startup so operators know what is bypassed
+ if _SKIP_TOOLS:
+ log.info("SPARC_SKIP_TOOLS: the following tools will be auto-approved without evaluation: %s", sorted(_SKIP_TOOLS))
+
@app.get("/healthz")
def healthz() -> dict[str, object]:
return {
@@ -75,15 +93,23 @@ def readyz() -> dict[str, object]:
@app.post("/reflect", response_model=ReflectResponse)
async def reflect(request: ReflectRequest) -> ReflectResponse:
+ # DEBUG: full request payload — only when SPARC_LOG_REQUESTS=true
if settings.log_requests:
- log.info("incoming reflect request: %s", request.model_dump_json())
+ log.debug("incoming reflect request: %s", request.model_dump_json())
if settings.strip_tool_arg_keys and request.tool_calls:
request = request.model_copy(
update={"tool_calls": _strip_tool_arg_keys(request.tool_calls, settings.strip_tool_arg_keys)}
)
if settings.log_requests:
- log.info("after strip (%s): tool_calls=%s", sorted(settings.strip_tool_arg_keys), request.tool_calls)
+ log.debug("after strip (%s): tool_calls=%s", sorted(settings.strip_tool_arg_keys), request.tool_calls)
+
+ if _SKIP_TOOLS and request.tool_calls:
+ tool_name = request.tool_calls[0].get("function", {}).get("name", "")
+ if tool_name in _SKIP_TOOLS:
+ # DEBUG: per-call skip entry — visible only at DEBUG level
+ log.debug("reflect tool=%s skipped (SPARC_SKIP_TOOLS)", tool_name)
+ return ReflectResponse(decision="approve", issues=[], overall_avg_score=None, execution_time_ms=None)
# SPARCReflectionComponent.process is synchronous (and CPU/IO bound on the
# LLM call); run it off the event loop so the service stays responsive.
diff --git a/authbridge/sparc-service/sparc_service/engine.py b/authbridge/sparc-service/sparc_service/engine.py
index 053fbf929..1880be2bc 100644
--- a/authbridge/sparc-service/sparc_service/engine.py
+++ b/authbridge/sparc-service/sparc_service/engine.py
@@ -8,9 +8,11 @@
from __future__ import annotations
+import json
import logging
import threading
from dataclasses import replace
+from datetime import datetime, timezone
from typing import Any, Callable
from .models import ReflectionIssue, ReflectRequest, ReflectResponse
@@ -118,14 +120,59 @@ def reflect(self, request: ReflectRequest) -> ReflectResponse:
decision = _decision_str(reflection.decision)
score = _extract_overall_score(raw_pipeline)
execution_ms = getattr(output, "execution_time_ms", None)
+
+ # Extract tool name + args from the first tool call for correlation.
+ first_tc = request.tool_calls[0] if request.tool_calls else {}
+ fn = first_tc.get("function", {})
+ if not fn:
+ log.warning("reflect: tool_calls[0] has no 'function' key; tool correlation unavailable. call=%s", first_tc)
+ tool_name = fn.get("name", "-")
+ raw_args = fn.get("arguments", "{}")
+ try:
+ tool_args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args
+ except (json.JSONDecodeError, TypeError):
+ tool_args = raw_args
+ try:
+ args_str = json.dumps(tool_args, separators=(",", ":"))
+ except (TypeError, ValueError):
+ args_str = repr(tool_args)
+
+ ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+ score_str = f"{score:.2f}" if score is not None else "-"
+ ms_str = f"{execution_ms:.1f}" if isinstance(execution_ms, (int, float)) else "-"
+
+ def _tok(*keys: str) -> str:
+ for k in keys:
+ v = raw_pipeline.get(k)
+ if v is not None:
+ return str(v)
+ return "-"
+
+ # INFO: safe correlation fields only. Tool arguments are caller-controlled
+ # (log-injection risk via embedded newlines — CodeQL alert #179) and can
+ # contain payload data (PII, payment ids, etc.), so they never appear at
+ # INFO. The session_id / track are correlation identifiers, not payload,
+ # and are logged as-is.
log.info(
- "reflect session=%s track=%s decision=%s score=%s ms=%s",
- request.session_id or "-",
+ "reflect ts=%s tool=%s decision=%s score=%s ms=%s track=%s session=%s",
+ ts, tool_name, decision, score_str, ms_str,
+ track, request.session_id or "-",
+ )
+ log.debug(
+ "reflect ts=%s tool=%s decision=%s score=%s ms=%s"
+ " track=%s session=%s tokens_in=%s tokens_out=%s messages=%s",
+ ts, tool_name, decision, score_str, ms_str,
track,
- decision,
- f"{score:.2f}" if score is not None else "-",
- f"{execution_ms:.1f}" if isinstance(execution_ms, (int, float)) else "-",
+ request.session_id or "-",
+ _tok("tokens_in", "input_tokens"),
+ _tok("tokens_out", "output_tokens"),
+ len(request.messages),
)
+ # Raw arguments are only logged when the operator explicitly opts in via
+ # SPARC_LOG_REQUESTS=true, and only at DEBUG. Args are still
+ # attacker-influenced text so DEBUG output should be treated as sensitive.
+ if self._settings.log_requests:
+ log.debug("reflect ts=%s tool=%s args=%s", ts, tool_name, args_str)
return ReflectResponse(
decision=decision,
diff --git a/authbridge/sparc-service/sparc_service/providers.py b/authbridge/sparc-service/sparc_service/providers.py
index 6489926b1..726e6d84c 100644
--- a/authbridge/sparc-service/sparc_service/providers.py
+++ b/authbridge/sparc-service/sparc_service/providers.py
@@ -7,11 +7,14 @@
from __future__ import annotations
+import logging
import os
from typing import Any
from .settings import Settings
+_debug_log = logging.getLogger("sparc_service.llm_debug")
+
# Map the configurable track names to altk Track enum members. Resolved lazily.
_TRACK_NAMES = {
"fast_track": "FAST_TRACK",
@@ -29,6 +32,150 @@ def resolve_track(name: str):
return getattr(Track, _TRACK_NAMES[name])
+def _patch_watsonx_for_reasoning_models(client_cls):
+ """Patch LLM client to inject schema via system prompt instead of response_format.
+
+ Some models don't support response_format structured output (WatsonX reasoning
+ models return output in reasoning_content not content; Haiku-4-5 via IBM LiteLLM
+ proxy also lacks response_format support). ALTK's _parse_llm_response only reads
+ content, so response_format mode always raises 'No content or tool calls found in
+ response'. Injecting the schema into the system prompt (same as ALTK's Ollama
+ provider) makes the model return valid JSON in content.
+ See: https://github.com/kagenti/kagenti-extensions/issues/676
+
+ Idempotent: uses a sentinel attribute on ``client_cls`` so repeated calls
+ (e.g. once per configured track when ``ReflectionEngine`` builds a new
+ component) don't stack wrappers on the shared ALTK class.
+ """
+ import functools
+
+ if getattr(client_cls, "_sparc_patched_reasoning", False):
+ return
+
+ original_generate = client_cls.generate
+ original_generate_async = client_cls.generate_async
+
+ @functools.wraps(original_generate)
+ def patched_generate(self, *args, **kwargs):
+ # Force-override (not setdefault) — ALTK passes schema_field="response_format"
+ # explicitly, so setdefault would be a no-op and the reasoning model would
+ # return output in reasoning_content instead of content, causing
+ # "No content or tool calls found in response". See ISSUE-676.
+ kwargs["schema_field"] = None
+ kwargs["include_schema_in_system_prompt"] = True
+ return original_generate(self, *args, **kwargs)
+
+ @functools.wraps(original_generate_async)
+ async def patched_generate_async(self, *args, **kwargs):
+ kwargs["schema_field"] = None
+ kwargs["include_schema_in_system_prompt"] = True
+ return await original_generate_async(self, *args, **kwargs)
+
+ client_cls.generate = patched_generate
+ client_cls.generate_async = patched_generate_async
+ client_cls._sparc_patched_reasoning = True
+
+
+def _patch_empty_response_retry(client_cls, max_retries: int = 3):
+ """Wrap generate_async to retry on 'No content or tool calls found in response'.
+
+ ALTK's ValidatingLLMClient.generate_async retry loop catches only
+ OutputValidationError. When the IBM LiteLLM proxy returns empty content,
+ _parse_llm_response raises ValueError (not OutputValidationError), so all 3
+ configured retries are bypassed and the metric fails immediately.
+
+ This wrapper catches that ValueError and retries up to max_retries times with
+ a short back-off before re-raising, giving the proxy a chance to succeed on a
+ subsequent attempt.
+
+ See ISSUE-019 in docs/open-issues.md and upstream ALTK tracker.
+
+ Idempotent: guarded by a sentinel attribute so retry layers don't stack
+ across repeated ``build_llm_client`` calls (per-track lazy build).
+ """
+ import asyncio
+ import functools
+
+ if getattr(client_cls, "_sparc_patched_retry", False):
+ return
+
+ original_generate_async = client_cls.generate_async
+
+ @functools.wraps(original_generate_async)
+ async def patched_generate_async(self, *args, **kwargs):
+ last_exc = None
+ for attempt in range(1, max_retries + 2): # max_retries extra attempts
+ try:
+ return await original_generate_async(self, *args, **kwargs)
+ except ValueError as exc:
+ if "No content or tool calls found in response" not in str(exc):
+ raise
+ last_exc = exc
+ if attempt <= max_retries:
+ _debug_log.debug(
+ "[LLM_DEBUG] empty-response retry %d/%d after ValueError: %s",
+ attempt, max_retries, exc,
+ )
+ await asyncio.sleep(0.5 * attempt)
+ else:
+ _debug_log.debug(
+ "[LLM_DEBUG] empty-response retry exhausted (%d attempts)", max_retries + 1
+ )
+ raise last_exc # type: ignore[misc]
+
+ client_cls.generate_async = patched_generate_async
+ client_cls._sparc_patched_retry = True
+
+
+def _patch_debug_logging(client_cls):
+ """Wrap generate_async to log the exact prompt and raw response.
+
+ Activated only when SPARC_DEBUG_LLM=true. Logs at DEBUG level so normal
+ runs are unaffected. Each log line is prefixed [LLM_DEBUG] for easy grep:
+
+ kubectl logs -n kagenti-system deploy/sparc-service | grep LLM_DEBUG
+
+ Idempotent: guarded by a sentinel attribute so debug wrappers don't stack
+ across repeated ``build_llm_client`` calls.
+ """
+ import functools
+ import json as _json
+
+ if getattr(client_cls, "_sparc_patched_debug", False):
+ return
+
+ original_generate_async = client_cls.generate_async
+
+ @functools.wraps(original_generate_async)
+ async def patched_generate_async(self, prompt, *args, **kwargs):
+ schema = kwargs.get("schema")
+ schema_field = kwargs.get("schema_field", "response_format")
+ retries = kwargs.get("retries", "?")
+
+ prompt_full = _json.dumps(prompt, ensure_ascii=False) if isinstance(prompt, list) else str(prompt)
+ schema_full = _json.dumps(schema, ensure_ascii=False) if isinstance(schema, dict) else str(type(schema))
+
+ _debug_log.debug(
+ "[LLM_DEBUG] >>> generate_async called\n"
+ " schema_field=%s retries=%s\n"
+ " schema=%s\n"
+ " prompt=%s",
+ schema_field, retries, schema_full, prompt_full,
+ )
+
+ try:
+ result = await original_generate_async(self, prompt, *args, **kwargs)
+ result_preview = _json.dumps(result, ensure_ascii=False)[:400] if isinstance(result, dict) else str(result)[:400]
+ _debug_log.debug("[LLM_DEBUG] <<< generate_async SUCCESS result=%s", result_preview)
+ return result
+ except Exception as exc:
+ _debug_log.debug("[LLM_DEBUG] <<< generate_async ERROR %s: %s", type(exc).__name__, exc)
+ raise
+
+ client_cls.generate_async = patched_generate_async
+ client_cls._sparc_patched_debug = True
+
+
def build_llm_client(settings: Settings):
"""Construct a validating ALTK LLM client for the configured provider.
@@ -50,14 +197,26 @@ def build_llm_client(settings: Settings):
# client needs comes from SPARC_LLM_KWARGS_JSON.
native = not settings.llm_registry_id
- if native and settings.provider == "watsonx":
- return client_cls(
+ debug = os.getenv("SPARC_DEBUG_LLM", "").strip().lower() in ("1", "true", "yes")
+
+ if native and settings.provider in ["watsonx", "litellm.watsonx"]:
+ client = client_cls(
model_name=settings.model,
api_key=settings.wx_api_key,
project_id=settings.wx_project_id,
api_base=settings.wx_url,
timeout=settings.llm_timeout_seconds,
)
+ # Only reasoning models (and the IBM LiteLLM proxy) need system-prompt
+ # schema injection — models like mistral-large-2512 support
+ # ``response_format`` natively, so applying it unconditionally would
+ # silently regress structured output. Gate on SPARC_SCHEMA_IN_PROMPT.
+ if settings.schema_in_prompt:
+ _patch_watsonx_for_reasoning_models(client_cls)
+ _patch_empty_response_retry(client_cls, max_retries=settings.retries)
+ if debug:
+ _patch_debug_logging(client_cls)
+ return client
if native and settings.provider == "ollama":
# Point every LiteLLM Ollama call at the configured server. We pass
@@ -67,11 +226,14 @@ def build_llm_client(settings: Settings):
# localhost:11434). The env vars are set too as a belt-and-suspenders.
os.environ["OLLAMA_API_BASE"] = settings.ollama_base_url
os.environ["OLLAMA_BASE_URL"] = settings.ollama_base_url
- return client_cls(
+ client = client_cls(
model_name=settings.model,
api_key=settings.ollama_api_key,
api_url=settings.ollama_base_url,
)
+ if debug:
+ _patch_debug_logging(client_cls)
+ return client
# Generic path (openai / azure / litellm, or any provider when
# SPARC_LLM_REGISTRY_ID is set). LiteLLM routes by the model string and reads
@@ -84,7 +246,21 @@ def build_llm_client(settings: Settings):
lite_kwargs.setdefault("timeout", settings.llm_timeout_seconds)
if settings.provider == "openai" and settings.openai_base_url and "api_base" not in lite_kwargs:
lite_kwargs["api_base"] = settings.openai_base_url
- return client_cls(model_name=settings.model, **lite_kwargs)
+ client = client_cls(model_name=settings.model, **lite_kwargs)
+ # IBM LiteLLM proxy (and other non-native providers) have the same empty-content
+ # issue as WatsonX reasoning models: ALTK passes schema_field="response_format"
+ # explicitly, but the proxy returns an empty content field when response_format
+ # is active, causing "No content or tool calls found in response". Apply the same
+ # system-prompt injection patch so the model returns JSON in content — but
+ # only when explicitly opted in via SPARC_SCHEMA_IN_PROMPT, since many
+ # models under the ``litellm`` provider support ``response_format`` natively.
+ if settings.provider == "litellm":
+ if settings.schema_in_prompt:
+ _patch_watsonx_for_reasoning_models(client_cls)
+ _patch_empty_response_retry(client_cls, max_retries=settings.retries)
+ if debug:
+ _patch_debug_logging(client_cls)
+ return client
def build_component(settings: Settings):
diff --git a/authbridge/sparc-service/sparc_service/settings.py b/authbridge/sparc-service/sparc_service/settings.py
index a894ca74b..ce1a7026a 100644
--- a/authbridge/sparc-service/sparc_service/settings.py
+++ b/authbridge/sparc-service/sparc_service/settings.py
@@ -26,6 +26,7 @@
# For full control you can also set SPARC_LLM_REGISTRY_ID to any ALTK registry id.
PROVIDER_REGISTRY_IDS: dict[str, str] = {
"watsonx": "litellm.watsonx.output_val",
+ "litellm.watsonx": "litellm.watsonx.output_val",
"ollama": "litellm.ollama.output_val",
"openai": "litellm.output_val",
"azure": "litellm.output_val",
@@ -35,6 +36,7 @@
# Per-provider default model id (empty → SPARC_MODEL is required).
PROVIDER_DEFAULT_MODELS: dict[str, str] = {
"watsonx": "mistral-large-2512",
+ "litellm.watsonx": "mistral-large-2512",
"ollama": "llama3.2:3b",
"openai": "gpt-4o-mini",
"azure": "",
@@ -102,6 +104,13 @@ class Settings:
log_requests: bool = False
strip_tool_arg_keys: frozenset = field(default_factory=frozenset)
+ # When true, inject the response schema via the system prompt instead of
+ # relying on the provider's native ``response_format`` parameter. Required
+ # for models that return output only in ``reasoning_content`` (WatsonX
+ # reasoning models, Haiku via IBM LiteLLM proxy). Leave false for models
+ # that support ``response_format`` natively (e.g. mistral-large-2512).
+ schema_in_prompt: bool = False
+
# Validation errors collected at load time (provider creds missing, etc.).
errors: tuple[str, ...] = field(default_factory=tuple)
@@ -147,8 +156,8 @@ def from_env(cls) -> "Settings":
errors.append(f"SPARC_LLM_KWARGS_JSON is not valid JSON: {exc}")
# Provider-specific credential / config validation.
- if provider == "watsonx" and (not wx_api_key or not wx_project_id):
- errors.append("provider=watsonx requires WX_API_KEY and WX_PROJECT_ID")
+ if provider in ("watsonx", "litellm.watsonx") and (not wx_api_key or not wx_project_id):
+ errors.append(f"provider={provider} requires WX_API_KEY and WX_PROJECT_ID")
if provider == "openai" and not (os.getenv("OPENAI_API_KEY") or "api_key" in llm_kwargs):
errors.append("provider=openai requires OPENAI_API_KEY (or api_key in SPARC_LLM_KWARGS_JSON)")
if provider in ("azure", "litellm") and not model:
@@ -183,5 +192,6 @@ def from_env(cls) -> "Settings":
port=_int_env("PORT", 8090),
log_requests=_truthy(os.getenv("SPARC_LOG_REQUESTS", "")),
strip_tool_arg_keys=strip_tool_arg_keys,
+ schema_in_prompt=_truthy(os.getenv("SPARC_SCHEMA_IN_PROMPT", "")),
errors=tuple(errors),
)
diff --git a/authbridge/sparc-service/tests/test_haiku_empty_response.py b/authbridge/sparc-service/tests/test_haiku_empty_response.py
new file mode 100644
index 000000000..06c614f9c
--- /dev/null
+++ b/authbridge/sparc-service/tests/test_haiku_empty_response.py
@@ -0,0 +1,308 @@
+"""Reproduce: IBM LiteLLM proxy + Haiku intermittently returns empty content
+when response_format (structured output) mode is used.
+
+Hypothesis: ALTK calls generate_async with schema_field="response_format",
+which tells LiteLLM to use structured output mode. The IBM proxy occasionally
+returns a response where choices[0].message.content is None or "" and
+tool_calls is also empty, causing:
+ ValueError: No content or tool calls found in response
+
+This test sends the exact same call N times and counts how many return
+empty content — proving the failure is intermittent and tied to response_format.
+
+Requirements:
+ pip install litellm python-dotenv
+ OAIKEY and OAIBASE must be set in /root/.env or env vars.
+
+Usage:
+ python -m pytest tests/test_haiku_empty_response.py -v -s
+ # or directly:
+ python tests/test_haiku_empty_response.py
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import asyncio
+from pathlib import Path
+
+import pytest
+
+# Live-LLM probes: opt-in only. Enable with RUN_HAIKU_TESTS=1 and OAIKEY/OAIBASE
+# in the environment. Otherwise the tests are skipped so `pytest` in CI (which
+# has no LLM credentials) does not attempt real network calls or fail on
+# missing config.
+_RUN_LIVE = os.environ.get("RUN_HAIKU_TESTS", "").strip().lower() in {"1", "true", "yes"}
+_live_only = pytest.mark.skipif(
+ not (_RUN_LIVE and os.environ.get("OAIKEY") and os.environ.get("OAIBASE")),
+ reason="Live LLM probe; set RUN_HAIKU_TESTS=1 with OAIKEY and OAIBASE to enable.",
+)
+
+# Load credentials from /root/.env if present
+_env_file = Path("/root/.env")
+if _env_file.exists():
+ for line in _env_file.read_text().splitlines():
+ line = line.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ k, _, v = line.partition("=")
+ k = k.strip().removeprefix("export").strip()
+ v = v.strip().strip("\"'")
+ os.environ.setdefault(k, v)
+
+API_KEY = os.environ.get("OAIKEY", "")
+API_BASE = os.environ.get("OAIBASE", "")
+MODEL = "claude-haiku-4-5-20251001"
+
+# The exact schema ALTK sends for general_hallucination_check (from production log)
+HALLUCINATION_SCHEMA = {
+ "title": "general_hallucination_check",
+ "description": "Assessment of tool call grounding accuracy, following the rubric defined in the task description.",
+ "type": "object",
+ "properties": {
+ "evidence": {"type": "string"},
+ "explanation": {"type": "string"},
+ "output": {"type": "integer", "minimum": 1, "maximum": 5},
+ "confidence": {"type": "number", "minimum": 0, "maximum": 1},
+ "correction": {"type": "object", "additionalProperties": True},
+ },
+ "required": ["confidence", "correction", "evidence", "explanation", "output"],
+ "additionalProperties": False,
+}
+
+# Full production prompt extracted from the failing call in zeus logs (2026-07-28T13:14:37Z)
+# This is the exact prompt that caused "No content or tool calls found in response"
+# for tool=update_reservation_flights on the airline benchmark.
+# The system message contains the full ALTK rubric + schema.
+# The user message contains the full airline policy + multi-turn conversation history.
+_SYSTEM_CONTENT = """### Task Description and Role:
+
+Evaluate whether each parameter value in the function call is correct and **directly supported** by the provided conversation history and adhere the API specification. Your assessment must be based **strictly on explicit evidence** from these sources and correctly formatted based on the API specifications. Do **not** assume or hallucinate any information that is not clearly documented and provided.
+
+---
+
+#### 1. Grounding Sources
+
+A parameter value is considered grounded if it originates from one of the following:
+
+- An explicit user message in the conversation
+- An assistant message that the user confirmed or acknowledged
+- The output of a previous tool call
+- A documented default value in the API specification
+
+---
+
+#### 2. Parameter Value Classification
+
+Each parameter value must be labeled using one of the following categories:
+
+- **CORRECT**
+ The value is explicitly mentioned, clearly implied in the conversation, or matches a documented default.
+
+- **MISSING_INFORMATION**
+ The value is underspecified or incomplete given the current context.
+
+- **FORMAT ERROR**
+ The value is conceptually correct but incorrectly formatted based on the API specifications.
+
+- **CONTRADICTORY_VALUES**
+ The value violates documented constraints or logical relationships between parameters.
+
+- **DEFAULT_ISSUE**
+ The default value of the parameter is not the correct value based on the conversation history.
+
+---
+
+#### 3. Handling Default Values
+
+- Optional parameters may only use default values that are **explicitly documented** in the API specification.
+- Defaults that are assumed or undocumented count as hallucinations.
+- If no value is provided and no default exists, the parameter should be omitted.
+
+---
+
+#### 4. Acceptable Transformations
+
+Some transformations are permitted, but only when grounded:
+
+- **Synonyms** are allowed only when normalized in the conversation.
+- **Unit conversions** are valid only if the conversation explicitly mentions the conversion.
+- **Format changes** are acceptable only if acknowledged by the assistant or supported by the specification.
+
+---
+
+#### 5. Multi-Call Context
+
+If the tool call appears in a sequence:
+
+- You may use outputs from earlier tool calls to justify parameter values.
+- Consider dependencies and ordering between calls when assessing grounding.
+
+---
+
+#### Conservative Judgment Principle
+
+When in doubt, err on the side of caution. If grounding cannot be clearly established, treat the parameter as incorrect.
+
+
+Your output must conform to the following JSON schema, in the same order as the fields appear in the schema:
+""" + json.dumps(HALLUCINATION_SCHEMA)
+
+_USER_CONTENT = """Conversation context:
+[{"content": "The task you are to complete is:\\nYou are a customer service agent that helps the user according to the provided below.\\n\\nContext:\\n- policy: # Airline Agent Policy\\n\\nThe current time is 2024-05-15 15:00:00 EST.\\n\\nAs an airline agent, you can help users **book**, **modify**, or **cancel** flight reservations. You also handle **refunds and compensation**.\\n\\nBefore taking any actions that update the booking database (booking, modifying flights, editing baggage, changing cabin class, or updating passenger information), you must list the action details and obtain explicit user confirmation (yes) to proceed.\\n\\nYou should not provide any information, knowledge, or procedures not provided by the user or available tools, or give subjective recommendations or comments.\\n\\nYou should only make one tool call at a time, and if you make a tool call, you should not respond to the user simultaneously. If you respond to the user, you should not make a tool call at the same time.\\n\\nYou should deny user requests that are against this policy.\\n\\nYou should transfer the user to a human agent if and only if the request cannot be handled within the scope of your actions.\\n\\n## Domain Basic\\n\\n### User\\nEach user has a profile containing:\\n- user id\\n- email\\n- addresses\\n- date of birth\\n- payment methods\\n- membership level\\n- reservation numbers\\n\\nThere are three types of payment methods: **credit card**, **gift card**, **travel certificate**.\\n\\nThere are three membership levels: **regular**, **silver**, **gold**.\\n\\n### Flight\\nEach flight has the following attributes:\\n- flight number\\n- origin\\n- destination\\n- scheduled departure and arrival time (local time)\\n\\nA flight can be available at multiple dates. For each date:\\n- If the status is **available**, the flight has not taken off, available seats and prices are listed.\\n- If the status is **delayed** or **on time**, the flight has not taken off, cannot be booked.\\n- If the status is **flying**, the flight has taken off but not landed, cannot be booked.\\n\\nThere are three cabin classes: **basic economy**, **economy**, **business**. **basic economy** is its own class, completely distinct from **economy**.\\n\\n## Book flight\\n\\nThe agent must first obtain the user id from the user.\\n\\nThe agent should then ask for the trip type, origin, destination.\\n\\nCabin:\\n- Cabin class must be the same across all the flights in a reservation.\\n\\nPassengers:\\n- Each reservation can have at most five passengers.\\n- The agent needs to collect the first name, last name, and date of birth for each passenger.\\n\\nPayment:\\n- Each reservation can use at most one travel certificate, at most one credit card, and at most three gift cards.\\n- The remaining amount of a travel certificate is not refundable.\\n- All payment methods must already be in user profile for safety reasons.\\n\\n## Modify flight\\n\\nFirst, the agent must obtain the user id and reservation id.\\n\\nChange flights:\\n- Basic economy flights cannot be modified.\\n- Other reservations can be modified without changing the origin, destination, and trip type.\\n\\nChange cabin:\\n- Cabin cannot be changed if any flight in the reservation has already been flown.\\n- In other cases, all reservations, including basic economy, can change cabin without changing the flights.\\n- Cabin class must remain the same across all the flights in the same reservation.\\n- If the price after cabin change is higher than the original price, the user is required to pay for the difference.\\n\\nChange passengers:\\n- The user can modify passengers but cannot modify the number of passengers.\\n\\nPayment:\\n- If the flights are changed, the user needs to provide a single gift card or credit card for payment or refund method.\\n\\n## Cancel flight\\n\\nFirst, the agent must obtain the user id and reservation id.\\n\\nThe agent must also obtain the reason for cancellation (change of plan, airline cancelled flight, or other reasons)\\n\\nIf any portion of the flight has already been flown, the agent cannot help and transfer is needed.\\n\\nOtherwise, flight can be cancelled if any of the following is true:\\n- The booking was made within the last 24 hrs\\n- The flight is cancelled by airline\\n- It is a business flight\\n- The user has travel insurance and the reason for cancellation is covered by insurance.\\n\\n## Refunds and Compensation\\nDo not proactively offer a compensation unless the user explicitly asks for one.", "role": "user"}, {"content": "Hello! How can I help you today? Please provide your user ID.", "role": "assistant"}, {"content": "sender='user' message='My user ID is test_user_0001. Please proceed with the cancellations.'", "role": "user"}, {"content": "To proceed with cancellations, I will need the reservation ID(s) and the reason for cancellation.", "role": "assistant"}, {"content": "sender='user' message='The reservation IDs are RSVN01 and RSVN02. Reason: change of plan. Cancel both.'", "role": "user"}, {"content": "", "role": "assistant"}, {"content": "\\"{\\\\\"reservation_id\\\\\": \\\\\"RSVN01\\\\\", \\\\\"user_id\\\\\": \\\\\"test_user_0001\\\\\", \\\\\"cabin\\\\\": \\\\\"basic_economy\\\\\", \\\\\"flights\\\\\": [{\\\\\"flight_number\\\\\": \\\\\"HAT005\\\\\", \\\\\"date\\\\\": \\\\\"2024-05-20\\\\\"}, {\\\\\"flight_number\\\\\": \\\\\"HAT178\\\\\", \\\\\"date\\\\\": \\\\\"2024-05-30\\\\\"}], \\\\\"passengers\\\\\": [{\\\\\"first_name\\\\\": \\\\\"Alex\\\\\", \\\\\"last_name\\\\\": \\\\\"Doe\\\\\", \\\\\"dob\\\\\": \\\\\"1980-01-01\\\\\"}, {\\\\\"first_name\\\\\": \\\\\"Sam\\\\\", \\\\\"last_name\\\\\": \\\\\"Roe\\\\\", \\\\\"dob\\\\\": \\\\\"1985-02-02\\\\\"}], \\\\\"payment_history\\\\\": [{\\\\\"payment_id\\\\\": \\\\\"credit_card_0000001\\\\\", \\\\\"amount\\\\\": 296}], \\\\\"insurance\\\\\": \\\\\"no\\\\\", \\\\\"status\\\\\": null}\\"", "role": "tool"}, {"content": "\\"{\\\\\"reservation_id\\\\\": \\\\\"RSVN02\\\\\", \\\\\"cabin\\\\\": \\\\\"economy\\\\\", \\\\\"flights\\\\\": [{\\\\\"flight_number\\\\\": \\\\\"HAT007\\\\\", \\\\\"date\\\\\": \\\\\"2024-05-19\\\\\"}, {\\\\\"flight_number\\\\\": \\\\\"HAT174\\\\\", \\\\\"date\\\\\": \\\\\"2024-05-29\\\\\"}], \\\\\"passengers\\\\\": [{\\\\\"first_name\\\\\": \\\\\"Alex\\\\\", \\\\\"last_name\\\\\": \\\\\"Doe\\\\\"}, {\\\\\"first_name\\\\\": \\\\\"Jane\\\\\", \\\\\"last_name\\\\\": \\\\\"Doe\\\\\"}], \\\\\"insurance\\\\\": \\\\\"yes\\\\\", \\\\\"status\\\\\": null}\\"", "role": "tool"}, {"content": "RSVN01 (Basic Economy, No Insurance): Not eligible for cancellation under change of plan. RSVN02 (Economy, With Insurance): Eligible for cancellation. Would you like to proceed with only RSVN02?", "role": "assistant"}, {"content": "sender='user' message='No, I want to upgrade RSVN01 to economy first, then cancel it. Proceed with that.'", "role": "user"}, {"content": "I will upgrade RSVN01 from basic economy to economy, then cancel both reservations. Please confirm.", "role": "assistant"}, {"content": "sender='user' message='Yes, upgrade RSVN01 to economy, no insurance. Cancel both RSVN01 and RSVN02 after.'", "role": "user"}, {"content": "", "role": "assistant"}, {"content": "\\"{\\\\\"reservation_id\\\\\": \\\\\"RSVN01\\\\\", \\\\\"cabin\\\\\": \\\\\"economy\\\\\", \\\\\"flights\\\\\": [{\\\\\"flight_number\\\\\": \\\\\"HAT005\\\\\", \\\\\"date\\\\\": \\\\\"2024-05-20\\\\\", \\\\\"price\\\\\": 166}, {\\\\\"flight_number\\\\\": \\\\\"HAT178\\\\\", \\\\\"date\\\\\": \\\\\"2024-05-30\\\\\", \\\\\"price\\\\\": 126}], \\\\\"payment_history\\\\\": [{\\\\\"payment_id\\\\\": \\\\\"credit_card_0000001\\\\\", \\\\\"amount\\\\\": 296}, {\\\\\"payment_id\\\\\": \\\\\"credit_card_0000001\\\\\", \\\\\"amount\\\\\": 288}], \\\\\"status\\\\\": null}\\"", "role": "tool"}]
+
+Tool Specification:
+{"name": "cancel_reservation", "description": "Cancel the whole reservation.", "parameters": {"properties": {"reservation_id": {"title": "Reservation Id", "type": "string"}}, "required": ["reservation_id"], "title": "cancel_reservationArgumentsWithoutSessionId", "type": "object"}}
+
+Proposed tool call:
+{"id": "9", "type": "function", "function": {"name": "cancel_reservation", "arguments": "{\\"reservation_id\\": \\"RSVN01\\"}"}}
+
+Return a JSON object as specified in the system prompt. You MUST keep the same order of fields in the JSON object as provided in the JSON schema and examples."""
+
+PROMPT = [
+ {"role": "system", "content": _SYSTEM_CONTENT},
+ {"role": "user", "content": _USER_CONTENT},
+]
+
+
+def _schema_to_pydantic(schema: dict):
+ """Reproduce ALTK's json_schema_to_pydantic_model conversion."""
+ from pydantic import BaseModel, Field, create_model
+ from typing import Optional, Any
+
+ type_mapping = {"string": str, "integer": int, "number": float, "boolean": bool, "array": list, "object": dict}
+ fields = {}
+ required_fields = set(schema.get("required", []))
+ for prop_name, prop_schema in schema.get("properties", {}).items():
+ field_type = type_mapping.get(prop_schema.get("type"), Any)
+ default = ... if prop_name in required_fields else None
+ desc = prop_schema.get("description")
+ fields[prop_name] = (field_type, Field(default, description=desc) if desc else (field_type, default))
+ return create_model(schema.get("title", "AutoModel"), **fields)
+
+
+def _call_with_response_format() -> dict | None:
+ """Single synchronous call reproducing ALTK's exact path:
+ schema dict -> pydantic model -> response_format kwarg -> litellm.acompletion
+ This is what ValidatingLLMClient.generate_async does when schema_field='response_format'.
+ """
+ import litellm
+
+ # ALTK converts schema dict to Pydantic model and passes as response_format
+ pydantic_schema = _schema_to_pydantic(HALLUCINATION_SCHEMA)
+
+ response = litellm.completion(
+ model=MODEL,
+ messages=PROMPT,
+ api_key=API_KEY,
+ api_base=API_BASE,
+ response_format=pydantic_schema, # Pydantic model, NOT a json_schema dict
+ timeout=30,
+ )
+ msg = response.choices[0].message
+ content = getattr(msg, "content", None)
+ tool_calls = getattr(msg, "tool_calls", None)
+ return {
+ "content": content,
+ "tool_calls": tool_calls,
+ "empty": not content and not tool_calls,
+ }
+
+
+def _call_with_system_prompt() -> dict | None:
+ """Single call with schema injected in system prompt — the proposed fix."""
+ import litellm
+
+ response = litellm.completion(
+ model=MODEL,
+ messages=PROMPT, # schema already embedded in system prompt above
+ api_key=API_KEY,
+ api_base=API_BASE,
+ timeout=30,
+ )
+ msg = response.choices[0].message
+ content = getattr(msg, "content", None)
+ tool_calls = getattr(msg, "tool_calls", None)
+ return {
+ "content": content,
+ "tool_calls": tool_calls,
+ "empty": not content and not tool_calls,
+ }
+
+
+@_live_only
+def test_response_format_intermittent_empty(n_calls: int = 10):
+ """Send the same call N times with response_format and count empty responses.
+
+ If the hypothesis is correct, at least some calls will return empty content,
+ proving the IBM proxy is intermittently broken in response_format mode.
+ """
+ empty_count = 0
+ results = []
+
+ for i in range(n_calls):
+ result = _call_with_response_format()
+ results.append(result)
+ status = "EMPTY" if result["empty"] else "OK"
+ print(f" call {i+1:02d}: {status} content_len={len(result['content'] or '')}")
+ if result["empty"]:
+ empty_count += 1
+
+ print(f"\nSummary: {empty_count}/{n_calls} calls returned empty content")
+
+ # The test proves the hypothesis if at least 1 call is empty.
+ # If 0 are empty, the IBM proxy may have been fixed or the prompt is too simple.
+ assert empty_count > 0, (
+ f"All {n_calls} calls succeeded — IBM proxy may be stable now, "
+ f"or the prompt needs to be longer to trigger the failure."
+ )
+
+
+@_live_only
+def test_system_prompt_mode_no_empty(n_calls: int = 10):
+ """Same N calls but with schema in system prompt instead of response_format.
+
+ If the fix works, zero calls should return empty content.
+ """
+ empty_count = 0
+
+ for i in range(n_calls):
+ result = _call_with_system_prompt()
+ status = "EMPTY" if result["empty"] else "OK"
+ print(f" call {i+1:02d}: {status} content_len={len(result['content'] or '')}")
+ if result["empty"]:
+ empty_count += 1
+
+ print(f"\nSummary: {empty_count}/{n_calls} calls returned empty content")
+
+ assert empty_count == 0, (
+ f"{empty_count}/{n_calls} calls returned empty content even with system-prompt mode"
+ )
+
+
+if __name__ == "__main__":
+ if not API_KEY or not API_BASE:
+ print("ERROR: OAIKEY and OAIBASE must be set in /root/.env or environment")
+ raise SystemExit(1)
+
+ N = 10
+ print(f"=== Test 1: response_format mode ({N} calls) ===")
+ empty = 0
+ for i in range(N):
+ r = _call_with_response_format()
+ status = "EMPTY" if r["empty"] else f"OK ({len(r['content'] or '')} chars)"
+ print(f" call {i+1:02d}: {status}")
+ if r["empty"]:
+ empty += 1
+ print(f"Result: {empty}/{N} empty\n")
+
+ print(f"=== Test 2: system-prompt mode ({N} calls) ===")
+ empty = 0
+ for i in range(N):
+ r = _call_with_system_prompt()
+ status = "EMPTY" if r["empty"] else f"OK ({len(r['content'] or '')} chars)"
+ print(f" call {i+1:02d}: {status}")
+ if r["empty"]:
+ empty += 1
+ print(f"Result: {empty}/{N} empty")
diff --git a/authbridge/sparc-service/tests/test_providers.py b/authbridge/sparc-service/tests/test_providers.py
index 89199a64d..c01f44170 100644
--- a/authbridge/sparc-service/tests/test_providers.py
+++ b/authbridge/sparc-service/tests/test_providers.py
@@ -20,6 +20,15 @@ class FakeClient:
def __init__(self, **kwargs: Any) -> None:
captured.update(kwargs)
+ # ``build_llm_client`` applies the retry/debug/reasoning wrappers to the
+ # class, so ``generate`` and ``generate_async`` must be present on the
+ # stub even though this test never invokes them.
+ def generate(self, *a: Any, **kw: Any) -> Any: # pragma: no cover
+ return None
+
+ async def generate_async(self, *a: Any, **kw: Any) -> Any: # pragma: no cover
+ return None
+
monkeypatch.setattr(llm_mod, "get_llm", lambda _registry_id: FakeClient)
return captured
@@ -50,3 +59,84 @@ def test_registry_override_skips_provider_kwargs(monkeypatch):
assert "api_base" not in captured
assert captured["model_name"] == "gpt-4o-mini"
assert captured["api_key"] == "sk-x"
+
+
+def test_empty_response_retry_recovers_after_two_failures():
+ """Deterministic guard for `_patch_empty_response_retry`.
+
+ Simulate two consecutive ``ValueError("No content or tool calls found in
+ response")`` failures followed by a success, then assert the wrapper caught
+ both, retried, and returned the successful result.
+ """
+ import asyncio
+
+ from sparc_service.providers import _patch_empty_response_retry
+
+ calls = {"n": 0}
+
+ class FakeClient:
+ async def generate_async(self, *args, **kwargs):
+ calls["n"] += 1
+ if calls["n"] < 3:
+ raise ValueError("No content or tool calls found in response")
+ return {"ok": True, "attempt": calls["n"]}
+
+ _patch_empty_response_retry(FakeClient, max_retries=3)
+ result = asyncio.run(FakeClient().generate_async())
+ assert result == {"ok": True, "attempt": 3}
+ assert calls["n"] == 3
+
+
+def test_empty_response_retry_reraises_unrelated_valueerror():
+ """A ValueError with a different message must NOT be swallowed by the wrapper."""
+ import asyncio
+ import pytest
+
+ from sparc_service.providers import _patch_empty_response_retry
+
+ class FakeClient:
+ async def generate_async(self, *args, **kwargs):
+ raise ValueError("some other error")
+
+ _patch_empty_response_retry(FakeClient, max_retries=3)
+ with pytest.raises(ValueError, match="some other error"):
+ asyncio.run(FakeClient().generate_async())
+
+
+def test_patches_are_idempotent():
+ """All three ``_patch_*`` helpers must no-op on a client class already patched.
+
+ Without the sentinel guards, ``ReflectionEngine``'s per-track lazy build
+ would wrap the same class N times, exploding retry counts and duplicating
+ debug lines. This asserts the sentinel path is taken.
+ """
+ from sparc_service.providers import (
+ _patch_debug_logging,
+ _patch_empty_response_retry,
+ _patch_watsonx_for_reasoning_models,
+ )
+
+ class FakeClient:
+ def generate(self, *a, **kw):
+ return "raw"
+
+ async def generate_async(self, *a, **kw):
+ return "raw"
+
+ # Apply each patch twice; second call must be a no-op (sentinel bit already set).
+ _patch_watsonx_for_reasoning_models(FakeClient)
+ first_generate = FakeClient.generate
+ first_generate_async = FakeClient.generate_async
+ _patch_watsonx_for_reasoning_models(FakeClient)
+ assert FakeClient.generate is first_generate
+ assert FakeClient.generate_async is first_generate_async
+
+ _patch_empty_response_retry(FakeClient, max_retries=1)
+ after_retry_wrapper = FakeClient.generate_async
+ _patch_empty_response_retry(FakeClient, max_retries=1)
+ assert FakeClient.generate_async is after_retry_wrapper
+
+ _patch_debug_logging(FakeClient)
+ after_debug_wrapper = FakeClient.generate_async
+ _patch_debug_logging(FakeClient)
+ assert FakeClient.generate_async is after_debug_wrapper