Reach the model the setup screen chose from the LlamaIndex Bot, not only an OpenAI one

.github/workflows/ci.yml

@@ -271,6 +271,13 @@ jobs:
           . .venv-langgraph-agui/bin/activate
           python -m pip install --requirement agent-langgraph-agui/requirements.txt --requirement agent-langgraph-agui/requirements-test.txt
           python -m pytest agent-langgraph-agui/tests -q
+      - name: LlamaIndex model-choice regression
+        run: |
+          set -euo pipefail
+          python -m venv .venv-llamaindex
+          . .venv-llamaindex/bin/activate
+          python -m pip install --requirement agent-llamaindex/requirements.txt --requirement agent-llamaindex/requirements-test.txt
+          python -m pytest agent-llamaindex/tests -q
       - run: bun install --frozen-lockfile
       - run: bun test tests/compose.test.ts
       - run: docker compose --env-file /dev/null --profile harness config --format json >/dev/null

CHANGELOG.md

@@ -8,6 +8,18 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 
 ## Unreleased
 
+### The LlamaIndex Bot answers with the model the setup screen chose
+
+The LlamaIndex Bot built an OpenAI client from `BOT_MODEL` and ignored `BOT_PROVIDER`, so it only
+worked with an OpenAI model name sent to OpenAI. Picked with an Anthropic key, every run failed with
+`Unknown model 'claude-sonnet-4-5'`; picked with an OpenAI-compatible endpoint, every run failed
+with `Unknown model` for that endpoint's model, and an OpenAI model name went to api.openai.com
+instead of the address given, because the client read `OPENAI_API_BASE` and not the
+`OPENAI_BASE_URL` Compose passes. It now reaches the model through LiteLLM as `provider/model`, the
+way the Agno Bot does, so all three choices answer. A model LiteLLM does not know is treated as able
+to call tools, which the AG-UI workflow requires, and parameters a model does not accept, such as
+the temperature LlamaIndex sends to a reasoning model, are dropped rather than refused.
+
 ### The Audit page says "not enforced" only under a dry-run refusal that went ahead
 
 On a deployment in `dry-run`, the Audit page printed "dry-run: recorded, not enforced" under every

agent-llamaindex/requirements-test.txt

@@ -0,0 +1,2 @@
+httpx==0.28.1
+pytest==9.0.2

agent-llamaindex/requirements.txt

@@ -1,6 +1,6 @@
 llama-index-core
 llama-index-protocols-ag-ui
-llama-index-llms-openai
+llama-index-llms-litellm
 fastapi
 python-multipart
 uvicorn[standard]

agent-llamaindex/src/main.py

@@ -6,9 +6,10 @@
 
 import os
 
+import litellm
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
-from llama_index.llms.openai import OpenAI
+from llama_index.llms.litellm import LiteLLM
 from llama_index.protocols.ag_ui.router import get_ag_ui_workflow_router
 
 TOKEN_HEADER = "x-openbot-agent-token"
@@ -20,6 +21,21 @@ def _model_id() -> str:
     return model if "/" in model else f"{provider}/{model}"
 
 
+def _llm() -> LiteLLM:
+    model = _model_id()
+    if not litellm.supports_function_calling(model):
+        litellm.register_model(
+            {
+                model: {
+                    "litellm_provider": model.split("/", 1)[0],
+                    "mode": "chat",
+                    "supports_function_calling": True,
+                }
+            }
+        )
+    return LiteLLM(model=model, additional_kwargs={"drop_params": True})
+
+
 app = FastAPI()
 
 
@@ -39,4 +55,4 @@ async def health():
     return {"ok": True, "harness": "llamaindex"}
 
 
-app.include_router(get_ag_ui_workflow_router(llm=OpenAI(model=(os.environ.get("BOT_MODEL") or "gpt-4o-mini"))))
+app.include_router(get_ag_ui_workflow_router(llm=_llm()))

agent-llamaindex/tests/test_main.py

@@ -0,0 +1,184 @@
+import importlib
+import json
+import socket
+import sys
+import threading
+import time
+from pathlib import Path
+
+import pytest
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.testclient import TestClient
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+TOKEN = "test-token"
+RUN = {
+    "threadId": "thread-1",
+    "runId": "run-1",
+    "state": {},
+    "messages": [{"id": "m1", "role": "user", "content": "Say hello"}],
+    "tools": [],
+    "context": [],
+    "forwardedProps": {},
+}
+
+
+def _sse(events):
+    async def stream():
+        for event in events:
+            yield event
+
+    return StreamingResponse(stream(), media_type="text/event-stream")
+
+
+def _provider_app(seen):
+    app = FastAPI()
+
+    @app.post("/v1/chat/completions")
+    async def openai_chat(request: Request):
+        body = await request.json()
+        seen.append(("openai", body["model"]))
+        if not body.get("stream"):
+            return JSONResponse(
+                {
+                    "id": "c",
+                    "object": "chat.completion",
+                    "created": 0,
+                    "model": body["model"],
+                    "choices": [
+                        {
+                            "index": 0,
+                            "finish_reason": "stop",
+                            "message": {"role": "assistant", "content": "hello"},
+                        }
+                    ],
+                }
+            )
+        chunk = {
+            "id": "c",
+            "object": "chat.completion.chunk",
+            "created": 0,
+            "model": body["model"],
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": {"role": "assistant", "content": "hello"},
+                    "finish_reason": None,
+                }
+            ],
+        }
+        done = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
+        return _sse(
+            [f"data: {json.dumps(chunk)}\n\n", f"data: {json.dumps(done)}\n\n", "data: [DONE]\n\n"]
+        )
+
+    @app.post("/v1/messages")
+    async def anthropic_messages(request: Request):
+        body = await request.json()
+        seen.append(("anthropic", body["model"]))
+        message = {
+            "id": "msg",
+            "type": "message",
+            "role": "assistant",
+            "model": body["model"],
+            "stop_sequence": None,
+        }
+        if not body.get("stream"):
+            return JSONResponse(
+                {
+                    **message,
+                    "content": [{"type": "text", "text": "hello"}],
+                    "stop_reason": "end_turn",
+                    "usage": {"input_tokens": 1, "output_tokens": 1},
+                }
+            )
+        events = [
+            ("message_start", {"type": "message_start", "message": {**message, "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}),
+            ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
+            ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello"}}),
+            ("content_block_stop", {"type": "content_block_stop", "index": 0}),
+            ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 1}}),
+            ("message_stop", {"type": "message_stop"}),
+        ]
+        return _sse([f"event: {name}\ndata: {json.dumps(data)}\n\n" for name, data in events])
+
+    return app
+
+
+@pytest.fixture
+def provider():
+    seen = []
+    with socket.socket() as probe:
+        probe.bind(("127.0.0.1", 0))
+        port = probe.getsockname()[1]
+    server = uvicorn.Server(
+        uvicorn.Config(_provider_app(seen), host="127.0.0.1", port=port, log_level="error")
+    )
+    thread = threading.Thread(target=server.run, daemon=True)
+    thread.start()
+    deadline = time.monotonic() + 10
+    while not server.started and time.monotonic() < deadline:
+        time.sleep(0.01)
+    yield f"http://127.0.0.1:{port}", seen
+    server.should_exit = True
+    thread.join(timeout=10)
+
+
+CHOICES = {
+    "an Anthropic key": (
+        lambda base: {
+            "BOT_PROVIDER": "anthropic",
+            "BOT_MODEL": "claude-sonnet-4-5",
+            "ANTHROPIC_API_KEY": "test-key",
+            "ANTHROPIC_BASE_URL": base,
+            "OPENAI_API_KEY": "",
+            "OPENAI_BASE_URL": "",
+        },
+        ("anthropic", "claude-sonnet-4-5"),
+    ),
+    "an OpenAI-compatible endpoint": (
+        lambda base: {
+            "BOT_PROVIDER": "",
+            "BOT_MODEL": "local-model",
+            "OPENAI_API_KEY": "no-key-needed",
+            "OPENAI_BASE_URL": f"{base}/v1",
+            "ANTHROPIC_API_KEY": "",
+        },
+        ("openai", "local-model"),
+    ),
+    "an OpenAI key": (
+        lambda base: {
+            "BOT_PROVIDER": "",
+            "BOT_MODEL": "gpt-5.5",
+            "OPENAI_API_KEY": "test-key",
+            "OPENAI_BASE_URL": f"{base}/v1",
+            "ANTHROPIC_API_KEY": "",
+        },
+        ("openai", "gpt-5.5"),
+    ),
+}
+
+
+@pytest.mark.parametrize("choice", list(CHOICES))
+def test_a_run_reaches_the_model_the_setup_screen_chose(monkeypatch, provider, choice):
+    base, seen = provider
+    environment, expected = CHOICES[choice]
+    monkeypatch.delenv("OPENAI_API_BASE", raising=False)
+    monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN)
+    for key, value in environment(base).items():
+        monkeypatch.setenv(key, value)
+
+    from src import main
+
+    main = importlib.reload(main)
+    response = TestClient(main.app).post(
+        "/run", json=RUN, headers={"x-openbot-agent-token": TOKEN}
+    )
+
+    assert response.status_code == 200
+    assert '"RUN_FINISHED"' in response.text
+    assert '"RUN_ERROR"' not in response.text
+    assert seen == [expected]