From 95041148c51f0fa147a424cff2f457a621fbf88a Mon Sep 17 00:00:00 2001 From: fengguo Date: Fri, 9 Oct 2026 23:26:48 +0800 Subject: [PATCH 1/3] fix: repair admin LLM test-connection endpoint POST /settings/test-connection calls LLMService.test_connection(), which has never existed, so the endpoint always raised AttributeError and the UI reported a generic failure regardless of whether the key was valid. Add the missing method. It probes the provider's models endpoint without spending completion tokens and without falling back to other providers, so a failure identifies this provider/base URL/key combination specifically. Credentials are sent only in headers, never in the URL. OpenRouter's /models is public and therefore cannot validate a key, so use /auth/key there and flag provisioning/management keys, which pass /auth/key but are rejected by chat completions. Also surface the template fallback from indicator aiGenerate through debug.generation_error: the built-in template passes validation, so previously a failed model call was indistinguishable from a successful response and the client reported success. Refs #280 --- backend_api_python/app/routes/indicator.py | 5 +++ backend_api_python/app/routes/settings.py | 7 +--- backend_api_python/app/services/llm.py | 49 ++++++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/backend_api_python/app/routes/indicator.py b/backend_api_python/app/routes/indicator.py index d29425a44..a32d2d281 100644 --- a/backend_api_python/app/routes/indicator.py +++ b/backend_api_python/app/routes/indicator.py @@ -1188,6 +1188,11 @@ def stream(): code_text, debug_info, edit_plan = _generate_final_code() + # Signal the template fallback to the client; otherwise a failed model + # call is indistinguishable from a successful template-shaped answer. + if edit_plan.get("error"): + debug_info["generation_error"] = str(edit_plan["error"])[:500] + if workspace_context: validation = _validate_indicator_code_internal(code_text) validation["edit_plan"] = edit_plan diff --git a/backend_api_python/app/routes/settings.py b/backend_api_python/app/routes/settings.py index 07a5ffc5c..ff5b7d342 100644 --- a/backend_api_python/app/routes/settings.py +++ b/backend_api_python/app/routes/settings.py @@ -2452,11 +2452,8 @@ def test_connection(): if service == 'openrouter': from app.services.llm import LLMService llm = LLMService() - result = llm.test_connection() - if result: - return jsonify({'code': 1, 'msg': 'OpenRouter connection successful'}) - else: - return jsonify({'code': 0, 'msg': 'OpenRouter connection failed'}) + ok, detail = llm.test_connection() + return jsonify({'code': 1 if ok else 0, 'msg': detail}) elif service == 'finnhub': import requests diff --git a/backend_api_python/app/services/llm.py b/backend_api_python/app/services/llm.py index 264463a35..3eee0b790 100644 --- a/backend_api_python/app/services/llm.py +++ b/backend_api_python/app/services/llm.py @@ -285,6 +285,55 @@ def is_configured(self, provider: LLMProvider = None) -> bool: return bool((self.get_base_url(p) or "").strip()) return p == LLMProvider.LITELLM + def test_connection(self, provider: LLMProvider = None) -> tuple[bool, str]: + """Probe the configured provider's models endpoint. + + Intentionally does not spend completion tokens and does not fall back to + other providers, so a failure here means *this* provider/base URL/key + combination is unreachable or rejected. Returns ``(ok, detail)``. + + Credentials are only ever sent in headers, never in the URL, so provider + errors and request exceptions cannot leak the key. + """ + p = provider or self.provider + model = self.get_default_model(p) + if not self.is_configured(p): + return False, f"{p.value}: API key is not configured" + key = (self.get_api_key(p) or "").strip() + base = (self.get_base_url(p) or "").rstrip("/") + headers: Dict[str, str] = {} + if p == LLMProvider.OPENROUTER: + # OpenRouter's /models is public, so it cannot validate the key. + # /auth/key requires the bearer token and 401s on a bad key. + url = f"{base}/auth/key" + headers["Authorization"] = f"Bearer {key}" + elif p == LLMProvider.GOOGLE: + url = f"{base}/models" + headers["x-goog-api-key"] = key + else: + url = f"{base}/models" + headers["Authorization"] = f"Bearer {key}" + try: + resp = requests.get(url, headers=headers, timeout=15) + except Exception as exc: + return False, f"{p.value}: {exc}" + if resp.status_code == 200: + if p == LLMProvider.OPENROUTER: + # A provisioning/management key passes /auth/key but is rejected + # (401 "User not found") by chat completions, so flag it here. + try: + info = (resp.json() or {}).get("data") or {} + except Exception: + info = {} + if info.get("is_provisioning_key") or info.get("is_management_key"): + return False, ( + f"{p.value}: this is a provisioning/management key, " + "which cannot run inference. Create a normal API key instead." + ) + return True, f"{p.value} reachable ยท model={model}" + body = resp.text[:200] if resp.text else "" + return False, f"{p.value}: HTTP {resp.status_code} {body}".strip() + # Legacy properties for backward compatibility @property def api_key(self): From c8114aecdb4e971a5ae00efe16c6c9cd0c520734 Mon Sep 17 00:00:00 2001 From: fengguo Date: Fri, 9 Oct 2026 23:27:55 +0800 Subject: [PATCH 2/3] test: cover LLM test_connection probe Cases: unconfigured key, OpenRouter authenticated endpoint selection, the provisioning/management key trap, HTTP failure surfacing, and that the Google key is sent as a header rather than in the URL. --- .../tests/test_llm_litellm_provider.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/backend_api_python/tests/test_llm_litellm_provider.py b/backend_api_python/tests/test_llm_litellm_provider.py index 4d4f43a54..d13019aae 100644 --- a/backend_api_python/tests/test_llm_litellm_provider.py +++ b/backend_api_python/tests/test_llm_litellm_provider.py @@ -701,3 +701,100 @@ def completion(**kwargs): assert out == "hello" assert captured["max_tokens"] == 16384 + + +class _FakeHttpGetResponse: + def __init__(self, status_code, payload=None, text=""): + self.status_code = status_code + self._payload = payload + self.text = text + + def json(self): + if self._payload is None: + raise ValueError("not json") + return self._payload + + +def test_test_connection_requires_configured_key(monkeypatch): + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "") + + ok, detail = service.test_connection() + + assert ok is False + assert "not configured" in detail + + +def test_test_connection_uses_authenticated_endpoint_for_openrouter(monkeypatch): + captured = {} + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "openrouter-key") + monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1") + + def fake_get(url, headers=None, timeout=None): + captured.update({"url": url, "headers": headers, "timeout": timeout}) + return _FakeHttpGetResponse(200, {"data": {}}) + + monkeypatch.setattr("app.services.llm.requests.get", fake_get) + + ok, detail = service.test_connection() + + assert ok is True + # /models is public on OpenRouter, so it cannot validate a key. + assert captured["url"] == "https://openrouter.ai/api/v1/auth/key" + assert captured["headers"]["Authorization"] == "Bearer openrouter-key" + assert "reachable" in detail + + +def test_test_connection_flags_openrouter_provisioning_key(monkeypatch): + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "provisioning-key") + monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1") + monkeypatch.setattr( + "app.services.llm.requests.get", + lambda *args, **kwargs: _FakeHttpGetResponse(200, {"data": {"is_provisioning_key": True}}), + ) + + ok, detail = service.test_connection() + + assert ok is False + # Such a key passes /auth/key but chat completions reject it with 401. + assert "provisioning/management" in detail + + +def test_test_connection_reports_http_failure(monkeypatch): + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "bad-key") + monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1") + monkeypatch.setattr( + "app.services.llm.requests.get", + lambda *args, **kwargs: _FakeHttpGetResponse(401, None, '{"error":"unauthorized"}'), + ) + + ok, detail = service.test_connection() + + assert ok is False + assert "HTTP 401" in detail + + +def test_test_connection_never_puts_google_key_in_url(monkeypatch): + captured = {} + service = LLMService(provider="google") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "google-key") + monkeypatch.setattr( + service, + "get_base_url", + lambda provider=None: "https://generativelanguage.googleapis.com/v1beta", + ) + + def fake_get(url, headers=None, timeout=None): + captured.update({"url": url, "headers": headers}) + return _FakeHttpGetResponse(200, {}) + + monkeypatch.setattr("app.services.llm.requests.get", fake_get) + + ok, _ = service.test_connection() + + assert ok is True + assert captured["headers"]["x-goog-api-key"] == "google-key" + assert "google-key" not in captured["url"] From 3c5442961dc3b2e3afd6ac71f7e7b3fcd12fcdff Mon Sep 17 00:00:00 2001 From: fengguo Date: Fri, 9 Oct 2026 23:49:00 +0800 Subject: [PATCH 3/3] feat: report the model that actually answered an LLM call A model-level fallback (requested model -> configured default -> static fallback) was only observable in the provider dashboard, so a misconfigured or unavailable model id silently produced answers from a different model while the client reported success. LLMService now records last_model_used / last_provider_used / model_fallback_used, the indicator generation path carries them in its edit plan, and aiGenerate returns them as generation_model / generation_provider / generation_model_fallback in the streamed debug frame. --- backend_api_python/app/routes/indicator.py | 4 ++ .../app/services/indicator_ai_generation.py | 10 +++- backend_api_python/app/services/llm.py | 24 +++++++- .../tests/test_llm_litellm_provider.py | 56 +++++++++++++++++++ 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/backend_api_python/app/routes/indicator.py b/backend_api_python/app/routes/indicator.py index a32d2d281..50d5a6593 100644 --- a/backend_api_python/app/routes/indicator.py +++ b/backend_api_python/app/routes/indicator.py @@ -1192,6 +1192,10 @@ def stream(): # call is indistinguishable from a successful template-shaped answer. if edit_plan.get("error"): debug_info["generation_error"] = str(edit_plan["error"])[:500] + if edit_plan.get("model"): + debug_info["generation_model"] = edit_plan["model"] + debug_info["generation_provider"] = edit_plan.get("provider") or "" + debug_info["generation_model_fallback"] = bool(edit_plan.get("model_fallback")) if workspace_context: validation = _validate_indicator_code_internal(code_text) diff --git a/backend_api_python/app/services/indicator_ai_generation.py b/backend_api_python/app/services/indicator_ai_generation.py index 3e9818589..495b7afdf 100644 --- a/backend_api_python/app/services/indicator_ai_generation.py +++ b/backend_api_python/app/services/indicator_ai_generation.py @@ -156,7 +156,7 @@ def generate_indicator_code_candidate( ) if use_patch_response: try: - return apply_model_code_edits(existing, content) + code, plan = apply_model_code_edits(existing, content) except CodeEditError as exc: logger.warning("indicator model patch rejected, retrying full candidate: %s", exc) fallback_prompt = ( @@ -182,7 +182,13 @@ def generate_indicator_code_candidate( "operation": "generate_candidate", "patch_error": str(exc), } + code = _strip_code_fences(content) or template_factory() else: plan = {"executor": "model", "operation": "generate_candidate"} + code = _strip_code_fences(content) or template_factory() - return _strip_code_fences(content) or template_factory(), plan + # A model-level fallback is otherwise only visible in the provider dashboard. + plan["model"] = llm.last_model_used + plan["provider"] = llm.last_provider_used + plan["model_fallback"] = llm.model_fallback_used + return code, plan diff --git a/backend_api_python/app/services/llm.py b/backend_api_python/app/services/llm.py index 3eee0b790..953e75bdb 100644 --- a/backend_api_python/app/services/llm.py +++ b/backend_api_python/app/services/llm.py @@ -164,6 +164,11 @@ def __init__(self, provider: str = None): provider: Override the default provider (openrouter, openai, google, deepseek, grok, atlascloud, custom, minimax) """ self._provider_override = provider + # Populated by call_llm_api() so callers can report what actually answered; + # a model-level fallback is otherwise only visible in provider dashboards. + self.last_model_used = "" + self.last_provider_used = "" + self.model_fallback_used = False @property def provider(self) -> LLMProvider: @@ -285,6 +290,17 @@ def is_configured(self, provider: LLMProvider = None) -> bool: return bool((self.get_base_url(p) or "").strip()) return p == LLMProvider.LITELLM + def _record_model_use( + self, + used_model: str, + requested_model: str, + provider: LLMProvider, + ) -> None: + """Remember which model/provider answered the most recent call.""" + self.last_model_used = used_model + self.last_provider_used = provider.value + self.model_fallback_used = used_model != requested_model + def test_connection(self, provider: LLMProvider = None) -> tuple[bool, str]: """Probe the configured provider's models endpoint. @@ -1260,23 +1276,25 @@ def call_llm_api(self, messages: list, model: str = None, temperature: float = 0 for current_model in models_to_try: try: if p == LLMProvider.LITELLM: - return self._call_litellm( + result = self._call_litellm( messages, current_model, temperature, api_key, base_url, timeout, use_json_mode=use_json_mode ) elif p == LLMProvider.GOOGLE: - return self._call_google_gemini( + result = self._call_google_gemini( messages, current_model, temperature, api_key, base_url, timeout ) else: # OpenAI-compatible providers - return self._call_openai_compatible( + result = self._call_openai_compatible( messages, current_model, temperature, api_key, base_url, timeout, use_json_mode=use_json_mode ) + self._record_model_use(current_model, models_to_try[0], p) + return result except LLMAPIError as e: status_code = e.status_code diff --git a/backend_api_python/tests/test_llm_litellm_provider.py b/backend_api_python/tests/test_llm_litellm_provider.py index d13019aae..18e3e8995 100644 --- a/backend_api_python/tests/test_llm_litellm_provider.py +++ b/backend_api_python/tests/test_llm_litellm_provider.py @@ -798,3 +798,59 @@ def fake_get(url, headers=None, timeout=None): assert ok is True assert captured["headers"]["x-goog-api-key"] == "google-key" assert "google-key" not in captured["url"] + + +def _openrouter_service_with(monkeypatch, default_model="deepseek/deepseek-v4.1-flash"): + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "openrouter-key") + monkeypatch.setattr( + service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1" + ) + monkeypatch.setattr(service, "get_default_model", lambda provider=None: default_model) + return service + + +def test_model_fallback_is_recorded_when_primary_model_fails(monkeypatch): + attempted = [] + service = _openrouter_service_with(monkeypatch) + + def fake_call(messages, model, *args, **kwargs): + attempted.append(model) + if model == "does/not-exist": + raise LLMAPIError( + "OpenRouter API 400 (model=does/not-exist): not a valid model ID", + status_code=400, + ) + return "generated code" + + monkeypatch.setattr(service, "_call_openai_compatible", fake_call) + + out = service.call_llm_api( + [{"role": "user", "content": "hi"}], + model="does/not-exist", + use_json_mode=False, + try_alternative_providers=False, + ) + + assert out == "generated code" + assert attempted == ["does/not-exist", "deepseek/deepseek-v4.1-flash"] + # The caller must be able to tell that a different model answered. + assert service.last_model_used == "deepseek/deepseek-v4.1-flash" + assert service.last_provider_used == "openrouter" + assert service.model_fallback_used is True + + +def test_model_fallback_flag_is_false_when_primary_model_answers(monkeypatch): + service = _openrouter_service_with(monkeypatch) + monkeypatch.setattr(service, "_call_openai_compatible", lambda *args, **kwargs: "generated code") + + out = service.call_llm_api( + [{"role": "user", "content": "hi"}], + model="deepseek/deepseek-v4.1-flash", + use_json_mode=False, + try_alternative_providers=False, + ) + + assert out == "generated code" + assert service.last_model_used == "deepseek/deepseek-v4.1-flash" + assert service.model_fallback_used is False