diff --git a/backend_api_python/app/routes/indicator.py b/backend_api_python/app/routes/indicator.py index d29425a44..50d5a6593 100644 --- a/backend_api_python/app/routes/indicator.py +++ b/backend_api_python/app/routes/indicator.py @@ -1188,6 +1188,15 @@ def stream(): code_text, debug_info, edit_plan = _generate_final_code() + # Signal the template fallback to the client; otherwise a failed model + # call is indistinguishable from a successful template-shaped answer. + if edit_plan.get("error"): + debug_info["generation_error"] = str(edit_plan["error"])[:500] + if edit_plan.get("model"): + debug_info["generation_model"] = edit_plan["model"] + debug_info["generation_provider"] = edit_plan.get("provider") or "" + debug_info["generation_model_fallback"] = bool(edit_plan.get("model_fallback")) + if workspace_context: validation = _validate_indicator_code_internal(code_text) validation["edit_plan"] = edit_plan diff --git a/backend_api_python/app/routes/settings.py b/backend_api_python/app/routes/settings.py index 07a5ffc5c..ff5b7d342 100644 --- a/backend_api_python/app/routes/settings.py +++ b/backend_api_python/app/routes/settings.py @@ -2452,11 +2452,8 @@ def test_connection(): if service == 'openrouter': from app.services.llm import LLMService llm = LLMService() - result = llm.test_connection() - if result: - return jsonify({'code': 1, 'msg': 'OpenRouter connection successful'}) - else: - return jsonify({'code': 0, 'msg': 'OpenRouter connection failed'}) + ok, detail = llm.test_connection() + return jsonify({'code': 1 if ok else 0, 'msg': detail}) elif service == 'finnhub': import requests diff --git a/backend_api_python/app/services/indicator_ai_generation.py b/backend_api_python/app/services/indicator_ai_generation.py index 3e9818589..495b7afdf 100644 --- a/backend_api_python/app/services/indicator_ai_generation.py +++ b/backend_api_python/app/services/indicator_ai_generation.py @@ -156,7 +156,7 @@ def generate_indicator_code_candidate( ) if use_patch_response: try: - return apply_model_code_edits(existing, content) + code, plan = apply_model_code_edits(existing, content) except CodeEditError as exc: logger.warning("indicator model patch rejected, retrying full candidate: %s", exc) fallback_prompt = ( @@ -182,7 +182,13 @@ def generate_indicator_code_candidate( "operation": "generate_candidate", "patch_error": str(exc), } + code = _strip_code_fences(content) or template_factory() else: plan = {"executor": "model", "operation": "generate_candidate"} + code = _strip_code_fences(content) or template_factory() - return _strip_code_fences(content) or template_factory(), plan + # A model-level fallback is otherwise only visible in the provider dashboard. + plan["model"] = llm.last_model_used + plan["provider"] = llm.last_provider_used + plan["model_fallback"] = llm.model_fallback_used + return code, plan diff --git a/backend_api_python/app/services/llm.py b/backend_api_python/app/services/llm.py index 264463a35..953e75bdb 100644 --- a/backend_api_python/app/services/llm.py +++ b/backend_api_python/app/services/llm.py @@ -164,6 +164,11 @@ def __init__(self, provider: str = None): provider: Override the default provider (openrouter, openai, google, deepseek, grok, atlascloud, custom, minimax) """ self._provider_override = provider + # Populated by call_llm_api() so callers can report what actually answered; + # a model-level fallback is otherwise only visible in provider dashboards. + self.last_model_used = "" + self.last_provider_used = "" + self.model_fallback_used = False @property def provider(self) -> LLMProvider: @@ -285,6 +290,66 @@ def is_configured(self, provider: LLMProvider = None) -> bool: return bool((self.get_base_url(p) or "").strip()) return p == LLMProvider.LITELLM + def _record_model_use( + self, + used_model: str, + requested_model: str, + provider: LLMProvider, + ) -> None: + """Remember which model/provider answered the most recent call.""" + self.last_model_used = used_model + self.last_provider_used = provider.value + self.model_fallback_used = used_model != requested_model + + def test_connection(self, provider: LLMProvider = None) -> tuple[bool, str]: + """Probe the configured provider's models endpoint. + + Intentionally does not spend completion tokens and does not fall back to + other providers, so a failure here means *this* provider/base URL/key + combination is unreachable or rejected. Returns ``(ok, detail)``. + + Credentials are only ever sent in headers, never in the URL, so provider + errors and request exceptions cannot leak the key. + """ + p = provider or self.provider + model = self.get_default_model(p) + if not self.is_configured(p): + return False, f"{p.value}: API key is not configured" + key = (self.get_api_key(p) or "").strip() + base = (self.get_base_url(p) or "").rstrip("/") + headers: Dict[str, str] = {} + if p == LLMProvider.OPENROUTER: + # OpenRouter's /models is public, so it cannot validate the key. + # /auth/key requires the bearer token and 401s on a bad key. + url = f"{base}/auth/key" + headers["Authorization"] = f"Bearer {key}" + elif p == LLMProvider.GOOGLE: + url = f"{base}/models" + headers["x-goog-api-key"] = key + else: + url = f"{base}/models" + headers["Authorization"] = f"Bearer {key}" + try: + resp = requests.get(url, headers=headers, timeout=15) + except Exception as exc: + return False, f"{p.value}: {exc}" + if resp.status_code == 200: + if p == LLMProvider.OPENROUTER: + # A provisioning/management key passes /auth/key but is rejected + # (401 "User not found") by chat completions, so flag it here. + try: + info = (resp.json() or {}).get("data") or {} + except Exception: + info = {} + if info.get("is_provisioning_key") or info.get("is_management_key"): + return False, ( + f"{p.value}: this is a provisioning/management key, " + "which cannot run inference. Create a normal API key instead." + ) + return True, f"{p.value} reachable ยท model={model}" + body = resp.text[:200] if resp.text else "" + return False, f"{p.value}: HTTP {resp.status_code} {body}".strip() + # Legacy properties for backward compatibility @property def api_key(self): @@ -1211,23 +1276,25 @@ def call_llm_api(self, messages: list, model: str = None, temperature: float = 0 for current_model in models_to_try: try: if p == LLMProvider.LITELLM: - return self._call_litellm( + result = self._call_litellm( messages, current_model, temperature, api_key, base_url, timeout, use_json_mode=use_json_mode ) elif p == LLMProvider.GOOGLE: - return self._call_google_gemini( + result = self._call_google_gemini( messages, current_model, temperature, api_key, base_url, timeout ) else: # OpenAI-compatible providers - return self._call_openai_compatible( + result = self._call_openai_compatible( messages, current_model, temperature, api_key, base_url, timeout, use_json_mode=use_json_mode ) + self._record_model_use(current_model, models_to_try[0], p) + return result except LLMAPIError as e: status_code = e.status_code diff --git a/backend_api_python/tests/test_llm_litellm_provider.py b/backend_api_python/tests/test_llm_litellm_provider.py index 4d4f43a54..18e3e8995 100644 --- a/backend_api_python/tests/test_llm_litellm_provider.py +++ b/backend_api_python/tests/test_llm_litellm_provider.py @@ -701,3 +701,156 @@ def completion(**kwargs): assert out == "hello" assert captured["max_tokens"] == 16384 + + +class _FakeHttpGetResponse: + def __init__(self, status_code, payload=None, text=""): + self.status_code = status_code + self._payload = payload + self.text = text + + def json(self): + if self._payload is None: + raise ValueError("not json") + return self._payload + + +def test_test_connection_requires_configured_key(monkeypatch): + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "") + + ok, detail = service.test_connection() + + assert ok is False + assert "not configured" in detail + + +def test_test_connection_uses_authenticated_endpoint_for_openrouter(monkeypatch): + captured = {} + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "openrouter-key") + monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1") + + def fake_get(url, headers=None, timeout=None): + captured.update({"url": url, "headers": headers, "timeout": timeout}) + return _FakeHttpGetResponse(200, {"data": {}}) + + monkeypatch.setattr("app.services.llm.requests.get", fake_get) + + ok, detail = service.test_connection() + + assert ok is True + # /models is public on OpenRouter, so it cannot validate a key. + assert captured["url"] == "https://openrouter.ai/api/v1/auth/key" + assert captured["headers"]["Authorization"] == "Bearer openrouter-key" + assert "reachable" in detail + + +def test_test_connection_flags_openrouter_provisioning_key(monkeypatch): + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "provisioning-key") + monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1") + monkeypatch.setattr( + "app.services.llm.requests.get", + lambda *args, **kwargs: _FakeHttpGetResponse(200, {"data": {"is_provisioning_key": True}}), + ) + + ok, detail = service.test_connection() + + assert ok is False + # Such a key passes /auth/key but chat completions reject it with 401. + assert "provisioning/management" in detail + + +def test_test_connection_reports_http_failure(monkeypatch): + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "bad-key") + monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1") + monkeypatch.setattr( + "app.services.llm.requests.get", + lambda *args, **kwargs: _FakeHttpGetResponse(401, None, '{"error":"unauthorized"}'), + ) + + ok, detail = service.test_connection() + + assert ok is False + assert "HTTP 401" in detail + + +def test_test_connection_never_puts_google_key_in_url(monkeypatch): + captured = {} + service = LLMService(provider="google") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "google-key") + monkeypatch.setattr( + service, + "get_base_url", + lambda provider=None: "https://generativelanguage.googleapis.com/v1beta", + ) + + def fake_get(url, headers=None, timeout=None): + captured.update({"url": url, "headers": headers}) + return _FakeHttpGetResponse(200, {}) + + monkeypatch.setattr("app.services.llm.requests.get", fake_get) + + ok, _ = service.test_connection() + + assert ok is True + assert captured["headers"]["x-goog-api-key"] == "google-key" + assert "google-key" not in captured["url"] + + +def _openrouter_service_with(monkeypatch, default_model="deepseek/deepseek-v4.1-flash"): + service = LLMService(provider="openrouter") + monkeypatch.setattr(service, "get_api_key", lambda provider=None: "openrouter-key") + monkeypatch.setattr( + service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1" + ) + monkeypatch.setattr(service, "get_default_model", lambda provider=None: default_model) + return service + + +def test_model_fallback_is_recorded_when_primary_model_fails(monkeypatch): + attempted = [] + service = _openrouter_service_with(monkeypatch) + + def fake_call(messages, model, *args, **kwargs): + attempted.append(model) + if model == "does/not-exist": + raise LLMAPIError( + "OpenRouter API 400 (model=does/not-exist): not a valid model ID", + status_code=400, + ) + return "generated code" + + monkeypatch.setattr(service, "_call_openai_compatible", fake_call) + + out = service.call_llm_api( + [{"role": "user", "content": "hi"}], + model="does/not-exist", + use_json_mode=False, + try_alternative_providers=False, + ) + + assert out == "generated code" + assert attempted == ["does/not-exist", "deepseek/deepseek-v4.1-flash"] + # The caller must be able to tell that a different model answered. + assert service.last_model_used == "deepseek/deepseek-v4.1-flash" + assert service.last_provider_used == "openrouter" + assert service.model_fallback_used is True + + +def test_model_fallback_flag_is_false_when_primary_model_answers(monkeypatch): + service = _openrouter_service_with(monkeypatch) + monkeypatch.setattr(service, "_call_openai_compatible", lambda *args, **kwargs: "generated code") + + out = service.call_llm_api( + [{"role": "user", "content": "hi"}], + model="deepseek/deepseek-v4.1-flash", + use_json_mode=False, + try_alternative_providers=False, + ) + + assert out == "generated code" + assert service.last_model_used == "deepseek/deepseek-v4.1-flash" + assert service.model_fallback_used is False