fix: repair LLM test-connection and make silent model/template fallbacks visible

backend_api_python/app/routes/indicator.py

@@ -1188,6 +1188,15 @@ def stream():
 
         code_text, debug_info, edit_plan = _generate_final_code()
 
+        # Signal the template fallback to the client; otherwise a failed model
+        # call is indistinguishable from a successful template-shaped answer.
+        if edit_plan.get("error"):
+            debug_info["generation_error"] = str(edit_plan["error"])[:500]
+        if edit_plan.get("model"):
+            debug_info["generation_model"] = edit_plan["model"]
+            debug_info["generation_provider"] = edit_plan.get("provider") or ""
+            debug_info["generation_model_fallback"] = bool(edit_plan.get("model_fallback"))
+
         if workspace_context:
             validation = _validate_indicator_code_internal(code_text)
             validation["edit_plan"] = edit_plan

backend_api_python/app/routes/settings.py

@@ -2452,11 +2452,8 @@ def test_connection():
         if service == 'openrouter':
             from app.services.llm import LLMService
             llm = LLMService()
-            result = llm.test_connection()
-            if result:
-                return jsonify({'code': 1, 'msg': 'OpenRouter connection successful'})
-            else:
-                return jsonify({'code': 0, 'msg': 'OpenRouter connection failed'})
+            ok, detail = llm.test_connection()
+            return jsonify({'code': 1 if ok else 0, 'msg': detail})
 
         elif service == 'finnhub':
             import requests

backend_api_python/app/services/indicator_ai_generation.py

@@ -156,7 +156,7 @@ def generate_indicator_code_candidate(
     )
     if use_patch_response:
         try:
-            return apply_model_code_edits(existing, content)
+            code, plan = apply_model_code_edits(existing, content)
         except CodeEditError as exc:
             logger.warning("indicator model patch rejected, retrying full candidate: %s", exc)
             fallback_prompt = (
@@ -182,7 +182,13 @@ def generate_indicator_code_candidate(
                 "operation": "generate_candidate",
                 "patch_error": str(exc),
             }
+            code = _strip_code_fences(content) or template_factory()
     else:
         plan = {"executor": "model", "operation": "generate_candidate"}
+        code = _strip_code_fences(content) or template_factory()
 
-    return _strip_code_fences(content) or template_factory(), plan
+    # A model-level fallback is otherwise only visible in the provider dashboard.
+    plan["model"] = llm.last_model_used
+    plan["provider"] = llm.last_provider_used
+    plan["model_fallback"] = llm.model_fallback_used
+    return code, plan

backend_api_python/app/services/llm.py

@@ -164,6 +164,11 @@ def __init__(self, provider: str = None):
             provider: Override the default provider (openrouter, openai, google, deepseek, grok, atlascloud, custom, minimax)
         """
         self._provider_override = provider
+        # Populated by call_llm_api() so callers can report what actually answered;
+        # a model-level fallback is otherwise only visible in provider dashboards.
+        self.last_model_used = ""
+        self.last_provider_used = ""
+        self.model_fallback_used = False
 
     @property
     def provider(self) -> LLMProvider:
@@ -285,6 +290,66 @@ def is_configured(self, provider: LLMProvider = None) -> bool:
             return bool((self.get_base_url(p) or "").strip())
         return p == LLMProvider.LITELLM
 
+    def _record_model_use(
+        self,
+        used_model: str,
+        requested_model: str,
+        provider: LLMProvider,
+    ) -> None:
+        """Remember which model/provider answered the most recent call."""
+        self.last_model_used = used_model
+        self.last_provider_used = provider.value
+        self.model_fallback_used = used_model != requested_model
+
+    def test_connection(self, provider: LLMProvider = None) -> tuple[bool, str]:
+        """Probe the configured provider's models endpoint.
+
+        Intentionally does not spend completion tokens and does not fall back to
+        other providers, so a failure here means *this* provider/base URL/key
+        combination is unreachable or rejected. Returns ``(ok, detail)``.
+
+        Credentials are only ever sent in headers, never in the URL, so provider
+        errors and request exceptions cannot leak the key.
+        """
+        p = provider or self.provider
+        model = self.get_default_model(p)
+        if not self.is_configured(p):
+            return False, f"{p.value}: API key is not configured"
+        key = (self.get_api_key(p) or "").strip()
+        base = (self.get_base_url(p) or "").rstrip("/")
+        headers: Dict[str, str] = {}
+        if p == LLMProvider.OPENROUTER:
+            # OpenRouter's /models is public, so it cannot validate the key.
+            # /auth/key requires the bearer token and 401s on a bad key.
+            url = f"{base}/auth/key"
+            headers["Authorization"] = f"Bearer {key}"
+        elif p == LLMProvider.GOOGLE:
+            url = f"{base}/models"
+            headers["x-goog-api-key"] = key
+        else:
+            url = f"{base}/models"
+            headers["Authorization"] = f"Bearer {key}"
+        try:
+            resp = requests.get(url, headers=headers, timeout=15)
+        except Exception as exc:
+            return False, f"{p.value}: {exc}"
+        if resp.status_code == 200:
+            if p == LLMProvider.OPENROUTER:
+                # A provisioning/management key passes /auth/key but is rejected
+                # (401 "User not found") by chat completions, so flag it here.
+                try:
+                    info = (resp.json() or {}).get("data") or {}
+                except Exception:
+                    info = {}
+                if info.get("is_provisioning_key") or info.get("is_management_key"):
+                    return False, (
+                        f"{p.value}: this is a provisioning/management key, "
+                        "which cannot run inference. Create a normal API key instead."
+                    )
+            return True, f"{p.value} reachable · model={model}"
+        body = resp.text[:200] if resp.text else ""
+        return False, f"{p.value}: HTTP {resp.status_code} {body}".strip()
+
     # Legacy properties for backward compatibility
     @property
     def api_key(self):
@@ -1211,23 +1276,25 @@ def call_llm_api(self, messages: list, model: str = None, temperature: float = 0
         for current_model in models_to_try:
             try:
                 if p == LLMProvider.LITELLM:
-                    return self._call_litellm(
+                    result = self._call_litellm(
                         messages, current_model, temperature,
                         api_key, base_url, timeout,
                         use_json_mode=use_json_mode
                     )
                 elif p == LLMProvider.GOOGLE:
-                    return self._call_google_gemini(
+                    result = self._call_google_gemini(
                         messages, current_model, temperature,
                         api_key, base_url, timeout
                     )
                 else:
                     # OpenAI-compatible providers
-                    return self._call_openai_compatible(
+                    result = self._call_openai_compatible(
                         messages, current_model, temperature,
                         api_key, base_url, timeout,
                         use_json_mode=use_json_mode
                     )
+                self._record_model_use(current_model, models_to_try[0], p)
+                return result
 
             except LLMAPIError as e:
                 status_code = e.status_code

backend_api_python/tests/test_llm_litellm_provider.py

@@ -701,3 +701,156 @@ def completion(**kwargs):
 
     assert out == "hello"
     assert captured["max_tokens"] == 16384
+
+
+class _FakeHttpGetResponse:
+    def __init__(self, status_code, payload=None, text=""):
+        self.status_code = status_code
+        self._payload = payload
+        self.text = text
+
+    def json(self):
+        if self._payload is None:
+            raise ValueError("not json")
+        return self._payload
+
+
+def test_test_connection_requires_configured_key(monkeypatch):
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "")
+
+    ok, detail = service.test_connection()
+
+    assert ok is False
+    assert "not configured" in detail
+
+
+def test_test_connection_uses_authenticated_endpoint_for_openrouter(monkeypatch):
+    captured = {}
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "openrouter-key")
+    monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1")
+
+    def fake_get(url, headers=None, timeout=None):
+        captured.update({"url": url, "headers": headers, "timeout": timeout})
+        return _FakeHttpGetResponse(200, {"data": {}})
+
+    monkeypatch.setattr("app.services.llm.requests.get", fake_get)
+
+    ok, detail = service.test_connection()
+
+    assert ok is True
+    # /models is public on OpenRouter, so it cannot validate a key.
+    assert captured["url"] == "https://openrouter.ai/api/v1/auth/key"
+    assert captured["headers"]["Authorization"] == "Bearer openrouter-key"
+    assert "reachable" in detail
+
+
+def test_test_connection_flags_openrouter_provisioning_key(monkeypatch):
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "provisioning-key")
+    monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1")
+    monkeypatch.setattr(
+        "app.services.llm.requests.get",
+        lambda *args, **kwargs: _FakeHttpGetResponse(200, {"data": {"is_provisioning_key": True}}),
+    )
+
+    ok, detail = service.test_connection()
+
+    assert ok is False
+    # Such a key passes /auth/key but chat completions reject it with 401.
+    assert "provisioning/management" in detail
+
+
+def test_test_connection_reports_http_failure(monkeypatch):
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "bad-key")
+    monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1")
+    monkeypatch.setattr(
+        "app.services.llm.requests.get",
+        lambda *args, **kwargs: _FakeHttpGetResponse(401, None, '{"error":"unauthorized"}'),
+    )
+
+    ok, detail = service.test_connection()
+
+    assert ok is False
+    assert "HTTP 401" in detail
+
+
+def test_test_connection_never_puts_google_key_in_url(monkeypatch):
+    captured = {}
+    service = LLMService(provider="google")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "google-key")
+    monkeypatch.setattr(
+        service,
+        "get_base_url",
+        lambda provider=None: "https://generativelanguage.googleapis.com/v1beta",
+    )
+
+    def fake_get(url, headers=None, timeout=None):
+        captured.update({"url": url, "headers": headers})
+        return _FakeHttpGetResponse(200, {})
+
+    monkeypatch.setattr("app.services.llm.requests.get", fake_get)
+
+    ok, _ = service.test_connection()
+
+    assert ok is True
+    assert captured["headers"]["x-goog-api-key"] == "google-key"
+    assert "google-key" not in captured["url"]
+
+
+def _openrouter_service_with(monkeypatch, default_model="deepseek/deepseek-v4.1-flash"):
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "openrouter-key")
+    monkeypatch.setattr(
+        service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1"
+    )
+    monkeypatch.setattr(service, "get_default_model", lambda provider=None: default_model)
+    return service
+
+
+def test_model_fallback_is_recorded_when_primary_model_fails(monkeypatch):
+    attempted = []
+    service = _openrouter_service_with(monkeypatch)
+
+    def fake_call(messages, model, *args, **kwargs):
+        attempted.append(model)
+        if model == "does/not-exist":
+            raise LLMAPIError(
+                "OpenRouter API 400 (model=does/not-exist): not a valid model ID",
+                status_code=400,
+            )
+        return "generated code"
+
+    monkeypatch.setattr(service, "_call_openai_compatible", fake_call)
+
+    out = service.call_llm_api(
+        [{"role": "user", "content": "hi"}],
+        model="does/not-exist",
+        use_json_mode=False,
+        try_alternative_providers=False,
+    )
+
+    assert out == "generated code"
+    assert attempted == ["does/not-exist", "deepseek/deepseek-v4.1-flash"]
+    # The caller must be able to tell that a different model answered.
+    assert service.last_model_used == "deepseek/deepseek-v4.1-flash"
+    assert service.last_provider_used == "openrouter"
+    assert service.model_fallback_used is True
+
+
+def test_model_fallback_flag_is_false_when_primary_model_answers(monkeypatch):
+    service = _openrouter_service_with(monkeypatch)
+    monkeypatch.setattr(service, "_call_openai_compatible", lambda *args, **kwargs: "generated code")
+
+    out = service.call_llm_api(
+        [{"role": "user", "content": "hi"}],
+        model="deepseek/deepseek-v4.1-flash",
+        use_json_mode=False,
+        try_alternative_providers=False,
+    )
+
+    assert out == "generated code"
+    assert service.last_model_used == "deepseek/deepseek-v4.1-flash"
+    assert service.model_fallback_used is False