diff --git a/.env.template b/.env.template index 66b06a0d..e0dc935f 100644 --- a/.env.template +++ b/.env.template @@ -68,9 +68,9 @@ # Allow optional /p/{provider}/v1/... passthrough aliases while keeping /p/{provider}/... canonical (default: true) # ALLOW_PASSTHROUGH_V1_ALIAS=true -# Comma-separated list of provider types enabled for /p/{provider}/... passthrough (default: openai,anthropic,openrouter,kilo,zai,vllm,deepseek) +# Comma-separated list of provider types enabled for /p/{provider}/... passthrough (default: openai,anthropic,openrouter,kilo,zai,vllm,llmd,deepseek) # Cohere native passthrough is opt-in; add cohere when those routes are needed. -# ENABLED_PASSTHROUGH_PROVIDERS=openai,anthropic,cohere,openrouter,kilo,zai,vllm,deepseek +# ENABLED_PASSTHROUGH_PROVIDERS=openai,anthropic,cohere,openrouter,kilo,zai,vllm,llmd,deepseek # Enable the realtime (speech-to-speech) endpoints (default: true): the /v1/realtime # websocket (and /p/{provider}/v1/realtime passthrough upgrade), the WebRTC SDP @@ -539,6 +539,18 @@ # Set base URL to enable (default: http://localhost:8000/v1) # VLLM_BASE_URL=http://localhost:8000/v1 +# llm-d Router / Endpoint Picker (EPP) +# The route is required and usually points to the EPP service. Include /v1. +# LLMD_BASE_URL=http://quickstart-epp.llm-d.svc.cluster.local/v1 +# Optional bearer token if the Gateway in front of llm-d requires one. +# LLMD_API_KEY=router-token +# Optional configured model list when GET /v1/models is not routed. +# LLMD_MODELS=Qwen/Qwen2.5-0.5B-Instruct +# Optional trusted InferenceObjective name injected by GoModel. +# LLMD_INFERENCE_OBJECTIVE=standard-traffic +# Derive the fairness ID from the effective user path (default: true). +# LLMD_FAIRNESS_FROM_USER_PATH=true + # SGLang (OpenAI-compatible server) # SGLANG_API_KEY is optional; set it only if launch_server uses --api-key. # SGLANG_API_KEY=token-abc123 diff --git a/CLAUDE.md b/CLAUDE.md index 04ff7aa9..46713faa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -138,5 +138,5 @@ Full reference: `.env.template` and `config/config.yaml` - **Guardrails:** Definitions are persisted in the `guardrail_definitions` store and managed via the admin API/dashboard; `config/config.yaml` entries are validated and upserted into that store at startup (a seed, not the source of truth). `GUARDRAILS_ENABLED` env var gates the feature. - **Provider API key rotation:** Any API-key provider accepts several keys: `[_SUFFIX]_API_KEY_` env vars (numbered from 2; `_1` is accepted as a synonym for the unsuffixed key) or `providers..api_keys` in `config.yaml` (merged after `api_key`, de-duplicated, unresolved `${...}` entries dropped; env replaces the whole YAML list). Identified sessions deterministically stay on one key by default, preserving provider prompt-cache affinity while spreading different sessions across the configured keys; sessionless requests remain round robin. Set `providers..session_sticky_keys: false` or untick **Session-sticky API keys** in the provider editor for strict per-request round robin. Realtime sessions use the same affinity. The trailing number names a key, not a provider: `OPENAI_API_KEY_2` is key 2 of `openai`, while `OPENAI_REGION_2_API_KEY` is the sole key of provider `openai-region-2`. Providers configured without keys are unaffected; Ollama is normally keyless, while SGLang and vLLM participate in rotation when optional API keys are configured. Non-API-key providers (Vertex, Bedrock) are unaffected. - **Provider credentials without env vars:** Every provider below can instead be configured from the admin dashboard's Providers page (or `/admin/provider-credentials` GET/PUT/DELETE), persisted to the `provider_credentials` store — the same declarative-shadows-store precedence as MCP servers: a provider name declared via env vars/`config.yaml` is read-only in the dashboard (`managed: true`), and a store row upsert/delete hot-registers or unregisters the provider into the live registry immediately, no restart. `GOMODEL` boots fine with zero providers configured (empty catalog) so this is a complete alternative to env-var credentials, not just a supplement. API keys (`api_keys`, an ordered rotation list, same semantics as `providers..api_keys`) and service-account secrets are redacted as `***********` on read; an upsert echoing any all-asterisk mask of at least three characters at a position preserves the stored value there (rejected if that position was never set). Disabling a row (`enabled: false`) unregisters it from routing without deleting the stored credentials. `GET /admin/provider-credentials/types` lists every constructible provider type with the credential form it accepts (`fields[]` of `name`/`required`/`advanced`/`options`, plus `default_base_url`), derived from each provider's `DiscoveryConfig` — the dashboard renders only those fields, so an OpenAI-type provider asks for an API key while a Vertex one asks for project/location/service account and no key at all. Upserts are validated against that form (and against Google's project-or-base-URL and service-account rules) *before* anything is persisted, so an unusable credential is rejected with a 400 naming the offending field in `error.param` rather than stored as a broken row. -- **Providers:** `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `ANTHROPIC_DEFAULT_MAX_TOKENS` (optional default `max_tokens` for Anthropic-translated requests that omit it; default 4096), `GEMINI_API_KEY`, `USE_GOOGLE_GEMINI_NATIVE_API` (true by default; false uses Gemini's OpenAI-compatible chat API), `XAI_API_KEY`, `GROQ_API_KEY`, `FIREWORKS_API_KEY`, `FIREWORKS_BASE_URL` (optional Fireworks AI endpoint override; default `https://api.fireworks.ai/inference/v1`), `META_API_KEY`, `META_BASE_URL` (optional Meta Model API endpoint override; default `https://api.meta.ai/v1`; Muse Spark models, e.g. `muse-spark-1.1`), `OPENROUTER_API_KEY`, `OPENROUTER_SITE_URL`/`OPENROUTER_APP_NAME` (optional OpenRouter attribution headers), `ZAI_API_KEY`, `ZAI_BASE_URL` (optional Z.ai endpoint override), `MINIMAX_API_KEY`, `MINIMAX_BASE_URL` (optional MiniMax endpoint override), `XIAOMI_API_KEY`, `XIAOMI_BASE_URL` (optional Xiaomi MiMo endpoint override), `OPENCODE_GO_API_KEY`, `OPENCODE_GO_BASE_URL` (optional OpenCode Go/Zen endpoint override; default `https://opencode.ai/zen/go/v1`), `OPENCODE_GO_MESSAGES_MODELS` (optional comma-separated model IDs routed to the Anthropic-native `/messages` endpoint instead of `/chat/completions`; default `qwen3.7-max`), `BAILIAN_API_KEY`, `BAILIAN_BASE_URL` (optional Bailian base URL for region switching; default `https://dashscope.aliyuncs.com/compatible-mode/v1`), `AZURE_API_KEY`, `AZURE_BASE_URL` (Azure OpenAI deployment base URL), `AZURE_API_VERSION` (optional Azure API version), `ORACLE_API_KEY` (Oracle API key), `ORACLE_BASE_URL` (Oracle OpenAI-compatible base URL), `BEDROCK_BASE_URL` (Bedrock Runtime region or endpoint), `BEDROCK_MANTLE_API_KEY`, `BEDROCK_MANTLE_BASE_URL` (Mantle region or endpoint), `BEDROCK_MANTLE_API_MODE` (`auto`, `openai`, or `standard`), `[_SUFFIX]_MODELS` (comma-separated configured model list for any provider type), `OLLAMA_BASE_URL`, `SGLANG_BASE_URL`, `SGLANG_API_KEY` (optional upstream SGLang bearer token), `VLLM_BASE_URL`, `VLLM_API_KEY` (optional upstream vLLM bearer token) +- **Providers:** `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `ANTHROPIC_DEFAULT_MAX_TOKENS` (optional default `max_tokens` for Anthropic-translated requests that omit it; default 4096), `GEMINI_API_KEY`, `USE_GOOGLE_GEMINI_NATIVE_API` (true by default; false uses Gemini's OpenAI-compatible chat API), `XAI_API_KEY`, `GROQ_API_KEY`, `FIREWORKS_API_KEY`, `FIREWORKS_BASE_URL` (optional Fireworks AI endpoint override; default `https://api.fireworks.ai/inference/v1`), `META_API_KEY`, `META_BASE_URL` (optional Meta Model API endpoint override; default `https://api.meta.ai/v1`; Muse Spark models, e.g. `muse-spark-1.1`), `OPENROUTER_API_KEY`, `OPENROUTER_SITE_URL`/`OPENROUTER_APP_NAME` (optional OpenRouter attribution headers), `ZAI_API_KEY`, `ZAI_BASE_URL` (optional Z.ai endpoint override), `MINIMAX_API_KEY`, `MINIMAX_BASE_URL` (optional MiniMax endpoint override), `XIAOMI_API_KEY`, `XIAOMI_BASE_URL` (optional Xiaomi MiMo endpoint override), `OPENCODE_GO_API_KEY`, `OPENCODE_GO_BASE_URL` (optional OpenCode Go/Zen endpoint override; default `https://opencode.ai/zen/go/v1`), `OPENCODE_GO_MESSAGES_MODELS` (optional comma-separated model IDs routed to the Anthropic-native `/messages` endpoint instead of `/chat/completions`; default `qwen3.7-max`), `BAILIAN_API_KEY`, `BAILIAN_BASE_URL` (optional Bailian base URL for region switching; default `https://dashscope.aliyuncs.com/compatible-mode/v1`), `AZURE_API_KEY`, `AZURE_BASE_URL` (Azure OpenAI deployment base URL), `AZURE_API_VERSION` (optional Azure API version), `ORACLE_API_KEY` (Oracle API key), `ORACLE_BASE_URL` (Oracle OpenAI-compatible base URL), `BEDROCK_BASE_URL` (Bedrock Runtime region or endpoint), `BEDROCK_MANTLE_API_KEY`, `BEDROCK_MANTLE_BASE_URL` (Mantle region or endpoint), `BEDROCK_MANTLE_API_MODE` (`auto`, `openai`, or `standard`), `[_SUFFIX]_MODELS` (comma-separated configured model list for any provider type), `OLLAMA_BASE_URL`, `SGLANG_BASE_URL`, `SGLANG_API_KEY` (optional upstream SGLang bearer token), `VLLM_BASE_URL`, `VLLM_API_KEY` (optional upstream vLLM bearer token), `LLMD_BASE_URL`, `LLMD_API_KEY` (optional Gateway bearer token), `LLMD_INFERENCE_OBJECTIVE`, `LLMD_FAIRNESS_FROM_USER_PATH` - **Provider model metadata:** `providers..models` accepts either model IDs (strings) or `{id, metadata}` objects. When `metadata` is supplied (`display_name`, `context_window`, `max_output_tokens`, `modes`, `capabilities`, `pricing`, …) it is merged onto the remote ai-model-list entry during enrichment, with operator values winning per-field. Primary use case: advertising context windows, capabilities, and pricing for local models (Ollama) and other custom endpoints whose IDs are not in the upstream registry. diff --git a/README.md b/README.md index 7d7d0dfc..77bb1b57 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,8 @@ const client = new Anthropic({ GoModel supports OpenAI, Anthropic, Cohere, Google Gemini, Vertex AI, DeepSeek, Groq, Fireworks AI, Meta (Muse Spark), OpenRouter, Z.ai, xAI (Grok), Alibaba Cloud Model Studio (Bailian), Kilo AI, MiniMax, Xiaomi MiMo, OpenCode Go, Azure -OpenAI, Oracle, Ollama, SGLang, vLLM, Amazon Bedrock Runtime, Amazon Bedrock Mantle, and -all OpenAI-compatible providers. +OpenAI, Oracle, Ollama, SGLang, vLLM, llm-d, Amazon Bedrock Runtime, Amazon +Bedrock Mantle, and all OpenAI-compatible providers. See the [Providers Overview](https://gomodel.enterpilot.io/docs/providers/overview?utm_source=readme) for the full per-provider feature matrix (chat, `/responses`, embeddings, files, batches, diff --git a/config/config.example.yaml b/config/config.example.yaml index 18747c09..dc687d5a 100644 --- a/config/config.example.yaml +++ b/config/config.example.yaml @@ -13,7 +13,7 @@ server: enable_passthrough_routes: true # expose /p/{provider}/{endpoint} passthrough routes allow_passthrough_v1_alias: true # allow /p/{provider}/v1/... while keeping /p/{provider}/... canonical user_path_header: "X-GoModel-User-Path" # env: USER_PATH_HEADER; inbound header used for user_path scoping - enabled_passthrough_providers: ["openai", "anthropic", "cohere", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek", "bailian"] # providers enabled on /p/{provider}/... + enabled_passthrough_providers: ["openai", "anthropic", "cohere", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek", "bailian"] # providers enabled on /p/{provider}/... realtime_enabled: true # env: REALTIME_ENABLED; expose /v1/realtime websocket and /p/{provider}/v1/realtime upgrades (OpenAI only) pid_file: "data/gomodel.pid" # env: PID_FILE; where the running gateway records its process id so `gomodel --reload` can find it. Set per instance when several gateways share a host; empty writes no pid file and disables --reload; changing it needs a restart, not a reload @@ -418,6 +418,18 @@ providers: # Optional: set this only when vllm serve was started with --api-key. # api_key: "token-abc123" + # llm-d Router / Endpoint Picker (EPP) + llmd: + type: llmd + base_url: "http://quickstart-epp.llm-d.svc.cluster.local/v1" + # Optional trusted request class injected as llm-d inference headers. + # inference_objective: "standard-traffic" + # Derive the llm-d fairness ID from GoModel's effective user path (default). + fairness_from_user_path: true + # Declare models when the route does not forward GET /v1/models. + models: + - id: "Qwen/Qwen2.5-0.5B-Instruct" + sglang: type: sglang base_url: "http://localhost:30000/v1" diff --git a/config/config.go b/config/config.go index da100f6c..b4d0b6e4 100644 --- a/config/config.go +++ b/config/config.go @@ -102,6 +102,7 @@ func buildDefaultConfig() *Config { "zai", "sglang", "vllm", + "llmd", "deepseek", }, }, diff --git a/config/config_test.go b/config/config_test.go index 6122cce3..1182bc28 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -30,6 +30,7 @@ func clearProviderEnvVars(t *testing.T) { "AZURE_API_KEY", "AZURE_BASE_URL", "AZURE_API_VERSION", "AZURE_MODELS", "ORACLE_API_KEY", "ORACLE_BASE_URL", "ORACLE_MODELS", "VLLM_API_KEY", "VLLM_BASE_URL", "VLLM_MODELS", + "LLMD_API_KEY", "LLMD_BASE_URL", "LLMD_MODELS", "LLMD_INFERENCE_OBJECTIVE", "LLMD_FAIRNESS_FROM_USER_PATH", "SGLANG_API_KEY", "SGLANG_BASE_URL", "SGLANG_MODELS", "OLLAMA_API_KEY", "OLLAMA_BASE_URL", "OLLAMA_MODELS", } { @@ -125,7 +126,7 @@ func TestBuildDefaultConfig(t *testing.T) { if !cfg.Server.AllowPassthroughV1Alias { t.Error("expected Server.AllowPassthroughV1Alias=true") } - if got, want := cfg.Server.EnabledPassthroughProviders, []string{"openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek"}; !reflect.DeepEqual(got, want) { + if got, want := cfg.Server.EnabledPassthroughProviders, []string{"openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek"}; !reflect.DeepEqual(got, want) { t.Errorf("expected Server.EnabledPassthroughProviders=%v, got %v", want, got) } if cfg.Models.ConfiguredProviderModelsMode != ConfiguredProviderModelsModeFallback { @@ -1202,7 +1203,7 @@ func TestLoad_ConfigExample_UsesNestedModelCacheSettings(t *testing.T) { t.Fatalf("expected Cache.Model.Redis to be nil in example config, got %+v", result.Config.Cache.Model.Redis) } gotProviders := result.Config.Server.EnabledPassthroughProviders - wantProviders := []string{"openai", "anthropic", "cohere", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek", "bailian"} + wantProviders := []string{"openai", "anthropic", "cohere", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek", "bailian"} if !reflect.DeepEqual(gotProviders, wantProviders) { t.Fatalf("Server.EnabledPassthroughProviders = %v, want %v", gotProviders, wantProviders) } diff --git a/config/providers.go b/config/providers.go index d9a6bcfb..05788eb4 100644 --- a/config/providers.go +++ b/config/providers.go @@ -13,18 +13,25 @@ type RawProviderConfig struct { APIKeys []string `yaml:"api_keys"` // SessionStickyKeys defaults to true. Set false to restore round-robin key // selection for every request, including requests carrying a session ID. - SessionStickyKeys *bool `yaml:"session_sticky_keys"` - BaseURL string `yaml:"base_url"` - APIVersion string `yaml:"api_version"` - Backend string `yaml:"backend"` - AuthType string `yaml:"auth_type"` - APIMode string `yaml:"api_mode"` - VertexProject string `yaml:"vertex_project"` - VertexLocation string `yaml:"vertex_location"` - ServiceAccountFile string `yaml:"service_account_file"` - ServiceAccountJSON string `yaml:"service_account_json"` - ServiceAccountJSONBase64 string `yaml:"service_account_json_base64"` - GCPScope string `yaml:"gcp_scope"` - Models []RawProviderModel `yaml:"models"` - Resilience *RawResilienceConfig `yaml:"resilience"` + SessionStickyKeys *bool `yaml:"session_sticky_keys"` + BaseURL string `yaml:"base_url"` + APIVersion string `yaml:"api_version"` + Backend string `yaml:"backend"` + AuthType string `yaml:"auth_type"` + APIMode string `yaml:"api_mode"` + VertexProject string `yaml:"vertex_project"` + VertexLocation string `yaml:"vertex_location"` + ServiceAccountFile string `yaml:"service_account_file"` + ServiceAccountJSON string `yaml:"service_account_json"` + ServiceAccountJSONBase64 string `yaml:"service_account_json_base64"` + GCPScope string `yaml:"gcp_scope"` + // InferenceObjective is the trusted llm-d InferenceObjective name injected + // into outbound requests. It is ignored by provider types other than llmd. + InferenceObjective string `yaml:"inference_objective"` + // FairnessFromUserPath controls whether the llmd provider derives its + // fairness ID from GoModel's effective (authenticated) user path. It + // defaults to true; nil preserves that default. + FairnessFromUserPath *bool `yaml:"fairness_from_user_path"` + Models []RawProviderModel `yaml:"models"` + Resilience *RawResilienceConfig `yaml:"resilience"` } diff --git a/config/server.go b/config/server.go index 69a97a05..e0af13d4 100644 --- a/config/server.go +++ b/config/server.go @@ -39,7 +39,7 @@ type ServerConfig struct { UserPathHeader string `yaml:"user_path_header" env:"USER_PATH_HEADER"` // EnabledPassthroughProviders lists the provider types enabled on // /p/{provider}/... passthrough routes. Default: - // ["openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek"]. + // ["openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek"]. EnabledPassthroughProviders []string `yaml:"enabled_passthrough_providers" env:"ENABLED_PASSTHROUGH_PROVIDERS"` // RealtimeEnabled exposes the realtime (speech-to-speech) websocket endpoint // at /v1/realtime and the /p/{provider}/v1/realtime passthrough upgrade. diff --git a/docs/advanced/config-yaml.mdx b/docs/advanced/config-yaml.mdx index 731e7dc9..4faf7479 100644 --- a/docs/advanced/config-yaml.mdx +++ b/docs/advanced/config-yaml.mdx @@ -34,7 +34,10 @@ must use the same suffix for each instance: for example, pair `gcp_adc`. Configured provider model lists can stay in env via `_MODELS`, for -example `OPENROUTER_MODELS`, `ORACLE_MODELS`, `AZURE_MODELS`, `SGLANG_MODELS`, or `VLLM_MODELS`. +example `OPENROUTER_MODELS`, `ORACLE_MODELS`, `AZURE_MODELS`, `SGLANG_MODELS`, +`VLLM_MODELS`, or `LLMD_MODELS`. llm-d instances also accept suffixed +`LLMD__INFERENCE_OBJECTIVE` and +`LLMD__FAIRNESS_FROM_USER_PATH` controls. Set `CONFIGURED_PROVIDER_MODELS_MODE=fallback` (default) to use those lists only when upstream `/models` fails or is empty, or `allowlist` to expose only the configured models for providers that define a list and skip their upstream diff --git a/docs/advanced/configuration.mdx b/docs/advanced/configuration.mdx index 9e1249b3..339ccc57 100644 --- a/docs/advanced/configuration.mdx +++ b/docs/advanced/configuration.mdx @@ -281,12 +281,13 @@ Set these to automatically register providers. No YAML configuration required. | `OLLAMA_BASE_URL` | Ollama (no API key needed) | | `SGLANG_BASE_URL` | SGLang (no API key needed unless upstream requires) | | `VLLM_BASE_URL` | vLLM (no API key needed unless upstream requires) | +| `LLMD_BASE_URL` | llm-d Router/EPP (no API key needed unless its Gateway requires one) | -Most providers can use a custom base URL via `_BASE_URL` (for example `OPENAI_BASE_URL`). DeepSeek defaults to `https://api.deepseek.com`; set `DEEPSEEK_BASE_URL` only for a compatible proxy or alternate DeepSeek endpoint. OpenRouter defaults to `https://openrouter.ai/api/v1` and can be overridden with `OPENROUTER_BASE_URL`. Kilo AI defaults to `https://api.kilo.ai/api/gateway` and can be overridden with `KILO_BASE_URL`. Z.ai defaults to `https://api.z.ai/api/paas/v4`; set `ZAI_BASE_URL=https://api.z.ai/api/coding/paas/v4` for the GLM Coding Plan endpoint. SGLang defaults to `http://localhost:30000/v1` when `SGLANG_API_KEY` is set, but keyless deployments should set `SGLANG_BASE_URL` explicitly to register the provider. vLLM follows the same pattern at `http://localhost:8000/v1`. Azure uses `AZURE_BASE_URL` for its deployment base URL and accepts an optional `AZURE_API_VERSION` override; otherwise it defaults to `2024-10-21`. Oracle requires `ORACLE_BASE_URL` because its OpenAI-compatible endpoint is region-specific. +Most providers can use a custom base URL via `_BASE_URL` (for example `OPENAI_BASE_URL`). DeepSeek defaults to `https://api.deepseek.com`; set `DEEPSEEK_BASE_URL` only for a compatible proxy or alternate DeepSeek endpoint. OpenRouter defaults to `https://openrouter.ai/api/v1` and can be overridden with `OPENROUTER_BASE_URL`. Kilo AI defaults to `https://api.kilo.ai/api/gateway` and can be overridden with `KILO_BASE_URL`. Z.ai defaults to `https://api.z.ai/api/paas/v4`; set `ZAI_BASE_URL=https://api.z.ai/api/coding/paas/v4` for the GLM Coding Plan endpoint. SGLang defaults to `http://localhost:30000/v1` when `SGLANG_API_KEY` is set, but keyless deployments should set `SGLANG_BASE_URL` explicitly to register the provider. vLLM follows the same pattern at `http://localhost:8000/v1`. llm-d has no universal endpoint, so `LLMD_BASE_URL` is always required; `LLMD_API_KEY` is optional. Azure uses `AZURE_BASE_URL` for its deployment base URL and accepts an optional `AZURE_API_VERSION` override; otherwise it defaults to `2024-10-21`. Oracle requires `ORACLE_BASE_URL` because its OpenAI-compatible endpoint is region-specific. Every provider type also accepts a comma-separated configured model list via `_MODELS`, for example `OPENROUTER_MODELS`, `ORACLE_MODELS`, -`AZURE_MODELS`, `SGLANG_MODELS`, or `VLLM_MODELS`. By default, +`AZURE_MODELS`, `SGLANG_MODELS`, `VLLM_MODELS`, or `LLMD_MODELS`. By default, `CONFIGURED_PROVIDER_MODELS_MODE=fallback` uses configured lists only when upstream `/models` fails, returns nil, or returns an empty list. Set `CONFIGURED_PROVIDER_MODELS_MODE=allowlist` to expose only configured models for @@ -425,6 +426,10 @@ export SGLANG_BASE_URL="http://localhost:30000/v1" # Registers keyless "sglang" # Optional: export SGLANG_API_KEY="token-abc123" export VLLM_BASE_URL="http://localhost:8000/v1" # Registers keyless "vllm" provider # Optional: export VLLM_API_KEY="token-abc123" +export LLMD_BASE_URL="http://quickstart-epp.llm-d.svc.cluster.local/v1" # Registers keyless "llmd" provider +export LLMD_MODELS="Qwen/Qwen2.5-0.5B-Instruct" +# Optional: export LLMD_INFERENCE_OBJECTIVE="standard-traffic" +# Optional: export LLMD_FAIRNESS_FROM_USER_PATH="true" ``` Use suffixed variables to register more than one instance of the same provider diff --git a/docs/docs.json b/docs/docs.json index e78b5207..5deab162 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -171,6 +171,7 @@ "providers/opencode-go", "providers/sglang", "providers/vllm", + "providers/llmd", "providers/multiple-ollama", "providers/kimicode", { diff --git a/docs/features/passthrough-api.mdx b/docs/features/passthrough-api.mdx index 64836f42..2f6e985d 100644 --- a/docs/features/passthrough-api.mdx +++ b/docs/features/passthrough-api.mdx @@ -131,7 +131,7 @@ from passthrough requests before forwarding them upstream. Passthrough is intentionally narrow while the API is in beta. -- `openai`, `anthropic`, `openrouter`, `kilo`, `zai`, `sglang`, `vllm`, and `deepseek` are enabled by +- `openai`, `anthropic`, `openrouter`, `kilo`, `zai`, `sglang`, `vllm`, `llmd`, and `deepseek` are enabled by default. - GoModel does not translate passthrough request bodies or response bodies. - Provider-native error bodies and status codes are proxied instead of converted @@ -146,7 +146,7 @@ Passthrough routes are enabled by default: ```env ENABLE_PASSTHROUGH_ROUTES=true ALLOW_PASSTHROUGH_V1_ALIAS=true -ENABLED_PASSTHROUGH_PROVIDERS=openai,anthropic,openrouter,kilo,zai,sglang,vllm,deepseek +ENABLED_PASSTHROUGH_PROVIDERS=openai,anthropic,openrouter,kilo,zai,sglang,vllm,llmd,deepseek ``` Set `ENABLED_PASSTHROUGH_PROVIDERS` to the provider types you want to expose. diff --git a/docs/providers/llmd.mdx b/docs/providers/llmd.mdx new file mode 100644 index 00000000..d67d6f85 --- /dev/null +++ b/docs/providers/llmd.mdx @@ -0,0 +1,144 @@ +--- +title: "llm-d" +description: "Route GoModel requests through the llm-d Inference Gateway and inject trusted scheduling metadata." +icon: "route" +keywords: ["llm-d", "Inference Gateway", "EPP", "Kubernetes", "inference scheduling"] +--- + +GoModel's `llmd` provider sends OpenAI-compatible requests to an +[llm-d Router](https://llm-d.ai/docs/dev/architecture/core/router) and adds +trusted scheduling metadata for the Endpoint Picker (EPP). Use this provider +when GoModel is the authenticated API gateway in front of llm-d. + +## Prerequisites + +- A running llm-d deployment with an HTTPRoute that reaches the Router/EPP. +- The model name exposed by that route. +- Network access from GoModel to the Router service. + +The [llm-d quickstart](https://llm-d.ai/docs/dev/getting-started/quickstart) +creates an EPP service named from the guide release. Use the service name from +your own deployment. + +## Configure with environment variables + +```bash +LLMD_BASE_URL=http://quickstart-epp.llm-d.svc.cluster.local/v1 +LLMD_MODELS=Qwen/Qwen2.5-0.5B-Instruct +GOMODEL_MASTER_KEY=change-me + +# Optional controls +LLMD_INFERENCE_OBJECTIVE=standard-traffic +LLMD_FAIRNESS_FROM_USER_PATH=true +# LLMD_API_KEY=router-token +``` + +`LLMD_BASE_URL` is required and should include `/v1`. `LLMD_API_KEY` is +optional; set it when the Gateway in front of llm-d requires bearer +authentication. + +When `LLMD_INFERENCE_OBJECTIVE` is omitted, GoModel does not send an inference +objective header. Set it when the llm-d deployment defines multiple inference +objectives and this GoModel provider should select one of them. + +Declare `LLMD_MODELS` when your route does not serve `GET /v1/models`. GoModel +uses that list as the provider's model inventory. + +## Configure with YAML + +```yaml +providers: + llmd: + type: llmd + base_url: "http://quickstart-epp.llm-d.svc.cluster.local/v1" + inference_objective: "standard-traffic" + fairness_from_user_path: true + models: + - id: "Qwen/Qwen2.5-0.5B-Instruct" +``` + +`fairness_from_user_path` defaults to `true`. GoModel derives the fairness ID +from the effective [`user_path`](/features/user-path), after authentication +and key policy have been applied. Set it to `false` if another trusted layer +sets fairness metadata. + + + A raw `X-GoModel-User-Path` supplied by a client is not enough to set the + fairness ID. Bind the path to a managed auth key or an extension identity so + GoModel can treat it as an effective authenticated path. + + + + Do not forward client-supplied llm-d scheduling headers from an untrusted + edge. A client could otherwise select another objective or fairness group. + The dedicated provider strips llm-d and Gateway API Inference Extension + control headers on passthrough requests, then injects the configured values. + + +For the objective and fairness ID, GoModel sends both the current `x-llm-d-*` +headers and the deprecated `x-gateway-*` aliases with the same trusted values. +This supports stable llm-d 0.8 deployments while using the canonical names +recognized by llm-d 0.9 and later. + +When EPP flow control returns a `429` on a translated inference route, GoModel +preserves the `x-llm-d-request-dropped-reason` response header so clients can +distinguish rejection from post-dispatch eviction. + +## Verify + +First create a managed GoModel API key in the dashboard, bind its `user_path` +to `/team/alpha`, and assign the returned value to `TEAM_ALPHA_GOMODEL_KEY`. +The key binding, rather than a client-asserted header, establishes the fairness +ID used by this request. + +```bash +curl -s http://localhost:8080/v1/chat/completions \ + -H "Authorization: Bearer ${TEAM_ALPHA_GOMODEL_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "llmd/Qwen/Qwen2.5-0.5B-Instruct", + "messages": [{"role": "user", "content": "Reply with exactly ok."}] + }' +``` + +GoModel removes the outer `llmd/` routing qualifier before sending the model +name upstream. Slash-shaped Hugging Face model IDs remain intact. + +## Supported routes + +The provider implements chat completions, Responses, embeddings, and model +listing through the OpenAI-compatible API. Passthrough is enabled by default +for the other routes in the +[llm-d HTTP API reference](https://llm-d.ai/docs/dev/api-reference/epp-http-apis), +including OpenAI completions, Anthropic Messages, and vLLM Generate: + +```bash +curl -s http://localhost:8080/p/llmd/completions \ + -H "Authorization: Bearer change-me" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen2.5-0.5B-Instruct", + "prompt": "Hello" + }' +``` + +The separate llm-d Batch Gateway is not exposed as GoModel's native batch API. +Files, audio, and Responses lifecycle utility endpoints are also not +advertised by this provider. + +## Multiple llm-d routes + +Use suffixed variables to register independent Router services: + +```bash +LLMD_PROD_BASE_URL=http://prod-epp.llm-d.svc.cluster.local/v1 +LLMD_PROD_MODELS=Qwen/Qwen3-32B +LLMD_PROD_INFERENCE_OBJECTIVE=production + +LLMD_DEV_BASE_URL=http://dev-epp.llm-d.svc.cluster.local/v1 +LLMD_DEV_MODELS=Qwen/Qwen2.5-0.5B-Instruct +LLMD_DEV_INFERENCE_OBJECTIVE=development +``` + +These variables register `llmd-prod` and `llmd-dev`. Select them with model +names such as `llmd-dev/Qwen/Qwen2.5-0.5B-Instruct`. diff --git a/docs/providers/overview.mdx b/docs/providers/overview.mdx index eed1ee1d..49e62a39 100644 --- a/docs/providers/overview.mdx +++ b/docs/providers/overview.mdx @@ -63,6 +63,7 @@ support, not every individual model capability exposed by an upstream provider. | Ollama | `OLLAMA_BASE_URL` | `llama3.2` | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | [Ollama](/providers/multiple-ollama) | | SGLang | `SGLANG_BASE_URL` (`SGLANG_API_KEY` optional) | `Qwen/Qwen2.5-0.5B-Instruct` | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | [SGLang](/providers/sglang) | | vLLM | `VLLM_BASE_URL` (`VLLM_API_KEY` optional) | `meta-llama/Llama-3.1-8B-Instruct` | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | [vLLM](/providers/vllm) | +| llm-d | `LLMD_BASE_URL` (`LLMD_API_KEY` optional) | `Qwen/Qwen2.5-0.5B-Instruct` | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | [llm-d](/providers/llmd) | | Amazon Bedrock | `BEDROCK_BASE_URL` (region or endpoint) + AWS credentials | `anthropic.claude-3-5-haiku-20241022-v1:0` | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | [Amazon Bedrock](/providers/bedrock) | | Amazon Bedrock Mantle | `BEDROCK_MANTLE_API_KEY` or AWS credentials | `openai.gpt-5.6-sol` | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | [Bedrock Mantle](/providers/bedrock-mantle) | @@ -70,6 +71,9 @@ support, not every individual model capability exposed by an upstream provider. ## Provider notes +- **llm-d** — point `LLMD_BASE_URL` at the Router/EPP service and declare + `LLMD_MODELS` when the route does not serve `/v1/models`. The dedicated + provider injects trusted inference-objective and user-path fairness headers. - **Z.ai GLM Coding Plan** — set `ZAI_BASE_URL=https://api.z.ai/api/coding/paas/v4`. - **MiniMax regions and current text models** — the default global endpoint is @@ -123,6 +127,8 @@ support, not every individual model capability exposed by an upstream provider. for providers that define a list, skipping their upstream `/models` calls. - **vLLM** — set `VLLM_API_KEY` only if the upstream server was started with `--api-key`. +- **llm-d** — `LLMD_BASE_URL` is required. `LLMD_API_KEY` is optional and is + sent only when the Gateway in front of the Router requires bearer auth. - **SGLang** — set `SGLANG_API_KEY` only if the upstream server was started with `--api-key`; otherwise `SGLANG_BASE_URL` is enough. - **Multiple API keys for one provider** — set `OPENAI_API_KEY_2`, diff --git a/helm/README.md b/helm/README.md index 17155aae..0110b58f 100644 --- a/helm/README.md +++ b/helm/README.md @@ -74,6 +74,10 @@ helm install gomodel ./helm \ | `providers.vllm.baseUrl` | vLLM OpenAI-compatible base URL mapped to `VLLM_BASE_URL`; required when vLLM is enabled | `""` | | `providers.sglang.enabled` | Enable SGLang | `false` | | `providers.sglang.baseUrl` | SGLang OpenAI-compatible base URL mapped to `SGLANG_BASE_URL`; required when enabled | `""` | +| `providers.llmd.enabled` | Enable llm-d | `false` | +| `providers.llmd.baseUrl` | llm-d Router/EPP base URL mapped to `LLMD_BASE_URL`; required when llm-d is enabled | `""` | +| `providers.llmd.inferenceObjective` | Trusted llm-d objective mapped to `LLMD_INFERENCE_OBJECTIVE` | `""` | +| `providers.llmd.fairnessFromUserPath` | Derive llm-d fairness ID from GoModel's effective user path | `true` | | `cache.type` | Cache type (local/redis) | `"redis"` | | `redis.enabled` | Deploy Redis subchart | `true` | | `metrics.enabled` | Enable Prometheus metrics | `true` | @@ -103,6 +107,7 @@ stringData: ORACLE_API_KEY: "..." SGLANG_API_KEY: "..." VLLM_API_KEY: "..." + LLMD_API_KEY: "..." ``` Oracle also requires a base URL in values. The chart maps `providers.oracle.baseUrl` @@ -112,6 +117,12 @@ vLLM does not require an API key unless the upstream server was started with `--api-key`. The chart maps `providers.vllm.baseUrl` to the container env var `VLLM_BASE_URL`. +llm-d also allows keyless access when its Gateway does not require bearer +authentication. The chart maps `providers.llmd.baseUrl` to `LLMD_BASE_URL` and +the optional scheduling controls to their `LLMD_*` environment variables. When +using `providers.existingSecret`, its `LLMD_API_KEY` entry may be omitted for a +keyless llm-d route. + SGLang does not require an API key unless the upstream server was started with `--api-key`. The chart maps `providers.sglang.baseUrl` to the container env var `SGLANG_BASE_URL`. @@ -141,6 +152,15 @@ helm install gomodel ./helm \ --set providers.vllm.baseUrl="http://vllm.default.svc.cluster.local:8000/v1" ``` +Example keyless llm-d setup: + +```bash +helm install gomodel ./helm \ + --set providers.llmd.enabled=true \ + --set providers.llmd.baseUrl="http://quickstart-epp.llm-d.svc.cluster.local/v1" \ + --set providers.llmd.inferenceObjective="standard-traffic" +``` + Example keyless SGLang setup: ```bash diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 1ac6472f..cf8f5888 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -166,6 +166,9 @@ Generate provider environment variables for the Deployment. secretKeyRef: name: {{ $secretName }} key: {{ upper $name }}_API_KEY +{{- if eq $name "llmd" }} + optional: true +{{- end }} {{- end }} {{- if or (or $hasAPIKey $enabledWithExistingSecret) $enabledWithBaseURL }} {{- if $config.baseUrl }} @@ -177,6 +180,16 @@ Generate provider environment variables for the Deployment. - name: USE_GOOGLE_GEMINI_NATIVE_API value: {{ $config.useNativeApi | quote }} {{- end }} +{{- if and (eq $name "llmd") (or (or $hasAPIKey $enabledWithExistingSecret) $enabledWithBaseURL) }} +{{- if $config.inferenceObjective }} +- name: LLMD_INFERENCE_OBJECTIVE + value: {{ $config.inferenceObjective | quote }} +{{- end }} +{{- if hasKey $config "fairnessFromUserPath" }} +- name: LLMD_FAIRNESS_FROM_USER_PATH + value: {{ $config.fairnessFromUserPath | quote }} +{{- end }} +{{- end }} {{- end }} {{- end }} {{- end }} diff --git a/helm/values.schema.json b/helm/values.schema.json index a2ed2a2f..6f58f6ae 100644 --- a/helm/values.schema.json +++ b/helm/values.schema.json @@ -110,6 +110,33 @@ } } }, + { + "if": { + "properties": { + "providers": { + "properties": { + "llmd": { + "properties": { "enabled": { "const": true } } + } + } + } + } + }, + "then": { + "properties": { + "providers": { + "properties": { + "llmd": { + "properties": { + "baseUrl": { "minLength": 1 } + }, + "required": ["baseUrl"] + } + } + } + } + } + }, { "if": { "properties": { @@ -262,6 +289,20 @@ } } }, + { + "properties": { + "providers": { + "properties": { + "llmd": { + "properties": { + "enabled": { "const": true }, + "baseUrl": { "minLength": 1 } + } + } + } + } + } + }, { "properties": { "providers": { @@ -378,6 +419,16 @@ "baseUrl": { "type": "string" } } }, + "llmd": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "apiKey": { "type": "string" }, + "baseUrl": { "type": "string" }, + "inferenceObjective": { "type": "string" }, + "fairnessFromUserPath": { "type": "boolean" } + } + }, "sglang": { "type": "object", "properties": { diff --git a/helm/values.yaml b/helm/values.yaml index 338f06a2..ecba99ca 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -42,7 +42,7 @@ auth: # LLM Provider configuration providers: # -- Use an existing secret for all provider API keys - # Secret should contain keys for enabled providers, such as OPENAI_API_KEY, ANTHROPIC_API_KEY, COHERE_API_KEY, GEMINI_API_KEY, GROQ_API_KEY, XAI_API_KEY, ZAI_API_KEY, KILO_API_KEY, ORACLE_API_KEY, SGLANG_API_KEY, or VLLM_API_KEY + # Secret should contain keys for enabled providers, such as OPENAI_API_KEY, ANTHROPIC_API_KEY, COHERE_API_KEY, GEMINI_API_KEY, GROQ_API_KEY, XAI_API_KEY, ZAI_API_KEY, KILO_API_KEY, ORACLE_API_KEY, SGLANG_API_KEY, VLLM_API_KEY, or LLMD_API_KEY existingSecret: "" openai: @@ -127,6 +127,18 @@ providers: # -- vLLM OpenAI-compatible base URL baseUrl: "" + llmd: + # -- Enable llm-d provider (API key optional; baseUrl required when enabled) + enabled: false + # -- Optional bearer token required by the Gateway in front of llm-d + apiKey: "" + # -- llm-d Router / EPP OpenAI-compatible base URL + baseUrl: "" + # -- Optional trusted llm-d InferenceObjective name + inferenceObjective: "" + # -- Derive the llm-d fairness ID from GoModel's effective user path + fairnessFromUserPath: true + sglang: # -- Enable SGLang provider (API key optional; baseUrl required when enabled) enabled: false diff --git a/internal/admin/dashboard/static/dist/assets/index-CiIXCM_n.js b/internal/admin/dashboard/static/dist/assets/index-SXsqspkt.js similarity index 99% rename from internal/admin/dashboard/static/dist/assets/index-CiIXCM_n.js rename to internal/admin/dashboard/static/dist/assets/index-SXsqspkt.js index 5de4cf58..8f0e3778 100644 --- a/internal/admin/dashboard/static/dist/assets/index-CiIXCM_n.js +++ b/internal/admin/dashboard/static/dist/assets/index-SXsqspkt.js @@ -5,7 +5,7 @@ `)>-1?e.split(` `):e}function Av(e,t){let{element:n,datasetIndex:r,index:i}=t,a=e.getDatasetMeta(r).controller,{label:o,value:s}=a.getLabelAndValue(i);return{chart:e,label:o,parsed:a.getParsed(i),raw:e.data.datasets[r].data[i],formattedValue:s,dataset:a.getDataset(),dataIndex:i,datasetIndex:r,element:n}}function jv(e,t){let n=e.chart.ctx,{body:r,footer:i,title:a}=e,{boxWidth:o,boxHeight:s}=t,c=_f(t.bodyFont),l=_f(t.titleFont),u=_f(t.footerFont),d=a.length,f=i.length,p=r.length,m=gf(t.padding),h=m.height,g=0,_=r.reduce((e,t)=>e+t.before.length+t.lines.length+t.after.length,0);if(_+=e.beforeBody.length+e.afterBody.length,d&&(h+=d*l.lineHeight+(d-1)*t.titleSpacing+t.titleMarginBottom),_){let e=t.displayColors?Math.max(s,c.lineHeight):c.lineHeight;h+=p*e+(_-p)*c.lineHeight+(_-1)*t.bodySpacing}f&&(h+=t.footerMarginTop+f*u.lineHeight+(f-1)*t.footerSpacing);let v=0,y=function(e){g=Math.max(g,n.measureText(e).width+v)};return n.save(),n.font=l.string,hu(e.title,y),n.font=c.string,hu(e.beforeBody.concat(e.afterBody),y),v=t.displayColors?o+2+t.boxPadding:0,hu(r,e=>{hu(e.before,y),hu(e.lines,y),hu(e.after,y)}),v=0,n.font=u.string,hu(e.footer,y),n.restore(),g+=m.width,{width:g,height:h}}function Mv(e,t){let{y:n,height:r}=t;return ne.height-r/2?`bottom`:`center`}function Nv(e,t,n,r){let{x:i,width:a}=r,o=n.caretSize+n.caretPadding;if(e===`left`&&i+a+o>t.width||e===`right`&&i-a-o<0)return!0}function Pv(e,t,n,r){let{x:i,width:a}=n,{width:o,chartArea:{left:s,right:c}}=e,l=`center`;return r===`center`?l=i<=(s+c)/2?`left`:`right`:i<=a/2?l=`left`:i>=o-a/2&&(l=`right`),Nv(l,e,t,n)&&(l=`center`),l}function Fv(e,t,n){let r=n.yAlign||t.yAlign||Mv(e,n);return{xAlign:n.xAlign||t.xAlign||Pv(e,t,n,r),yAlign:r}}function Iv(e,t){let{x:n,width:r}=e;return t===`right`?n-=r:t===`center`&&(n-=r/2),n}function Lv(e,t,n){let{y:r,height:i}=e;return t===`top`?r+=n:t===`bottom`?r-=i+n:r-=i/2,r}function Rv(e,t,n,r){let{caretSize:i,caretPadding:a,cornerRadius:o}=e,{xAlign:s,yAlign:c}=n,l=i+a,{topLeft:u,topRight:d,bottomLeft:f,bottomRight:p}=hf(o),m=Iv(t,s),h=Lv(t,c,l);return c===`center`?s===`left`?m+=l:s===`right`&&(m-=l):s===`left`?m-=Math.max(u,f)+i:s===`right`&&(m+=Math.max(d,p)+i),{x:rd(m,0,r.width-t.width),y:rd(h,0,r.height-t.height)}}function zv(e,t,n){let r=gf(n.padding);return t===`center`?e.x+e.width/2:t===`right`?e.x+e.width-r.right:e.x+r.left}function Bv(e){return Ov([],kv(e))}function Vv(e,t,n){return bf(e,{tooltip:t,tooltipItems:n,type:`tooltip`})}function Hv(e,t){let n=t&&t.dataset&&t.dataset.tooltip&&t.dataset.tooltip.callbacks;return n?e.override(n):e}var Uv={beforeTitle:iu,title(e){if(e.length>0){let t=e[0],n=t.chart.data.labels,r=n?n.length:0;if(this&&this.options&&this.options.mode===`dataset`)return t.dataset.label||``;if(t.label)return t.label;if(r>0&&t.dataIndex{let t={before:[],lines:[],after:[]},i=Hv(n,e);Ov(t.before,kv(Wv(i,`beforeLabel`,this,e))),Ov(t.lines,Wv(i,`label`,this,e)),Ov(t.after,kv(Wv(i,`afterLabel`,this,e))),r.push(t)}),r}getAfterBody(e,t){return Bv(Wv(t.callbacks,`afterBody`,this,e))}getFooter(e,t){let{callbacks:n}=t,r=Wv(n,`beforeFooter`,this,e),i=Wv(n,`footer`,this,e),a=Wv(n,`afterFooter`,this,e),o=[];return o=Ov(o,kv(r)),o=Ov(o,kv(i)),o=Ov(o,kv(a)),o}_createItems(e){let t=this._active,n=this.chart.data,r=[],i=[],a=[],o=[],s,c;for(s=0,c=t.length;se.filter(t,r,i,n))),e.itemSort&&(o=o.sort((t,r)=>e.itemSort(t,r,n))),hu(o,t=>{let n=Hv(e.callbacks,t);r.push(Wv(n,`labelColor`,this,t)),i.push(Wv(n,`labelPointStyle`,this,t)),a.push(Wv(n,`labelTextColor`,this,t))}),this.labelColors=r,this.labelPointStyles=i,this.labelTextColors=a,this.dataPoints=o,o}update(e,t){let n=this.options.setContext(this.getContext()),r=this._active,i,a=[];if(!r.length)this.opacity!==0&&(i={opacity:0});else{let e=Dv[n.position].call(this,r,this._eventPosition);a=this._createItems(n),this.title=this.getTitle(a,n),this.beforeBody=this.getBeforeBody(a,n),this.body=this.getBody(a,n),this.afterBody=this.getAfterBody(a,n),this.footer=this.getFooter(a,n);let t=this._size=jv(this,n),o=Object.assign({},e,t),s=Fv(this.chart,n,o),c=Rv(n,o,s,this.chart);this.xAlign=s.xAlign,this.yAlign=s.yAlign,i={opacity:1,x:c.x,y:c.y,width:t.width,height:t.height,caretX:e.x,caretY:e.y}}this._tooltipItems=a,this.$context=void 0,i&&this._resolveAnimations().update(this,i),e&&n.external&&n.external.call(this,{chart:this.chart,tooltip:this,replay:t})}drawCaret(e,t,n,r){let i=this.getCaretPosition(e,n,r);t.lineTo(i.x1,i.y1),t.lineTo(i.x2,i.y2),t.lineTo(i.x3,i.y3)}getCaretPosition(e,t,n){let{xAlign:r,yAlign:i}=this,{caretSize:a,cornerRadius:o}=n,{topLeft:s,topRight:c,bottomLeft:l,bottomRight:u}=hf(o),{x:d,y:f}=e,{width:p,height:m}=t,h,g,_,v,y,b;return i===`center`?(y=f+m/2,r===`left`?(h=d,g=h-a,v=y+a,b=y-a):(h=d+p,g=h+a,v=y-a,b=y+a),_=h):(g=r===`left`?d+Math.max(s,l)+a:r===`right`?d+p-Math.max(c,u)-a:this.caretX,i===`top`?(v=f,y=v-a,h=g-a,_=g+a):(v=f+m,y=v+a,h=g+a,_=g-a),b=v),{x1:h,x2:g,x3:_,y1:v,y2:y,y3:b}}drawTitle(e,t,n){let r=this.title,i=r.length,a,o,s;if(i){let c=vp(n.rtl,this.x,this.width);for(e.x=zv(this,n.titleAlign,n),t.textAlign=c.textAlign(n.titleAlign),t.textBaseline=`middle`,a=_f(n.titleFont),o=n.titleSpacing,t.fillStyle=n.titleColor,t.font=a.string,s=0;se!==0)?(e.beginPath(),e.fillStyle=i.multiKeyBackground,cf(e,{x:t,y:p,w:c,h:s,radius:o}),e.fill(),e.stroke(),e.fillStyle=a.backgroundColor,e.beginPath(),cf(e,{x:n,y:p+1,w:c-2,h:s-2,radius:o}),e.fill()):(e.fillStyle=i.multiKeyBackground,e.fillRect(t,p,c,s),e.strokeRect(t,p,c,s),e.fillStyle=a.backgroundColor,e.fillRect(n,p+1,c-2,s-2))}e.fillStyle=this.labelTextColors[n]}drawBody(e,t,n){let{body:r}=this,{bodySpacing:i,bodyAlign:a,displayColors:o,boxHeight:s,boxWidth:c,boxPadding:l}=n,u=_f(n.bodyFont),d=u.lineHeight,f=0,p=vp(n.rtl,this.x,this.width),m=function(n){t.fillText(n,p.x(e.x+f),e.y+d/2),e.y+=d+i},h=p.textAlign(a),g,_,v,y,b,x,S;for(t.textAlign=a,t.textBaseline=`middle`,t.font=u.string,e.x=zv(this,h,n),t.fillStyle=n.bodyColor,hu(this.beforeBody,m),f=o&&h!==`right`?a===`center`?c/2+l:c+2+l:0,y=0,x=r.length;y0&&t.stroke()}_updateAnimationTarget(e){let t=this.chart,n=this.$animations,r=n&&n.x,i=n&&n.y;if(r||i){let n=Dv[e.position].call(this,this._active,this._eventPosition);if(!n)return;let a=this._size=jv(this,e),o=Object.assign({},n,this._size),s=Fv(t,e,o),c=Rv(e,o,s,t);(r._to!==c.x||i._to!==c.y)&&(this.xAlign=s.xAlign,this.yAlign=s.yAlign,this.width=a.width,this.height=a.height,this.caretX=n.x,this.caretY=n.y,this._resolveAnimations().update(this,c))}}_willRender(){return!!this.opacity}draw(e){let t=this.options.setContext(this.getContext()),n=this.opacity;if(!n)return;this._updateAnimationTarget(t);let r={width:this.width,height:this.height},i={x:this.x,y:this.y};n=Math.abs(n)<.001?0:n;let a=gf(t.padding),o=this.title.length||this.beforeBody.length||this.body.length||this.afterBody.length||this.footer.length;t.enabled&&o&&(e.save(),e.globalAlpha=n,this.drawBackground(i,e,r,t),yp(e,t.textDirection),i.y+=a.top,this.drawTitle(i,e,t),this.drawBody(i,e,t),this.drawFooter(i,e,t),bp(e,t.textDirection),e.restore())}getActiveElements(){return this._active||[]}setActiveElements(e,t){let n=this._active,r=e.map(({datasetIndex:e,index:t})=>{let n=this.chart.getDatasetMeta(e);if(!n)throw Error(`Cannot find a dataset at index `+e);return{datasetIndex:e,element:n.data[t],index:t}}),i=!gu(n,r),a=this._positionChanged(r,t);(i||a)&&(this._active=r,this._eventPosition=t,this._ignoreReplayEvents=!0,this.update(!0))}handleEvent(e,t,n=!0){if(t&&this._ignoreReplayEvents)return!1;this._ignoreReplayEvents=!1;let r=this.options,i=this._active||[],a=this._getActiveElements(e,i,t,n),o=this._positionChanged(a,e),s=t||!gu(a,i)||o;return s&&(this._active=a,(r.enabled||r.external)&&(this._eventPosition={x:e.x,y:e.y},this.update(!0,t))),s}_getActiveElements(e,t,n,r){let i=this.options;if(e.type===`mouseout`)return[];if(!r)return t.filter(e=>this.chart.data.datasets[e.datasetIndex]&&this.chart.getDatasetMeta(e.datasetIndex).controller.getParsed(e.index)!==void 0);let a=this.chart.getElementsAtEventForMode(e,i.mode,i,n);return i.reverse&&a.reverse(),a}_positionChanged(e,t){let{caretX:n,caretY:r,options:i}=this,a=Dv[i.position].call(this,e,t);return a!==!1&&(n!==a.x||r!==a.y)}},Kv=Object.freeze({__proto__:null,Colors:k_,Decimation:F_,Filler:fv,Legend:xv,SubTitle:Ev,Title:wv,Tooltip:{id:`tooltip`,_element:Gv,positioners:Dv,afterInit(e,t,n){n&&(e.tooltip=new Gv({chart:e,options:n}))},beforeUpdate(e,t,n){e.tooltip&&e.tooltip.initialize(n)},reset(e,t,n){e.tooltip&&e.tooltip.initialize(n)},afterDraw(e){let t=e.tooltip;if(t&&t._willRender()){let n={tooltip:t};if(e.notifyPlugins(`beforeTooltipDraw`,{...n,cancelable:!0})===!1)return;t.draw(e.ctx),e.notifyPlugins(`afterTooltipDraw`,n)}},afterEvent(e,t){if(e.tooltip){let n=t.replay;e.tooltip.handleEvent(t.event,n,t.inChartArea)&&(t.changed=!0)}},defaults:{enabled:!0,external:null,position:`average`,backgroundColor:`rgba(0,0,0,0.8)`,titleColor:`#fff`,titleFont:{weight:`bold`},titleSpacing:2,titleMarginBottom:6,titleAlign:`left`,bodyColor:`#fff`,bodySpacing:2,bodyFont:{},bodyAlign:`left`,footerColor:`#fff`,footerSpacing:2,footerMarginTop:6,footerFont:{weight:`bold`},footerAlign:`left`,padding:6,caretPadding:2,caretSize:5,cornerRadius:6,boxHeight:(e,t)=>t.bodyFont.size,boxWidth:(e,t)=>t.bodyFont.size,multiKeyBackground:`#fff`,displayColors:!0,boxPadding:0,borderColor:`rgba(0,0,0,0)`,borderWidth:0,animation:{duration:400,easing:`easeOutQuart`},animations:{numbers:{type:`number`,properties:[`x`,`y`,`width`,`height`,`caretX`,`caretY`]},opacity:{easing:`linear`,duration:200}},callbacks:Uv},defaultRoutes:{bodyFont:`font`,footerFont:`font`,titleFont:`font`},descriptors:{_scriptable:e=>e!==`filter`&&e!==`itemSort`&&e!==`external`,_indexable:!1,callbacks:{_scriptable:!1,_indexable:!1},animation:{_fallback:!1},animations:{_fallback:`animation`}},additionalOptionScopes:[`interaction`]}}),qv=(e,t,n,r)=>(typeof t==`string`?(n=e.push(t)-1,r.unshift({index:n,label:t})):isNaN(t)&&(n=null),n);function Jv(e,t,n,r){let i=e.indexOf(t);return i===-1?qv(e,t,n,r):i===e.lastIndexOf(t)?i:n}var Yv=(e,t)=>e===null?null:rd(Math.round(e),0,t);function Xv(e){let t=this.getLabels();return e>=0&&et.length-1?null:this.getPixelForValue(t[e].value)}getValueForPixel(e){return Math.round(this._startValue+this.getDecimalForPixel(e)*this._valueRange)}getBasePixel(){return this.bottom}};function Qv(e,t){let n=[],{bounds:r,step:i,min:a,max:o,precision:s,count:c,maxTicks:l,maxDigits:u,includeBounds:d}=e,f=i||1,p=l-1,{min:m,max:h}=t,g=!ou(a),_=!ou(o),v=!ou(c),y=(h-m)/(u+1),b=Uu((h-m)/p/f)*f,x,S,C,w;if(b<1e-14&&!g&&!_)return[{value:m},{value:h}];w=Math.ceil(h/b)-Math.floor(m/b),w>p&&(b=Uu(w*b/p/f)*f),ou(s)||(x=10**s,b=Math.ceil(b*x)/x),r===`ticks`?(S=Math.floor(m/b)*b,C=Math.ceil(h/b)*b):(S=m,C=h),g&&_&&i&&qu((o-a)/i,b/1e3)?(w=Math.round(Math.min((o-a)/b,l)),b=(o-a)/w,S=a,C=o):v?(S=g?a:S,C=_?o:C,w=c-1,b=(C-S)/w):(w=(C-S)/b,w=Hu(w,Math.round(w),b/1e3)?Math.round(w):Math.ceil(w));let T=Math.max(Zu(b),Zu(S));x=10**(ou(s)?T:s),S=Math.round(S*x)/x,C=Math.round(C*x)/x;let ee=0;for(g&&(d&&S!==a?(n.push({value:a}),So)break;n.push({value:e})}return _&&d&&C!==o?n.length&&Hu(n[n.length-1].value,o,$v(o,y,e))?n[n.length-1].value=o:n.push({value:o}):(!_||C===o)&&n.push({value:C}),n}function $v(e,t,{horizontal:n,minRotation:r}){let i=Yu(r),a=(n?Math.sin(i):Math.cos(i))||.001,o=.75*t*(``+e).length;return Math.min(t/a,o)}var ey=class extends Jh{constructor(e){super(e),this.start=void 0,this.end=void 0,this._startValue=void 0,this._endValue=void 0,this._valueRange=0}parse(e,t){return ou(e)||(typeof e==`number`||e instanceof Number)&&!isFinite(+e)?null:+e}handleTickRangeOptions(){let{beginAtZero:e}=this.options,{minDefined:t,maxDefined:n}=this.getUserBounds(),{min:r,max:i}=this,a=e=>r=t?r:e,o=e=>i=n?i:e;if(e){let e=Vu(r),t=Vu(i);e<0&&t<0?o(0):e>0&&t>0&&a(0)}if(r===i){let t=i===0?1:Math.abs(i*.05);o(i+t),e||a(r-t)}this.min=r,this.max=i}getTickLimit(){let{maxTicksLimit:e,stepSize:t}=this.options.ticks,n;return t?(n=Math.ceil(this.max/t)-Math.floor(this.min/t)+1,n>1e3&&(console.warn(`scales.${this.id}.ticks.stepSize: ${t} would result generating up to ${n} ticks. Limiting to 1000.`),n=1e3)):(n=this.computeTickLimit(),e||=11),e&&(n=Math.min(e,n)),n}computeTickLimit(){return 1/0}buildTicks(){let e=this.options,t=e.ticks,n=this.getTickLimit();n=Math.max(2,n);let r=Qv({maxTicks:n,bounds:e.bounds,min:e.min,max:e.max,precision:t.precision,step:t.stepSize,count:t.count,maxDigits:this._maxDigits(),horizontal:this.isHorizontal(),minRotation:t.minRotation||0,includeBounds:t.includeBounds!==!1},this._range||this);return e.bounds===`ticks`&&Ju(r,this,`value`),e.reverse?(r.reverse(),this.start=this.max,this.end=this.min):(this.start=this.min,this.end=this.max),r}configure(){let e=this.ticks,t=this.min,n=this.max;if(super.configure(),this.options.offset&&e.length){let r=(n-t)/Math.max(e.length-1,1)/2;t-=r,n+=r}this._startValue=t,this._endValue=n,this._valueRange=n-t}getLabelForValue(e){return Fd(e,this.chart.options.locale,this.options.ticks.format)}},ty=class extends ey{static id=`linear`;static defaults={ticks:{callback:Rd.formatters.numeric}};determineDataLimits(){let{min:e,max:t}=this.getMinMax(!0);this.min=lu(e)?e:0,this.max=lu(t)?t:1,this.handleTickRangeOptions()}computeTickLimit(){let e=this.isHorizontal(),t=e?this.width:this.height,n=Yu(this.options.ticks.minRotation),r=(e?Math.sin(n):Math.cos(n))||.001,i=this._resolveTickFontOptions(0);return Math.ceil(t/Math.min(40,i.lineHeight/r))}getPixelForValue(e){return e===null?NaN:this.getPixelForDecimal((e-this._startValue)/this._valueRange)}getValueForPixel(e){return this._startValue+this.getDecimalForPixel(e)*this._valueRange}},ny=e=>Math.floor(Bu(e)),ry=(e,t)=>10**(ny(e)+t);function iy(e){return e/10**ny(e)==1}function ay(e,t,n){let r=10**n,i=Math.floor(e/r);return Math.ceil(t/r)-i}function oy(e,t){let n=ny(t-e);for(;ay(e,t,n)>10;)n++;for(;ay(e,t,n)<10;)n--;return Math.min(n,ny(e))}function sy(e,{min:t,max:n}){t=uu(e.min,t);let r=[],i=ny(t),a=oy(t,n),o=a<0?10**Math.abs(a):1,s=10**a,c=i>a?10**i:0,l=Math.round((t-c)*o)/o,u=Math.floor((t-c)/s/10)*s*10,d=Math.floor((l-u)/10**a),f=uu(e.min,Math.round((c+u+d*10**a)*o)/o);for(;f=10?d=d<15?15:20:d++,d>=20&&(a++,d=2,o=a>=0?1:o),f=Math.round((c+u+d*10**a)*o)/o;let p=uu(e.max,f);return r.push({value:p,major:iy(p),significand:d}),r}var cy=class extends Jh{static id=`logarithmic`;static defaults={ticks:{callback:Rd.formatters.logarithmic,major:{enabled:!0}}};constructor(e){super(e),this.start=void 0,this.end=void 0,this._startValue=void 0,this._valueRange=0}parse(e,t){let n=ey.prototype.parse.apply(this,[e,t]);if(n===0){this._zero=!0;return}return lu(n)&&n>0?n:null}determineDataLimits(){let{min:e,max:t}=this.getMinMax(!0);this.min=lu(e)?Math.max(0,e):null,this.max=lu(t)?Math.max(0,t):null,this.options.beginAtZero&&(this._zero=!0),this._zero&&this.min!==this._suggestedMin&&!lu(this._userMin)&&(this.min=e===ry(this.min,0)?ry(this.min,-1):ry(this.min,0)),this.handleTickRangeOptions()}handleTickRangeOptions(){let{minDefined:e,maxDefined:t}=this.getUserBounds(),n=this.min,r=this.max,i=t=>n=e?n:t,a=e=>r=t?r:e;n===r&&(n<=0?(i(1),a(10)):(i(ry(n,-1)),a(ry(r,1)))),n<=0&&i(ry(r,-1)),r<=0&&a(ry(n,1)),this.min=n,this.max=r}buildTicks(){let e=this.options,t=sy({min:this._userMin,max:this._userMax},this);return e.bounds===`ticks`&&Ju(t,this,`value`),e.reverse?(t.reverse(),this.start=this.max,this.end=this.min):(this.start=this.min,this.end=this.max),t}getLabelForValue(e){return e===void 0?`0`:Fd(e,this.chart.options.locale,this.options.ticks.format)}configure(){let e=this.min;super.configure(),this._startValue=Bu(e),this._valueRange=Bu(this.max)-Bu(e)}getPixelForValue(e){return(e===void 0||e===0)&&(e=this.min),e===null||isNaN(e)?NaN:this.getPixelForDecimal(e===this.min?0:(Bu(e)-this._startValue)/this._valueRange)}getValueForPixel(e){let t=this.getDecimalForPixel(e);return 10**(this._startValue+t*this._valueRange)}};function ly(e){let t=e.ticks;if(t.display&&e.display){let e=gf(t.backdropPadding);return du(t.font&&t.font.size,Wd.font.size)+e.height}return 0}function uy(e,t,n){return n=su(n)?n:[n],{w:qd(e,t.string,n),h:n.length*t.lineHeight}}function dy(e,t,n,r,i){return e===r||e===i?{start:t-n/2,end:t+n/2}:ei?{start:t-n,end:t}:{start:t,end:t+n}}function fy(e){let t={l:e.left+e._padding.left,r:e.right-e._padding.right,t:e.top+e._padding.top,b:e.bottom-e._padding.bottom},n=Object.assign({},t),r=[],i=[],a=e._pointLabels.length,o=e.options.pointLabels,s=o.centerPointLabels?Mu/a:0;for(let c=0;ct.r&&(s=(r.end-t.r)/a,e.r=Math.max(e.r,t.r+s)),i.startt.b&&(c=(i.end-t.b)/o,e.b=Math.max(e.b,t.b+c))}function my(e,t,n){let r=e.drawingArea,{extra:i,additionalAngle:a,padding:o,size:s}=n,c=e.getPointPosition(t,r+i+o,a),l=Math.round(Xu(td(c.angle+Lu))),u=yy(c.y,s.h,l),d=_y(l),f=vy(c.x,s.w,d);return{visible:!0,x:c.x,y:u,textAlign:d,left:f,top:u,right:f+s.w,bottom:u+s.h}}function hy(e,t){if(!t)return!0;let{left:n,top:r,right:i,bottom:a}=e;return!(Qd({x:n,y:r},t)||Qd({x:n,y:a},t)||Qd({x:i,y:r},t)||Qd({x:i,y:a},t))}function gy(e,t,n){let r=[],i=e._pointLabels.length,a=e.options,{centerPointLabels:o,display:s}=a.pointLabels,c={extra:ly(a)/2,additionalAngle:o?Mu/i:0},l;for(let a=0;a270||n<90)&&(e-=t),e}function by(e,t,n){let{left:r,top:i,right:a,bottom:o}=n,{backdropColor:s}=t;if(!ou(s)){let n=hf(t.borderRadius),c=gf(t.backdropPadding);e.fillStyle=s;let l=r-c.left,u=i-c.top,d=a-r+c.width,f=o-i+c.height;Object.values(n).some(e=>e!==0)?(e.beginPath(),cf(e,{x:l,y:u,w:d,h:f,radius:n}),e.fill()):e.fillRect(l,u,d,f)}}function xy(e,t){let{ctx:n,options:{pointLabels:r}}=e;for(let i=t-1;i>=0;i--){let t=e._pointLabelItems[i];if(!t.visible)continue;let a=r.setContext(e.getPointLabelContext(i));by(n,a,t);let o=_f(a.font),{x:s,y:c,textAlign:l}=t;sf(n,e._pointLabels[i],s,c+o.lineHeight/2,o,{color:a.color,textAlign:l,textBaseline:`middle`})}}function Sy(e,t,n,r){let{ctx:i}=e;if(n)i.arc(e.xCenter,e.yCenter,t,0,Nu);else{let n=e.getPointPosition(0,t);i.moveTo(n.x,n.y);for(let a=1;a{let n=mu(this.options.pointLabels.callback,[e,t],this);return n||n===0?n:``}).filter((e,t)=>this.chart.getDataVisibility(t))}fit(){let e=this.options;e.display&&e.pointLabels.display?fy(this):this.setCenterPoint(0,0,0,0)}setCenterPoint(e,t,n,r){this.xCenter+=Math.floor((e-t)/2),this.yCenter+=Math.floor((n-r)/2),this.drawingArea-=Math.min(this.drawingArea/2,Math.max(e,t,n,r))}getIndexAngle(e){let t=Nu/(this._pointLabels.length||1),n=this.options.startAngle||0;return td(e*t+Yu(n))}getDistanceFromCenterForValue(e){if(ou(e))return NaN;let t=this.drawingArea/(this.max-this.min);return this.options.reverse?(this.max-e)*t:(e-this.min)*t}getValueForDistanceFromCenter(e){if(ou(e))return NaN;let t=e/(this.drawingArea/(this.max-this.min));return this.options.reverse?this.max-t:this.min+t}getPointLabelContext(e){let t=this._pointLabels||[];if(e>=0&&e{if(t!==0||t===0&&this.min<0){s=this.getDistanceFromCenterForValue(e.value);let n=this.getContext(t),o=r.setContext(n),c=i.setContext(n);Cy(this,o,s,a,c)}}),n.display){for(e.save(),o=a-1;o>=0;o--){let r=n.setContext(this.getPointLabelContext(o)),{color:i,lineWidth:a}=r;!a||!i||(e.lineWidth=a,e.strokeStyle=i,e.setLineDash(r.borderDash),e.lineDashOffset=r.borderDashOffset,s=this.getDistanceFromCenterForValue(t.reverse?this.min:this.max),c=this.getPointPosition(o,s),e.beginPath(),e.moveTo(this.xCenter,this.yCenter),e.lineTo(c.x,c.y),e.stroke())}e.restore()}}drawBorder(){}drawLabels(){let e=this.ctx,t=this.options,n=t.ticks;if(!n.display)return;let r=this.getIndexAngle(0),i,a;e.save(),e.translate(this.xCenter,this.yCenter),e.rotate(r),e.textAlign=`center`,e.textBaseline=`middle`,this.ticks.forEach((r,o)=>{if(o===0&&this.min>=0&&!t.reverse)return;let s=n.setContext(this.getContext(o)),c=_f(s.font);if(i=this.getDistanceFromCenterForValue(this.ticks[o].value),s.showLabelBackdrop){e.font=c.string,a=e.measureText(r.label).width,e.fillStyle=s.backdropColor;let t=gf(s.backdropPadding);e.fillRect(-a/2-t.left,-i-c.size/2-t.top,a+t.width,c.size+t.height)}sf(e,r.label,0,-i,c,{color:s.color,strokeColor:s.textStrokeColor,strokeWidth:s.textStrokeWidth})}),e.restore()}drawTitle(){}},Ey={millisecond:{common:!0,size:1,steps:1e3},second:{common:!0,size:1e3,steps:60},minute:{common:!0,size:6e4,steps:60},hour:{common:!0,size:36e5,steps:24},day:{common:!0,size:864e5,steps:30},week:{common:!1,size:6048e5,steps:4},month:{common:!0,size:2628e6,steps:12},quarter:{common:!1,size:7884e6,steps:4},year:{common:!0,size:3154e7}},Dy=Object.keys(Ey);function Oy(e,t){return e-t}function ky(e,t){if(ou(t))return null;let n=e._adapter,{parser:r,round:i,isoWeekday:a}=e._parseOpts,o=t;return typeof r==`function`&&(o=r(o)),lu(o)||(o=typeof r==`string`?n.parse(o,r):n.parse(o)),o===null?null:(i&&(o=i===`week`&&(Ku(a)||a===!0)?n.startOf(o,`isoWeek`,a):n.startOf(o,i)),+o)}function Ay(e,t,n,r){let i=Dy.length;for(let a=Dy.indexOf(e);a=Dy.indexOf(n);a--){let n=Dy[a];if(Ey[n].common&&e._adapter.diff(i,r,n)>=t-1)return n}return Dy[n?Dy.indexOf(n):0]}function My(e){for(let t=Dy.indexOf(e)+1,n=Dy.length;t=t?n[r]:n[i];e[a]=!0}}function Py(e,t,n,r){let i=e._adapter,a=+i.startOf(t[0].value,r),o=t[t.length-1].value,s,c;for(s=a;s<=o;s=+i.add(s,1,r))c=n[s],c>=0&&(t[c].major=!0);return t}function Fy(e,t,n){let r=[],i={},a=t.length,o,s;for(o=0;o+e.value))}initOffsets(e=[]){let t=0,n=0,r,i;this.options.offset&&e.length&&(r=this.getDecimalForValue(e[0]),t=e.length===1?1-r:(this.getDecimalForValue(e[1])-r)/2,i=this.getDecimalForValue(e[e.length-1]),n=e.length===1?i:(i-this.getDecimalForValue(e[e.length-2]))/2);let a=e.length<3?.5:.25;t=rd(t,0,a),n=rd(n,0,a),this._offsets={start:t,end:n,factor:1/(t+1+n)}}_generate(){let e=this._adapter,t=this.min,n=this.max,r=this.options,i=r.time,a=i.unit||Ay(i.minUnit,t,n,this._getLabelCapacity(t)),o=du(r.ticks.stepSize,1),s=a===`week`&&i.isoWeekday,c=Ku(s)||s===!0,l={},u=t,d,f;if(c&&(u=+e.startOf(u,`isoWeek`,s)),u=+e.startOf(u,c?`day`:a),e.diff(n,t,a)>1e5*o)throw Error(t+` and `+n+` are too far apart with stepSize of `+o+` `+a);let p=r.ticks.source===`data`&&this.getDataTimestamps();for(d=u,f=0;d+e)}getLabelForValue(e){let t=this._adapter,n=this.options.time;return n.tooltipFormat?t.format(e,n.tooltipFormat):t.format(e,n.displayFormats.datetime)}format(e,t){let n=this.options.time.displayFormats,r=this._unit,i=t||n[r];return this._adapter.format(e,i)}_tickFormatFunction(e,t,n,r){let i=this.options,a=i.ticks.callback;if(a)return mu(a,[e,t,n],this);let o=i.time.displayFormats,s=this._unit,c=this._majorUnit,l=s&&o[s],u=c&&o[c],d=n[t],f=c&&u&&d&&d.major;return this._adapter.format(e,r||(f?u:l))}generateTickLabels(e){let t,n,r;for(t=0,n=e.length;t0?o:1}getDataTimestamps(){let e=this._cache.data||[],t,n;if(e.length)return e;let r=this.getMatchingVisibleMetas();if(this._normalized&&r.length)return this._cache.data=r[0].controller.getAllParsedValues(this);for(t=0,n=r.length;t=e[r].pos&&t<=e[i].pos&&({lo:r,hi:i}=sd(e,`pos`,t)),{pos:a,time:s}=e[r],{pos:o,time:c}=e[i]):(t>=e[r].time&&t<=e[i].time&&({lo:r,hi:i}=sd(e,`time`,t)),{time:a,pos:s}=e[r],{time:o,pos:c}=e[i]);let l=o-a;return l?s+(c-s)*(t-a)/l:s}var Ry=class extends Iy{static id=`timeseries`;static defaults=Iy.defaults;constructor(e){super(e),this._table=[],this._minPos=void 0,this._tableRange=void 0}initOffsets(){let e=this._getTimestampsForTable(),t=this._table=this.buildLookupTable(e);this._minPos=Ly(t,this.min),this._tableRange=Ly(t,this.max)-this._minPos,super.initOffsets(e)}buildLookupTable(e){let{min:t,max:n}=this,r=[],i=[],a,o,s,c,l;for(a=0,o=e.length;a=t&&c<=n&&r.push(c);if(r.length<2)return[{time:t,pos:0},{time:n,pos:1}];for(a=0,o=r.length;ae-t)}_getTimestampsForTable(){let e=this._cache.all||[];if(e.length)return e;let t=this.getDataTimestamps(),n=this.getLabelTimestamps();return e=t.length&&n.length?this.normalize(t.concat(n)):t.length?t:n,e=this._cache.all=e,e}getDecimalForValue(e){return(Ly(this._table,e)-this._minPos)/this._tableRange}getValueForPixel(e){let t=this._offsets,n=this.getDecimalForPixel(e)/t.factor-t.end;return Ly(this._table,n*this._tableRange+this._minPos,!0)}},zy=[Am,__,Kv,Object.freeze({__proto__:null,CategoryScale:Zv,LinearScale:ty,LogarithmicScale:cy,RadialLinearScale:Ty,TimeScale:Iy,TimeSeriesScale:Ry})];Ig.register(...zy);var By=Ig,Vy=z(``);function Hy(e,t){D(t,!0);let n=K(t,`class`,3,``),r=K(t,`ariaLabel`,3,``);function i(e){if(Da.tick,typeof t.build!=`function`)return;let n=t.build();if(!n)return;let r=new By(e.getContext(`2d`),n);return()=>r.destroy()}var a=Vy();ji(a,()=>i),I(()=>{W(a,1,Pi(n())),G(a,`aria-label`,r())}),B(e,a),O()}var Uy=z(``),Wy=z(`
`);function Gy(e,t){D(t,!0);let n=K(t,`options`,19,()=>[]),r=K(t,`ariaLabel`,3,``),i=K(t,`class`,3,``);var a=Wy();U(a,21,n,e=>e.value,(e,n)=>{var r=Uy();let i;var a=N(r,!0);E(r),I(()=>{i=W(r,1,`segmented-btn svelte-92fh5i`,null,i,{active:t.value===L(n).value}),G(r,`aria-pressed`,t.value===L(n).value),V(a,L(n).label)}),R(`click`,r,()=>t.onchange?.(L(n).value)),B(e,r)}),E(a),I(()=>{W(a,1,Pi([`segmented-control`,i()]),`svelte-92fh5i`),G(a,`aria-label`,r())}),B(e,a),O()}Vr([`click`]);function Ky(e){return getComputedStyle(document.documentElement).getPropertyValue(e).trim()}function qy(){return{grid:Ky(`--chart-grid`),text:Ky(`--chart-text`),dayMarker:Ky(`--chart-day-marker`),tooltipBg:Ky(`--chart-tooltip-bg`),tooltipBorder:Ky(`--chart-tooltip-border`),tooltipText:Ky(`--chart-tooltip-text`)}}function Jy(){return{size:11,family:`'SF Mono', Menlo, Consolas, monospace`}}function Yy(e,t){return{backgroundColor:e.tooltipBg,borderColor:e.tooltipBorder,borderWidth:1,titleColor:e.tooltipText,bodyColor:e.tooltipText,callbacks:t}}function Xy(e){if(typeof document>`u`||!document.body)return e;let t=document.createElement(`span`);t.style.display=`none`,t.style.color=e,document.body.appendChild(t);let n=getComputedStyle(t).color;return document.body.removeChild(t),n||e}var Zy=[`#c2845a`,`#7a9e7e`,`#d4a574`,`#b8a98e`,`#8b9e6b`,`#7d8a97`,`#c47a5a`,`#6b8e6b`,`#a09486`,`#9b7ea4`,`#c49a6c`];function Qy(){return[...Zy]}function $y(e){let t=5381,n=String(e||``);for(let e=0;exc(e)}}var ab={seconds:{apiName:`second`,windowLabel:`Last 60 seconds`,refreshMs:2e3},minutes:{apiName:`minute`,windowLabel:`Last 60 minutes`,refreshMs:5e3},hours:{apiName:`hour`,windowLabel:`Last 24 hours`,refreshMs:2e4},days:{apiName:`day`,windowLabel:`Last 30 days`,refreshMs:6e4}},ob=[{value:`seconds`,label:`Seconds`},{value:`minutes`,label:`Minutes`},{value:`hours`,label:`Hours`},{value:`days`,label:`Days`}];function sb(){return{input:0,output:0,prompt:0,local:0}}function cb(e){return String(e).padStart(2,`0`)}function lb(e){let t=Number(e);return Number.isFinite(t)&&t>0?t:0}function ub(e,t){if(!Number.isFinite(t))return``;let n=new Date(t);switch(e){case`seconds`:return cb(n.getHours())+`:`+cb(n.getMinutes())+`:`+cb(n.getSeconds());case`minutes`:return cb(n.getHours())+`:`+cb(n.getMinutes());case`hours`:return cb(n.getHours())+`:00`;default:return cb(n.getMonth()+1)+`-`+cb(n.getDate())}}function db(e,t){let n=[],r=[],i={input:[],output:[],prompt:[],local:[]},a=sb();for(let o of e||[]){let e=Date.parse(o&&o.start),s=lb(o&&o.input_tokens),c=lb(o&&o.output_tokens),l=lb(o&&o.prompt_cached_tokens),u=lb(o&&o.locally_cached_tokens);n.push(ub(t,e)),r.push(Number.isFinite(e)?e:null),i.input.push(s),i.output.push(c),i.prompt.push(l),i.local.push(u),a.input+=s,a.output+=c,a.prompt+=l,a.local+=u}return{labels:n,stamps:r,cols:i,totals:a}}function fb(e){let t=e||sb();return t.input+t.output+t.prompt+t.local>0}function pb(e,t){return xc(Math.max(0,Math.round(e&&e[t]||0)))}function mb(e){return(ab[e]||ab.minutes).windowLabel}function hb(e,t){return`Live token throughput, `+mb(t).toLowerCase()+`. Input `+pb(e,`input`)+`, output `+pb(e,`output`)+`, prompt cached `+pb(e,`prompt`)+`, locally cached `+pb(e,`local`)+` tokens.`}function gb(e,t,n,r){let i=e=>xc(Math.max(0,Math.round(e))),a=n.stamps,o=(e,t,n)=>({label:e,data:t,backgroundColor:n,borderWidth:0,borderRadius:0,categoryPercentage:1,barPercentage:1,stack:`tokens`});return{type:`bar`,plugins:[{id:`liveTokensDayMarks`,afterDatasetsDraw:t=>{if(r===`days`)return;let n=t.getDatasetMeta(0),i=t.chartArea;if(!n||!n.data||!i)return;let o=t.ctx;o.save(),o.font=`10px 'SF Mono', Menlo, Consolas, monospace`;let s=null;for(let t=0;t{if(!e.length)return``;let t=a[e[0].dataIndex];if(!t)return e[0].label;let n=new Date(t);return r===`days`?n.toLocaleDateString():n.toLocaleString()},label:e=>e.dataset.label+`: `+i(e.parsed.y),footer:e=>{let t=0;return e.forEach(e=>{t+=Number(e.parsed.y)||0}),`Total: `+i(t)}})}}}}var _b=900,vb=new class{#e=A(`minutes`);get granularity(){return L(this.#e)}set granularity(e){j(this.#e,e,!0)}#t=A(M([]));get buckets(){return L(this.#t)}set buckets(e){j(this.#t,e,!0)}#n=A(!1);get active(){return L(this.#n)}set active(e){j(this.#n,e,!0)}#r=null;#i=null;#a=null;#o=0;#s=null;#c=!1;start(){this.stop(),this.active=!0,this.fetch(),this.#l(),this.#u()}stop(){this.active=!1,this.#r&&=(clearInterval(this.#r),null),this.#i&&=(clearTimeout(this.#i),null),this.#a&&=(clearTimeout(this.#a),null),this.#o=0,this.#s&&=(this.#s.abort(),null),this.buckets=[]}setGranularity(e){!ab[e]||e===this.granularity||(this.granularity=e,this.buckets=[],this.#l(),this.fetch())}#l(){this.#r&&=(clearInterval(this.#r),null);let e=ab[this.granularity]||ab.minutes;this.#r=setInterval(()=>{this.active&&this.fetch()},e.refreshMs)}noteUsageEvent(e){!this.active||e!==`usage.flushed`||(this.#i||=setTimeout(()=>{this.#i=null,this.fetch()},_b))}async fetch(){if(!this.active||this.#c)return;this.#c=!0;let e=this.granularity;try{let t=await Ts(`/admin/usage/throughput?granularity=`+(ab[e]||ab.minutes).apiName,{label:`token throughput`});if(t.stale||!t.ok||this.granularity!==e)return;this.buckets=t.data&&Array.isArray(t.data.buckets)?t.data.buckets:[]}catch(e){if(Ds(e))return;console.error(`Failed to fetch token throughput:`,e)}finally{this.#c=!1,this.active&&this.granularity!==e&&this.fetch()}}async#u(){await As.ensureLoaded(),this.active&&As.liveLogsVisible()&&(typeof ReadableStream>`u`||(this.#s&&this.#s.abort(),this.#s=new AbortController,this.#d(this.#s)))}async#d(e){let t=()=>{e.signal.aborted||this.#s!==e||this.#p()};try{let n=await Cs(`/admin/live/logs?types=usage`,{signal:e.signal});if(!n.ok||!n.body||typeof n.body.getReader!=`function`){t();return}this.#o=0,await tb(n.body.getReader(),e=>this.#f(e)),t()}catch(n){if(Ds(n)||e.signal.aborted)return;console.error(`Live usage stream failed:`,n),t()}}#f(e){if(!e||typeof e!=`object`)return;let t=String(e.type||``).trim();t.indexOf(`usage.`)===0&&this.noteUsageEvent(t)}#p(){if(!this.active||this.#a)return;let{attempt:e,delay:t}=rb(this.#o);this.#o=e,this.#a=setTimeout(()=>{this.#a=null,this.#u()},t)}},yb=z(`
`),bb=z(`
Waiting for live requests…
`),xb=z(`

Live Token Throughput

`);function Sb(e,t){D(t,!0);let n=k(()=>db(vb.buckets,vb.granularity)),r=k(()=>L(n).totals);function i(){return{input:Xy(`var(--token-input)`),output:Xy(`var(--token-output)`),prompt:Xy(`var(--token-prompt)`),local:Xy(`var(--token-local)`)}}let a=[{metric:`input`,label:`Input Tokens`,colorVar:`--token-input`},{metric:`output`,label:`Output Tokens`,colorVar:`--token-output`},{metric:`prompt`,label:`Prompt (Input) Cached`,colorVar:`--token-prompt`},{metric:`local`,label:`Locally Cached`,colorVar:`--token-local`}];var o=xb(),s=N(o),c=N(s),l=F(N(c),2),u=N(l);let d;var f=F(u,2),p=N(f,!0);E(f),E(l),E(c),Gy(F(c,2),{ariaLabel:`Live token throughput granularity`,get options(){return ob},get value(){return vb.granularity},onchange:e=>vb.setGranularity(e)}),E(s);var m=F(s,2);U(m,21,()=>a,e=>e.metric,(e,t)=>{var n=yb(),i=N(n),a=F(i,2),o=N(a,!0);E(a);var s=F(a,2),c=N(s,!0);E(s),E(n),I(e=>{Vi(i,`background: var(${L(t).colorVar??``})`),V(o,L(t).label),V(c,e)},[()=>pb(L(r),L(t).metric)]),B(e,n)}),E(m);var h=F(m,2),g=N(h);{let e=k(()=>hb(L(r),vb.granularity));Hy(g,{get ariaLabel(){return L(e)},build:()=>gb(qy(),i(),L(n),vb.granularity)})}var _=F(g,2),v=e=>{B(e,bb())},y=k(()=>!fb(L(r)));H(_,e=>{L(y)&&e(v)}),E(h),E(o),I(e=>{d=W(u,1,`live-dot`,null,d,{"is-streaming":vb.active}),V(p,e)},[()=>mb(vb.granularity)]),B(e,o),O()}function Cb(e){let t=e||{};if(t.total_tokens!==null&&t.total_tokens!==void 0){let e=Number(t.total_tokens);if(Number.isFinite(e))return e}let n=Number(t.total_input_tokens||0),r=Number(t.total_output_tokens||0);return(Number.isFinite(n)?n:0)+(Number.isFinite(r)?r:0)}function wb(e,t){if(!t)return 0;let n=e&&e.summary?e.summary:{},r=Number(n.total_hits||0);return Number.isFinite(r)&&r>0?r:0}function Tb(e,t,n){let r=Number(e&&e.total_requests||0);return(Number.isFinite(r)?r:0)+wb(t,n)}function Eb(e,t,n){let r=wb(t,n);return r<=0?``:_c(Tb(e,t,n)-r)+` to providers + `+_c(r)+` from cache`}function Db(e){let t=e&&e.summary?e.summary:{},n=Number(t.total_input_tokens||0),r=Number(t.total_output_tokens||0);return(Number.isFinite(n)?n:0)+(Number.isFinite(r)?r:0)}function Ob(e,t,n){let r=e=>{let t=Number(e||0);return Number.isFinite(t)&&t>0?t:0},i=e||{},a=r(i.uncached_input_tokens),o=r(i.cached_input_tokens),s=r(i.cache_write_input_tokens),c=t&&t.summary?t.summary:{},l=n?r(c.total_input_tokens):0;return[{key:`uncached`,label:`Regular`,tokens:a+s,colorVar:`--cache-meter-uncached`,note:s>0?`Includes `+_c(s)+` cache-write tokens`:``},{key:`prompt`,label:`Prompt cached`,tokens:o,colorVar:`--cache-meter-prompt`,note:`Provider prompt-cache reads`},{key:`local`,label:`Locally cached`,tokens:l,colorVar:`--cache-meter-local`,note:`Served from GoModel response cache`}]}function kb(e,t,n){return Ob(e,t,n).reduce((e,t)=>e+t.tokens,0)}function Ab(e,t,n){return kb(e,t,n)>0}function jb(e,t,n){let r=Ob(e,t,n),i=r.reduce((e,t)=>e+t.tokens,0);if(i<=0)return r.map(e=>Object.assign({},e,{pct:0}));let a=r.map(e=>{let t=e.tokens/i*100,n=Math.floor(t);return Object.assign({},e,{pct:n,remainder:t-n})}),o=100-a.reduce((e,t)=>e+t.pct,0);return a.map((e,t)=>({index:t,remainder:e.remainder,tokens:e.tokens})).filter(e=>e.tokens>0).sort((e,t)=>t.remainder-e.remainder).forEach(e=>{o>0&&(a[e.index].pct+=1,--o)}),a}function Mb(e,t,n){return jb(e,t,n).filter(e=>e.tokens>0)}function Nb(e){let t=[e.label+`: `+_c(e.tokens)+` input tokens (`+e.pct+`%)`];return e.note&&t.push(e.note),t.join(` -`)}function Pb(e){let t=(e||[]).map(e=>e.label+` `+e.pct+`%`);return`Cache breakdown of input tokens — `+(t.length?t.join(`, `):`no data`)}function Fb(e){return e.getUTCFullYear()+`-`+String(e.getUTCMonth()+1).padStart(2,`0`)+`-`+String(e.getUTCDate()).padStart(2,`0`)}function Ib(e,t,n,r){if(t!==`daily`||!n||!r)return e;let i={};(e||[]).forEach(e=>{i[e.date]=e});let a=[];for(let e=new Date(n);e<=r;e.setUTCDate(e.getUTCDate()+1)){let t=Fb(e);a.push(i[t]||{date:t,input_tokens:0,output_tokens:0,total_tokens:0,requests:0,input_cost:null,output_cost:null,total_cost:null})}return a}function Lb(e,t){let n=e=>Number(e)||0,r=e.map(e=>e.date),i=e.map(e=>n(e.uncached_input_tokens)+n(e.cache_write_input_tokens)+n(e.cached_input_tokens)>0?n(e.uncached_input_tokens)+n(e.cache_write_input_tokens):n(e.input_tokens)),a=e.map(e=>n(e.output_tokens)),o=e.map(e=>n(e.cached_input_tokens)),s={};return(t||[]).forEach(e=>{s[e.date]=e}),{labels:r,inputPaid:i,output:a,prompt:o,local:r.map(e=>{let t=s[e];return t?n(t.input_tokens)+n(t.output_tokens):0})}}function Rb(e){let t=e||{},n=Math.max(0,Number(t.uncached_input_tokens)||0),r=Math.max(0,Number(t.cached_input_tokens)||0),i=Math.max(0,Number(t.cache_write_input_tokens)||0),a=n+r+i;return a>0?r/a*100:0}function zb(e){let t=e||{};return(Number(t.uncached_input_tokens)||0)+(Number(t.cached_input_tokens)||0)+(Number(t.cache_write_input_tokens)||0)>0}function Bb(e){return zb(e)?Math.round(Rb(e))+`%`:`—`}function Vb(e,t,n={}){let r=!!n.cacheEnabled,i=n.resolve||(e=>e),a=(e,t)=>i(`color-mix(in srgb, `+e+` `+t+`%, transparent)`),o=(e,t,n,r)=>Object.assign({label:e,data:t,borderColor:n,backgroundColor:n,fill:!1,tension:.3,borderWidth:2,pointRadius:0,pointHoverRadius:4},r||{}),s=[o(`Input Tokens`,t.inputPaid,i(`var(--token-input)`),{fill:`origin`}),o(`Output Tokens`,t.output,i(`var(--token-output)`),{fill:`-1`}),o(`Prompt (Input) Cached`,t.prompt,i(`var(--token-prompt)`),{fill:`-1`,borderDash:[6,4]})];return r&&s.push(o(`Locally Cached`,t.local,a(`var(--info)`,35),{fill:`-1`,borderDash:[2,3]})),{type:`line`,data:{labels:t.labels,datasets:s},options:{responsive:!0,maintainAspectRatio:!1,animation:{duration:0},interaction:{mode:`index`,intersect:!1},plugins:{legend:{labels:{color:e.text,font:{size:12}}},tooltip:Yy(e,{label:e=>e.dataset.label+`: `+e.parsed.y.toLocaleString(),footer:e=>{let t=0;return e.forEach(e=>{t+=Number(e.parsed.y)||0}),`Total: `+t.toLocaleString()}})},scales:{x:{stacked:!0,grid:{color:e.grid},border:{display:!1},ticks:{color:e.text,font:Jy(),maxRotation:0,autoSkip:!0,maxTicksLimit:10}},y:{stacked:!0,beginAtZero:!0,grid:{color:e.grid},border:{display:!1},ticks:ib(e)}}}}}function Hb(e,t,n){let r=Math.max(0,Math.min(100,e));return{type:`doughnut`,data:{datasets:[{data:[r,100-r],backgroundColor:[t,n],borderWidth:0,spacing:0}]},options:{rotation:-90,circumference:180,cutout:`84%`,responsive:!0,maintainAspectRatio:!1,animation:{duration:0},layout:{padding:1},events:[],plugins:{legend:{display:!1},tooltip:{enabled:!1}}}}}var Ub=`gomodel_provider_status_details_expanded`,Wb=`gomodel_provider_card_expanded_overrides`,Gb=3e3,Kb=`https://gomodel.enterpilot.io/docs/providers/`,qb={anthropic:`anthropic`,azure:`azure`,bailian:`bailian`,bedrock:`bedrock`,"bedrock-mantle":`bedrock-mantle`,cohere:`cohere`,deepseek:`deepseek`,gemini:`gemini`,opencode_go:`opencode-go`,oracle:`oracle`,sglang:`sglang`,vertex:`vertex`,vllm:`vllm`,xiaomi:`xiaomi`};function Jb(){return{summary:{total:0,healthy:0,degraded:0,unhealthy:0,overall_status:`degraded`},providers:[]}}function Yb(e){let t={detailsExpanded:!1,cardOverrides:{}};try{if(e){let n=e.getItem(Ub);n===`true`||n===`false`?t.detailsExpanded=n===`true`:e.setItem(Ub,`false`);let r=JSON.parse(e.getItem(Wb)||`{}`);r&&typeof r==`object`&&!Array.isArray(r)&&(t.cardOverrides=r)}}catch{}return t}function Xb(e,t){if(e)try{e.setItem(Ub,t?`true`:`false`)}catch{}}function Zb(e,t){if(e)try{e.setItem(Wb,JSON.stringify(t))}catch{}}function Qb(e,t,n){let r=n&&n.name?String(n.name):``;return r&&Object.prototype.hasOwnProperty.call(e,r)?e[r]===!0:t}function $b(e){return`is-`+(String(e&&e.overall_status||`degraded`).trim()||`degraded`)}function ex(e){return`is-`+(String(e||`degraded`).trim()||`degraded`)}function tx(e){let t=e||{};return String(t.healthy||0)+`/`+String(t.total||0)}function nx(e){let t=e||{},n=Number(t.total||0),r=Number(t.healthy||0);return n>0&&rString(e&&e.status_label||``).trim().toLowerCase()===`starting`)}function ax(e){if(!e||!e.runtime)return``;let t=e.runtime.last_model_fetch_at||``,n=e.runtime.last_availability_check_at||``;return t?n&&Date.parse(n)>Date.parse(t)?n:t:n}function ox(e,t){let n=ax(e);if(!n||typeof t!=`function`)return`-`;let r=t(n);if(!r||r===`-`)return`-`;let i=String(r).split(` `);return i.length>1?i.slice(1).join(` `):r}function sx(e,t){let n=ax(e);return n?typeof t==`function`?t(n):String(n):``}function cx(e){if(!e)return``;let t=String(e.name||``).trim(),n=String(e.type||e.config&&e.config.type||``).trim();return!n||n===t?``:n}function lx(e){let t=String(e&&(e.type||e.config&&e.config.type)||``).trim().toLowerCase(),n=t?qb[t]:``;return n?Kb+n+`?utm_source=gomodel_dashboard`:``}function ux(e){let t=e&&e.config&&e.config.resilience?e.config.resilience.retry:null;return t?String(t.max_retries)+` retries, `+t.initial_backoff+` initial, `+t.max_backoff+` max, factor `+t.backoff_factor+`, jitter `+t.jitter_factor:`-`}function dx(e){let t=e&&e.config&&e.config.resilience?e.config.resilience.circuit_breaker:null;return t?String(t.failure_threshold)+` fail, `+String(t.success_threshold)+` success, `+t.timeout+` timeout`:`-`}function fx(e){let t=e&&e.config&&Array.isArray(e.config.models)?e.config.models.filter(Boolean):[];return t.length===0?`Automatic`:t.join(`, `)}function px(e){if(!e)return``;let t=[];return e.status_reason&&t.push(String(e.status_reason)),e.last_error&&t.push(`Last error: `+String(e.last_error)),t.join(` +`)}function Pb(e){let t=(e||[]).map(e=>e.label+` `+e.pct+`%`);return`Cache breakdown of input tokens — `+(t.length?t.join(`, `):`no data`)}function Fb(e){return e.getUTCFullYear()+`-`+String(e.getUTCMonth()+1).padStart(2,`0`)+`-`+String(e.getUTCDate()).padStart(2,`0`)}function Ib(e,t,n,r){if(t!==`daily`||!n||!r)return e;let i={};(e||[]).forEach(e=>{i[e.date]=e});let a=[];for(let e=new Date(n);e<=r;e.setUTCDate(e.getUTCDate()+1)){let t=Fb(e);a.push(i[t]||{date:t,input_tokens:0,output_tokens:0,total_tokens:0,requests:0,input_cost:null,output_cost:null,total_cost:null})}return a}function Lb(e,t){let n=e=>Number(e)||0,r=e.map(e=>e.date),i=e.map(e=>n(e.uncached_input_tokens)+n(e.cache_write_input_tokens)+n(e.cached_input_tokens)>0?n(e.uncached_input_tokens)+n(e.cache_write_input_tokens):n(e.input_tokens)),a=e.map(e=>n(e.output_tokens)),o=e.map(e=>n(e.cached_input_tokens)),s={};return(t||[]).forEach(e=>{s[e.date]=e}),{labels:r,inputPaid:i,output:a,prompt:o,local:r.map(e=>{let t=s[e];return t?n(t.input_tokens)+n(t.output_tokens):0})}}function Rb(e){let t=e||{},n=Math.max(0,Number(t.uncached_input_tokens)||0),r=Math.max(0,Number(t.cached_input_tokens)||0),i=Math.max(0,Number(t.cache_write_input_tokens)||0),a=n+r+i;return a>0?r/a*100:0}function zb(e){let t=e||{};return(Number(t.uncached_input_tokens)||0)+(Number(t.cached_input_tokens)||0)+(Number(t.cache_write_input_tokens)||0)>0}function Bb(e){return zb(e)?Math.round(Rb(e))+`%`:`—`}function Vb(e,t,n={}){let r=!!n.cacheEnabled,i=n.resolve||(e=>e),a=(e,t)=>i(`color-mix(in srgb, `+e+` `+t+`%, transparent)`),o=(e,t,n,r)=>Object.assign({label:e,data:t,borderColor:n,backgroundColor:n,fill:!1,tension:.3,borderWidth:2,pointRadius:0,pointHoverRadius:4},r||{}),s=[o(`Input Tokens`,t.inputPaid,i(`var(--token-input)`),{fill:`origin`}),o(`Output Tokens`,t.output,i(`var(--token-output)`),{fill:`-1`}),o(`Prompt (Input) Cached`,t.prompt,i(`var(--token-prompt)`),{fill:`-1`,borderDash:[6,4]})];return r&&s.push(o(`Locally Cached`,t.local,a(`var(--info)`,35),{fill:`-1`,borderDash:[2,3]})),{type:`line`,data:{labels:t.labels,datasets:s},options:{responsive:!0,maintainAspectRatio:!1,animation:{duration:0},interaction:{mode:`index`,intersect:!1},plugins:{legend:{labels:{color:e.text,font:{size:12}}},tooltip:Yy(e,{label:e=>e.dataset.label+`: `+e.parsed.y.toLocaleString(),footer:e=>{let t=0;return e.forEach(e=>{t+=Number(e.parsed.y)||0}),`Total: `+t.toLocaleString()}})},scales:{x:{stacked:!0,grid:{color:e.grid},border:{display:!1},ticks:{color:e.text,font:Jy(),maxRotation:0,autoSkip:!0,maxTicksLimit:10}},y:{stacked:!0,beginAtZero:!0,grid:{color:e.grid},border:{display:!1},ticks:ib(e)}}}}}function Hb(e,t,n){let r=Math.max(0,Math.min(100,e));return{type:`doughnut`,data:{datasets:[{data:[r,100-r],backgroundColor:[t,n],borderWidth:0,spacing:0}]},options:{rotation:-90,circumference:180,cutout:`84%`,responsive:!0,maintainAspectRatio:!1,animation:{duration:0},layout:{padding:1},events:[],plugins:{legend:{display:!1},tooltip:{enabled:!1}}}}}var Ub=`gomodel_provider_status_details_expanded`,Wb=`gomodel_provider_card_expanded_overrides`,Gb=3e3,Kb=`https://gomodel.enterpilot.io/docs/providers/`,qb={anthropic:`anthropic`,azure:`azure`,bailian:`bailian`,bedrock:`bedrock`,"bedrock-mantle":`bedrock-mantle`,cohere:`cohere`,deepseek:`deepseek`,gemini:`gemini`,llmd:`llmd`,opencode_go:`opencode-go`,oracle:`oracle`,sglang:`sglang`,vertex:`vertex`,vllm:`vllm`,xiaomi:`xiaomi`};function Jb(){return{summary:{total:0,healthy:0,degraded:0,unhealthy:0,overall_status:`degraded`},providers:[]}}function Yb(e){let t={detailsExpanded:!1,cardOverrides:{}};try{if(e){let n=e.getItem(Ub);n===`true`||n===`false`?t.detailsExpanded=n===`true`:e.setItem(Ub,`false`);let r=JSON.parse(e.getItem(Wb)||`{}`);r&&typeof r==`object`&&!Array.isArray(r)&&(t.cardOverrides=r)}}catch{}return t}function Xb(e,t){if(e)try{e.setItem(Ub,t?`true`:`false`)}catch{}}function Zb(e,t){if(e)try{e.setItem(Wb,JSON.stringify(t))}catch{}}function Qb(e,t,n){let r=n&&n.name?String(n.name):``;return r&&Object.prototype.hasOwnProperty.call(e,r)?e[r]===!0:t}function $b(e){return`is-`+(String(e&&e.overall_status||`degraded`).trim()||`degraded`)}function ex(e){return`is-`+(String(e||`degraded`).trim()||`degraded`)}function tx(e){let t=e||{};return String(t.healthy||0)+`/`+String(t.total||0)}function nx(e){let t=e||{},n=Number(t.total||0),r=Number(t.healthy||0);return n>0&&rString(e&&e.status_label||``).trim().toLowerCase()===`starting`)}function ax(e){if(!e||!e.runtime)return``;let t=e.runtime.last_model_fetch_at||``,n=e.runtime.last_availability_check_at||``;return t?n&&Date.parse(n)>Date.parse(t)?n:t:n}function ox(e,t){let n=ax(e);if(!n||typeof t!=`function`)return`-`;let r=t(n);if(!r||r===`-`)return`-`;let i=String(r).split(` `);return i.length>1?i.slice(1).join(` `):r}function sx(e,t){let n=ax(e);return n?typeof t==`function`?t(n):String(n):``}function cx(e){if(!e)return``;let t=String(e.name||``).trim(),n=String(e.type||e.config&&e.config.type||``).trim();return!n||n===t?``:n}function lx(e){let t=String(e&&(e.type||e.config&&e.config.type)||``).trim().toLowerCase(),n=t?qb[t]:``;return n?Kb+n+`?utm_source=gomodel_dashboard`:``}function ux(e){let t=e&&e.config&&e.config.resilience?e.config.resilience.retry:null;return t?String(t.max_retries)+` retries, `+t.initial_backoff+` initial, `+t.max_backoff+` max, factor `+t.backoff_factor+`, jitter `+t.jitter_factor:`-`}function dx(e){let t=e&&e.config&&e.config.resilience?e.config.resilience.circuit_breaker:null;return t?String(t.failure_threshold)+` fail, `+String(t.success_threshold)+` success, `+t.timeout+` timeout`:`-`}function fx(e){let t=e&&e.config&&Array.isArray(e.config.models)?e.config.models.filter(Boolean):[];return t.length===0?`Automatic`:t.join(`, `)}function px(e){if(!e)return``;let t=[];return e.status_reason&&t.push(String(e.status_reason)),e.last_error&&t.push(`Last error: `+String(e.last_error)),t.join(` `)}function mx(e){let t=e&&e.request_health;return t&&typeof t==`object`?t:null}function hx(e){let t=mx(e);return t?String(t.circuit_state||``).trim():``}function gx(e){let t=hx(e);return t?t.charAt(0).toUpperCase()+t.slice(1):``}function _x(e){let t=hx(e);return t===`open`?`is-unhealthy`:t===`half-open`?`is-degraded`:`is-healthy`}function vx(e){let t=mx(e);if(!t)return``;let n=Number(t.requests||0),r=Number(t.errors||0),i=Math.round(Number(t.window_seconds||0)/60),a=i>0?`last `+i+` min`:`recent`;return String(n)+` request`+(n===1?``:`s`)+` · `+String(r)+` error`+(r===1?``:`s`)+` (`+a+`)`}function yx(e){let t=mx(e);return t&&Array.isArray(t.models)?t.models:[]}function bx(e){return e?String(Number(e.errors||0))+`/`+String(Number(e.requests||0))+` failed`:``}function xx(e){let t=e&&e.last_error;return!t||!t.message?``:(t.status_code?`HTTP `+String(t.status_code)+`: `:``)+t.message}function Sx(){return{name:``,slug:``,url:``,transport:`http`,description:``,enabled:!0,headers:[],allowed_tools:``,disallowed_tools:``,user_paths:``,tool_timeout_seconds:``}}function Cx(){return{server:``,status:``,instructions:``,tools:[],prompts:[],resources:[],templates:[]}}function wx(e){return String(e&&(e.slug||e.name)||``).trim()}function Tx(e){return String(e&&e.status||``).trim()||`connecting`}function Ex(e){switch(Tx(e)){case`connected`:return`status-success`;case`degraded`:return String(e&&e.last_error||``).trim()?`status-error`:`status-warning`;case`connecting`:return`status-neutral`;default:return`status-unknown`}}function Dx(e,t){let n=Tx(e),r=String(e&&e.last_error||``).trim();return r&&n!==`connected`?r:n===`connected`&&e&&e.connected_at?`Connected since `+(typeof t==`function`?t:String)(e.connected_at):``}function Ox(e){return String(e&&e.transport||``)===`stdio`?`local command`:String(e&&e.url||``).trim()||`—`}function kx(e){let t=Number(e&&e.prompt_count||0),n=Number(e&&e.resource_count||0);return t+` prompts · `+n+` resources`}function Ax(e){let t=String(e||``).normalize(`NFKD`).toLowerCase(),n=t.replace(/[\u0300-\u036f]/g,``).replace(/[^a-z0-9]+/g,`-`).replace(/^-+|-+$/g,``).slice(0,64).replace(/-+$/g,``);if(n)return n;let r=2166136261;for(let e of t)r=Math.imul((r^e.codePointAt(0))>>>0,16777619)>>>0;return`mcp-`+r.toString(16).padStart(8,`0`)}function jx(e){return String(e||``).split(` `).map(e=>e.trim()).filter(e=>e)}function Mx(e){return!e||typeof e!=`object`||Array.isArray(e)?[]:Object.keys(e).sort().map(t=>({name:t,value:String(e[t]||``)}))}function Nx(e){let t={};return(Array.isArray(e)?e:[]).forEach(e=>{let n=String(e&&e.name||``).trim();n&&(t[n]=String(e&&e.value||``))}),t}function Px(e,t){let n=Array.isArray(e)?e:[];if(!t)return n;let r=String(t).toLowerCase();return n.filter(e=>[e.name,e.slug,e.url,e.transport,e.description,e.status].some(e=>String(e||``).toLowerCase().includes(r)))}function Fx(e){return{name:String(e.name||``).trim(),slug:wx(e),url:String(e.url||``).trim(),transport:e.transport===`sse`?`sse`:`http`,description:String(e.description||``).trim(),enabled:e.enabled!==!1,headers:Mx(e.headers),allowed_tools:(Array.isArray(e.allowed_tools)?e.allowed_tools:[]).join(`, `),disallowed_tools:(Array.isArray(e.disallowed_tools)?e.disallowed_tools:[]).join(`, `),user_paths:(Array.isArray(e.user_paths)?e.user_paths:[]).join(` diff --git a/internal/admin/dashboard/static/dist/index.html b/internal/admin/dashboard/static/dist/index.html index d57678b3..e391351b 100644 --- a/internal/admin/dashboard/static/dist/index.html +++ b/internal/admin/dashboard/static/dist/index.html @@ -7,7 +7,7 @@ GoModel Dashboard - + diff --git a/internal/providers/config.go b/internal/providers/config.go index a946f765..b3ed2ccc 100644 --- a/internal/providers/config.go +++ b/internal/providers/config.go @@ -36,6 +36,8 @@ type ProviderConfig struct { ServiceAccountJSON string ServiceAccountJSONBase64 string GCPScope string + InferenceObjective string + FairnessFromUserPath bool Models []string // ModelMetadataOverrides holds operator-supplied metadata keyed by raw model // ID (as it appears in the provider's /models response). The registry merges @@ -143,6 +145,8 @@ const ( providerEnvFieldServiceAccountJSONBase64 providerEnvFieldGCPScope providerEnvFieldSessionStickyKeys + providerEnvFieldInferenceObjective + providerEnvFieldFairnessFromUserPath ) type providerEnvSource struct { @@ -169,8 +173,10 @@ type providerEnvValues struct { ServiceAccountJSON string ServiceAccountJSONBase64 string GCPScope string + InferenceObjective string Models []string SessionStickyKeys *bool + FairnessFromUserPath *bool } // apiKeys returns the ordered key set this env group declares: the unsuffixed @@ -236,7 +242,9 @@ func (v providerEnvValues) empty() bool { strings.TrimSpace(v.ServiceAccountJSON) == "" && strings.TrimSpace(v.ServiceAccountJSONBase64) == "" && strings.TrimSpace(v.GCPScope) == "" && + strings.TrimSpace(v.InferenceObjective) == "" && v.SessionStickyKeys == nil && + v.FairnessFromUserPath == nil && len(v.Models) == 0 } @@ -307,6 +315,12 @@ func collectProviderEnvValues(prefix string, spec DiscoveryConfig, environ []str if parsed, err := strconv.ParseBool(strings.TrimSpace(value)); err == nil { values.SessionStickyKeys = &parsed } + case providerEnvFieldInferenceObjective: + values.InferenceObjective = value + case providerEnvFieldFairnessFromUserPath: + if parsed, err := strconv.ParseBool(strings.TrimSpace(value)); err == nil { + values.FairnessFromUserPath = &parsed + } } groups[suffix] = values } @@ -376,6 +390,15 @@ func parseProviderEnvKey(prefix, key string, spec DiscoveryConfig) (string, prov {name: "GCP_SCOPE", field: providerEnvFieldGCPScope}, }, fields...) } + if strings.EqualFold(prefix, "LLMD") { + fields = append([]struct { + name string + field providerEnvField + }{ + {name: "FAIRNESS_FROM_USER_PATH", field: providerEnvFieldFairnessFromUserPath}, + {name: "INFERENCE_OBJECTIVE", field: providerEnvFieldInferenceObjective}, + }, fields...) + } for _, candidate := range fields { if candidate.field == providerEnvFieldAPIVersion && !spec.SupportsAPIVersion { @@ -516,6 +539,8 @@ func (v providerEnvValues) rawConfig(providerType string, spec DiscoveryConfig) ServiceAccountJSON: v.ServiceAccountJSON, ServiceAccountJSONBase64: v.ServiceAccountJSONBase64, GCPScope: v.GCPScope, + InferenceObjective: v.InferenceObjective, + FairnessFromUserPath: v.FairnessFromUserPath, Models: rawProviderModelsFromIDs(v.Models), SessionStickyKeys: v.SessionStickyKeys, } @@ -572,6 +597,12 @@ func overlayProviderEnvValues(existing config.RawProviderConfig, values provider if values.GCPScope != "" { existing.GCPScope = values.GCPScope } + if values.InferenceObjective != "" { + existing.InferenceObjective = values.InferenceObjective + } + if values.FairnessFromUserPath != nil { + existing.FairnessFromUserPath = values.FairnessFromUserPath + } if values.SessionStickyKeys != nil { existing.SessionStickyKeys = values.SessionStickyKeys } @@ -845,6 +876,10 @@ func buildProviderConfig(raw config.RawProviderConfig, global config.ResilienceC ModelMetadataOverrides: config.ProviderModelMetadataOverrides(raw.Models), Resilience: global, } + if resolved.Type == "llmd" { + resolved.InferenceObjective = raw.InferenceObjective + resolved.FairnessFromUserPath = enabledByDefault(raw.FairnessFromUserPath) + } if raw.Resilience == nil { return resolved @@ -884,6 +919,10 @@ func buildProviderConfig(raw config.RawProviderConfig, global config.ResilienceC } func sessionStickyKeysEnabled(value *bool) bool { + return enabledByDefault(value) +} + +func enabledByDefault(value *bool) bool { return value == nil || *value } diff --git a/internal/providers/config_test.go b/internal/providers/config_test.go index 953cd888..9f4cea48 100644 --- a/internal/providers/config_test.go +++ b/internal/providers/config_test.go @@ -54,6 +54,10 @@ var testDiscoveryConfigs = map[string]DiscoveryConfig{ DefaultBaseURL: "http://localhost:8000/v1", AllowAPIKeyless: true, }, + "llmd": { + RequireBaseURL: true, + AllowAPIKeyless: true, + }, "sglang": { DefaultBaseURL: "http://localhost:30000/v1", AllowAPIKeyless: true, @@ -97,6 +101,43 @@ func TestBuildProviderConfig_InheritsGlobal(t *testing.T) { } } +func TestBuildProviderConfig_LLMDControlDefaults(t *testing.T) { + disabled := false + tests := []struct { + name string + raw config.RawProviderConfig + wantObjective string + wantFair bool + }{ + { + name: "defaults fairness to effective user path", + raw: config.RawProviderConfig{Type: "llmd", InferenceObjective: "premium"}, + wantObjective: "premium", + wantFair: true, + }, + { + name: "explicitly disables fairness derivation", + raw: config.RawProviderConfig{Type: "llmd", FairnessFromUserPath: &disabled}, + wantFair: false, + }, + { + name: "ignores llmd-only fields for another provider type", + raw: config.RawProviderConfig{Type: "openai", InferenceObjective: "premium"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildProviderConfig(tt.raw, globalResilience) + if got.InferenceObjective != tt.wantObjective { + t.Errorf("InferenceObjective = %q, want %q", got.InferenceObjective, tt.wantObjective) + } + if got.FairnessFromUserPath != tt.wantFair { + t.Errorf("FairnessFromUserPath = %v, want %v", got.FairnessFromUserPath, tt.wantFair) + } + }) + } +} + func TestBuildProviderConfig_CanDisableSessionStickyKeys(t *testing.T) { disabled := false got := buildProviderConfig(config.RawProviderConfig{ @@ -464,6 +505,21 @@ func TestFilterEmptyProviders_RemovesOracleByTypeWithoutBaseURL(t *testing.T) { } } +func TestFilterEmptyProviders_LLMDRequiresBaseURLButNotAPIKey(t *testing.T) { + raw := map[string]config.RawProviderConfig{ + "missing-endpoint": {Type: "llmd"}, + "router": {Type: "llmd", BaseURL: "http://llmd-epp.default.svc/v1"}, + } + + got := filterEmptyProviders(raw, testDiscoveryConfigs) + if _, exists := got["missing-endpoint"]; exists { + t.Fatal("llmd without base URL must be removed") + } + if _, exists := got["router"]; !exists { + t.Fatal("keyless llmd with base URL must be retained") + } +} + // --- applyProviderEnvVars --- func TestApplyProviderEnvVars_DiscoversFromAPIKey(t *testing.T) { @@ -483,6 +539,40 @@ func TestApplyProviderEnvVars_DiscoversFromAPIKey(t *testing.T) { } } +func TestApplyProviderEnvVars_LLMDControls(t *testing.T) { + t.Setenv("LLMD_BASE_URL", "http://llmd-epp.default.svc/v1") + t.Setenv("LLMD_INFERENCE_OBJECTIVE", "premium-traffic") + t.Setenv("LLMD_FAIRNESS_FROM_USER_PATH", "false") + t.Setenv("LLMD_CANARY_BASE_URL", "http://llmd-canary.default.svc/v1") + t.Setenv("LLMD_CANARY_INFERENCE_OBJECTIVE", "canary-traffic") + t.Setenv("LLMD_CANARY_FAIRNESS_FROM_USER_PATH", "false") + + got := applyProviderEnvVars(map[string]config.RawProviderConfig{}, testDiscoveryConfigs) + tests := []struct { + name string + provider string + baseURL string + objective string + }{ + {name: "primary", provider: "llmd", baseURL: "http://llmd-epp.default.svc/v1", objective: "premium-traffic"}, + {name: "suffixed", provider: "llmd-canary", baseURL: "http://llmd-canary.default.svc/v1", objective: "canary-traffic"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := got[tt.provider] + if provider.BaseURL != tt.baseURL { + t.Errorf("BaseURL = %q, want %q", provider.BaseURL, tt.baseURL) + } + if provider.InferenceObjective != tt.objective { + t.Errorf("InferenceObjective = %q, want %q", provider.InferenceObjective, tt.objective) + } + if provider.FairnessFromUserPath == nil || *provider.FairnessFromUserPath { + t.Fatalf("FairnessFromUserPath = %v, want false", provider.FairnessFromUserPath) + } + }) + } +} + func TestApplyProviderEnvVars_SessionStickyKeys(t *testing.T) { disabled := false tests := []struct { diff --git a/internal/providers/llmd/llmd.go b/internal/providers/llmd/llmd.go new file mode 100644 index 00000000..c4a51a33 --- /dev/null +++ b/internal/providers/llmd/llmd.go @@ -0,0 +1,283 @@ +// Package llmd provides integration with the llm-d Router's OpenAI-compatible API. +package llmd + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + + "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" + "github.com/enterpilot/gomodel/internal/providers" + "github.com/enterpilot/gomodel/internal/providers/openai" +) + +const ( + canonicalObjectiveHeader = "X-Llm-D-Inference-Objective" + legacyObjectiveHeader = "X-Gateway-Inference-Objective" + canonicalFairnessHeader = "X-Llm-D-Inference-Fairness-Id" + legacyFairnessHeader = "X-Gateway-Inference-Fairness-Id" + droppedReasonHeader = "X-Llm-D-Request-Dropped-Reason" +) + +// Registration provides factory registration for the llm-d provider. An +// llm-d deployment has no universal endpoint, so base_url is required. Router +// authentication is optional and depends on the Gateway in front of llm-d. +var Registration = providers.Registration{ + Type: "llmd", + New: New, + PassthroughSemanticEnricher: passthroughSemanticEnricher, + Discovery: providers.DiscoveryConfig{ + RequireBaseURL: true, + AllowAPIKeyless: true, + }, +} + +// ControlConfig contains trusted llm-d request-classification settings. +type ControlConfig struct { + InferenceObjective string + FairnessFromUserPath bool +} + +// Provider implements the OpenAI-compatible inference subset understood by +// llm-d's openai-parser. It intentionally does not advertise native files, +// batches, audio, or Responses lifecycle interfaces. +type Provider struct { + compatible *openai.CompatibleProvider + rootClient *llmclient.Client + controls ControlConfig +} + +var ( + _ core.Provider = (*Provider)(nil) + _ core.PassthroughProvider = (*Provider)(nil) +) + +// New creates an llm-d provider. +func New(cfg providers.ProviderConfig, opts providers.ProviderOptions) core.Provider { + return newProvider(cfg.APIKey, cfg.BaseURL, ControlConfig{ + InferenceObjective: cfg.InferenceObjective, + FairnessFromUserPath: cfg.FairnessFromUserPath, + }, opts, nil) +} + +// NewWithHTTPClient creates an llm-d provider with a custom HTTP client. +// If httpClient is nil, http.DefaultClient is used. +func NewWithHTTPClient(apiKey, baseURL string, controls ControlConfig, httpClient *http.Client, hooks llmclient.Hooks) *Provider { + return newProvider(apiKey, baseURL, controls, providers.ProviderOptions{Hooks: hooks}, httpClient) +} + +func newProvider(apiKey, baseURL string, controls ControlConfig, opts providers.ProviderOptions, httpClient *http.Client) *Provider { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + p := &Provider{controls: controls} + keys := opts.Keyring(apiKey) + opts.Keys = keys + compatibleCfg := openai.CompatibleProviderConfig{ + ProviderName: "llmd", + BaseURL: baseURL, + HTTPClient: httpClient, + SetHeaders: p.setHeaders, + } + rootCfg := llmclient.Config{ + ProviderName: "llmd", + BaseURL: passthroughBaseURL(baseURL), + Retry: opts.Resilience.Retry, + Hooks: opts.Hooks, + CircuitBreaker: opts.Resilience.CircuitBreaker, + } + p.compatible = openai.NewCompatibleProvider(apiKey, opts, compatibleCfg) + setRootHeaders := func(req *http.Request) { + p.setHeaders(req, keys.NextForContext(req.Context())) + } + if httpClient == nil { + p.rootClient = llmclient.New(rootCfg, setRootHeaders) + return p + } + + p.rootClient = llmclient.NewWithHTTPClient(httpClient, rootCfg, setRootHeaders) + return p +} + +// SetBaseURL changes the llm-d Router endpoint. +func (p *Provider) SetBaseURL(baseURL string) { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + p.compatible.SetBaseURL(baseURL) + p.rootClient.SetBaseURL(passthroughBaseURL(baseURL)) +} + +// ChatCompletion sends a chat completion request to llm-d. +func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (*core.ChatResponse, error) { + resp, err := p.compatible.ChatCompletion(ctx, req) + return resp, exposeDroppedReason(err) +} + +// StreamChatCompletion streams a chat completion request through llm-d. +func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatRequest) (io.ReadCloser, error) { + resp, err := p.compatible.StreamChatCompletion(ctx, req) + return resp, exposeDroppedReason(err) +} + +// ListModels asks the llm-d route for its OpenAI-compatible model inventory. +// Operators should configure providers..models when their HTTPRoute does +// not forward GET /v1/models to a model server. +func (p *Provider) ListModels(ctx context.Context) (*core.ModelsResponse, error) { + resp, err := p.compatible.ListModels(ctx) + return resp, exposeDroppedReason(err) +} + +// Responses sends a Responses API request to llm-d. +func (p *Provider) Responses(ctx context.Context, req *core.ResponsesRequest) (*core.ResponsesResponse, error) { + resp, err := p.compatible.Responses(ctx, req) + return resp, exposeDroppedReason(err) +} + +// StreamResponses streams a Responses API request through llm-d. +func (p *Provider) StreamResponses(ctx context.Context, req *core.ResponsesRequest) (io.ReadCloser, error) { + resp, err := p.compatible.StreamResponses(ctx, req) + return resp, exposeDroppedReason(err) +} + +// Embeddings sends an embeddings request to llm-d. +func (p *Provider) Embeddings(ctx context.Context, req *core.EmbeddingRequest) (*core.EmbeddingResponse, error) { + resp, err := p.compatible.Embeddings(ctx, req) + return resp, exposeDroppedReason(err) +} + +// Passthrough routes an opaque request through llm-d after removing all +// client-supplied credentials and llm-d control headers. +func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest) (*core.PassthroughResponse, error) { + if req == nil { + return nil, core.NewInvalidRequestError("passthrough request is required", nil) + } + clean := *req + clean.Headers = cloneWithoutSensitiveHeaders(req.Headers) + endpoint := providers.PassthroughEndpoint(clean.Endpoint) + if usesV1PassthroughBase(endpoint) { + resp, err := p.compatible.Passthrough(ctx, &clean) + return resp, exposeDroppedReason(err) + } + + resp, err := p.rootClient.DoPassthrough(ctx, llmclient.Request{ + Method: clean.Method, + Endpoint: endpoint, + RawBodyReader: clean.Body, + Headers: clean.Headers, + }) + if err != nil { + return nil, exposeDroppedReason(err) + } + return &core.PassthroughResponse{ + StatusCode: resp.StatusCode, + Headers: providers.CloneHTTPHeaders(resp.Header), + Body: resp.Body, + }, nil +} + +func (p *Provider) setHeaders(req *http.Request, apiKey string) { + providers.SetAuthHeaders(req, apiKey, providers.AuthHeaderConfig{ + AuthScheme: "Bearer ", + RequestIDHeader: "X-Request-Id", + OptionalAPIKey: true, + }) + + // Send both spellings during llm-d's header transition. Stable v0.8 uses + // the x-gateway aliases; v0.9+ gives the canonical x-llm-d value priority. + if objective := trustedValue(p.controls.InferenceObjective); objective != "" { + req.Header.Set(canonicalObjectiveHeader, objective) + req.Header.Set(legacyObjectiveHeader, objective) + } + if p.controls.FairnessFromUserPath { + // Only an authenticated/authorized identity override is a safe fairness + // key. The request snapshot may contain a client-asserted user-path + // header, so do not use UserPathFromContext's snapshot fallback here. + if fairnessID := trustedValue(core.GetEffectiveUserPath(req.Context())); fairnessID != "" { + req.Header.Set(canonicalFairnessHeader, fairnessID) + req.Header.Set(legacyFairnessHeader, fairnessID) + } + } +} + +func trustedValue(value string) string { + value = strings.TrimSpace(value) + if value == "" || strings.ContainsAny(value, "\r\n") || strings.Contains(value, "${") { + return "" + } + return value +} + +func cloneWithoutSensitiveHeaders(src http.Header) http.Header { + if len(src) == 0 { + return nil + } + dst := make(http.Header, len(src)) + for key, values := range src { + if core.IsCredentialHeader(key) || isControlHeader(key) { + continue + } + dst[key] = append([]string(nil), values...) + } + if len(dst) == 0 { + return nil + } + return dst +} + +func isControlHeader(key string) bool { + key = strings.ToLower(strings.TrimSpace(key)) + return strings.HasPrefix(key, "x-llm-d-") || + strings.HasPrefix(key, "x-gateway-inference-") || + key == "x-gateway-model-name-rewrite" || + strings.HasPrefix(key, "x-gateway-destination-endpoint") || + strings.HasPrefix(key, "x-slo-") +} + +func passthroughBaseURL(baseURL string) string { + if before, ok := strings.CutSuffix(strings.TrimRight(strings.TrimSpace(baseURL), "/"), "/v1"); ok { + return before + } + return strings.TrimRight(strings.TrimSpace(baseURL), "/") +} + +func usesV1PassthroughBase(endpoint string) bool { + endpoint = providers.PassthroughEndpoint(endpoint) + if strings.HasPrefix(endpoint, "/v1/") { + return false + } + for _, prefix := range []string{ + "/models", "/chat/completions", "/responses", "/completions", "/embeddings", "/messages", + } { + if endpoint == prefix || strings.HasPrefix(endpoint, prefix+"/") { + return true + } + } + return false +} + +func exposeDroppedReason(err error) error { + if err == nil { + return nil + } + var gatewayErr *core.GatewayError + if !errors.As(err, &gatewayErr) || gatewayErr == nil || gatewayErr.StatusCode != http.StatusTooManyRequests { + return err + } + reason := trustedValue(gatewayErr.ResponseHeaders.Get(droppedReasonHeader)) + if reason == "" { + return err + } + return &responseHeaderError{ + err: err, + headers: http.Header{droppedReasonHeader: []string{reason}}, + } +} + +type responseHeaderError struct { + err error + headers http.Header +} + +func (e *responseHeaderError) Error() string { return e.err.Error() } +func (e *responseHeaderError) Unwrap() error { return e.err } +func (e *responseHeaderError) ResponseHeaders() http.Header { return e.headers.Clone() } diff --git a/internal/providers/llmd/llmd_test.go b/internal/providers/llmd/llmd_test.go new file mode 100644 index 00000000..eb0f8f90 --- /dev/null +++ b/internal/providers/llmd/llmd_test.go @@ -0,0 +1,346 @@ +package llmd + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" + "github.com/enterpilot/gomodel/internal/providers" +) + +func TestRegistrationRequiresEndpointAndAllowsKeyless(t *testing.T) { + if Registration.Type != "llmd" { + t.Fatalf("Registration.Type = %q, want llmd", Registration.Type) + } + if !Registration.Discovery.RequireBaseURL { + t.Fatal("llmd base URL must be required") + } + if !Registration.Discovery.AllowAPIKeyless { + t.Fatal("llmd must allow a keyless Gateway") + } +} + +func TestChatCompletionInjectsTrustedLLMDHeaders(t *testing.T) { + trustedCtx := core.WithEffectiveUserPath(context.Background(), "/team/alpha") + trustedCtx = core.WithRequestID(trustedCtx, "req-llmd-1") + tests := []struct { + name string + apiKey string + controls ControlConfig + ctx context.Context + wantHeaders map[string]string + }{ + { + name: "configured controls", + apiKey: "router-token", + controls: ControlConfig{ + InferenceObjective: "premium-traffic", + FairnessFromUserPath: true, + }, + ctx: trustedCtx, + wantHeaders: map[string]string{ + "Authorization": "Bearer router-token", + "X-Request-Id": "req-llmd-1", + canonicalObjectiveHeader: "premium-traffic", + legacyObjectiveHeader: "premium-traffic", + canonicalFairnessHeader: "/team/alpha", + legacyFairnessHeader: "/team/alpha", + }, + }, + { + name: "keyless without controls", + ctx: context.Background(), + controls: ControlConfig{}, + wantHeaders: map[string]string{ + "Authorization": "", + canonicalObjectiveHeader: "", + legacyObjectiveHeader: "", + canonicalFairnessHeader: "", + legacyFairnessHeader: "", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.Header.Clone() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id":"chatcmpl-llmd", + "created":1677652288, + "model":"Qwen/Qwen2.5-0.5B-Instruct", + "choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}] + }`)) + })) + defer server.Close() + + provider := NewWithHTTPClient(tt.apiKey, server.URL, tt.controls, server.Client(), llmclient.Hooks{}) + resp, err := provider.ChatCompletion(tt.ctx, &core.ChatRequest{ + Model: "Qwen/Qwen2.5-0.5B-Instruct", + Messages: []core.Message{{Role: "user", Content: "hello"}}, + }) + if err != nil { + t.Fatalf("ChatCompletion() error = %v", err) + } + if resp.ID != "chatcmpl-llmd" || resp.Model != "Qwen/Qwen2.5-0.5B-Instruct" { + t.Errorf("response identity = (%q, %q), want normalized ID and model", resp.ID, resp.Model) + } + if len(resp.Choices) != 1 || resp.Choices[0].Message.Role != "assistant" || resp.Choices[0].Message.Content != "ok" { + t.Errorf("response choices = %#v, want one assistant choice containing ok", resp.Choices) + } + for key, want := range tt.wantHeaders { + assertHeader(t, got, key, want) + } + }) + } +} + +func TestPassthroughReplacesClientSuppliedControlHeaders(t *testing.T) { + var got []http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = append(got, r.Header.Clone()) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"tokens":[1,2,3]}`)) + })) + defer server.Close() + + provider := NewWithHTTPClient("router-token", server.URL+"/v1", ControlConfig{ + InferenceObjective: "trusted-objective", + FairnessFromUserPath: true, + }, server.Client(), llmclient.Hooks{}) + ctx := core.WithEffectiveUserPath(context.Background(), "/trusted/tenant") + + for _, endpoint := range []string{"tokenize", "chat/completions"} { + resp, err := provider.Passthrough(ctx, &core.PassthroughRequest{ + Method: http.MethodPost, + Endpoint: endpoint, + Body: io.NopCloser(strings.NewReader(`{}`)), + Headers: http.Header{ + "Content-Type": {"application/json"}, + "Authorization": {"Bearer client-token"}, + "X-Api-Key": {"client-api-key"}, + "Api-Key": {"client-azure-key"}, + "X-Goog-Api-Key": {"client-google-key"}, + canonicalObjectiveHeader: {"attacker-objective"}, + legacyFairnessHeader: {"attacker-tenant"}, + "X-Llm-D-Slo-Ttft-Ms": {"1"}, + "X-Gateway-Model-Name-Rewrite": {"other-model"}, + "X-Gateway-Destination-Endpoint-Served": {"10.0.0.1:8000"}, + }, + }) + if err != nil { + t.Fatalf("Passthrough(%q) error = %v", endpoint, err) + } + _ = resp.Body.Close() + } + + if len(got) != 2 { + t.Fatalf("upstream requests = %d, want root and /v1 requests", len(got)) + } + for i, headers := range got { + assertHeader(t, headers, "Authorization", "Bearer router-token") + assertHeader(t, headers, canonicalObjectiveHeader, "trusted-objective") + assertHeader(t, headers, legacyObjectiveHeader, "trusted-objective") + assertHeader(t, headers, canonicalFairnessHeader, "/trusted/tenant") + assertHeader(t, headers, legacyFairnessHeader, "/trusted/tenant") + for _, key := range []string{ + "X-Api-Key", + "Api-Key", + "X-Goog-Api-Key", + "X-Llm-D-Slo-Ttft-Ms", + "X-Gateway-Model-Name-Rewrite", + "X-Gateway-Destination-Endpoint-Served", + } { + if value := headers.Get(key); value != "" { + t.Errorf("request %d: %s = %q, want stripped", i, key, value) + } + } + } +} + +func TestPassthroughPreservesDroppedReasonOnRawErrorResponses(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set(droppedReasonHeader, "rejected-saturated") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":{"message":"request dropped"}}`)) + })) + defer server.Close() + + provider := NewWithHTTPClient("", server.URL+"/v1", ControlConfig{}, server.Client(), llmclient.Hooks{}) + for _, endpoint := range []string{"tokenize", "chat/completions"} { + resp, err := provider.Passthrough(context.Background(), &core.PassthroughRequest{ + Method: http.MethodPost, + Endpoint: endpoint, + Body: io.NopCloser(strings.NewReader(`{}`)), + }) + if err != nil { + t.Fatalf("Passthrough(%q) error = %v", endpoint, err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusTooManyRequests { + t.Errorf("Passthrough(%q) status = %d, want %d", endpoint, resp.StatusCode, http.StatusTooManyRequests) + } + assertHeader(t, http.Header(resp.Headers), droppedReasonHeader, "rejected-saturated") + } +} + +func TestPassthroughSelectsV1AndRouterRootPaths(t *testing.T) { + var gotPaths []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPaths = append(gotPaths, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + provider := NewWithHTTPClient("", server.URL+"/v1", ControlConfig{}, server.Client(), llmclient.Hooks{}) + for _, endpoint := range []string{"completions", "messages", "inference/v1/generate", "tokenize"} { + resp, err := provider.Passthrough(context.Background(), &core.PassthroughRequest{ + Method: http.MethodPost, + Endpoint: endpoint, + Body: io.NopCloser(strings.NewReader(`{}`)), + }) + if err != nil { + t.Fatalf("Passthrough(%q) error = %v", endpoint, err) + } + _ = resp.Body.Close() + } + + want := []string{"/v1/completions", "/v1/messages", "/inference/v1/generate", "/tokenize"} + if len(gotPaths) != len(want) { + t.Fatalf("paths = %v, want %v", gotPaths, want) + } + for i := range want { + if gotPaths[i] != want[i] { + t.Errorf("path[%d] = %q, want %q", i, gotPaths[i], want[i]) + } + } +} + +func TestCompatibleAndRootClientsShareKeyRotation(t *testing.T) { + var gotAuth []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = append(gotAuth, r.Header.Get("Authorization")) + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/v1/chat/completions" { + _, _ = w.Write([]byte(`{"id":"chatcmpl-1","model":"test","choices":[]}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + provider := newProvider("", server.URL+"/v1", ControlConfig{}, providers.ProviderOptions{ + Keys: providers.NewKeyring("router-a", "router-b"), + }, server.Client()) + passthrough, err := provider.Passthrough(context.Background(), &core.PassthroughRequest{ + Method: http.MethodPost, + Endpoint: "tokenize", + Body: io.NopCloser(strings.NewReader(`{}`)), + }) + if err != nil { + t.Fatalf("Passthrough() error = %v", err) + } + _ = passthrough.Body.Close() + if _, err := provider.ChatCompletion(context.Background(), &core.ChatRequest{ + Model: "test", + Messages: []core.Message{{Role: "user", Content: "hello"}}, + }); err != nil { + t.Fatalf("ChatCompletion() error = %v", err) + } + + want := []string{"Bearer router-a", "Bearer router-b"} + if len(gotAuth) != len(want) { + t.Fatalf("Authorization headers = %v, want %v", gotAuth, want) + } + for i := range want { + if gotAuth[i] != want[i] { + t.Errorf("Authorization[%d] = %q, want %q", i, gotAuth[i], want[i]) + } + } +} + +func TestFairnessHeaderCanBeDisabled(t *testing.T) { + request := httptest.NewRequest(http.MethodPost, "http://llmd.invalid/v1/chat/completions", nil) + request = request.WithContext(core.WithEffectiveUserPath(request.Context(), "/team/alpha")) + provider := &Provider{controls: ControlConfig{FairnessFromUserPath: false}} + + provider.setHeaders(request, "") + + if value := request.Header.Get(canonicalFairnessHeader); value != "" { + t.Fatalf("canonical fairness header = %q, want empty", value) + } + if value := request.Header.Get(legacyFairnessHeader); value != "" { + t.Fatalf("legacy fairness header = %q, want empty", value) + } +} + +func TestFairnessHeaderIgnoresClientAssertedSnapshotPath(t *testing.T) { + request := httptest.NewRequest(http.MethodPost, "http://llmd.invalid/v1/chat/completions", nil) + snapshot := core.NewRequestSnapshot( + http.MethodPost, "/v1/chat/completions", nil, nil, nil, "application/json", nil, false, "", nil, + "/untrusted/client", + ) + request = request.WithContext(core.WithRequestSnapshot(request.Context(), snapshot)) + provider := &Provider{controls: ControlConfig{FairnessFromUserPath: true}} + + provider.setHeaders(request, "") + + if value := request.Header.Get(canonicalFairnessHeader); value != "" { + t.Fatalf("canonical fairness header = %q, want empty", value) + } + if value := request.Header.Get(legacyFairnessHeader); value != "" { + t.Fatalf("legacy fairness header = %q, want empty", value) + } +} + +func TestExposeDroppedReasonReturnsOnlySafeLLMDHeader(t *testing.T) { + upstream := core.NewRateLimitError("llmd", "request dropped") + upstream.ResponseHeaders = http.Header{ + droppedReasonHeader: {"rejected-saturated"}, + "Set-Cookie": {"secret=value"}, + } + + wrapped := exposeDroppedReason(upstream) + if !errors.Is(wrapped, upstream) { + t.Fatal("wrapped error must preserve the upstream error chain") + } + headerErr, ok := wrapped.(interface{ ResponseHeaders() http.Header }) + if !ok { + t.Fatalf("wrapped error type %T does not expose response headers", wrapped) + } + headers := headerErr.ResponseHeaders() + assertHeader(t, headers, droppedReasonHeader, "rejected-saturated") + if value := headers.Get("Set-Cookie"); value != "" { + t.Fatalf("Set-Cookie = %q, want filtered", value) + } +} + +func TestProviderDoesNotAdvertiseUnsupportedNativeSurfaces(t *testing.T) { + provider := NewWithHTTPClient("", "http://llmd.invalid/v1", ControlConfig{}, nil, llmclient.Hooks{}) + if _, ok := any(provider).(core.NativeBatchProvider); ok { + t.Fatal("llmd provider must not advertise the separate llm-d Batch Gateway") + } + if _, ok := any(provider).(core.NativeFileProvider); ok { + t.Fatal("llmd provider must not advertise native files") + } + if _, ok := any(provider).(core.NativeResponseLifecycleProvider); ok { + t.Fatal("llmd provider must not advertise Responses lifecycle operations") + } +} + +func assertHeader(t *testing.T, headers http.Header, key, want string) { + t.Helper() + if got := headers.Get(key); got != want { + t.Errorf("%s = %q, want %q", key, got, want) + } +} diff --git a/internal/providers/llmd/passthrough_semantics.go b/internal/providers/llmd/passthrough_semantics.go new file mode 100644 index 00000000..e761683b --- /dev/null +++ b/internal/providers/llmd/passthrough_semantics.go @@ -0,0 +1,15 @@ +package llmd + +import "github.com/enterpilot/gomodel/internal/providers" + +var passthroughSemanticEnricher = providers.NewSemanticEnricher("llmd", map[string]providers.PassthroughEndpointSemantics{ + "/chat/completions": {Operation: "llmd.chat_completions", AuditPath: "/v1/chat/completions"}, + "/responses": {Operation: "llmd.responses", AuditPath: "/v1/responses"}, + "/embeddings": {Operation: "llmd.embeddings", AuditPath: "/v1/embeddings"}, + "/completions": {Operation: "llmd.completions", AuditPath: "/v1/completions"}, + "/messages": {Operation: "llmd.messages", AuditPath: "/v1/messages"}, + "/inference/v1/generate": { + Operation: "llmd.vllm_generate", + AuditPath: "/inference/v1/generate", + }, +}) diff --git a/internal/server/handlers_test.go b/internal/server/handlers_test.go index 666042ed..1b9335ef 100644 --- a/internal/server/handlers_test.go +++ b/internal/server/handlers_test.go @@ -6659,6 +6659,53 @@ func TestProviderPassthrough_NormalizesErrorResponse(t *testing.T) { } } +func TestProviderPassthrough_LLMDDroppedReasonOnNormalizedError(t *testing.T) { + for _, path := range []string{ + "/p/llmd/tokenize", + "/p/llmd/v1/chat/completions", + } { + t.Run(path, func(t *testing.T) { + provider := &mockProvider{ + passthroughResponse: &core.PassthroughResponse{ + StatusCode: http.StatusTooManyRequests, + Headers: http.Header{ + "Content-Type": {"application/json"}, + llmdDroppedReasonHeader: {"rejected-saturated"}, + "X-Upstream": {"must-not-leak"}, + "Set-Cookie": {"session=secret"}, + }, + Body: io.NopCloser(strings.NewReader(`{"error":{"message":"request dropped","type":"rate_limit_error"}}`)), + }, + } + + e := echo.New() + handler := NewHandler(provider, nil, nil, nil) + e.POST("/p/:provider/*", handler.ProviderPassthrough) + + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{}`)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusTooManyRequests) + } + if got := rec.Header().Get(llmdDroppedReasonHeader); got != "rejected-saturated" { + t.Fatalf("%s = %q, want rejected-saturated", llmdDroppedReasonHeader, got) + } + if got := rec.Header().Get("X-Upstream"); got != "" { + t.Fatalf("X-Upstream should not be forwarded, got %q", got) + } + if got := rec.Header().Get("Set-Cookie"); got != "" { + t.Fatalf("Set-Cookie should not be forwarded, got %q", got) + } + if body := rec.Body.String(); !strings.Contains(body, `"message":"request dropped"`) { + t.Fatalf("unexpected error body: %s", body) + } + }) + } +} + func TestProviderPassthrough_OpenAIV1AliasNormalizesByDefault(t *testing.T) { provider := &mockProvider{ passthroughResponse: &core.PassthroughResponse{ @@ -7131,7 +7178,7 @@ func TestProviderPassthrough_RejectsUnsupportedProvider(t *testing.T) { if !strings.Contains(rec.Body.String(), `provider passthrough for \"groq\" is not enabled`) { t.Fatalf("unexpected error body: %s", rec.Body.String()) } - if !strings.Contains(rec.Body.String(), "anthropic, deepseek, kilo, openai, openrouter, sglang, vllm, zai") { + if !strings.Contains(rec.Body.String(), "anthropic, deepseek, kilo, llmd, openai, openrouter, sglang, vllm, zai") { t.Fatalf("unexpected error body: %s", rec.Body.String()) } } diff --git a/internal/server/passthrough_support.go b/internal/server/passthrough_support.go index d63190a8..5bce2f74 100644 --- a/internal/server/passthrough_support.go +++ b/internal/server/passthrough_support.go @@ -16,7 +16,9 @@ import ( "github.com/enterpilot/gomodel/internal/usage" ) -var defaultEnabledPassthroughProviders = []string{"openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek"} +var defaultEnabledPassthroughProviders = []string{"openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek"} + +const llmdDroppedReasonHeader = "X-Llm-D-Request-Dropped-Reason" func (h *Handler) setEnabledPassthroughProviders(providerTypes []string) { h.enabledPassthroughProviders = normalizeEnabledPassthroughProviders(providerTypes) @@ -253,7 +255,12 @@ func (s *passthroughService) proxyPassthroughResponse(c *echo.Context, providerT if err != nil { return handleError(c, core.NewProviderError(providerType, http.StatusBadGateway, "failed to read provider passthrough error response", err)) } - return handleError(c, core.ParseProviderError(providerType, resp.StatusCode, body, nil)) + gatewayErr := core.ParseProviderError(providerType, resp.StatusCode, body, nil) + headers := passthroughErrorResponseHeaders(providerType, resp.StatusCode, http.Header(resp.Headers)) + if len(headers) == 0 { + return handleError(c, gatewayErr) + } + return handleError(c, &gatewayErrorWithResponseHeaders{GatewayError: gatewayErr, headers: headers}) } copyPassthroughResponseHeaders(c.Response().Header(), http.Header(resp.Headers)) @@ -326,3 +333,14 @@ func (s *passthroughService) proxyPassthroughResponse(c *echo.Context, providerT } return nil } + +func passthroughErrorResponseHeaders(providerType string, statusCode int, src http.Header) http.Header { + if providerType != "llmd" || statusCode != http.StatusTooManyRequests { + return nil + } + reason := strings.TrimSpace(src.Get(llmdDroppedReasonHeader)) + if reason == "" || strings.ContainsAny(reason, "\r\n") { + return nil + } + return http.Header{llmdDroppedReasonHeader: []string{reason}} +} diff --git a/internal/usage/cost_test.go b/internal/usage/cost_test.go index fadea1aa..597048af 100644 --- a/internal/usage/cost_test.go +++ b/internal/usage/cost_test.go @@ -747,7 +747,8 @@ func TestCalculateGranularCost_OutputOnlyPricing(t *testing.T) { // TestCalculateGranularCost_OpenAICompatibleProvidersDefaultMappings is a // regression test for issue #435. Providers that speak the OpenAI usage schema // but are not explicitly listed in providerMappings (xiaomi, deepseek, zai, -// minimax, bailian, oracle, azure, sglang, vllm, ollama, opencode_go) must still apply +// minimax, bailian, oracle, azure, sglang, vllm, llmd, ollama, opencode_go) must +// still apply // the cached-input discount instead of billing cached tokens at the full input // rate. Previously these providers produced a much higher cost than "openai" // for the same response, over-charging cache-heavy workloads. @@ -768,7 +769,7 @@ func TestCalculateGranularCost_OpenAICompatibleProvidersDefaultMappings(t *testi for _, provider := range []string{ "xiaomi", "deepseek", "zai", "minimax", "bailian", - "oracle", "azure", "sglang", "vllm", "ollama", "opencode_go", + "oracle", "azure", "sglang", "vllm", "llmd", "ollama", "opencode_go", } { got := CalculateGranularCost(1_000_000, 200_000, rawData, provider, pricing) assertCostNear(t, provider+" InputCost", got.InputCost, *want.InputCost) diff --git a/run/providers.go b/run/providers.go index 2a4e2ea2..b0331baf 100644 --- a/run/providers.go +++ b/run/providers.go @@ -16,6 +16,7 @@ import ( "github.com/enterpilot/gomodel/internal/providers/groq" "github.com/enterpilot/gomodel/internal/providers/kilo" "github.com/enterpilot/gomodel/internal/providers/kimicode" + "github.com/enterpilot/gomodel/internal/providers/llmd" "github.com/enterpilot/gomodel/internal/providers/meta" "github.com/enterpilot/gomodel/internal/providers/minimax" "github.com/enterpilot/gomodel/internal/providers/ollama" @@ -56,6 +57,7 @@ func defaultProviderFactory(cfg *config.Config) *providers.ProviderFactory { factory.Add(groq.Registration) factory.Add(kilo.Registration) factory.Add(kimicode.Registration) + factory.Add(llmd.Registration) factory.Add(meta.Registration) factory.Add(minimax.Registration) factory.Add(ollama.Registration) diff --git a/run/providers_test.go b/run/providers_test.go index c7ff5a67..4aa00429 100644 --- a/run/providers_test.go +++ b/run/providers_test.go @@ -46,6 +46,12 @@ func TestDefaultProviderFactoryCredentialForms(t *testing.T) { fields: []string{"api_keys", "base_url", "session_sticky_keys", "models"}, required: nil, }, + { + // llm-d can be keyless, but it has no meaningful universal endpoint. + providerType: "llmd", + fields: []string{"api_keys", "base_url", "session_sticky_keys", "models"}, + required: []string{"base_url"}, + }, { // SGLang supports both unauthenticated and --api-key deployments. providerType: "sglang", @@ -150,7 +156,7 @@ var credentialPayloadFields = []string{ func TestDefaultProviderFactoryRegistersAllProviderTypes(t *testing.T) { expected := []string{ "anthropic", "azure", "bailian", "bedrock", "bedrock-mantle", "cohere", "deepseek", "fireworks", - "gemini", "groq", "kilo", "kimicode", "meta", "minimax", "ollama", "openai", "opencode_go", + "gemini", "groq", "kilo", "kimicode", "llmd", "meta", "minimax", "ollama", "openai", "opencode_go", "openrouter", "oracle", "sglang", "vertex", "vllm", "xai", "xiaomi", "zai", } diff --git a/web/dashboard/src/pages/overview/providersLogic.js b/web/dashboard/src/pages/overview/providersLogic.js index 4fce397d..b2bb462c 100644 --- a/web/dashboard/src/pages/overview/providersLogic.js +++ b/web/dashboard/src/pages/overview/providersLogic.js @@ -22,6 +22,7 @@ const PROVIDER_DOC_SLUGS = { cohere: "cohere", deepseek: "deepseek", gemini: "gemini", + llmd: "llmd", opencode_go: "opencode-go", oracle: "oracle", sglang: "sglang", diff --git a/web/dashboard/tests/overview-providers.test.js b/web/dashboard/tests/overview-providers.test.js index 3059a9e5..f75cd86e 100644 --- a/web/dashboard/tests/overview-providers.test.js +++ b/web/dashboard/tests/overview-providers.test.js @@ -157,6 +157,10 @@ test("providerDocUrl links provider types with docs and stays empty otherwise", providerDocUrl({ name: "gemini", type: "GEMINI" }), "https://gomodel.enterpilot.io/docs/providers/gemini?utm_source=gomodel_dashboard", ); + assert.equal( + providerDocUrl({ type: "llmd" }), + "https://gomodel.enterpilot.io/docs/providers/llmd?utm_source=gomodel_dashboard", + ); assert.equal( providerDocUrl({ type: "sglang" }), "https://gomodel.enterpilot.io/docs/providers/sglang?utm_source=gomodel_dashboard",