feat(llmd): add dedicated llm-d provider

.env.template

@@ -68,9 +68,9 @@
 # Allow optional /p/{provider}/v1/... passthrough aliases while keeping /p/{provider}/... canonical (default: true)
 # ALLOW_PASSTHROUGH_V1_ALIAS=true
 
-# Comma-separated list of provider types enabled for /p/{provider}/... passthrough (default: openai,anthropic,openrouter,kilo,zai,vllm,deepseek)
+# Comma-separated list of provider types enabled for /p/{provider}/... passthrough (default: openai,anthropic,openrouter,kilo,zai,vllm,llmd,deepseek)
 # Cohere native passthrough is opt-in; add cohere when those routes are needed.
-# ENABLED_PASSTHROUGH_PROVIDERS=openai,anthropic,cohere,openrouter,kilo,zai,vllm,deepseek
+# ENABLED_PASSTHROUGH_PROVIDERS=openai,anthropic,cohere,openrouter,kilo,zai,vllm,llmd,deepseek
 
 # Enable the realtime (speech-to-speech) endpoints (default: true): the /v1/realtime
 # websocket (and /p/{provider}/v1/realtime passthrough upgrade), the WebRTC SDP
@@ -539,6 +539,18 @@
 # Set base URL to enable (default: http://localhost:8000/v1)
 # VLLM_BASE_URL=http://localhost:8000/v1
 
+# llm-d Router / Endpoint Picker (EPP)
+# The route is required and usually points to the EPP service. Include /v1.
+# LLMD_BASE_URL=http://quickstart-epp.llm-d.svc.cluster.local/v1
+# Optional bearer token if the Gateway in front of llm-d requires one.
+# LLMD_API_KEY=router-token
+# Optional configured model list when GET /v1/models is not routed.
+# LLMD_MODELS=Qwen/Qwen2.5-0.5B-Instruct
+# Optional trusted InferenceObjective name injected by GoModel.
+# LLMD_INFERENCE_OBJECTIVE=standard-traffic
+# Derive the fairness ID from the effective user path (default: true).
+# LLMD_FAIRNESS_FROM_USER_PATH=true
+
 # SGLang (OpenAI-compatible server)
 # SGLANG_API_KEY is optional; set it only if launch_server uses --api-key.
 # SGLANG_API_KEY=token-abc123

CLAUDE.md

@@ -138,5 +138,5 @@ Full reference: `.env.template` and `config/config.yaml`
 - **Guardrails:** Definitions are persisted in the `guardrail_definitions` store and managed via the admin API/dashboard; `config/config.yaml` entries are validated and upserted into that store at startup (a seed, not the source of truth). `GUARDRAILS_ENABLED` env var gates the feature.
 - **Provider API key rotation:** Any API-key provider accepts several keys: `<PROVIDER>[_SUFFIX]_API_KEY_<n>` env vars (numbered from 2; `_1` is accepted as a synonym for the unsuffixed key) or `providers.<name>.api_keys` in `config.yaml` (merged after `api_key`, de-duplicated, unresolved `${...}` entries dropped; env replaces the whole YAML list). Identified sessions deterministically stay on one key by default, preserving provider prompt-cache affinity while spreading different sessions across the configured keys; sessionless requests remain round robin. Set `providers.<name>.session_sticky_keys: false` or untick **Session-sticky API keys** in the provider editor for strict per-request round robin. Realtime sessions use the same affinity. The trailing number names a key, not a provider: `OPENAI_API_KEY_2` is key 2 of `openai`, while `OPENAI_REGION_2_API_KEY` is the sole key of provider `openai-region-2`. Providers configured without keys are unaffected; Ollama is normally keyless, while SGLang and vLLM participate in rotation when optional API keys are configured. Non-API-key providers (Vertex, Bedrock) are unaffected.
 - **Provider credentials without env vars:** Every provider below can instead be configured from the admin dashboard's Providers page (or `/admin/provider-credentials` GET/PUT/DELETE), persisted to the `provider_credentials` store — the same declarative-shadows-store precedence as MCP servers: a provider name declared via env vars/`config.yaml` is read-only in the dashboard (`managed: true`), and a store row upsert/delete hot-registers or unregisters the provider into the live registry immediately, no restart. `GOMODEL` boots fine with zero providers configured (empty catalog) so this is a complete alternative to env-var credentials, not just a supplement. API keys (`api_keys`, an ordered rotation list, same semantics as `providers.<name>.api_keys`) and service-account secrets are redacted as `***********` on read; an upsert echoing any all-asterisk mask of at least three characters at a position preserves the stored value there (rejected if that position was never set). Disabling a row (`enabled: false`) unregisters it from routing without deleting the stored credentials. `GET /admin/provider-credentials/types` lists every constructible provider type with the credential form it accepts (`fields[]` of `name`/`required`/`advanced`/`options`, plus `default_base_url`), derived from each provider's `DiscoveryConfig` — the dashboard renders only those fields, so an OpenAI-type provider asks for an API key while a Vertex one asks for project/location/service account and no key at all. Upserts are validated against that form (and against Google's project-or-base-URL and service-account rules) *before* anything is persisted, so an unusable credential is rejected with a 400 naming the offending field in `error.param` rather than stored as a broken row.
-- **Providers:** `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `ANTHROPIC_DEFAULT_MAX_TOKENS` (optional default `max_tokens` for Anthropic-translated requests that omit it; default 4096), `GEMINI_API_KEY`, `USE_GOOGLE_GEMINI_NATIVE_API` (true by default; false uses Gemini's OpenAI-compatible chat API), `XAI_API_KEY`, `GROQ_API_KEY`, `FIREWORKS_API_KEY`, `FIREWORKS_BASE_URL` (optional Fireworks AI endpoint override; default `https://api.fireworks.ai/inference/v1`), `META_API_KEY`, `META_BASE_URL` (optional Meta Model API endpoint override; default `https://api.meta.ai/v1`; Muse Spark models, e.g. `muse-spark-1.1`), `OPENROUTER_API_KEY`, `OPENROUTER_SITE_URL`/`OPENROUTER_APP_NAME` (optional OpenRouter attribution headers), `ZAI_API_KEY`, `ZAI_BASE_URL` (optional Z.ai endpoint override), `MINIMAX_API_KEY`, `MINIMAX_BASE_URL` (optional MiniMax endpoint override), `XIAOMI_API_KEY`, `XIAOMI_BASE_URL` (optional Xiaomi MiMo endpoint override), `OPENCODE_GO_API_KEY`, `OPENCODE_GO_BASE_URL` (optional OpenCode Go/Zen endpoint override; default `https://opencode.ai/zen/go/v1`), `OPENCODE_GO_MESSAGES_MODELS` (optional comma-separated model IDs routed to the Anthropic-native `/messages` endpoint instead of `/chat/completions`; default `qwen3.7-max`), `BAILIAN_API_KEY`, `BAILIAN_BASE_URL` (optional Bailian base URL for region switching; default `https://dashscope.aliyuncs.com/compatible-mode/v1`), `AZURE_API_KEY`, `AZURE_BASE_URL` (Azure OpenAI deployment base URL), `AZURE_API_VERSION` (optional Azure API version), `ORACLE_API_KEY` (Oracle API key), `ORACLE_BASE_URL` (Oracle OpenAI-compatible base URL), `BEDROCK_BASE_URL` (Bedrock Runtime region or endpoint), `BEDROCK_MANTLE_API_KEY`, `BEDROCK_MANTLE_BASE_URL` (Mantle region or endpoint), `BEDROCK_MANTLE_API_MODE` (`auto`, `openai`, or `standard`), `<PROVIDER>[_SUFFIX]_MODELS` (comma-separated configured model list for any provider type), `OLLAMA_BASE_URL`, `SGLANG_BASE_URL`, `SGLANG_API_KEY` (optional upstream SGLang bearer token), `VLLM_BASE_URL`, `VLLM_API_KEY` (optional upstream vLLM bearer token)
+- **Providers:** `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `ANTHROPIC_DEFAULT_MAX_TOKENS` (optional default `max_tokens` for Anthropic-translated requests that omit it; default 4096), `GEMINI_API_KEY`, `USE_GOOGLE_GEMINI_NATIVE_API` (true by default; false uses Gemini's OpenAI-compatible chat API), `XAI_API_KEY`, `GROQ_API_KEY`, `FIREWORKS_API_KEY`, `FIREWORKS_BASE_URL` (optional Fireworks AI endpoint override; default `https://api.fireworks.ai/inference/v1`), `META_API_KEY`, `META_BASE_URL` (optional Meta Model API endpoint override; default `https://api.meta.ai/v1`; Muse Spark models, e.g. `muse-spark-1.1`), `OPENROUTER_API_KEY`, `OPENROUTER_SITE_URL`/`OPENROUTER_APP_NAME` (optional OpenRouter attribution headers), `ZAI_API_KEY`, `ZAI_BASE_URL` (optional Z.ai endpoint override), `MINIMAX_API_KEY`, `MINIMAX_BASE_URL` (optional MiniMax endpoint override), `XIAOMI_API_KEY`, `XIAOMI_BASE_URL` (optional Xiaomi MiMo endpoint override), `OPENCODE_GO_API_KEY`, `OPENCODE_GO_BASE_URL` (optional OpenCode Go/Zen endpoint override; default `https://opencode.ai/zen/go/v1`), `OPENCODE_GO_MESSAGES_MODELS` (optional comma-separated model IDs routed to the Anthropic-native `/messages` endpoint instead of `/chat/completions`; default `qwen3.7-max`), `BAILIAN_API_KEY`, `BAILIAN_BASE_URL` (optional Bailian base URL for region switching; default `https://dashscope.aliyuncs.com/compatible-mode/v1`), `AZURE_API_KEY`, `AZURE_BASE_URL` (Azure OpenAI deployment base URL), `AZURE_API_VERSION` (optional Azure API version), `ORACLE_API_KEY` (Oracle API key), `ORACLE_BASE_URL` (Oracle OpenAI-compatible base URL), `BEDROCK_BASE_URL` (Bedrock Runtime region or endpoint), `BEDROCK_MANTLE_API_KEY`, `BEDROCK_MANTLE_BASE_URL` (Mantle region or endpoint), `BEDROCK_MANTLE_API_MODE` (`auto`, `openai`, or `standard`), `<PROVIDER>[_SUFFIX]_MODELS` (comma-separated configured model list for any provider type), `OLLAMA_BASE_URL`, `SGLANG_BASE_URL`, `SGLANG_API_KEY` (optional upstream SGLang bearer token), `VLLM_BASE_URL`, `VLLM_API_KEY` (optional upstream vLLM bearer token), `LLMD_BASE_URL`, `LLMD_API_KEY` (optional Gateway bearer token), `LLMD_INFERENCE_OBJECTIVE`, `LLMD_FAIRNESS_FROM_USER_PATH`
 - **Provider model metadata:** `providers.<name>.models` accepts either model IDs (strings) or `{id, metadata}` objects. When `metadata` is supplied (`display_name`, `context_window`, `max_output_tokens`, `modes`, `capabilities`, `pricing`, …) it is merged onto the remote ai-model-list entry during enrichment, with operator values winning per-field. Primary use case: advertising context windows, capabilities, and pricing for local models (Ollama) and other custom endpoints whose IDs are not in the upstream registry.

README.md

@@ -157,8 +157,8 @@ const client = new Anthropic({
 GoModel supports OpenAI, Anthropic, Cohere, Google Gemini, Vertex AI, DeepSeek,
 Groq, Fireworks AI, Meta (Muse Spark), OpenRouter, Z.ai, xAI (Grok), Alibaba
 Cloud Model Studio (Bailian), Kilo AI, MiniMax, Xiaomi MiMo, OpenCode Go, Azure
-OpenAI, Oracle, Ollama, SGLang, vLLM, Amazon Bedrock Runtime, Amazon Bedrock Mantle, and
-all OpenAI-compatible providers.
+OpenAI, Oracle, Ollama, SGLang, vLLM, llm-d, Amazon Bedrock Runtime, Amazon
+Bedrock Mantle, and all OpenAI-compatible providers.
 
 See the [Providers Overview](https://gomodel.enterpilot.io/docs/providers/overview?utm_source=readme) for the full
 per-provider feature matrix (chat, `/responses`, embeddings, files, batches,

config/config.example.yaml

@@ -13,7 +13,7 @@ server:
   enable_passthrough_routes: true # expose /p/{provider}/{endpoint} passthrough routes
   allow_passthrough_v1_alias: true # allow /p/{provider}/v1/... while keeping /p/{provider}/... canonical
   user_path_header: "X-GoModel-User-Path" # env: USER_PATH_HEADER; inbound header used for user_path scoping
-  enabled_passthrough_providers: ["openai", "anthropic", "cohere", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek", "bailian"] # providers enabled on /p/{provider}/...
+  enabled_passthrough_providers: ["openai", "anthropic", "cohere", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek", "bailian"] # providers enabled on /p/{provider}/...
   realtime_enabled: true # env: REALTIME_ENABLED; expose /v1/realtime websocket and /p/{provider}/v1/realtime upgrades (OpenAI only)
   pid_file: "data/gomodel.pid" # env: PID_FILE; where the running gateway records its process id so `gomodel --reload` can find it. Set per instance when several gateways share a host; empty writes no pid file and disables --reload; changing it needs a restart, not a reload
 
@@ -418,6 +418,18 @@ providers:
     # Optional: set this only when vllm serve was started with --api-key.
     # api_key: "token-abc123"
 
+  # llm-d Router / Endpoint Picker (EPP)
+  llmd:
+    type: llmd
+    base_url: "http://quickstart-epp.llm-d.svc.cluster.local/v1"
+    # Optional trusted request class injected as llm-d inference headers.
+    # inference_objective: "standard-traffic"
+    # Derive the llm-d fairness ID from GoModel's effective user path (default).
+    fairness_from_user_path: true
+    # Declare models when the route does not forward GET /v1/models.
+    models:
+      - id: "Qwen/Qwen2.5-0.5B-Instruct"
+
   sglang:
     type: sglang
     base_url: "http://localhost:30000/v1"

config/config.go

@@ -102,6 +102,7 @@ func buildDefaultConfig() *Config {
 				"zai",
 				"sglang",
 				"vllm",
+				"llmd",
 				"deepseek",
 			},
 		},

config/config_test.go

@@ -30,6 +30,7 @@ func clearProviderEnvVars(t *testing.T) {
 		"AZURE_API_KEY", "AZURE_BASE_URL", "AZURE_API_VERSION", "AZURE_MODELS",
 		"ORACLE_API_KEY", "ORACLE_BASE_URL", "ORACLE_MODELS",
 		"VLLM_API_KEY", "VLLM_BASE_URL", "VLLM_MODELS",
+		"LLMD_API_KEY", "LLMD_BASE_URL", "LLMD_MODELS", "LLMD_INFERENCE_OBJECTIVE", "LLMD_FAIRNESS_FROM_USER_PATH",
 		"SGLANG_API_KEY", "SGLANG_BASE_URL", "SGLANG_MODELS",
 		"OLLAMA_API_KEY", "OLLAMA_BASE_URL", "OLLAMA_MODELS",
 	} {
@@ -125,7 +126,7 @@ func TestBuildDefaultConfig(t *testing.T) {
 	if !cfg.Server.AllowPassthroughV1Alias {
 		t.Error("expected Server.AllowPassthroughV1Alias=true")
 	}
-	if got, want := cfg.Server.EnabledPassthroughProviders, []string{"openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek"}; !reflect.DeepEqual(got, want) {
+	if got, want := cfg.Server.EnabledPassthroughProviders, []string{"openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek"}; !reflect.DeepEqual(got, want) {
 		t.Errorf("expected Server.EnabledPassthroughProviders=%v, got %v", want, got)
 	}
 	if cfg.Models.ConfiguredProviderModelsMode != ConfiguredProviderModelsModeFallback {
@@ -1202,7 +1203,7 @@ func TestLoad_ConfigExample_UsesNestedModelCacheSettings(t *testing.T) {
 			t.Fatalf("expected Cache.Model.Redis to be nil in example config, got %+v", result.Config.Cache.Model.Redis)
 		}
 		gotProviders := result.Config.Server.EnabledPassthroughProviders
-		wantProviders := []string{"openai", "anthropic", "cohere", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek", "bailian"}
+		wantProviders := []string{"openai", "anthropic", "cohere", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek", "bailian"}
 		if !reflect.DeepEqual(gotProviders, wantProviders) {
 			t.Fatalf("Server.EnabledPassthroughProviders = %v, want %v", gotProviders, wantProviders)
 		}

config/providers.go

@@ -13,18 +13,25 @@ type RawProviderConfig struct {
 	APIKeys []string `yaml:"api_keys"`
 	// SessionStickyKeys defaults to true. Set false to restore round-robin key
 	// selection for every request, including requests carrying a session ID.
-	SessionStickyKeys        *bool                `yaml:"session_sticky_keys"`
-	BaseURL                  string               `yaml:"base_url"`
-	APIVersion               string               `yaml:"api_version"`
-	Backend                  string               `yaml:"backend"`
-	AuthType                 string               `yaml:"auth_type"`
-	APIMode                  string               `yaml:"api_mode"`
-	VertexProject            string               `yaml:"vertex_project"`
-	VertexLocation           string               `yaml:"vertex_location"`
-	ServiceAccountFile       string               `yaml:"service_account_file"`
-	ServiceAccountJSON       string               `yaml:"service_account_json"`
-	ServiceAccountJSONBase64 string               `yaml:"service_account_json_base64"`
-	GCPScope                 string               `yaml:"gcp_scope"`
-	Models                   []RawProviderModel   `yaml:"models"`
-	Resilience               *RawResilienceConfig `yaml:"resilience"`
+	SessionStickyKeys        *bool  `yaml:"session_sticky_keys"`
+	BaseURL                  string `yaml:"base_url"`
+	APIVersion               string `yaml:"api_version"`
+	Backend                  string `yaml:"backend"`
+	AuthType                 string `yaml:"auth_type"`
+	APIMode                  string `yaml:"api_mode"`
+	VertexProject            string `yaml:"vertex_project"`
+	VertexLocation           string `yaml:"vertex_location"`
+	ServiceAccountFile       string `yaml:"service_account_file"`
+	ServiceAccountJSON       string `yaml:"service_account_json"`
+	ServiceAccountJSONBase64 string `yaml:"service_account_json_base64"`
+	GCPScope                 string `yaml:"gcp_scope"`
+	// InferenceObjective is the trusted llm-d InferenceObjective name injected
+	// into outbound requests. It is ignored by provider types other than llmd.
+	InferenceObjective string `yaml:"inference_objective"`
+	// FairnessFromUserPath controls whether the llmd provider derives its
+	// fairness ID from GoModel's effective (authenticated) user path. It
+	// defaults to true; nil preserves that default.
+	FairnessFromUserPath *bool                `yaml:"fairness_from_user_path"`
+	Models               []RawProviderModel   `yaml:"models"`
+	Resilience           *RawResilienceConfig `yaml:"resilience"`
 }

config/server.go

@@ -39,7 +39,7 @@ type ServerConfig struct {
 	UserPathHeader string `yaml:"user_path_header" env:"USER_PATH_HEADER"`
 	// EnabledPassthroughProviders lists the provider types enabled on
 	// /p/{provider}/... passthrough routes. Default:
-	// ["openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "deepseek"].
+	// ["openai", "anthropic", "openrouter", "kilo", "zai", "sglang", "vllm", "llmd", "deepseek"].
 	EnabledPassthroughProviders []string `yaml:"enabled_passthrough_providers" env:"ENABLED_PASSTHROUGH_PROVIDERS"`
 	// RealtimeEnabled exposes the realtime (speech-to-speech) websocket endpoint
 	// at /v1/realtime and the /p/{provider}/v1/realtime passthrough upgrade.

docs/advanced/config-yaml.mdx

@@ -34,7 +34,10 @@ must use the same suffix for each instance: for example, pair
 `gcp_adc`.
 
 Configured provider model lists can stay in env via `<PROVIDER>_MODELS`, for
-example `OPENROUTER_MODELS`, `ORACLE_MODELS`, `AZURE_MODELS`, `SGLANG_MODELS`, or `VLLM_MODELS`.
+example `OPENROUTER_MODELS`, `ORACLE_MODELS`, `AZURE_MODELS`, `SGLANG_MODELS`,
+`VLLM_MODELS`, or `LLMD_MODELS`. llm-d instances also accept suffixed
+`LLMD_<SUFFIX>_INFERENCE_OBJECTIVE` and
+`LLMD_<SUFFIX>_FAIRNESS_FROM_USER_PATH` controls.
 Set `CONFIGURED_PROVIDER_MODELS_MODE=fallback` (default) to use those lists only
 when upstream `/models` fails or is empty, or `allowlist` to expose only the
 configured models for providers that define a list and skip their upstream