From 1a72ad3ca16494fd4b38d05ba3d596fa1f95db18 Mon Sep 17 00:00:00 2001 From: Tanguille <91473554+Tanguille@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:40:52 +0200 Subject: [PATCH] feat(router): compile dataPlane: AgentGateway to AgentgatewayModel agentGatewayRef.routing: Model compiles a ModelRouter onto agentgateway's own model router (agentgateway v1.6.0+): one HTTPRoute whose single rule at "/" hands requests to AgentgatewayModel "*", and one AgentgatewayModel per inferenceServiceRef backend that matches the backend's served name and points a custom provider at the InferenceService's Service. For single-replica backends this drops the Inference Extension CRDs, the Endpoint Picker and the Gateway-wide PreRouting policy, and with them the MultipleInferencePoolsUnsupported and ConflictingPreRoutingPolicy refusals. routing defaults to InferencePool, so existing routers compile as before. Model mode requires sectionName or hostnames, so its catch-all rule cannot take over a shared Gateway. It refuses inputs it cannot express (rules, defaultRoute, external backends, resolution: endpoint, pickers, unmapped runtimes) and gates on the AgentgatewayModel CRD. It never adopts objects it did not create, prunes the models of removed backends, and cleans up the other mode's objects when routing changes. Part of #2151 (phase 1). Signed-off-by: Tanguille <91473554+Tanguille@users.noreply.github.com> --- api/v1alpha1/modelrouter_types.go | 46 +- api/v1alpha1/zz_generated.deepcopy.go | 7 +- charts/llmkube/templates/clusterrole.yaml | 1 + .../llmkube/templates/crds/modelrouters.yaml | 46 + .../inference.llmkube.dev_modelrouters.yaml | 46 + config/rbac/role.yaml | 1 + docs/site/concepts/model-router.md | 57 + .../gateway_resources_agentgateway.go | 28 +- .../modelrouter_agentgateway_controller.go | 254 +- .../modelrouter_agentgateway_model.go | 314 + .../modelrouter_agentgateway_model_test.go | 718 +++ .../modelrouter_agentgateway_picker.go | 6 +- .../modelrouter_agentgateway_rbac_test.go | 8 +- .../modelrouter_agentgateway_test.go | 20 +- internal/controller/modelrouter_webhook.go | 3 + test/crd/inference-extension/README.md | 5 +- .../agentgatewaymodel.yaml | 5646 +++++++++++++++++ test/crd/inference-extension/httproute.yaml | 2 + test/e2e-rbac/crds/README.md | 7 +- test/e2e-rbac/crds/agentgatewaymodel.yaml | 1 + test/e2e-rbac/fixtures/50-agentgateway.yaml | 21 + 21 files changed, 7114 insertions(+), 123 deletions(-) create mode 100644 internal/controller/modelrouter_agentgateway_model.go create mode 100644 internal/controller/modelrouter_agentgateway_model_test.go create mode 100644 test/crd/inference-extension/agentgatewaymodel.yaml create mode 120000 test/e2e-rbac/crds/agentgatewaymodel.yaml diff --git a/api/v1alpha1/modelrouter_types.go b/api/v1alpha1/modelrouter_types.go index e130a72c1..8e4005c7d 100644 --- a/api/v1alpha1/modelrouter_types.go +++ b/api/v1alpha1/modelrouter_types.go @@ -766,19 +766,63 @@ type MCPServerSpec struct { Enabled bool `json:"enabled,omitempty"` } +// AgentGatewayRouting selects what a dataPlane: AgentGateway ModelRouter +// compiles to. +type AgentGatewayRouting string + +const ( + AgentGatewayRoutingInferencePool AgentGatewayRouting = "InferencePool" + AgentGatewayRoutingModel AgentGatewayRouting = "Model" +) + // AgentGatewayReference identifies the pre-installed agentgateway data plane a // dataPlane: AgentGateway ModelRouter compiles onto. The Gateway is the // gateway.networking.k8s.io Gateway whose GatewayClass is agentgateway; the // EndpointPicker is the Endpoint Picker (EPP) Service that selects a model // server endpoint per request for each generated InferencePool. The Gateway, // the Gateway API Inference Extension CRDs, and a running EPP are documented -// prerequisites that LLMKube does not install or own. +// prerequisites that LLMKube does not install or own. With routing: Model only +// the Gateway and agentgateway v1.6.0 or later are. // +kubebuilder:validation:XValidation:rule="has(self.endpointPicker) == has(self.endpointPickerPort)",message="endpointPicker and endpointPickerPort must be set together" +// +kubebuilder:validation:XValidation:rule="!(has(self.routing) && self.routing == 'Model') || !has(self.endpointPicker)",message="endpointPicker and endpointPickerPort apply only to routing: InferencePool" +// +kubebuilder:validation:XValidation:rule="!(has(self.routing) && self.routing == 'Model') || has(self.sectionName) || has(self.hostnames)",message="routing: Model requires sectionName or hostnames: its HTTPRoute matches every path, so it must not take over a whole Gateway" +// +kubebuilder:validation:XValidation:rule="(has(self.routing) && self.routing == 'Model') || (!has(self.sectionName) && !has(self.hostnames))",message="sectionName and hostnames apply only to routing: Model" type AgentGatewayReference struct { // Name is the agentgateway Gateway's name. // +kubebuilder:validation:Required Name string `json:"name"` + // Routing selects the compile target. "InferencePool" (the default) is the + // Gateway API Inference Extension path described on spec.agentGatewayRef. + // "Model" uses agentgateway's model router (v1.6.0+): one AgentgatewayModel + // per backend, matched on its displayName (else name), behind one + // catch-all HTTPRoute. It needs no Inference Extension CRDs or Endpoint + // Picker. spec.rules, spec.defaultRoute, external backends, resolution: + // endpoint, and per-backend endpoint pickers are refused in this mode. + // +kubebuilder:validation:Enum=InferencePool;Model + // +kubebuilder:default=InferencePool + // +optional + Routing AgentGatewayRouting `json:"routing,omitempty"` + + // SectionName names the Gateway listener the generated HTTPRoute attaches + // to. routing: Model only, where it or hostnames is required, because the + // route takes every path on its listeners. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +optional + SectionName string `json:"sectionName,omitempty"` + + // Hostnames limit the generated HTTPRoute to these hosts. routing: Model + // only, where it or sectionName is required. + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=253 + // +kubebuilder:validation:items:Pattern=`^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +optional + Hostnames []string `json:"hostnames,omitempty"` + // Namespace is the Gateway's namespace. Empty means the ModelRouter's own // namespace. A different namespace is supported: the generated // model-extraction AgentgatewayPolicy then lives in the Gateway's diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 02926a4e8..fe3b38143 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -54,6 +54,11 @@ func (in *AccelerationStatus) DeepCopy() *AccelerationStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AgentGatewayReference) DeepCopyInto(out *AgentGatewayReference) { *out = *in + if in.Hostnames != nil { + in, out := &in.Hostnames, &out.Hostnames + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentGatewayReference. @@ -1483,7 +1488,7 @@ func (in *ModelRouterSpec) DeepCopyInto(out *ModelRouterSpec) { if in.AgentGatewayRef != nil { in, out := &in.AgentGatewayRef, &out.AgentGatewayRef *out = new(AgentGatewayReference) - **out = **in + (*in).DeepCopyInto(*out) } if in.Backends != nil { in, out := &in.Backends, &out.Backends diff --git a/charts/llmkube/templates/clusterrole.yaml b/charts/llmkube/templates/clusterrole.yaml index dbf79836c..d783cc8b9 100644 --- a/charts/llmkube/templates/clusterrole.yaml +++ b/charts/llmkube/templates/clusterrole.yaml @@ -225,6 +225,7 @@ rules: - apiGroups: - agentgateway.dev resources: + - agentgatewaymodels - agentgatewaypolicies verbs: - create diff --git a/charts/llmkube/templates/crds/modelrouters.yaml b/charts/llmkube/templates/crds/modelrouters.yaml index 568e3f722..c578dd6ec 100644 --- a/charts/llmkube/templates/crds/modelrouters.yaml +++ b/charts/llmkube/templates/crds/modelrouters.yaml @@ -110,6 +110,18 @@ spec: maximum: 65535 minimum: 1 type: integer + hostnames: + description: |- + Hostnames limit the generated HTTPRoute to these hosts. routing: Model + only, where it or sectionName is required. + items: + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + minItems: 1 + type: array name: description: Name is the agentgateway Gateway's name. type: string @@ -124,12 +136,46 @@ spec: EndpointPicker Service all live in this ModelRouter's namespace regardless. type: string + routing: + default: InferencePool + description: |- + Routing selects the compile target. "InferencePool" (the default) is the + Gateway API Inference Extension path described on spec.agentGatewayRef. + "Model" uses agentgateway's model router (v1.6.0+): one AgentgatewayModel + per backend, matched on its displayName (else name), behind one + catch-all HTTPRoute. It needs no Inference Extension CRDs or Endpoint + Picker. spec.rules, spec.defaultRoute, external backends, resolution: + endpoint, and per-backend endpoint pickers are refused in this mode. + enum: + - InferencePool + - Model + type: string + sectionName: + description: |- + SectionName names the Gateway listener the generated HTTPRoute attaches + to. routing: Model only, where it or hostnames is required, because the + route takes every path on its listeners. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string required: - name type: object x-kubernetes-validations: - message: endpointPicker and endpointPickerPort must be set together rule: has(self.endpointPicker) == has(self.endpointPickerPort) + - message: 'endpointPicker and endpointPickerPort apply only to routing: + InferencePool' + rule: '!(has(self.routing) && self.routing == ''Model'') || !has(self.endpointPicker)' + - message: 'routing: Model requires sectionName or hostnames: its + HTTPRoute matches every path, so it must not take over a whole + Gateway' + rule: '!(has(self.routing) && self.routing == ''Model'') || has(self.sectionName) + || has(self.hostnames)' + - message: 'sectionName and hostnames apply only to routing: Model' + rule: (has(self.routing) && self.routing == 'Model') || (!has(self.sectionName) + && !has(self.hostnames)) backends: description: |- Backends are the candidate destinations the router can dispatch to. diff --git a/config/crd/bases/inference.llmkube.dev_modelrouters.yaml b/config/crd/bases/inference.llmkube.dev_modelrouters.yaml index 69888fa3d..6a2bcb55c 100644 --- a/config/crd/bases/inference.llmkube.dev_modelrouters.yaml +++ b/config/crd/bases/inference.llmkube.dev_modelrouters.yaml @@ -106,6 +106,18 @@ spec: maximum: 65535 minimum: 1 type: integer + hostnames: + description: |- + Hostnames limit the generated HTTPRoute to these hosts. routing: Model + only, where it or sectionName is required. + items: + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + minItems: 1 + type: array name: description: Name is the agentgateway Gateway's name. type: string @@ -120,12 +132,46 @@ spec: EndpointPicker Service all live in this ModelRouter's namespace regardless. type: string + routing: + default: InferencePool + description: |- + Routing selects the compile target. "InferencePool" (the default) is the + Gateway API Inference Extension path described on spec.agentGatewayRef. + "Model" uses agentgateway's model router (v1.6.0+): one AgentgatewayModel + per backend, matched on its displayName (else name), behind one + catch-all HTTPRoute. It needs no Inference Extension CRDs or Endpoint + Picker. spec.rules, spec.defaultRoute, external backends, resolution: + endpoint, and per-backend endpoint pickers are refused in this mode. + enum: + - InferencePool + - Model + type: string + sectionName: + description: |- + SectionName names the Gateway listener the generated HTTPRoute attaches + to. routing: Model only, where it or hostnames is required, because the + route takes every path on its listeners. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string required: - name type: object x-kubernetes-validations: - message: endpointPicker and endpointPickerPort must be set together rule: has(self.endpointPicker) == has(self.endpointPickerPort) + - message: 'endpointPicker and endpointPickerPort apply only to routing: + InferencePool' + rule: '!(has(self.routing) && self.routing == ''Model'') || !has(self.endpointPicker)' + - message: 'routing: Model requires sectionName or hostnames: its + HTTPRoute matches every path, so it must not take over a whole + Gateway' + rule: '!(has(self.routing) && self.routing == ''Model'') || has(self.sectionName) + || has(self.hostnames)' + - message: 'sectionName and hostnames apply only to routing: Model' + rule: (has(self.routing) && self.routing == 'Model') || (!has(self.sectionName) + && !has(self.hostnames)) backends: description: |- Backends are the candidate destinations the router can dispatch to. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 67edd30a3..708e5f54e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -81,6 +81,7 @@ rules: - apiGroups: - agentgateway.dev resources: + - agentgatewaymodels - agentgatewaypolicies verbs: - create diff --git a/docs/site/concepts/model-router.md b/docs/site/concepts/model-router.md index b9f974fd0..d34ccccad 100644 --- a/docs/site/concepts/model-router.md +++ b/docs/site/concepts/model-router.md @@ -227,6 +227,63 @@ Each `InferencePool` is named after its `InferenceService`, and a router never a `primary-fallback` routes each rule to its first healthy backend. When that backend's `InferenceService` has no ready replica, the next reconcile moves the rule to the next backend. Failover follows replica readiness, not individual request errors: a primary that is up but failing requests keeps its traffic. +## Model routing on dataPlane: AgentGateway + +`agentGatewayRef.routing: Model` compiles the router onto agentgateway's own model router instead of the Inference Extension. It needs agentgateway v1.6.0 or later, which ships the `AgentgatewayModel` CRD, and a Gateway. It needs no Gateway API Inference Extension CRDs, no Endpoint Picker, and no Gateway-wide policy, which suits single-replica `InferenceService`s reached through their Service. + +```yaml +apiVersion: inference.llmkube.dev/v1alpha1 +kind: ModelRouter +metadata: + name: llm-router +spec: + dataPlane: AgentGateway + agentGatewayRef: + name: ai-gateway + routing: Model + sectionName: llm + backends: + - name: chat + displayName: chat-large + inferenceServiceRef: + name: chat-vllm + - name: chat-small + inferenceServiceRef: + name: chat-llama + - name: reranker + inferenceServiceRef: + name: reranker +``` + +The router generates, all in its own namespace and owned by it: + +- one `HTTPRoute` named after the router, attached to the Gateway, with a single rule named `models` at `/` whose only backend is `AgentgatewayModel` `*`. The rule sits at `/` because agentgateway strips the rule's path prefix before it classifies the endpoint: under `/v1`, `/v1/embeddings` would parse as a chat completion; +- one `AgentgatewayModel` per backend, named `-` (cut and hashed past 63 characters), attached to that rule. Its `match.model` is the backend's `displayName`, else its `name`, so served names such as `Qwen3.5-2B` or `org/model` work. It reaches the `InferenceService`'s Service through a custom provider, and the request keeps its model name, so the served name must be one the runtime answers to: llama.cpp ignores it, and vLLM needs a matching `--served-model-name`. + +The provider and API formats come from the `InferenceService`: + +| Runtime | `spec.mode` (or inferred) | `providerOverride` | Formats | +|---|---|---|---| +| `llamacpp` (or unset) | chat | `llamacpp` | `Completions` | +| `vllm` | chat | `vllm` | `Completions`, `Messages`, `AnthropicTokenCount`, `Responses` | +| `llamacpp`, `vllm` | embedding | as above | `Embeddings` | +| `llamacpp`, `vllm` | rerank | as above | `Rerank` | + +agentgateway then routes each request on the body's `model`, answers `GET /v1/models` itself, and returns a JSON 404 `model_not_found` for a model no backend serves. + +A rule at `/` takes every path on the listeners it attaches to, so this mode requires `agentGatewayRef.sectionName` (one listener), `agentGatewayRef.hostnames`, or both. Your own routes still share the listener: a rule with a longer match, such as a `PathPrefix` `/v1` with a header match, outranks the router's `/`. Use one `routing: Model` router per listener and hostname. Two routers' `/` rules tie, Gateway API gives the tie to the older route, and the newer router's models get no traffic. + +The router reports `AgentGatewayReady=False` and generates nothing when: + +- the `AgentgatewayModel` CRD is not installed (`AgentgatewayModelCRDNotInstalled`); +- the spec has `rules`, a `defaultRoute`, an `external` backend, `resolution: endpoint`, a backend `endpointPicker`, a served name containing `*`, or two backends serving one name (`UnsupportedInModelRouting`; the webhook rejects these at apply time). Fallback between backends and external backends need an `AgentgatewayBackend`, which this mode does not generate yet; +- a backend's `InferenceService` runs a runtime other than `llamacpp` or `vllm` (`UnsupportedRuntimeInModelRouting`); +- a generated name is held by an object LLMKube did not create (`GeneratedResourceNameTaken`); delete or rename that object. + +When you replace hand-written `AgentgatewayModel`s with a router, remove the hand-written ones for the same model names on that listener, so each name has one model. + +Removing a backend deletes its model. Switching `routing` in either direction deletes the other mode's objects, including the `InferencePool` mode's extraction policy in the Gateway's namespace. + ## Composition with LiteLLM LLMKube does not replace [LiteLLM](https://github.com/BerriAI/litellm). For organizations already running a LiteLLM proxy as their cloud-provider abstraction, point a `ModelRouter` external backend at it: diff --git a/internal/controller/gateway_resources_agentgateway.go b/internal/controller/gateway_resources_agentgateway.go index 7863fea60..c837ff484 100644 --- a/internal/controller/gateway_resources_agentgateway.go +++ b/internal/controller/gateway_resources_agentgateway.go @@ -209,15 +209,6 @@ func newAgentGatewayHTTPRoute( rules []agentGatewayRuleResource, backends []agentGatewayBackendResource, ) *unstructured.Unstructured { - parentRef := map[string]interface{}{ - metadataNameField: ref.Name, - "kind": "Gateway", - "group": gatewayBackendRefGroupAPI, - } - if ref.Namespace != "" { - parentRef["namespace"] = ref.Namespace - } - compiledRules := make([]interface{}, 0, len(rules)) poolFor := make(map[string]string, len(backends)) for _, b := range backends { @@ -235,12 +226,29 @@ func newAgentGatewayHTTPRoute( // leaving-AgentGateway sweep can find it (mirrors the extraction policy). u.SetLabels(map[string]string{agentGatewayRouterUIDLabel: string(mr.UID)}) u.Object["spec"] = map[string]interface{}{ - "parentRefs": []interface{}{parentRef}, + "parentRefs": []interface{}{agentGatewayRouteParentRef(ref)}, "rules": compiledRules, } return u } +// agentGatewayRouteParentRef only carries a sectionName with routing: Model, +// where the CRD allows it, so the InferencePool route renders unchanged. +func agentGatewayRouteParentRef(ref *inferencev1alpha1.AgentGatewayReference) map[string]interface{} { + parentRef := map[string]interface{}{ + metadataNameField: ref.Name, + "kind": "Gateway", + "group": gatewayBackendRefGroupAPI, + } + if ref.Namespace != "" { + parentRef["namespace"] = ref.Namespace + } + if ref.SectionName != "" { + parentRef["sectionName"] = ref.SectionName + } + return parentRef +} + // compileAgentGatewayRouteRule builds one HTTPRoute rule: the matches block (a // match per model name, ANDed with header matches; a header-only match when the // rule declares no models, e.g. a catch-all) and the backendRefs block. diff --git a/internal/controller/modelrouter_agentgateway_controller.go b/internal/controller/modelrouter_agentgateway_controller.go index b9b0ec12e..a289cdbf4 100644 --- a/internal/controller/modelrouter_agentgateway_controller.go +++ b/internal/controller/modelrouter_agentgateway_controller.go @@ -174,10 +174,15 @@ type ModelRouterAgentGatewayReconciler struct { // InferencePool kind and agentgateway.dev's AgentgatewayPolicy kind. detectorOnce sync.Once detector *crdDetector + + // modelDetector gates routing: Model on the AgentgatewayModel CRD alone. + modelDetectorOnce sync.Once + modelDetector *crdDetector } // +kubebuilder:rbac:groups=inference.networking.k8s.io,resources=inferencepools,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=agentgateway.dev,resources=agentgatewaypolicies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=agentgateway.dev,resources=agentgatewaymodels,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch @@ -224,7 +229,7 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c // status.agentGateway block (set exactly when generation succeeded), or a // TRUE AgentGatewayReady condition. A FALSE condition is not evidence: // refusals set it without generating anything, and counting it would make - // every refusal reconcile re-run the three Lists below - the cost this + // every refusal reconcile re-run the four Lists below - the cost this // gate exists to remove. readyCond := apimeta.FindStatusCondition(mr.Status.Conditions, ModelRouterAgentGatewayConditionReady) usedAgentGateway := hasFinalizer || mr.Status.AgentGateway != nil || @@ -240,10 +245,11 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{}, err } if deleting || leaving { - // Pools and routes are owner-referenced, but owner-ref GC only - // fires when the router object goes away: a router that leaves - // AgentGateway mode must stop attaching to the Gateway now. - if err := r.cleanupGeneratedPoolsAndRoutes(ctx, mr); err != nil { + // Pools, routes and models are owner-referenced, but owner-ref GC + // only fires when the router object goes away: a router that + // leaves AgentGateway mode must stop attaching to the Gateway now. + if err := r.deleteGeneratedObjects(ctx, mr, nil, + inferencePoolGVK(), gatewayHTTPRouteGVK(), agentGatewayModelGVK()); err != nil { return ctrl.Result{}, err } } @@ -309,7 +315,8 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c // kind is meant, rather than having to read the detector; #1725). A // transient discovery error (not a missing kind) is returned so we requeue // rather than disabling. - present, missing, err := r.requiredCRDsMissing(log) + modelRouting := agentGatewayModelRouting(mr) + present, missing, err := r.requiredCRDsMissing(log, modelRouting) if err != nil { return ctrl.Result{}, err } @@ -318,6 +325,11 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c for _, gvk := range missing { names = append(names, fmt.Sprintf("%s (%s/%s)", gvk.Kind, gvk.Group, gvk.Version)) } + if modelRouting { + return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonModelCRDMissing, + "agentGatewayRef.routing: Model needs agentgateway v1.6.0 or later, and its AgentgatewayModel CRD is not "+ + "installed; agentgateway exposure is disabled; missing: "+strings.Join(names, ", ")) + } return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonCRDsMissing, "Gateway API Inference Extension and agentgateway CRDs are not installed; agentgateway exposure is disabled; missing: "+ strings.Join(names, ", ")) @@ -373,50 +385,53 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil } - // Fail loud when the AgentgatewayPolicy this router would generate already - // exists and LLMKube did not create it. Runs BEFORE the conflict check so - // a foreign policy that both holds our name and conflicts reports the more - // specific name-taken reason, and before any generation so a name-taken - // router yields no partial resources. The pools and the route are checked - // the same way inside reconcileAgentGatewayResources, once their names are - // known from the resolved backends. - if err := r.checkGeneratedNameAvailable(ctx, mr, newAgentGatewayModelExtractionPolicy(mr, mr.Spec.AgentGatewayRef)); err != nil { - var taken *agentGatewayNameTakenError - if errors.As(err, &taken) { - // Whether the name is free depends on objects outside the router's - // spec; requeue so deleting or renaming the foreign object takes - // effect without an unrelated event. - if serr := r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonResourceNameTaken, err.Error()); serr != nil { - return ctrl.Result{}, serr + // routing: Model writes no extraction policy, so these checks don't apply. + if !modelRouting { + // Fail loud when the AgentgatewayPolicy this router would generate already + // exists and LLMKube did not create it. Runs BEFORE the conflict check so + // a foreign policy that both holds our name and conflicts reports the more + // specific name-taken reason, and before any generation so a name-taken + // router yields no partial resources. The pools and the route are checked + // the same way inside reconcileAgentGatewayResources, once their names are + // known from the resolved backends. + if err := r.checkGeneratedNameAvailable(ctx, mr, newAgentGatewayModelExtractionPolicy(mr, mr.Spec.AgentGatewayRef)); err != nil { + var taken *agentGatewayNameTakenError + if errors.As(err, &taken) { + // Whether the name is free depends on objects outside the router's + // spec; requeue so deleting or renaming the foreign object takes + // effect without an unrelated event. + if serr := r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonResourceNameTaken, err.Error()); serr != nil { + return ctrl.Result{}, serr + } + return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil } - return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil + return ctrl.Result{}, err } - return ctrl.Result{}, err - } - // Fail loud when the target Gateway already carries a PreRouting - // transformation policy this router does not own. agentgateway merges - // equal-specificity policies oldest-wins per whole field and never - // combines list entries (and no condition reports the dropped field), so a - // second transformation on the same Gateway either silently disables the - // generated model extraction or is silently disabled by it. Refuse before - // any generation so a conflicted router yields no partial resources. - conflicts, err := r.agentGatewayPreRoutingConflicts(ctx, mr, gateway.GetLabels()) - if err != nil { - return ctrl.Result{}, err - } - if len(conflicts) > 0 { - if err := r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonPolicyConflict, - fmt.Sprintf("Gateway %s/%s already carries PreRouting transformation policy(ies) that LLMKube does not own: %s. "+ - "agentgateway merges same-specificity policies oldest-wins per field without combining list entries, "+ - "so the generated model extraction could be silently dropped, or silently shadow the existing policy. "+ - "Remove or scope the other policy", agentGatewayPolicyNamespace(mr), mr.Spec.AgentGatewayRef.Name, - strings.Join(conflicts, ", "))); err != nil { + // Fail loud when the target Gateway already carries a PreRouting + // transformation policy this router does not own. agentgateway merges + // equal-specificity policies oldest-wins per whole field and never + // combines list entries (and no condition reports the dropped field), so a + // second transformation on the same Gateway either silently disables the + // generated model extraction or is silently disabled by it. Refuse before + // any generation so a conflicted router yields no partial resources. + conflicts, err := r.agentGatewayPreRoutingConflicts(ctx, mr, gateway.GetLabels()) + if err != nil { return ctrl.Result{}, err } - // Recheck on the conflict cadence so the condition clears by itself - // when the foreign policy is deleted. - return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil + if len(conflicts) > 0 { + if err := r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonPolicyConflict, + fmt.Sprintf("Gateway %s/%s already carries PreRouting transformation policy(ies) that LLMKube does not own: %s. "+ + "agentgateway merges same-specificity policies oldest-wins per field without combining list entries, "+ + "so the generated model extraction could be silently dropped, or silently shadow the existing policy. "+ + "Remove or scope the other policy", agentGatewayPolicyNamespace(mr), mr.Spec.AgentGatewayRef.Name, + strings.Join(conflicts, ", "))); err != nil { + return ctrl.Result{}, err + } + // Recheck on the conflict cadence so the condition clears by itself + // when the foreign policy is deleted. + return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil + } } // Fail-loud on matches the agentgateway data plane cannot express. We @@ -424,7 +439,10 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c // user expects. The agentgateway plane matches on the OpenAI "model" field // (via the extracted agentGatewayHTTPModelHeader) and on headers (via // HTTPRoute header matches); everything the Gateway plane rejects is - // rejected here too. + // rejected here too. routing: Model has its own boundary, checked first. + if msg := agentGatewayModelRoutingMessage(mr); msg != "" { + return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonUnsupportedModelRouting, msg) + } if msg := agentGatewayUnsupportedMatchMessage(mr); msg != "" { return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonUnsupported, msg) } @@ -480,7 +498,12 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, reason, msg) } - ejected, err := r.reconcileAgentGatewayResources(ctx, mr) + var ejected []string + if modelRouting { + err = r.reconcileAgentGatewayModels(ctx, mr) + } else { + ejected, err = r.reconcileAgentGatewayResources(ctx, mr) + } if err != nil { var taken *agentGatewayNameTakenError if errors.As(err, &taken) { @@ -492,6 +515,11 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c } return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil } + // No requeue: the InferenceService watch re-reconciles on a fix. + var refusal *agentGatewayRefusalError + if errors.As(err, &refusal) { + return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, refusal.reason, refusal.message) + } _ = r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonReconcile, err.Error()) return ctrl.Result{}, err } @@ -577,6 +605,10 @@ func (r *ModelRouterAgentGatewayReconciler) reconcileAgentGatewayResources( return nil, fmt.Errorf("%s/%s: %w", obj.GetKind(), obj.GetName(), err) } } + // Remove what routing: Model left behind. + if err := r.deleteGeneratedObjects(ctx, mr, nil, agentGatewayModelGVK()); err != nil { + return nil, err + } return ejected, nil } @@ -595,27 +627,9 @@ func (r *ModelRouterAgentGatewayReconciler) resolveAgentGatewayBackends( pickers, _, _ := agentGatewayResolvePoolPickers(mr) resolved := make([]agentGatewayBackendResource, 0, len(mr.Spec.Backends)) for _, b := range mr.Spec.Backends { - if b.External != nil { - return nil, fmt.Errorf("backend %q is External; dataPlane: AgentGateway routes to an InferencePool "+ - "that selects model-server pods, so off-cluster backends cannot be attached", b.Name) - } - if b.InferenceServiceRef == nil { - return nil, fmt.Errorf("backend %q has no inferenceServiceRef", b.Name) - } - - isvc := &inferencev1alpha1.InferenceService{} - key := types.NamespacedName{Name: b.InferenceServiceRef.Name, Namespace: mr.Namespace} - if err := r.Get(ctx, key, isvc); err != nil { - if apierrors.IsNotFound(err) { - return nil, fmt.Errorf("backend %q references InferenceService %q which does not exist in namespace %q", - b.Name, b.InferenceServiceRef.Name, mr.Namespace) - } - return nil, fmt.Errorf("backend %q: looking up InferenceService %q: %w", b.Name, b.InferenceServiceRef.Name, err) - } - - port := int64(8080) - if isvc.Spec.Endpoint != nil && isvc.Spec.Endpoint.Port > 0 { - port = int64(isvc.Spec.Endpoint.Port) + isvc, err := r.backendInferenceService(ctx, mr, b) + if err != nil { + return nil, err } resolved = append(resolved, agentGatewayBackendResource{ Name: b.Name, @@ -623,7 +637,7 @@ func (r *ModelRouterAgentGatewayReconciler) resolveAgentGatewayBackends( // InferenceService name so one pool serves one model's pods. PoolName: sanitizeDNSName(isvc.Name), FQDN: fmt.Sprintf("%s.%s.svc.cluster.local", sanitizeDNSName(isvc.Name), isvc.Namespace), - Port: port, + Port: gatewayServicePort(isvc), Picker: pickers[sanitizeDNSName(isvc.Name)], // A backend is healthy iff its InferenceService has at least one ready // replica. An unhealthy backend is ejected from the route's @@ -635,6 +649,33 @@ func (r *ModelRouterAgentGatewayReconciler) resolveAgentGatewayBackends( return resolved, nil } +// backendInferenceService fetches a backend's InferenceService. External +// backends and missing InferenceServices are errors. +func (r *ModelRouterAgentGatewayReconciler) backendInferenceService( + ctx context.Context, + mr *inferencev1alpha1.ModelRouter, + b inferencev1alpha1.RouterBackend, +) (*inferencev1alpha1.InferenceService, error) { + if b.External != nil { + return nil, fmt.Errorf("backend %q is External; dataPlane: AgentGateway routes to an InferencePool "+ + "that selects model-server pods, so off-cluster backends cannot be attached", b.Name) + } + if b.InferenceServiceRef == nil { + return nil, fmt.Errorf("backend %q has no inferenceServiceRef", b.Name) + } + + isvc := &inferencev1alpha1.InferenceService{} + key := types.NamespacedName{Name: b.InferenceServiceRef.Name, Namespace: mr.Namespace} + if err := r.Get(ctx, key, isvc); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf("backend %q references InferenceService %q which does not exist in namespace %q", + b.Name, b.InferenceServiceRef.Name, mr.Namespace) + } + return nil, fmt.Errorf("backend %q: looking up InferenceService %q: %w", b.Name, b.InferenceServiceRef.Name, err) + } + return isvc, nil +} + // applyAgentGatewayResource creates-or-updates desired. Objects in the // ModelRouter's namespace (pools, route, same-namespace extraction policy) are // controller-referenced to the ModelRouter so owner-ref GC removes them. The @@ -695,7 +736,13 @@ func (r *ModelRouterAgentGatewayReconciler) applyAgentGatewayResource( // requiredCRDsMissing reports whether the CRDs this slice needs are registered, // and, when they are not, exactly which GVKs are missing (so the disabled // condition can name them). Delegates to the shared crdDetector. -func (r *ModelRouterAgentGatewayReconciler) requiredCRDsMissing(log logr.Logger) (bool, []schema.GroupVersionKind, error) { +func (r *ModelRouterAgentGatewayReconciler) requiredCRDsMissing(log logr.Logger, modelRouting bool) (bool, []schema.GroupVersionKind, error) { + if modelRouting { + r.modelDetectorOnce.Do(func() { + r.modelDetector = newCRDDetector("agentgateway-model", []schema.GroupVersionKind{agentGatewayModelGVK()}) + }) + return r.modelDetector.PresentMissing(r.Client, log) + } r.detectorOnce.Do(func() { r.detector = newCRDDetector("agentgateway", modelRouterAgentGatewayGVKs()) }) @@ -756,6 +803,7 @@ func (r *ModelRouterAgentGatewayReconciler) agentGatewayAllowedRoutesViolation( mr *inferencev1alpha1.ModelRouter, ) (string, error) { policyNS := agentGatewayPolicyNamespace(mr) + sectionName := mr.Spec.AgentGatewayRef.SectionName listeners, _, _ := unstructured.NestedSlice(gateway.Object, "spec", "listeners") nsLabels := map[string]string{} @@ -805,6 +853,13 @@ func (r *ModelRouterAgentGatewayReconciler) agentGatewayAllowedRoutesViolation( kinds, _, _ := unstructured.NestedSlice(listener, "allowedRoutes", "kinds") summary = append(summary, describeListener(from, proto, kinds)) + // A sectionName route attaches to that listener only. + if sectionName != "" { + if name, _, _ := unstructured.NestedString(listener, metadataNameField); name != sectionName { + continue + } + } + // An HTTPRoute attaches only to HTTP/HTTPS listeners, and only to // kinds the listener lists: an unset kinds list defaults by protocol // (HTTP/HTTPS with no kinds admits HTTPRoute), while a non-empty list @@ -862,10 +917,17 @@ func (r *ModelRouterAgentGatewayReconciler) agentGatewayAllowedRoutesViolation( } } } - return fmt.Sprintf("Gateway %s/%s has no listener that admits namespace %q for an HTTPRoute (listeners: %s); "+ - "the generated HTTPRoute would be refused (NotAllowedByListeners), and the model-extraction AgentgatewayPolicy "+ - "must not be written without that consent", - policyNS, mr.Spec.AgentGatewayRef.Name, mr.Namespace, strings.Join(summary, ", ")), nil + listener := "no listener" + if sectionName != "" { + listener = fmt.Sprintf("no listener named %q", sectionName) + } + msg := fmt.Sprintf("Gateway %s/%s has %s that admits namespace %q for an HTTPRoute (listeners: %s); "+ + "the generated HTTPRoute would be refused (NotAllowedByListeners)", + policyNS, mr.Spec.AgentGatewayRef.Name, listener, mr.Namespace, strings.Join(summary, ", ")) + if agentGatewayModelRouting(mr) { + return msg, nil + } + return msg + ", and the model-extraction AgentgatewayPolicy must not be written without that consent", nil } // labelSelectorFromUnstructured converts an unstructured LabelSelector @@ -1104,21 +1166,24 @@ func (r *ModelRouterAgentGatewayReconciler) cleanupExtractionPolicies( return nil } -// cleanupGeneratedPoolsAndRoutes deletes the InferencePools and HTTPRoute this -// router generated, matched by the router-UID label in the router's namespace. -// It runs when the router leaves AgentGateway mode (dataPlane change or ref -// removal) and on deletion: owner-ref GC only fires when the router object -// goes away, but a router that leaves the mode must stop attaching to the -// Gateway immediately. The sweep is label-based, not owner-ref-based, because -// the Gateway plane also owner-references objects in the router's namespace; -// the UID label is what makes it mode-precise. A missing CRD (NoMatch, or the -// cached-mapping 404 after CRD removal) means the API server removed the CRs -// with it, so there is nothing to clean. -func (r *ModelRouterAgentGatewayReconciler) cleanupGeneratedPoolsAndRoutes( +// deleteGeneratedObjects deletes the objects of each given kind this router +// generated, matched by the router-UID label in the router's namespace, except +// the names in keep. It runs when the router leaves AgentGateway mode +// (dataPlane change or ref removal) and on deletion: owner-ref GC only fires +// when the router object goes away, but a router that leaves the mode must +// stop attaching to the Gateway immediately. Each routing mode also runs it to +// clear the other mode's kinds. The sweep is label-based, not owner-ref-based, +// because the Gateway plane also owner-references objects in the router's +// namespace; the UID label is what makes it mode-precise. A missing CRD +// (NoMatch, or the cached-mapping 404 after CRD removal) means the API server +// removed the CRs with it, so there is nothing to clean. +func (r *ModelRouterAgentGatewayReconciler) deleteGeneratedObjects( ctx context.Context, mr *inferencev1alpha1.ModelRouter, + keep map[string]bool, + gvks ...schema.GroupVersionKind, ) error { - for _, gvk := range []schema.GroupVersionKind{inferencePoolGVK(), gatewayHTTPRouteGVK()} { + for _, gvk := range gvks { list := &unstructured.UnstructuredList{} list.SetGroupVersionKind(gvk) if err := r.List(ctx, list, client.InNamespace(mr.Namespace), @@ -1130,6 +1195,9 @@ func (r *ModelRouterAgentGatewayReconciler) cleanupGeneratedPoolsAndRoutes( } for i := range list.Items { obj := &list.Items[i] + if keep[obj.GetName()] { + continue + } if err := r.Delete(ctx, obj); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("delete %s %s/%s: %w", gvk.Kind, obj.GetNamespace(), obj.GetName(), err) } @@ -1167,7 +1235,7 @@ func (r *ModelRouterAgentGatewayReconciler) setAgentGatewayReady( ) error { patch := client.MergeFrom(mr.DeepCopy()) mr.Status.AgentGateway = &inferencev1alpha1.AgentGatewayStatus{ - PoolReady: true, + PoolReady: !agentGatewayModelRouting(mr), Endpoint: agentGatewayEndpointAddress(mr.Spec.AgentGatewayRef), } message := agentGatewayReadyMessage(mr) @@ -1184,8 +1252,12 @@ func (r *ModelRouterAgentGatewayReconciler) setAgentGatewayReady( } // agentGatewayReadyMessage builds the success condition message: the rule count -// and the resolved endpoint. +// (with routing: Model, the model count) and the resolved endpoint. func agentGatewayReadyMessage(mr *inferencev1alpha1.ModelRouter) string { + if agentGatewayModelRouting(mr) { + return fmt.Sprintf("compiled %d model(s) onto agentgateway %s", len(mr.Spec.Backends), + agentGatewayEndpointAddress(mr.Spec.AgentGatewayRef)) + } return fmt.Sprintf("compiled %d rule(s) onto agentgateway %s", len(mr.Spec.Rules), agentGatewayEndpointAddress(mr.Spec.AgentGatewayRef)) } @@ -1337,12 +1409,8 @@ func compileAgentGatewayRules(mr *inferencev1alpha1.ModelRouter) ([]agentGateway // defaultRoute catch-all. First-match ordering mirrors the proxy. if mr.Spec.DefaultRouteStrategy == inferencev1alpha1.DefaultRouteStrategyBackendNameMatch { for _, b := range mr.Spec.Backends { - modelID := b.Name - if b.DisplayName != "" { - modelID = b.DisplayName - } rules = append(rules, agentGatewayRuleResource{ - Models: []string{modelID}, + Models: []string{agentGatewayServedName(b)}, BackendRefs: []agentGatewayBackendRef{{Name: b.Name}}, }) } diff --git a/internal/controller/modelrouter_agentgateway_model.go b/internal/controller/modelrouter_agentgateway_model.go new file mode 100644 index 000000000..9dd11ff77 --- /dev/null +++ b/internal/controller/modelrouter_agentgateway_model.go @@ -0,0 +1,314 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + inferencev1alpha1 "github.com/defilantech/llmkube/api/v1alpha1" +) + +// routing: Model compiles a router to agentgateway's own model router +// (v1.6.0+): one HTTPRoute whose single rule hands every request to +// AgentgatewayModel "*", and one AgentgatewayModel per backend. agentgateway +// matches the body's model, and answers GET /v1/models and unknown models +// itself. + +const ( + agentGatewayModelKind = "AgentgatewayModel" + + // Models name the route's rule as sectionName: agentgateway lets a + // parentRef omit it only while the route has exactly one rule. + agentGatewayModelRuleName = "models" + + // A DNS-1123 label, so a generated name also fits a label value. + agentGatewayModelNameMaxLength = 63 + + // The AgentgatewayModel CRD's match.model maxLength. + agentGatewayModelMatchMaxLength = 1024 + + modelRouterAgentGatewayReasonModelCRDMissing = "AgentgatewayModelCRDNotInstalled" + modelRouterAgentGatewayReasonUnsupportedModelRouting = "UnsupportedInModelRouting" + modelRouterAgentGatewayReasonUnsupportedRuntime = "UnsupportedRuntimeInModelRouting" +) + +func agentGatewayModelGVK() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: agentGatewayPolicyGroup, Version: agentGatewayPolicyVersion, Kind: agentGatewayModelKind} +} + +// agentGatewayModelRouting reports routing: Model; unset means InferencePool. +func agentGatewayModelRouting(mr *inferencev1alpha1.ModelRouter) bool { + ref := mr.Spec.AgentGatewayRef + return ref != nil && ref.Routing == inferencev1alpha1.AgentGatewayRoutingModel +} + +func agentGatewayServedName(b inferencev1alpha1.RouterBackend) string { + if b.DisplayName != "" { + return b.DisplayName + } + return b.Name +} + +// agentGatewayModelName derives the object name from router and backend, not +// from the served name, which need not be a valid name ("org/model"). Long or +// dash-ending names are cut and hashed. Collisions ("a-b"+"c" vs "a"+"b-c", or +// routers "a.b" and "a-b") are refused as GeneratedResourceNameTaken. +func agentGatewayModelName(mr *inferencev1alpha1.ModelRouter, backend string) string { + name := sanitizeDNSName(mr.Name) + "-" + backend + if len(name) <= agentGatewayModelNameMaxLength && !strings.HasSuffix(name, "-") { + return name + } + sum := sha256.Sum256([]byte(mr.Name + "/" + backend)) + tag := "-" + hex.EncodeToString(sum[:])[:8] + keep := min(len(name), agentGatewayModelNameMaxLength-len(tag)) + return strings.TrimRight(name[:keep], "-") + tag +} + +// agentGatewayModelProvider maps an InferenceService's runtime and serving mode +// to agentgateway's provider identity and API formats. ok is false for other +// runtimes: a guessed format would fail every request. +func agentGatewayModelProvider(isvc *inferencev1alpha1.InferenceService) (provider string, formats []string, ok bool) { + var chat []string + switch runtimeNameLabel(isvc) { + case "llamacpp": + provider, chat = "llamacpp", []string{"Completions"} + case RuntimeVLLM: + // vLLM serves Messages and Responses itself. + provider, chat = "vllm", []string{"Completions", "Messages", "AnthropicTokenCount", "Responses"} + default: + return "", nil, false + } + switch resolveServingMode(isvc) { + case servingModeEmbedding: + return provider, []string{"Embeddings"}, true + case servingModeRerank: + return provider, []string{"Rerank"}, true + } + return provider, chat, true +} + +// agentGatewayModelRoutingMessage returns why a routing: Model spec cannot +// compile, or "". The reconciler and the webhook share it. +func agentGatewayModelRoutingMessage(mr *inferencev1alpha1.ModelRouter) string { + if !agentGatewayModelRouting(mr) { + return "" + } + if len(mr.Spec.Rules) > 0 { + return fmt.Sprintf("rule %q does not compile with agentGatewayRef.routing: Model: agentgateway's model router "+ + "sends each request to the backend whose displayName (else name) equals its model field. Remove "+ + "spec.rules, or use routing: InferencePool", mr.Spec.Rules[0].Name) + } + if mr.Spec.DefaultRoute != "" { + return "spec.defaultRoute does not compile with agentGatewayRef.routing: Model: agentgateway answers an " + + "unknown model with 404 model_not_found. Remove it, or use routing: InferencePool" + } + servedBy := make(map[string]string, len(mr.Spec.Backends)) + for _, b := range mr.Spec.Backends { + switch { + case b.External != nil: + return fmt.Sprintf("backend %q is external; agentGatewayRef.routing: Model compiles only "+ + "inferenceServiceRef backends", b.Name) + case b.Resolution == inferencev1alpha1.RouterBackendResolutionEndpoint: + return fmt.Sprintf("backend %q sets resolution: endpoint; agentGatewayRef.routing: Model reaches the "+ + "InferenceService through its Service (resolution: service)", b.Name) + case b.EndpointPicker != nil: + return fmt.Sprintf("backend %q sets endpointPicker, which applies only to agentGatewayRef.routing: "+ + "InferencePool", b.Name) + } + served := agentGatewayServedName(b) + if len(served) > agentGatewayModelMatchMaxLength { + return fmt.Sprintf("backend %q serves a model name of %d characters; agentgateway's match.model accepts at most %d", + b.Name, len(served), agentGatewayModelMatchMaxLength) + } + if strings.Contains(served, "*") { + return fmt.Sprintf("backend %q serves model %q; agentgateway reads '*' in a model name as a wildcard", b.Name, served) + } + if other, taken := servedBy[served]; taken { + return fmt.Sprintf("backends %q and %q both serve model %q; agentgateway's model router needs one "+ + "backend per model name", other, b.Name, served) + } + servedBy[served] = b.Name + } + return "" +} + +// agentGatewayRefusalError is a refusal found during generation, reported as a +// condition rather than a reconcile error. +type agentGatewayRefusalError struct { + reason, message string +} + +func (e *agentGatewayRefusalError) Error() string { return e.message } + +type agentGatewayModelBackend struct { + ObjectName string + ServedName string + Service string + Port int64 + Provider string + Formats []string +} + +// newAgentGatewayModelRoute builds the router's HTTPRoute. The rule sits at "/" +// because agentgateway strips the rule's path prefix before it classifies the +// endpoint: under /v1, an embeddings request would parse as a completion. It +// reuses the InferencePool route's name, so a mode switch rewrites one route. +func newAgentGatewayModelRoute(mr *inferencev1alpha1.ModelRouter) *unstructured.Unstructured { + ref := mr.Spec.AgentGatewayRef + spec := map[string]interface{}{ + "parentRefs": []interface{}{agentGatewayRouteParentRef(ref)}, + "rules": []interface{}{ + map[string]interface{}{ + metadataNameField: agentGatewayModelRuleName, + "backendRefs": []interface{}{ + map[string]interface{}{ + metadataNameField: "*", + "kind": agentGatewayModelKind, + "group": agentGatewayPolicyGroup, + }, + }, + }, + }, + } + if len(ref.Hostnames) > 0 { + spec["hostnames"] = toInterfaceSlice(ref.Hostnames) + } + + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gatewayHTTPRouteGVK()) + u.SetName(modelRouterAgentGatewayResourceName(mr)) + u.SetNamespace(mr.Namespace) + u.SetLabels(map[string]string{agentGatewayRouterUIDLabel: string(mr.UID)}) + u.Object["spec"] = spec + return u +} + +// newAgentGatewayModel sets group and kind where the CRD would default them, +// so an unchanged spec causes no update. +func newAgentGatewayModel(mr *inferencev1alpha1.ModelRouter, route string, b agentGatewayModelBackend) *unstructured.Unstructured { + formats := make([]interface{}, 0, len(b.Formats)) + for _, f := range b.Formats { + formats = append(formats, map[string]interface{}{"type": f}) + } + + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(agentGatewayModelGVK()) + u.SetName(b.ObjectName) + u.SetNamespace(mr.Namespace) + u.SetLabels(map[string]string{agentGatewayRouterUIDLabel: string(mr.UID)}) + u.Object["spec"] = map[string]interface{}{ + "parentRefs": []interface{}{ + map[string]interface{}{ + metadataNameField: route, + "kind": "HTTPRoute", + "group": gatewayBackendRefGroupAPI, + "sectionName": agentGatewayModelRuleName, + }, + }, + "match": map[string]interface{}{"model": b.ServedName}, + "provider": "Custom", + "custom": map[string]interface{}{ + "backendRef": map[string]interface{}{ + metadataNameField: b.Service, + "port": b.Port, + }, + "providerOverride": b.Provider, + "formats": formats, + }, + } + return u +} + +// resolveAgentGatewayModelBackends returns an agentGatewayRefusalError for an +// unmapped runtime. +func (r *ModelRouterAgentGatewayReconciler) resolveAgentGatewayModelBackends( + ctx context.Context, + mr *inferencev1alpha1.ModelRouter, +) ([]agentGatewayModelBackend, error) { + resolved := make([]agentGatewayModelBackend, 0, len(mr.Spec.Backends)) + for _, b := range mr.Spec.Backends { + isvc, err := r.backendInferenceService(ctx, mr, b) + if err != nil { + return nil, err + } + provider, formats, ok := agentGatewayModelProvider(isvc) + if !ok { + return nil, &agentGatewayRefusalError{ + reason: modelRouterAgentGatewayReasonUnsupportedRuntime, + message: fmt.Sprintf("backend %q: InferenceService %q runs runtime %q, which agentGatewayRef.routing: "+ + "Model has no agentgateway provider mapping for (supported: llamacpp, vllm); use routing: "+ + "InferencePool for it", b.Name, isvc.Name, runtimeNameLabel(isvc)), + } + } + resolved = append(resolved, agentGatewayModelBackend{ + ObjectName: agentGatewayModelName(mr, b.Name), + ServedName: agentGatewayServedName(b), + Service: sanitizeDNSName(isvc.Name), + Port: gatewayServicePort(isvc), + Provider: provider, + Formats: formats, + }) + } + return resolved, nil +} + +// reconcileAgentGatewayModels applies the models and route, then deletes the +// models of removed backends and whatever routing: InferencePool left behind. +// Deleting last means a mode switch never points the route at deleted pools. +func (r *ModelRouterAgentGatewayReconciler) reconcileAgentGatewayModels(ctx context.Context, mr *inferencev1alpha1.ModelRouter) error { + backends, err := r.resolveAgentGatewayModelBackends(ctx, mr) + if err != nil { + return err + } + + // Models first, so a rejected model leaves the previous route in place. + route := newAgentGatewayModelRoute(mr) + desired := make([]*unstructured.Unstructured, 0, len(backends)+1) + keep := make(map[string]bool, len(backends)) + for _, b := range backends { + desired = append(desired, newAgentGatewayModel(mr, route.GetName(), b)) + keep[b.ObjectName] = true + } + desired = append(desired, route) + + // Refuse before any write if a generated name belongs to someone else. + for _, obj := range desired { + if err := r.checkGeneratedNameAvailable(ctx, mr, obj); err != nil { + return err + } + } + + for _, obj := range desired { + if err := r.applyAgentGatewayResource(ctx, mr, obj); err != nil { + return fmt.Errorf("%s/%s: %w", obj.GetKind(), obj.GetName(), err) + } + } + if err := r.deleteGeneratedObjects(ctx, mr, keep, agentGatewayModelGVK()); err != nil { + return err + } + if err := r.deleteGeneratedObjects(ctx, mr, nil, inferencePoolGVK()); err != nil { + return err + } + return r.withdrawAgentGatewayPolicy(ctx, mr) +} diff --git a/internal/controller/modelrouter_agentgateway_model_test.go b/internal/controller/modelrouter_agentgateway_model_test.go new file mode 100644 index 000000000..3c3a29a24 --- /dev/null +++ b/internal/controller/modelrouter_agentgateway_model_test.go @@ -0,0 +1,718 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "path/filepath" + "reflect" + "strings" + "testing" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + inferencev1alpha1 "github.com/defilantech/llmkube/api/v1alpha1" +) + +func TestAgentGatewayModelProvider(t *testing.T) { + chatVLLM := []string{"Completions", "Messages", "AnthropicTokenCount", "Responses"} + tests := []struct { + name string + spec inferencev1alpha1.InferenceServiceSpec + wantProvider string + wantFormats []string + wantOK bool + }{ + {"unset runtime is llama.cpp chat", inferencev1alpha1.InferenceServiceSpec{}, "llamacpp", []string{"Completions"}, true}, + {"llamacpp chat", inferencev1alpha1.InferenceServiceSpec{Runtime: "llamacpp"}, "llamacpp", []string{"Completions"}, true}, + {"llamacpp embedding mode", inferencev1alpha1.InferenceServiceSpec{Mode: "embedding"}, "llamacpp", []string{"Embeddings"}, true}, + {"llamacpp rerank inferred from --reranking", inferencev1alpha1.InferenceServiceSpec{ + ExtraArgs: []string{"--reranking", "--embedding"}, + }, "llamacpp", []string{"Rerank"}, true}, + {"llamacpp embedding inferred from endpoint path", inferencev1alpha1.InferenceServiceSpec{ + Endpoint: &inferencev1alpha1.EndpointSpec{Path: "/v1/embeddings"}, + }, "llamacpp", []string{"Embeddings"}, true}, + {"vllm chat", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeVLLM}, "vllm", chatVLLM, true}, + {"vllm embedding", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeVLLM, Mode: "embedding"}, "vllm", []string{"Embeddings"}, true}, + {"vllm rerank", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeVLLM, Mode: "rerank"}, "vllm", []string{"Rerank"}, true}, + {"sglang is unmapped", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeSGLANG}, "", nil, false}, + {"tgi is unmapped", inferencev1alpha1.InferenceServiceSpec{Runtime: "tgi"}, "", nil, false}, + {"llamacpp-router is unmapped", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeLlamaCppRouter}, "", nil, false}, + {"generic is unmapped", inferencev1alpha1.InferenceServiceSpec{Runtime: "generic"}, "", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider, formats, ok := agentGatewayModelProvider(&inferencev1alpha1.InferenceService{Spec: tt.spec}) + if ok != tt.wantOK || provider != tt.wantProvider || !reflect.DeepEqual(formats, tt.wantFormats) { + t.Errorf("agentGatewayModelProvider = (%q, %v, %v), want (%q, %v, %v)", + provider, formats, ok, tt.wantProvider, tt.wantFormats, tt.wantOK) + } + }) + } +} + +func TestAgentGatewayModelName(t *testing.T) { + router := func(name string) *inferencev1alpha1.ModelRouter { + return &inferencev1alpha1.ModelRouter{ObjectMeta: metav1.ObjectMeta{Name: name}} + } + long := strings.Repeat("r", 60) + + tests := []struct { + name, router, backend, want string + }{ + {"short names join", "llm-router", "chat-llama", "llm-router-chat-llama"}, + {"dots in the router name are sanitized", "ai.models", "chat-llama", "ai-models-chat-llama"}, + {"a long pair is cut and hashed", long, "chat-llama", ""}, + {"a trailing dash is hashed away", "llm-router", "chat-", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := agentGatewayModelName(router(tt.router), tt.backend) + if errs := validation.IsDNS1123Label(got); len(errs) > 0 { + t.Fatalf("agentGatewayModelName(%q, %q) = %q, not a DNS-1123 label: %v", tt.router, tt.backend, got, errs) + } + if tt.want != "" && got != tt.want { + t.Errorf("agentGatewayModelName(%q, %q) = %q, want %q", tt.router, tt.backend, got, tt.want) + } + if again := agentGatewayModelName(router(tt.router), tt.backend); again != got { + t.Errorf("name is not deterministic: %q then %q", got, again) + } + }) + } + + // Two routers in one namespace serving the same backend name get two + // objects; so do two long pairs that only differ past the cut. + if a, b := agentGatewayModelName(router("team-a"), "chat-llama"), agentGatewayModelName(router("team-b"), "chat-llama"); a == b { + t.Errorf("two routers derived the same model name %q", a) + } + if a, b := agentGatewayModelName(router(long), "backend-one"), agentGatewayModelName(router(long), "backend-two"); a == b { + t.Errorf("two long pairs that differ past the cut derived the same name %q", a) + } + // "chat" and "chat-" are distinct backend names and must stay distinct. + if a, b := agentGatewayModelName(router("llm-router"), "chat"), agentGatewayModelName(router("llm-router"), "chat-"); a == b { + t.Errorf("backends chat and chat- derived the same name %q", a) + } +} + +// modelRoutingRouter attaches to ai-gateway's http listener, one backend per +// InferenceService. +func modelRoutingRouter(ns, name string, isvcs ...string) *inferencev1alpha1.ModelRouter { + mr := &inferencev1alpha1.ModelRouter{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: inferencev1alpha1.ModelRouterSpec{ + DataPlane: inferencev1alpha1.ModelRouterDataPlaneAgentGateway, + AgentGatewayRef: &inferencev1alpha1.AgentGatewayReference{ + Name: "ai-gateway", + Routing: inferencev1alpha1.AgentGatewayRoutingModel, + SectionName: "http", + }, + }, + } + for _, isvc := range isvcs { + mr.Spec.Backends = append(mr.Spec.Backends, + inferencev1alpha1.RouterBackend{Name: isvc, InferenceServiceRef: corev1LocalRef(isvc)}) + } + return mr +} + +func TestAgentGatewayModelRoutingMessage(t *testing.T) { + tests := []struct { + name string + mutate func(*inferencev1alpha1.ModelRouter) + want string + }{ + {"clean", func(*inferencev1alpha1.ModelRouter) {}, ""}, + {"rules", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Rules = []inferencev1alpha1.RouterRule{{ + Name: "by-name", + Match: &inferencev1alpha1.RuleMatch{Models: []string{"a"}}, + Route: inferencev1alpha1.RuleRoute{Backends: []string{"a"}}, + }} + }, `rule "by-name"`}, + {"defaultRoute", func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.DefaultRoute = "a" }, "spec.defaultRoute"}, + {"external backend", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Backends = append(mr.Spec.Backends, inferencev1alpha1.RouterBackend{ + Name: "cloud", External: &inferencev1alpha1.ExternalProvider{Provider: "openai", Model: "gpt-5"}, + }) + }, `backend "cloud" is external`}, + {"resolution endpoint", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Backends[0].Resolution = inferencev1alpha1.RouterBackendResolutionEndpoint + }, "resolution: endpoint"}, + {"backend endpointPicker", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Backends[0].EndpointPicker = &inferencev1alpha1.RouterEndpointPicker{ + Service: &inferencev1alpha1.EndpointPickerService{Name: "epp", Port: 9002}, + } + }, "sets endpointPicker"}, + {"wildcard served name", func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.Backends[0].DisplayName = "chat-*" }, "wildcard"}, + {"served name over match.model's limit", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Backends[0].DisplayName = strings.Repeat("m", agentGatewayModelMatchMaxLength+1) + }, "at most 1024"}, + {"duplicate served name", func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.Backends[1].DisplayName = "a" }, `both serve model "a"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mr := modelRoutingRouter("ns", "router", "a", "b") + tt.mutate(mr) + got := agentGatewayModelRoutingMessage(mr) + if tt.want == "" && got != "" { + t.Fatalf("clean router refused: %s", got) + } + if !strings.Contains(got, tt.want) { + t.Errorf("agentGatewayModelRoutingMessage = %q, want it to contain %q", got, tt.want) + } + }) + } +} + +func TestModelRouterValidator_AgentGatewayModelRouting(t *testing.T) { + v := &ModelRouterValidator{} + + if _, err := v.ValidateCreate(context.Background(), modelRoutingRouter("ns", "router", "a", "b")); err != nil { + t.Errorf("a routing: Model router without pickers must be admitted: %v", err) + } + + pool := modelRoutingRouter("ns", "router", "a", "b") + pool.Spec.AgentGatewayRef.Routing = inferencev1alpha1.AgentGatewayRoutingInferencePool + pool.Spec.AgentGatewayRef.SectionName = "" + if _, err := v.ValidateCreate(context.Background(), pool); err == nil || !strings.Contains(err.Error(), "Endpoint Picker") { + t.Errorf("routing: InferencePool without a picker: got %v, want the EndpointPickerRequired message", err) + } + + withRule := modelRoutingRouter("ns", "router", "a") + withRule.Spec.Rules = []inferencev1alpha1.RouterRule{{ + Name: "r", + Match: &inferencev1alpha1.RuleMatch{Models: []string{"a"}}, + Route: inferencev1alpha1.RuleRoute{Backends: []string{"a"}}, + }} + if _, err := v.ValidateCreate(context.Background(), withRule); err == nil || !strings.Contains(err.Error(), "routing: Model") { + t.Errorf("routing: Model with rules: got %v, want the routing: Model refusal", err) + } +} + +// An empty want means the router must be admitted. +func TestModelRouterAgentGateway_ModelRoutingCEL(t *testing.T) { + c, _, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + tests := []struct { + name, want string + mutate func(*inferencev1alpha1.AgentGatewayReference) + }{ + {"sectionName", "", func(*inferencev1alpha1.AgentGatewayReference) {}}, + {"hostnames only", "", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.SectionName = "" + ref.Hostnames = []string{"llm.example.com"} + }}, + {"no listener and no hostnames", "requires sectionName or hostnames", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.SectionName = "" + }}, + {"router-wide picker", "apply only to routing: InferencePool", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.EndpointPicker, ref.EndpointPickerPort = "epp", 9002 + }}, + {"sectionName on InferencePool", "apply only to routing: Model", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.Routing = inferencev1alpha1.AgentGatewayRoutingInferencePool + }}, + {"hostnames on InferencePool", "apply only to routing: Model", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.Routing, ref.SectionName = inferencev1alpha1.AgentGatewayRoutingInferencePool, "" + ref.Hostnames = []string{"llm.example.com"} + }}, + {"sectionName that is not a Gateway API section name", "spec.agentGatewayRef.sectionName", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.SectionName = "HTTP" + }}, + {"hostname that is not a Gateway API hostname", "spec.agentGatewayRef.hostnames[0]", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.Hostnames = []string{"LLM.example.com"} + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mr := modelRoutingRouter(ns, "", "a") + mr.GenerateName = "cel-" + tt.mutate(mr.Spec.AgentGatewayRef) + err := c.Create(context.Background(), mr) + if tt.want == "" { + if err != nil { + t.Errorf("create must be admitted: %v", err) + } + } else if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Errorf("create: got %v, want an error containing %q", err, tt.want) + } + }) + } +} + +// A router written before routing existed compiles exactly as before. +func TestModelRouterAgentGateway_InferencePoolStaysDefault(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "legacy-cuda") + mr := sameNamespaceRouterFixture(ns, "legacy", "legacy-cuda") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("a router without routing must be admitted: %v", err) + } + if got := mr.Spec.AgentGatewayRef.Routing; got != inferencev1alpha1.AgentGatewayRoutingInferencePool { + t.Errorf("routing defaulted to %q, want InferencePool", got) + } + reconcileAgentGatewayRouter(t, newModelRouterAgentGatewayReconciler(t, cfg), mr) + + getUnstructuredInNamespace(t, c, inferencePoolGVK(), ns, "legacy-cuda") + getPolicyInNamespace(t, c, ns, "legacy") + route := getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "legacy") + parentRefs, _, _ := unstructured.NestedSlice(route.Object, "spec", "parentRefs") + wantParents := []interface{}{map[string]interface{}{ + "name": "ai-gateway", "kind": "Gateway", "group": "gateway.networking.k8s.io", + }} + if !reflect.DeepEqual(parentRefs, wantParents) { + t.Errorf("route parentRefs = %v, want %v", parentRefs, wantParents) + } + if _, has, _ := unstructured.NestedSlice(route.Object, "spec", "hostnames"); has { + t.Errorf("route grew hostnames: %v", route.Object["spec"]) + } + if models := listGeneratedNames(t, c, mr, agentGatewayModelGVK()); len(models) != 0 { + t.Errorf("routing: InferencePool generated AgentgatewayModels %v", models) + } +} + +func makeModelISvc(t *testing.T, c client.Client, ns, name, runtime, mode string, port int32) { + t.Helper() + isvc := &inferencev1alpha1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: inferencev1alpha1.InferenceServiceSpec{ + ModelRef: name, + Runtime: runtime, + Mode: mode, + Endpoint: &inferencev1alpha1.EndpointSpec{Port: port}, + }, + } + if err := c.Create(context.Background(), isvc); err != nil { + t.Fatalf("create isvc %s: %v", name, err) + } + setISvcReadyReplicasInNamespace(t, c, ns, name, 1) +} + +func agentGatewayReadyCondition(t *testing.T, c client.Client, mr *inferencev1alpha1.ModelRouter) (*metav1.Condition, *inferencev1alpha1.ModelRouter) { + t.Helper() + fresh := &inferencev1alpha1.ModelRouter{} + if err := c.Get(context.Background(), types.NamespacedName{Name: mr.Name, Namespace: mr.Namespace}, fresh); err != nil { + t.Fatalf("get modelrouter: %v", err) + } + return apimeta.FindStatusCondition(fresh.Status.Conditions, ModelRouterAgentGatewayConditionReady), fresh +} + +func listGeneratedNames(t *testing.T, c client.Client, mr *inferencev1alpha1.ModelRouter, gvk schema.GroupVersionKind) []string { + t.Helper() + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk) + if err := c.List(context.Background(), list, client.InNamespace(mr.Namespace), + client.MatchingLabels{agentGatewayRouterUIDLabel: string(mr.UID)}); err != nil { + t.Fatalf("list %s: %v", gvk.Kind, err) + } + names := make([]string, 0, len(list.Items)) + for _, item := range list.Items { + names = append(names, item.GetName()) + } + return names +} + +// One model per runtime and format, validated by the upstream CRD. +func TestModelRouterAgentGateway_ModelRoutingCompiles(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeModelISvc(t, c, ns, "chat-vllm", RuntimeVLLM, "", 8000) + makeBackendISvcInNamespace(t, c, ns, "chat-llama") + makeModelISvc(t, c, ns, "reranker", "llamacpp", "rerank", 8080) + makeModelISvc(t, c, ns, "embedder", "", "embedding", 8080) + + mr := modelRoutingRouter(ns, "llm-router", "chat-vllm", "chat-llama", "reranker", "embedder") + mr.Spec.Backends[0].DisplayName = "chat-large" + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + + r := newModelRouterAgentGatewayReconciler(t, cfg) + reconcileAgentGatewayRouter(t, r, mr) + + cond, fresh := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Status != metav1.ConditionTrue || !strings.Contains(cond.Message, "compiled 4 model(s)") { + t.Fatalf("expected AgentGatewayReady=True compiling 4 models, got %+v", cond) + } + if fresh.Status.AgentGateway == nil || fresh.Status.AgentGateway.PoolReady { + t.Errorf("status.agentGateway = %+v, want set with poolReady false (no pool in routing: Model)", fresh.Status.AgentGateway) + } + + route := getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "llm-router") + assertOwnedByRouter(t, route, fresh) + wantRoute := map[string]interface{}{ + "parentRefs": []interface{}{map[string]interface{}{ + "name": "ai-gateway", "kind": "Gateway", "group": "gateway.networking.k8s.io", "sectionName": "http", + }}, + "rules": []interface{}{map[string]interface{}{ + "name": "models", + "backendRefs": []interface{}{map[string]interface{}{ + "name": "*", "kind": "AgentgatewayModel", "group": "agentgateway.dev", + }}, + }}, + } + if !reflect.DeepEqual(route.Object["spec"], wantRoute) { + t.Errorf("route spec = %#v\nwant %#v", route.Object["spec"], wantRoute) + } + + model := func(served, service string, port int64, provider string, formats ...string) map[string]interface{} { + fs := make([]interface{}, 0, len(formats)) + for _, f := range formats { + fs = append(fs, map[string]interface{}{"type": f}) + } + return map[string]interface{}{ + "parentRefs": []interface{}{map[string]interface{}{ + "name": "llm-router", "kind": "HTTPRoute", "group": "gateway.networking.k8s.io", "sectionName": "models", + }}, + "match": map[string]interface{}{"model": served}, + "provider": "Custom", + "custom": map[string]interface{}{ + "backendRef": map[string]interface{}{"name": service, "port": port}, + "providerOverride": provider, + "formats": fs, + }, + } + } + want := map[string]map[string]interface{}{ + "llm-router-chat-vllm": model("chat-large", "chat-vllm", 8000, "vllm", + "Completions", "Messages", "AnthropicTokenCount", "Responses"), + "llm-router-chat-llama": model("chat-llama", "chat-llama", 8080, "llamacpp", "Completions"), + "llm-router-reranker": model("reranker", "reranker", 8080, "llamacpp", "Rerank"), + "llm-router-embedder": model("embedder", "embedder", 8080, "llamacpp", "Embeddings"), + } + for name, spec := range want { + got := getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, name) + assertOwnedByRouter(t, got, fresh) + if label := got.GetLabels()[agentGatewayRouterUIDLabel]; label != string(fresh.UID) { + t.Errorf("%s uid label = %q, want %q", name, label, fresh.UID) + } + if !reflect.DeepEqual(got.Object["spec"], spec) { + t.Errorf("%s spec = %#v\nwant %#v", name, got.Object["spec"], spec) + } + } + + if pools := listGeneratedNames(t, c, fresh, inferencePoolGVK()); len(pools) != 0 { + t.Errorf("routing: Model generated InferencePools %v", pools) + } + assertNotExistsInNamespace(t, c, agentGatewayPolicyGVK(), ns, "llm-router") +} + +// Served names that are not object names, from two routers in one namespace. +func TestModelRouterAgentGateway_ModelRoutingServedNames(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "chat-llama") + makeModelISvc(t, c, ns, "org-model", RuntimeVLLM, "", 8000) + + r := newModelRouterAgentGatewayReconciler(t, cfg) + for _, name := range []string{"team-a", "team-b"} { + mr := modelRoutingRouter(ns, name, "chat-llama", "org-model") + mr.Spec.Backends[0].DisplayName = "Qwen3.5-2B" + mr.Spec.Backends[1].DisplayName = "org/model" + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter %s: %v", name, err) + } + reconcileAgentGatewayRouter(t, r, mr) + if cond, _ := agentGatewayReadyCondition(t, c, mr); cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("router %s: expected AgentGatewayReady=True, got %+v", name, cond) + } + + for backend, served := range map[string]string{"chat-llama": "Qwen3.5-2B", "org-model": "org/model"} { + m := getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, name+"-"+backend) + if got, _, _ := unstructured.NestedString(m.Object, "spec", "match", "model"); got != served { + t.Errorf("%s match.model = %q, want %q", m.GetName(), got, served) + } + } + } +} + +func TestModelRouterAgentGateway_ModelRoutingHostnames(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "chat-llama") + mr := modelRoutingRouter(ns, "by-host", "chat-llama") + mr.Spec.AgentGatewayRef.SectionName = "" + mr.Spec.AgentGatewayRef.Hostnames = []string{"llm.example.com", "*.llm.example.com"} + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, newModelRouterAgentGatewayReconciler(t, cfg), mr) + + route := getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "by-host") + hostnames, _, _ := unstructured.NestedStringSlice(route.Object, "spec", "hostnames") + if want := []string{"llm.example.com", "*.llm.example.com"}; !reflect.DeepEqual(hostnames, want) { + t.Errorf("route hostnames = %v, want %v", hostnames, want) + } + parentRefs, _, _ := unstructured.NestedSlice(route.Object, "spec", "parentRefs") + if len(parentRefs) != 1 || parentRefs[0].(map[string]interface{})["sectionName"] != nil { + t.Errorf("route parentRefs = %v, want one parentRef without sectionName", parentRefs) + } +} + +func TestModelRouterAgentGateway_ModelRoutingPrunesRemovedBackend(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "keep") + makeBackendISvcInNamespace(t, c, ns, "drop") + + mr := modelRoutingRouter(ns, "prune", "keep", "drop") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + r := newModelRouterAgentGatewayReconciler(t, cfg) + reconcileAgentGatewayRouter(t, r, mr) + getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, "prune-drop") + + _, fresh := agentGatewayReadyCondition(t, c, mr) + fresh.Spec.Backends = fresh.Spec.Backends[:1] + if err := c.Update(context.Background(), fresh); err != nil { + t.Fatalf("remove backend: %v", err) + } + reconcileAgentGatewayRouter(t, r, fresh) + + if got := listGeneratedNames(t, c, fresh, agentGatewayModelGVK()); !reflect.DeepEqual(got, []string{"prune-keep"}) { + t.Errorf("models after removing backend drop = %v, want [prune-keep]", got) + } +} + +// InferencePool -> Model -> InferencePool -> Model -> Proxy, with a +// cross-namespace Gateway so the extraction policy carries a finalizer. +func TestModelRouterAgentGateway_ModelRoutingSwitchesCleanUp(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + gwNS := newTestNamespace(t, c) + ctx := context.Background() + + makeGateway(t, c, gwNS, "ai-gateway", nil, []interface{}{gatewayListenerAllowingAll()}) + makeBackendISvcInNamespace(t, c, ns, "switch-cuda") + + mr := crossNamespaceRouterFixture(ns, "switcher", gwNS, "switch-cuda") + if err := c.Create(ctx, mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + r := newModelRouterAgentGatewayReconciler(t, cfg) + reconcileAgentGatewayRouter(t, r, mr) + policyName := ns + "-switcher" + getPolicyInNamespace(t, c, gwNS, policyName) + getUnstructuredInNamespace(t, c, inferencePoolGVK(), ns, "switch-cuda") + + update := func(mutate func(*inferencev1alpha1.ModelRouter)) *inferencev1alpha1.ModelRouter { + t.Helper() + _, fresh := agentGatewayReadyCondition(t, c, mr) + mutate(fresh) + if err := c.Update(ctx, fresh); err != nil { + t.Fatalf("update modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, r, fresh) + return fresh + } + + // InferencePool -> Model. + toModel := func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Rules = nil + mr.Spec.AgentGatewayRef.Routing = inferencev1alpha1.AgentGatewayRoutingModel + mr.Spec.AgentGatewayRef.SectionName = "http" + mr.Spec.AgentGatewayRef.EndpointPicker, mr.Spec.AgentGatewayRef.EndpointPickerPort = "", 0 + } + update(toModel) + cond, fresh := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("after switching to routing: Model: expected AgentGatewayReady=True, got %+v", cond) + } + assertNoPolicyInNamespace(t, c, gwNS, policyName) + assertNotExistsInNamespace(t, c, inferencePoolGVK(), ns, "switch-cuda") + if controllerutil.ContainsFinalizer(fresh, modelRouterAgentGatewayFinalizer) { + t.Errorf("routing: Model kept the cross-namespace policy finalizer") + } + getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, "switcher-switch-cuda") + route := getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "switcher") + if rules, _, _ := unstructured.NestedSlice(route.Object, "spec", "rules"); len(rules) != 1 || + rules[0].(map[string]interface{})["name"] != agentGatewayModelRuleName { + t.Errorf("route was not rewritten to the model router rule: %v", rules) + } + + // Model -> InferencePool. + update(func(mr *inferencev1alpha1.ModelRouter) { + back := crossNamespaceRouterFixture(ns, "switcher", gwNS, "switch-cuda") + mr.Spec.AgentGatewayRef = back.Spec.AgentGatewayRef + mr.Spec.Rules = back.Spec.Rules + }) + if cond, _ := agentGatewayReadyCondition(t, c, mr); cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("after switching back to InferencePool: expected AgentGatewayReady=True, got %+v", cond) + } + assertNotExistsInNamespace(t, c, agentGatewayModelGVK(), ns, "switcher-switch-cuda") + getPolicyInNamespace(t, c, gwNS, policyName) + getUnstructuredInNamespace(t, c, inferencePoolGVK(), ns, "switch-cuda") + + // Model -> Proxy: leaving the plane sweeps the route and the models. + update(toModel) + getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, "switcher-switch-cuda") + update(func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.DataPlane = inferencev1alpha1.ModelRouterDataPlaneProxy + }) + assertNotExistsInNamespace(t, c, agentGatewayModelGVK(), ns, "switcher-switch-cuda") + assertNotExistsInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "switcher") +} + +func TestModelRouterAgentGateway_ModelRoutingRefusesUnownedModel(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "chat-llama") + + handmade := &unstructured.Unstructured{} + handmade.SetGroupVersionKind(agentGatewayModelGVK()) + handmade.SetName("llm-router-chat-llama") + handmade.SetNamespace(ns) + handmade.Object["spec"] = map[string]interface{}{ + "parentRefs": []interface{}{map[string]interface{}{"kind": "HTTPRoute", "name": "hand-written"}}, + "provider": "Custom", + "custom": map[string]interface{}{ + "backendRef": map[string]interface{}{"name": "chat-llama", "port": int64(8080)}, + "formats": []interface{}{map[string]interface{}{"type": "Completions"}}, + }, + } + if err := c.Create(context.Background(), handmade); err != nil { + t.Fatalf("create handmade model: %v", err) + } + + mr := modelRoutingRouter(ns, "llm-router", "chat-llama") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + r := newModelRouterAgentGatewayReconciler(t, cfg) + res, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: mr.Name, Namespace: ns}}) + if err != nil { + t.Fatalf("reconcile must refuse, not error: %v", err) + } + if res.RequeueAfter != agentGatewayConflictRequeueAfter { + t.Errorf("requeue after = %v, want %v (the name-taken recheck)", res.RequeueAfter, agentGatewayConflictRequeueAfter) + } + cond, _ := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Reason != modelRouterAgentGatewayReasonResourceNameTaken || + !strings.Contains(cond.Message, "AgentgatewayModel "+ns+"/llm-router-chat-llama") { + t.Fatalf("expected AgentGatewayReady=False/%s naming the model, got %+v", modelRouterAgentGatewayReasonResourceNameTaken, cond) + } + assertForeignObjectUntouched(t, c, handmade) + assertNotExistsInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "llm-router") +} + +func TestModelRouterAgentGateway_ModelRoutingRefusals(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "llama") + makeModelISvc(t, c, ns, "sglang", RuntimeSGLANG, "", 8080) + r := newModelRouterAgentGatewayReconciler(t, cfg) + + tests := []struct { + name, isvc, reason, message string + mutate func(*inferencev1alpha1.ModelRouter) + }{ + {"default-route", "llama", modelRouterAgentGatewayReasonUnsupportedModelRouting, "spec.defaultRoute", + func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.DefaultRoute = "llama" }}, + {"unmapped-runtime", "sglang", modelRouterAgentGatewayReasonUnsupportedRuntime, `runtime "sglang"`, + func(*inferencev1alpha1.ModelRouter) {}}, + {"unknown-listener", "llama", modelRouterAgentGatewayReasonNotAllowed, `no listener named "models"`, + func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.AgentGatewayRef.SectionName = "models" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mr := modelRoutingRouter(ns, "refused-"+tt.name, tt.isvc) + tt.mutate(mr) + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, r, mr) + cond, _ := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != tt.reason || !strings.Contains(cond.Message, tt.message) { + t.Fatalf("expected AgentGatewayReady=False/%s containing %q, got %+v", tt.reason, tt.message, cond) + } + assertNotExistsInNamespace(t, c, gatewayHTTPRouteGVK(), ns, mr.Name) + assertNotExistsInNamespace(t, c, agentGatewayModelGVK(), ns, mr.Name+"-"+tt.isvc) + }) + } +} + +// routing: Model needs the AgentgatewayModel CRD and nothing else. +func TestModelRouterAgentGateway_ModelRoutingCRDGate(t *testing.T) { + reconcileGate := func(t *testing.T, extraCRDs ...string) (client.Client, *metav1.Condition) { + t.Helper() + crds := append([]string{ + filepath.Join(inferenceExtensionTestCRDDir, "gateway.yaml"), + filepath.Join(inferenceExtensionTestCRDDir, "httproute.yaml"), + }, extraCRDs...) + c, cfg, stop := startAgentGatewayTestEnvWithCRDs(t, crds...) + t.Cleanup(stop) + makeGateway(t, c, testNS, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, testNS, "gate-llama") + mr := modelRoutingRouter(testNS, "gate", "gate-llama") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, newModelRouterAgentGatewayReconciler(t, cfg), mr) + cond, _ := agentGatewayReadyCondition(t, c, mr) + return c, cond + } + + t.Run("without the AgentgatewayModel CRD", func(t *testing.T) { + c, cond := reconcileGate(t) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != modelRouterAgentGatewayReasonModelCRDMissing || + !strings.Contains(cond.Message, "AgentgatewayModel (agentgateway.dev/v1alpha1)") { + t.Errorf("expected AgentGatewayReady=False/%s naming the kind, got %+v", modelRouterAgentGatewayReasonModelCRDMissing, cond) + } + assertNotExistsInNamespace(t, c, gatewayHTTPRouteGVK(), testNS, "gate") + }) + + t.Run("with only the AgentgatewayModel CRD", func(t *testing.T) { + c, cond := reconcileGate(t, filepath.Join(inferenceExtensionTestCRDDir, "agentgatewaymodel.yaml")) + if cond == nil || cond.Status != metav1.ConditionTrue { + t.Errorf("expected AgentGatewayReady=True, got %+v", cond) + } + getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), testNS, "gate-gate-llama") + }) +} diff --git a/internal/controller/modelrouter_agentgateway_picker.go b/internal/controller/modelrouter_agentgateway_picker.go index df2522425..eab9b6acb 100644 --- a/internal/controller/modelrouter_agentgateway_picker.go +++ b/internal/controller/modelrouter_agentgateway_picker.go @@ -37,8 +37,12 @@ type agentGatewayPicker struct { // every pool resolves exactly one picker and no picker serves two pools. A // non-empty reason names the refusal; the reconciler and the webhook both call // this, so admission and reconcile cannot diverge. External backends are not -// counted: the reconciler refuses them separately. +// counted: the reconciler refuses them separately. routing: Model compiles no +// pool, so it resolves none. func agentGatewayResolvePoolPickers(mr *inferencev1alpha1.ModelRouter) (map[string]agentGatewayPicker, string, string) { + if agentGatewayModelRouting(mr) { + return nil, "", "" + } var def *agentGatewayPicker if ref := mr.Spec.AgentGatewayRef; ref != nil && ref.EndpointPicker != "" { def = &agentGatewayPicker{Name: ref.EndpointPicker, Port: ref.EndpointPickerPort} diff --git a/internal/controller/modelrouter_agentgateway_rbac_test.go b/internal/controller/modelrouter_agentgateway_rbac_test.go index b0c8750ab..9128dd0e6 100644 --- a/internal/controller/modelrouter_agentgateway_rbac_test.go +++ b/internal/controller/modelrouter_agentgateway_rbac_test.go @@ -43,9 +43,10 @@ func TestModelRouterAgentGateway_RBACCoversClientVerbs(t *testing.T) { // resource: the guard chain Gets the referenced Gateway and, for // allowedRoutes Selector matching, the router's namespace; backend // resolution Gets InferenceServices; the apply creates/updates pools, the - // extraction policy, and the HTTPRoute; the conflict detector Lists - // policies; the deletion, leaving, and consent-withdrawal sweeps List and - // Delete policies, pools, and routes; the pre-flight name check Gets each + // extraction policy, the HTTPRoute, and (routing: Model) the + // AgentgatewayModels; the conflict detector Lists policies; the deletion, + // leaving, consent-withdrawal, and mode-switch sweeps and the stale-model + // prune List and Delete policies, pools, routes, and models; the pre-flight name check Gets each // generated object; status and finalizer writes update and patch the // router; the Recorder emits events. // @@ -66,6 +67,7 @@ func TestModelRouterAgentGateway_RBACCoversClientVerbs(t *testing.T) { {"inference.networking.k8s.io/inferencepools", []string{"get", "list", "create", "update", "patch", "delete"}}, {"gateway.networking.k8s.io/httproutes", []string{"get", "list", "create", "update", "patch", "delete"}}, {"agentgateway.dev/agentgatewaypolicies", []string{"get", "list", "create", "update", "patch", "delete"}}, + {"agentgateway.dev/agentgatewaymodels", []string{"get", "list", "create", "update", "patch", "delete"}}, {"gateway.networking.k8s.io/gateways", []string{"get"}}, {"/namespaces", []string{"get", "list", "watch"}}, {"/events", []string{"create", "patch"}}, diff --git a/internal/controller/modelrouter_agentgateway_test.go b/internal/controller/modelrouter_agentgateway_test.go index 3b1e635f0..5042bf610 100644 --- a/internal/controller/modelrouter_agentgateway_test.go +++ b/internal/controller/modelrouter_agentgateway_test.go @@ -2568,8 +2568,8 @@ func newCountingModelRouterAgentGatewayReconciler(t *testing.T, cfg *rest.Config } // TestModelRouterAgentGateway_SweepOnlyForUsedRouters pins the evidence gate -// on the leave-AgentGateway sweep: the sweep runs three uncached Lists -// (cluster-wide policies, pools, routes), so a router that never used +// on the leave-AgentGateway sweep: the sweep runs four uncached Lists +// (cluster-wide policies, pools, routes, models), so a router that never used // AgentGateway mode must make none, and a router that switches away must be // swept exactly once - after the sweep clears the plane's status, later // reconciles find no evidence and skip it. @@ -2629,14 +2629,14 @@ func TestModelRouterAgentGateway_SweepOnlyForUsedRouters(t *testing.T) { } // The switch reconcile sweeps: one List each for policies - // (cluster-wide), pools, and routes. + // (cluster-wide), pools, routes, and models. if _, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{Name: mr.Name, Namespace: ns}, }); err != nil { t.Fatalf("reconcile after switch: %v", err) } - if got := counter.lists - before; got != 3 { - t.Errorf("the switch reconcile made %d List calls, want exactly 3 (one per swept kind)", got) + if got := counter.lists - before; got != 4 { + t.Errorf("the switch reconcile made %d List calls, want exactly 4 (one per swept kind)", got) } assertAllGoneLike(t, c, ns, mr.Name, "sweep1-cuda") @@ -2649,7 +2649,7 @@ func TestModelRouterAgentGateway_SweepOnlyForUsedRouters(t *testing.T) { t.Fatalf("reconcile %d after sweep: %v", i+1, err) } } - if got := counter.lists - before - 3; got != 0 { + if got := counter.lists - before - 4; got != 0 { t.Errorf("later reconciles made %d List calls after the sweep, want 0", got) } }) @@ -2674,14 +2674,14 @@ func TestModelRouterAgentGateway_SweepOnlyForUsedRouters(t *testing.T) { } // The ref-removal reconcile sweeps: one List each for policies - // (cluster-wide), pools, and routes. + // (cluster-wide), pools, routes, and models. if _, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{Name: mr.Name, Namespace: ns}, }); err != nil { t.Fatalf("reconcile after ref removal: %v", err) } - if got := counter.lists - before; got != 3 { - t.Errorf("the ref-removal reconcile made %d List calls, want exactly 3 (one per swept kind)", got) + if got := counter.lists - before; got != 4 { + t.Errorf("the ref-removal reconcile made %d List calls, want exactly 4 (one per swept kind)", got) } assertAllGoneLike(t, c, ns, mr.Name, "sweep2-cuda") @@ -2694,7 +2694,7 @@ func TestModelRouterAgentGateway_SweepOnlyForUsedRouters(t *testing.T) { t.Fatalf("reconcile %d after sweep: %v", i+1, err) } } - if got := counter.lists - before - 3; got != 0 { + if got := counter.lists - before - 4; got != 0 { t.Errorf("later reconciles made %d List calls after the sweep, want 0", got) } refless := &inferencev1alpha1.ModelRouter{} diff --git a/internal/controller/modelrouter_webhook.go b/internal/controller/modelrouter_webhook.go index a4dba0e0f..52460778d 100644 --- a/internal/controller/modelrouter_webhook.go +++ b/internal/controller/modelrouter_webhook.go @@ -177,6 +177,9 @@ func agentGatewayModeViolations(mr *inferencev1alpha1.ModelRouter, specPath *fie if _, reason, msg := agentGatewayResolvePoolPickers(mr); reason != "" { errs = append(errs, field.Invalid(specPath.Child("backends"), nil, msg)) } + if msg := agentGatewayModelRoutingMessage(mr); msg != "" { + errs = append(errs, field.Invalid(specPath, nil, msg)) + } if msg := agentGatewayUnsupportedMatchMessage(mr); msg != "" { errs = append(errs, field.Invalid(specPath.Child("rules"), nil, msg)) } diff --git a/test/crd/inference-extension/README.md b/test/crd/inference-extension/README.md index e1470f51a..c9c6018ef 100644 --- a/test/crd/inference-extension/README.md +++ b/test/crd/inference-extension/README.md @@ -9,12 +9,15 @@ schemas on purpose: the generated InferencePool spec must pass server-side validation (a flat `selector` map was once emitted here and only the real CRD's `matchLabels` requirement catches it), and the AgentgatewayPolicy CRD's CEL rules (PreRouting targets a Gateway/ListenerSet, same-namespace targetRefs) are -why the generated extraction policy lives in the Gateway's namespace. +why the generated extraction policy lives in the Gateway's namespace. The +AgentgatewayModel CRD's CEL rules (a custom provider's backendRef kind, a +match.model wildcard shape) check what agentGatewayRef.routing: Model generates. | File | Source | Pinned | License | |--------------------------|----------------------------------------------------------------------|----------|------------| | inferencepool.yaml | kubernetes-sigs/gateway-api-inference-extension config/crd/bases | v1.6.2 | Apache-2.0 | | agentgatewaypolicy.yaml | agentgateway/agentgateway agentgateway-crds chart template | v1.6.0 | Apache-2.0 | +| agentgatewaymodel.yaml | agentgateway/agentgateway agentgateway-crds chart template | v1.6.0 | Apache-2.0 | | gateway.yaml | Hand-made minimal stub (Gateway API v1 Gateway; permissive spec) | - | - | | httproute.yaml | Hand-made minimal stub (Gateway API v1 HTTPRoute) | - | - | diff --git a/test/crd/inference-extension/agentgatewaymodel.yaml b/test/crd/inference-extension/agentgatewaymodel.yaml new file mode 100644 index 000000000..2833859e7 --- /dev/null +++ b/test/crd/inference-extension/agentgatewaymodel.yaml @@ -0,0 +1,5646 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + labels: + app: agentgateway + app.kubernetes.io/name: agentgateway + name: agentgatewaymodels.agentgateway.dev +spec: + group: agentgateway.dev + names: + categories: + - agentgateway + kind: AgentgatewayModel + listKind: AgentgatewayModelList + plural: agentgatewaymodels + shortNames: + - agmodel + singular: agentgatewaymodel + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Model name matched from client requests + jsonPath: .spec.match.model + name: Model Match + type: string + - description: The age of the model. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Desired model configuration. + properties: + azure: + description: Provider-specific settings for Azure AI. + properties: + apiVersion: + description: |- + The version of the Azure OpenAI API to use. + If unset, defaults to `v1`. + maxLength: 64 + minLength: 1 + type: string + projectName: + description: |- + The Foundry project name, required when `resourceType` is `Foundry`. + Used to construct paths: /api/projects/{projectName}/openai/v1/... + maxLength: 256 + minLength: 1 + type: string + resourceName: + description: |- + The Azure resource name used to construct the endpoint host. + For OpenAI: {resourceName}.openai.azure.com + For Foundry: {resourceName}.services.ai.azure.com + Note: when the Azure portal "Foundry legacy" template was used, the + generated resource name may end in "-resource" (e.g. "myproject-resource"); + that suffix is part of the resource name as the user configured it, not + part of the hostname suffix agentgateway should append. + maxLength: 256 + minLength: 1 + type: string + resourceType: + description: The type of Azure endpoint. Determines the host suffix. + enum: + - Foundry + - OpenAI + type: string + required: + - resourceName + - resourceType + type: object + x-kubernetes-validations: + - message: projectName is required when resourceType is Foundry + rule: self.resourceType != 'Foundry' || has(self.projectName) + baseURL: + description: |- + BaseURL overrides the provider address and base path prefix. It must use the + http or https scheme. Backend policies may override the default TLS + configuration. Query parameters, fragments, and user info are not supported. + The URL path is the upstream base path and defaults to / when omitted. + Provider-specific endpoint paths are appended to this base path. + For example, `https://api.openai.com/v1` sends completions to `/v1/chat/completions`, + while `https://api.openai.com` sends them to `/chat/completions`. + format: uri + maxLength: 1024 + minLength: 1 + type: string + bedrock: + description: Provider-specific settings for Amazon Bedrock. + properties: + endpointPreference: + description: |- + EndpointPreference selects which Bedrock API surface to prefer. + Defaults to `RuntimePreferred`, preferring runtime over mantle. + Decides which endpoint to pick mainly based on the catalog tags + `mantle` and `runtime`. + enum: + - MantleOnly + - MantlePreferred + - RuntimeOnly + - RuntimePreferred + type: string + guardrail: + description: |- + Guardrail policy to use for the backend. See + . + If not specified, the AWS Guardrail policy will not be used. + properties: + identifier: + description: Identifier of the Guardrail policy to use for + the backend. + maxLength: 256 + minLength: 1 + type: string + version: + description: Version of the Guardrail policy to use for the + backend. + maxLength: 256 + minLength: 1 + type: string + required: + - identifier + - version + type: object + region: + description: |- + AWS region to use for the backend. + Defaults to `us-east-1` if not specified. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9-]+$ + type: string + type: object + x-kubernetes-validations: + - message: Bedrock guardrails cannot be used with MantlePreferred + or MantleOnly + rule: '!has(self.guardrail) || !has(self.endpointPreference) || + !(self.endpointPreference in [''MantlePreferred'', ''MantleOnly''])' + custom: + description: Provider-specific settings for a custom provider. + properties: + backendRef: + description: |- + Kubernetes backend that serves this provider. + `backendRef` may target only a namespace-local Service or InferencePool. + If unset, host and port must be set on the parent provider. + properties: + group: + description: |- + API group of the referenced resource. For example, `gateway.networking.k8s.io`. + Defaults to the empty string, which identifies the core API group. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: |- + Kind of the referenced resource. For example, `Service`. + Defaults to "Service" when not specified. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name of the referenced resource. + maxLength: 253 + minLength: 1 + type: string + port: + description: |- + Destination port number to use for this resource. + Required when the referenced resource is a Kubernetes Service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '((!has(self.group) || size(self.group) == 0) && (!has(self.kind) + || self.kind == ''Service'')) ? has(self.port) : true' + formats: + description: Provider-native API formats this provider supports. + items: + description: Provider-native LLM API format settings. + properties: + path: + description: |- + Default upstream path override for this format. + If unset, agentgateway uses the default path for the format. + maxLength: 1024 + minLength: 1 + type: string + type: + description: Provider-native API format. + enum: + - AnthropicTokenCount + - Completions + - Embeddings + - Messages + - Realtime + - Rerank + - Responses + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: path must start with / + rule: '!has(self.path) || self.path.startsWith(''/'')' + maxItems: 6 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + providerOverride: + description: |- + Provider identity used for cost-catalog lookup and telemetry. + Defaults to "custom" when unset. + maxLength: 256 + minLength: 1 + type: string + required: + - formats + type: object + x-kubernetes-validations: + - message: custom provider backendRef may target only Service or InferencePool + rule: '!has(self.backendRef) || (((!has(self.backendRef.group) || + self.backendRef.group == "") && (!has(self.backendRef.kind) || + self.backendRef.kind == ''Service'')) || (has(self.backendRef.group) + && self.backendRef.group == ''inference.networking.k8s.io'' && + has(self.backendRef.kind) && self.backendRef.kind == ''InferencePool''))' + match: + description: Conditions for selecting this model from client requests. + properties: + model: + description: |- + Model name matched against client requests. It may be exact, a suffix + wildcard such as `gpt-*`, a prefix wildcard such as `*-latest`, or `*`. + When omitted, the model matches metadata.name exactly. + maxLength: 1024 + minLength: 1 + type: string + x-kubernetes-validations: + - message: model wildcards must be '*', a suffix like 'gpt-*', + or a prefix like '*-latest' + rule: '!self.contains(''*'') || (self.indexOf(''*'') == self.lastIndexOf(''*'') + && (self.indexOf(''*'') == 0 || self.indexOf(''*'') == size(self) + - 1))' + type: object + parentRefs: + description: |- + Parent resources to which this model attaches. Supported parent kinds are + Gateway, ListenerSet, and HTTPRoute. + + A Gateway or ListenerSet parent attaches the model directly to its + listeners. An HTTPRoute parent attaches the model to the referenced rule; + sectionName selects a named rule, or the HTTPRoute must contain exactly + one rule when sectionName is omitted. The selected rule must use exactly + one AgentgatewayModel backend with name "*". If the rule has path matches, + they must use PathPrefix matching. + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: 'Namespace of the referent. Defaults to the Route''s + local namespace. Cross-namespace references must be explicitly + allowed, for example via ReferenceGrant. Support: Core' + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: 'Port this Route targets on the parent, interpreted + per parent kind (for example a Gateway listener port or Service + port). Support: Extended' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: 'Name of a section within the target resource, + for example a Gateway Listener name or Service port name. + Empty references the entire resource. Support: Core' + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 16 + minItems: 1 + type: array + policies: + description: Policies applied to this concrete model. + properties: + auth: + description: Credentials used to authenticate requests to this + model provider. + properties: + aws: + description: |- + Explicit AWS authentication method for the model provider. When omitted, + default AWS SDK credential discovery is used. + properties: + assumeRole: + description: |- + AWS STS AssumeRole settings to use before signing backend requests. + Ambient AWS credentials are used as the source credentials for STS. + properties: + externalId: + description: ExternalID is set when the role's trust + policy requires sts:ExternalId. + maxLength: 1224 + minLength: 2 + pattern: ^[\w+=,.@:/-]+$ + type: string + roleArn: + description: AWS IAM role ARN to assume. + minLength: 1 + pattern: ^arn:aws[a-z-]*:iam::[0-9]{12}:role/.+$ + type: string + sessionName: + description: |- + SessionName is a custom session name (RoleSessionName) for CloudTrail and + Cost & Usage Report attribution. If unset, AWS generates a random name. + pattern: ^[\w+=,.@-]{2,64}$ + type: string + sessionNameExpression: + description: |- + SessionNameExpression is a CEL expression evaluated against each request + to produce the session name (RoleSessionName), for example `jwt.sub` or + `request.headers["x-team"]`. If the expression does not produce a valid + session name at request time, the request is rejected. + maxLength: 16384 + minLength: 1 + type: string + tags: + description: |- + Session tags passed to STS AssumeRole for cost attribution in the AWS Cost + & Usage Report, once activated. STS allows at most 50 per role session. + items: + description: |- + AwsSessionTag is an AWS STS session tag passed to AssumeRole for cost + attribution. Exactly one of value and expression must be set. + properties: + expression: + description: |- + CEL expression evaluated against each request to produce the tag value, + for example `jwt.sub` or `request.headers["x-app"]`. Requests with invalid + tag values are rejected. + maxLength: 16384 + minLength: 1 + type: string + key: + description: Key is the tag key. + maxLength: 128 + minLength: 1 + type: string + value: + description: Value is a static tag value. + maxLength: 256 + type: string + required: + - key + type: object + x-kubernetes-validations: + - message: exactly one of value or expression must + be set + rule: has(self.value) != has(self.expression) + maxItems: 50 + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + required: + - roleArn + type: object + x-kubernetes-validations: + - message: at most one of the fields in [sessionName sessionNameExpression] + may be set + rule: (has(self.sessionName)?1:0)+(has(self.sessionNameExpression)?1:0) + <= 1 + region: + description: |- + AWS SigV4 signing region, for example `us-east-1`. Set this when the + target AWS service is in a different region than the gateway. If unset, + typed AWS backends may provide this automatically; otherwise the ambient + AWS region is used. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for AWS credentials, defaulting to a Kubernetes `Secret`. + The default Secret resolver expects `accessKey`, `secretKey`, and optional + `sessionToken` keys. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == 0) ? + (!has(self.kind) || size(self.kind) == 0 || self.kind + == ''Secret'') : (has(self.kind) && size(self.kind) + > 0)' + serviceName: + description: |- + AWS SigV4 signing service name, for example + `bedrock`, `bedrock-agentcore`, or `execute-api`). If unset, typed AWS + backends may provide this automatically. + maxLength: 256 + minLength: 1 + type: string + type: object + x-kubernetes-validations: + - message: secretRef and assumeRole are mutually exclusive + rule: '!(has(self.secretRef) && has(self.assumeRole))' + azure: + description: Azure authentication method for the model provider. + properties: + managedIdentity: + description: |- + Managed identity authentication settings. Leave this object empty to use + the system-assigned identity. To use a user-assigned identity, set one of + `clientId`, `objectId`, or `resourceId`. + properties: + clientId: + description: Client ID of the user-assigned managed + identity. + type: string + objectId: + description: Object ID of the user-assigned managed + identity. + type: string + resourceId: + description: Resource ID of the user-assigned managed + identity. + type: string + type: object + x-kubernetes-validations: + - message: at most one of the fields in [clientId objectId + resourceId] may be set + rule: (has(self.clientId)?1:0)+(has(self.objectId)?1:0)+(has(self.resourceId)?1:0) + <= 1 + scopes: + description: |- + Scopes requested for the Azure access token. When omitted, the scope is + inferred from the backend hostname. Managed Identity supports exactly one + scope. + items: + type: string + maxItems: 64 + minItems: 1 + type: array + secretRef: + description: |- + Credential source for Azure credentials, defaulting to a Kubernetes + `Secret`. The default Secret resolver expects `clientID`, `tenantID`, and + `clientSecret` keys. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == 0) ? + (!has(self.kind) || size(self.kind) == 0 || self.kind + == ''Secret'') : (has(self.kind) && size(self.kind) + > 0)' + workloadIdentity: + description: |- + Workload identity authentication settings. Uses the federated token and + Azure env vars projected into the data plane pod. Recommended on AKS with + Workload Identity enabled. + type: object + type: object + x-kubernetes-validations: + - message: managedIdentity supports exactly one scope + rule: '!has(self.managedIdentity) || !has(self.scopes) || + self.scopes.size() == 1' + - message: at most one of the fields in [secretRef managedIdentity + workloadIdentity] may be set + rule: (has(self.secretRef)?1:0)+(has(self.managedIdentity)?1:0)+(has(self.workloadIdentity)?1:0) + <= 1 + credentials: + description: |- + Credentials is a list of additional credentials to inject on the backend + request. Each entry resolves a Secret key and writes its value to the + entry's location. `credentials` is independent of the primary + `key`/`secretRef`/`passthrough` mechanism and may be set on its own or + alongside it. + items: + description: |- + BackendAuthCredential specifies one additional credential to inject on the + backend request. + properties: + location: + description: Where the credential is inserted on the + backend request. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 + pseudo-headers (names beginning with `:`) + are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + secretRef: + description: |- + SecretRef references a Kubernetes Secret holding the credential value, and + optionally overrides the key read from it. Defaults to `Authorization`, + matching the key convention used by the top-level `secretRef`. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. If omitted, + a location-specific default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; + empty defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == 0) + ? (!has(self.kind) || size(self.kind) == 0 || self.kind + == ''Secret'') : (has(self.kind) && size(self.kind) + > 0)' + required: + - location + - secretRef + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + gcp: + description: |- + Google authentication method for the model provider. When omitted, + default Google credential discovery is used. + properties: + audience: + description: |- + Explicit `aud` value for the ID token. Only + valid with `IdToken` type. If not set, the `aud` is automatically + derived from the backend hostname. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for ADC-compatible Google credential JSON, defaulting to + a Kubernetes `Secret`. By default, the value is read from + `credentials.json`; set `secretRef.key` to override it. When omitted, + ambient credentials are used. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. If omitted, + a location-specific default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == 0) ? + (!has(self.kind) || size(self.kind) == 0 || self.kind + == ''Secret'') : (has(self.kind) && size(self.kind) + > 0)' + type: + description: |- + The type of token to generate. To authenticate to GCP services, + generally an `AccessToken` is used. To authenticate to Cloud Run, an + `IdToken` is used. + enum: + - AccessToken + - IdToken + type: string + type: object + x-kubernetes-validations: + - message: audience is only valid with IdToken + rule: 'has(self.audience) ? self.type == ''IdToken'' : true' + key: + description: |- + Inline key to use as the value of the `Authorization` header. This option + is the least secure; usage of a `Secret` is preferred. + maxLength: 2048 + type: string + location: + description: |- + Where backend credentials are inserted. If omitted, credentials are + written to the `Authorization` header with the `Bearer ` prefix. This + applies to `key`, `secretRef`, and `passthrough`. Entries in + `credentials` carry their own location. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 pseudo-headers + (names beginning with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + oauthTokenExchange: + description: |- + OAuth 2.0 token exchange (RFC 8693) / jwt-bearer (RFC 7523) + authentication. + properties: + actorToken: + description: RFC 8693 delegation actor token. TokenExchange + grant only. + properties: + mayAct: + description: may_act claim validation mode. When omitted, + may_act is not enforced. + enum: + - Required + type: string + source: + description: Where to read the actor token. Actor + tokens have no default source. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + expression: + description: CEL expression that extracts the + credential from the request. + maxLength: 16384 + minLength: 1 + type: string + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 + pseudo-headers (names beginning with `:`) + are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie expression] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0)+(has(self.expression)?1:0) + == 1 + tokenType: + description: |- + OAuth token type. Empty defaults to AccessToken. Custom absolute URI values + are supported for actor tokens. + type: string + required: + - source + type: object + x-kubernetes-validations: + - message: mayAct Required requires tokenType Jwt + rule: '!has(self.mayAct) || self.mayAct != ''Required'' + || (has(self.tokenType) && self.tokenType == ''Jwt'')' + additionalParams: + additionalProperties: + description: A Common Expression Language (CEL) expression. + maxLength: 16384 + minLength: 1 + type: string + description: Extra form params; values are CEL expressions + over the incoming request. + maxProperties: 64 + type: object + audiences: + description: Audiences sent to the token endpoint. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 64 + minItems: 1 + type: array + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind of the referent, + for example `Service`. Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. Defaults to + the local namespace. A cross-namespace reference + requires a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. Required when + the referent is a Kubernetes `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') + ? has(self.port) : true' + cache: + description: Response cache configuration. + properties: + inMemory: + properties: + defaultTtl: + description: TTL used when the token endpoint + omits expires_in. Default 300s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: defaultTtl must be at least 1 second + rule: duration(self) >= duration('1s') + maxEntries: + description: Default 8192; 0 disables the cache. + format: int32 + type: integer + type: object + type: object + clientAuth: + description: Client authentication for the token endpoint. + When unset, none is sent. + properties: + clientId: + description: Client ID sent to the token endpoint. + minLength: 1 + type: string + method: + description: Client authentication method. Defaults + to ClientSecretBasic. + enum: + - ClientSecretBasic + - ClientSecretPost + - PrivateKeyJwt + type: string + privateKeyJwt: + description: Client assertion settings. Required when + method is PrivateKeyJwt. + properties: + alg: + description: JWS signing algorithm. Defaults to + RS256. + enum: + - ES256 + - ES384 + - PS256 + - RS256 + - RS384 + - RS512 + type: string + assertionAudience: + description: Audience for the client assertion, + typically the token endpoint URL. + minLength: 1 + type: string + certificateHeader: + description: JWS certificate header. Required + when certificateRef is set. + enum: + - x5c + - x5t#S256 + type: string + certificateRef: + description: |- + PEM-encoded X.509 certificate chain, leaf first, for certificateHeader. The + leaf public key should match signingKeyRef; a mismatch only logs a warning + but the token endpoint will reject the assertions. Required when + certificateHeader is set. The key defaults to `certificate`. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; + empty defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both + group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + kid: + description: Optional JWS key ID header. + type: string + signingKeyRef: + description: PEM-encoded RSA or EC private key; + key defaults to `signingKey`. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; + empty defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both + group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + required: + - assertionAudience + - signingKeyRef + type: object + x-kubernetes-validations: + - message: certificateRef and certificateHeader must + be set together + rule: has(self.certificateRef) == has(self.certificateHeader) + secretRef: + description: |- + Secret providing the `clientSecret` key by default; override via + `secretRef.key`. When omitted, client_id is sent without a secret, which + is only valid with ClientSecretPost. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. If + omitted, a location-specific default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; + empty defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == + 0) ? (!has(self.kind) || size(self.kind) == 0 + || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + required: + - clientId + type: object + x-kubernetes-validations: + - message: privateKeyJwt settings require method PrivateKeyJwt + rule: has(self.privateKeyJwt) == (has(self.method) && + self.method == 'PrivateKeyJwt') + - message: secretRef is not valid with method PrivateKeyJwt + rule: '!has(self.secretRef) || !has(self.method) || + self.method != ''PrivateKeyJwt''' + - message: clientAuth without secretRef requires method + ClientSecretPost or PrivateKeyJwt + rule: has(self.secretRef) || has(self.privateKeyJwt) + || (has(self.method) && self.method == 'ClientSecretPost') + grantType: + description: RFC followed by the request. Defaults to + TokenExchange (RFC 8693). + enum: + - JwtBearer + - TokenExchange + type: string + location: + description: |- + Where the exchanged token is written to the backend request. + Defaults to Authorization: Bearer. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 pseudo-headers + (names beginning with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + path: + description: Token endpoint path; defaults to "/". Must + start with "/". + pattern: ^/ + type: string + requestedTokenType: + description: |- + RFC 8693 requested_token_type. Unlike subject/actor token types, only the + built-in values may be requested; custom URIs are not supported here. + enum: + - AccessToken + - Jwt + - IdToken + - IdJag + type: string + resources: + description: Resources sent to the token endpoint. + items: + type: string + maxItems: 64 + minItems: 1 + type: array + scopes: + description: Scopes sent to the token endpoint. + items: + type: string + maxItems: 64 + minItems: 1 + type: array + subjectToken: + description: |- + Subject token / assertion source and type. Defaults to Authorization Bearer, AccessToken. + The token type may be a built-in value or a custom absolute URI for providers + that support custom token exchange profiles. + properties: + source: + description: Where to read the token. CEL `expression` + variant is permitted. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + expression: + description: CEL expression that extracts the + credential from the request. + maxLength: 16384 + minLength: 1 + type: string + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 + pseudo-headers (names beginning with `:`) + are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie expression] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0)+(has(self.expression)?1:0) + == 1 + tokenType: + description: |- + OAuth token type. Empty defaults to AccessToken. Custom absolute URI values + are supported for subject tokens. + type: string + type: object + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: actorToken is only valid with TokenExchange grantType + rule: '!(has(self.actorToken) && has(self.grantType) && + self.grantType == ''JwtBearer'')' + - message: requestedTokenType is only valid with TokenExchange + grantType + rule: '!(has(self.requestedTokenType) && has(self.grantType) + && self.grantType == ''JwtBearer'')' + - message: requestedTokenType IdJag is only supported by crossAppAccess + rule: '!has(self.requestedTokenType) || self.requestedTokenType + != ''IdJag''' + - message: path may not be set when url is set + rule: 'has(self.url) ? !has(self.path) : true' + - message: exactly one of the fields in [backendRef url] must + be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) == + 1 + passthrough: + description: |- + Reuses a client token already validated by another policy. Those policies + may strip client credentials; passthrough adds the original token back to + the backend request. Without client auth policies, this has no effect. + type: object + secretRef: + description: |- + Credential source for the authorization value, defaulting to a Kubernetes + `Secret`. By default, the value is read from the `Authorization` key; set + `secretRef.key` to override it. A `Bearer ` prefix is stripped only from + the default `Authorization` key. + properties: + group: + description: API group of the referenced credential; empty + selects the core API group + type: string + key: + description: Key in the referenced Secret. If omitted, + a location-specific default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group and + kind + rule: '(!has(self.group) || size(self.group) == 0) ? (!has(self.kind) + || size(self.kind) == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) > 0)' + type: object + x-kubernetes-validations: + - message: must specify credentials, or at most one of key/secretRef/passthrough/aws/azure/gcp/oauthTokenExchange + (credentials may be combined with a primary auth kind) + rule: has(self.credentials) || has(self.key) || has(self.secretRef) + || has(self.passthrough) || has(self.aws) || has(self.azure) + || has(self.gcp) || has(self.oauthTokenExchange) + - message: location may only be set for key, secretRef, or passthrough + auth + rule: 'has(self.location) ? has(self.key) || has(self.secretRef) + || has(self.passthrough) : true' + - message: at most one of the fields in [key secretRef passthrough + aws azure gcp oauthTokenExchange] may be set + rule: (has(self.key)?1:0)+(has(self.secretRef)?1:0)+(has(self.passthrough)?1:0)+(has(self.aws)?1:0)+(has(self.azure)?1:0)+(has(self.gcp)?1:0)+(has(self.oauthTokenExchange)?1:0) + <= 1 + authorization: + description: Authorization rules that clients must satisfy to + use this model. + properties: + action: + description: |- + The effect of this rule when it matches. + If unspecified, defaults to `Allow`. + `Require` rules are cumulative: all require rules must match. + enum: + - Allow + - Deny + - Require + type: string + policy: + description: |- + The authorization rule to evaluate. + + * `Allow`: any matching allow rule allows the request. + * `Require`: every require rule must match for the request to be allowed. + * `Deny`: any matching deny rule denies the request. + + `Deny` is not recommended because expression failures fail to deny; prefer + `Allow` or `Require`. If used, design expressions defensively against evaluation errors. + + If at least one `Allow` rule is configured, requests are denied unless at + least one allow rule matches. + properties: + matchExpressions: + description: CEL expressions that must all evaluate to + true for the rule to match. + items: + description: A Common Expression Language (CEL) expression. + maxLength: 16384 + minLength: 1 + type: string + maxItems: 256 + minItems: 1 + type: array + required: + - matchExpressions + type: object + required: + - policy + type: object + finalTransformations: + description: |- + CEL transformations applied to fields in the provider request body. + After conversion from one provider format to another, these transformations are applied to the converted request body. + items: + description: |- + Maps a request JSON field to a CEL expression. + The expression is evaluated against the current request body and its result + is assigned to the configured field. + properties: + expression: + description: CEL expression used to compute the field value. + maxLength: 16384 + minLength: 1 + type: string + field: + description: Name of the field to set. + maxLength: 256 + minLength: 1 + type: string + required: + - expression + - field + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - field + x-kubernetes-list-type: map + headers: + description: Request and response header changes applied to provider + traffic. + properties: + request: + description: Header changes to apply before forwarding a request. + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name + and value as defined by RFC 7230. + properties: + name: + description: Name of the HTTP header, case-insensitive + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value for the HTTP header + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name + and value as defined by RFC 7230. + properties: + name: + description: Name of the HTTP header, case-insensitive + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value for the HTTP header + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + response: + description: Header changes to apply before returning a response. + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name + and value as defined by RFC 7230. + properties: + name: + description: Name of the HTTP header, case-insensitive + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value for the HTTP header + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name + and value as defined by RFC 7230. + properties: + name: + description: Name of the HTTP header, case-insensitive + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value for the HTTP header + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + x-kubernetes-validations: + - message: at least one of the fields in [request response] must + be set + rule: '[has(self.request),has(self.response)].filter(x,x==true).size() + >= 1' + health: + description: Health checking and eviction behavior for this model + provider. + properties: + eviction: + description: Settings for evicting unhealthy backends. + properties: + consecutiveFailures: + description: |- + Number of consecutive unhealthy responses required before the backend is evicted. + For example, a value of 5 means the backend must receive 5 unhealthy responses in a row before being evicted. + When both consecutiveFailures and healthThreshold are set, the backend is evicted when either condition is met. + When neither is set, a single unhealthy response can trigger eviction. + format: int32 + minimum: 0 + type: integer + duration: + description: |- + Base time a backend should be evicted after being marked unhealthy. + Subsequent evictions use multiplicative backoff (duration * times_evicted). + If all endpoints are evicted, the load balancer falls back to returning evicted endpoints + rather than failing entirely. + If unset, defaults to `3s`. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: duration must be at least 1 second + rule: duration(self) >= duration('1s') + healthThreshold: + description: |- + EWMA health score threshold, from 0 to 100. When set, a backend is evicted + only if its computed health drops below this value after an unhealthy + response (e.g. 50 evicts when EWMA health falls below 50%). Unlike + consecutiveFailures, this sliding-window average lets a single success delay + eviction. If both are set, either condition evicts; if neither, a single + unhealthy response evicts. + format: int32 + maximum: 100 + minimum: 0 + type: integer + restoreHealth: + description: |- + Health score from 0 to 100 assigned to a backend when it returns from eviction. + For gradual recovery, set below 100; for full recovery immediately, set 100. + If unset, the backend resumes with the health it had when evicted. + format: int32 + maximum: 100 + minimum: 0 + type: integer + type: object + unhealthyCondition: + description: |- + CEL expression that determines whether a response indicates an unhealthy backend. + When the expression evaluates to true, the backend is considered unhealthy and may be evicted. + + For example, to evict on 5xx responses: `response.code >= 500`. + + When unset, any 5xx response, or a connection failure, is treated as unhealthy. + This default lowers the backend's health score but does not trigger eviction on its own. + maxLength: 16384 + minLength: 1 + type: string + type: object + promptGuard: + description: Guardrails for requests and responses sent to this + model provider. + properties: + request: + description: Prompt guards to apply to requests sent by the + client. + items: + description: Prompt guards to apply to requests sent by + the client. + properties: + bedrockGuardrails: + description: |- + AWS Bedrock Guardrails settings for prompt + guarding. + properties: + action: + description: |- + Action controls whether the guardrail's verdict is enforced or only + observed. `Reject` (the default) enforces the guardrail: a blocked + assessment rejects the request/response and an anonymized assessment masks + the matched content. `Audit` runs the guardrail in observe mode: it is + invoked and its assessment recorded (metrics + structured log), but the + request/response is never blocked or masked. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + identifier: + description: Identifier of the Guardrail policy + to use for the backend. + maxLength: 256 + minLength: 1 + type: string + policies: + description: Policies for communicating with AWS + Bedrock Guardrails. + properties: + auth: + description: Settings for authenticating to + AWS Bedrock Guardrails. + properties: + aws: + description: |- + AWS authentication method for Bedrock Guardrails. Use `aws: {}` for + default AWS SDK credential discovery. + properties: + assumeRole: + description: |- + AWS STS AssumeRole settings to use before signing backend requests. + Ambient AWS credentials are used as the source credentials for STS. + properties: + externalId: + description: ExternalID is set when + the role's trust policy requires + sts:ExternalId. + maxLength: 1224 + minLength: 2 + pattern: ^[\w+=,.@:/-]+$ + type: string + roleArn: + description: AWS IAM role ARN to + assume. + minLength: 1 + pattern: ^arn:aws[a-z-]*:iam::[0-9]{12}:role/.+$ + type: string + sessionName: + description: |- + SessionName is a custom session name (RoleSessionName) for CloudTrail and + Cost & Usage Report attribution. If unset, AWS generates a random name. + pattern: ^[\w+=,.@-]{2,64}$ + type: string + sessionNameExpression: + description: |- + SessionNameExpression is a CEL expression evaluated against each request + to produce the session name (RoleSessionName), for example `jwt.sub` or + `request.headers["x-team"]`. If the expression does not produce a valid + session name at request time, the request is rejected. + maxLength: 16384 + minLength: 1 + type: string + tags: + description: |- + Session tags passed to STS AssumeRole for cost attribution in the AWS Cost + & Usage Report, once activated. STS allows at most 50 per role session. + items: + description: |- + AwsSessionTag is an AWS STS session tag passed to AssumeRole for cost + attribution. Exactly one of value and expression must be set. + properties: + expression: + description: |- + CEL expression evaluated against each request to produce the tag value, + for example `jwt.sub` or `request.headers["x-app"]`. Requests with invalid + tag values are rejected. + maxLength: 16384 + minLength: 1 + type: string + key: + description: Key is the tag + key. + maxLength: 128 + minLength: 1 + type: string + value: + description: Value is a static + tag value. + maxLength: 256 + type: string + required: + - key + type: object + x-kubernetes-validations: + - message: exactly one of value + or expression must be set + rule: has(self.value) != has(self.expression) + maxItems: 50 + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + required: + - roleArn + type: object + x-kubernetes-validations: + - message: at most one of the fields + in [sessionName sessionNameExpression] + may be set + rule: (has(self.sessionName)?1:0)+(has(self.sessionNameExpression)?1:0) + <= 1 + region: + description: |- + AWS SigV4 signing region, for example `us-east-1`. Set this when the + target AWS service is in a different region than the gateway. If unset, + typed AWS backends may provide this automatically; otherwise the ambient + AWS region is used. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for AWS credentials, defaulting to a Kubernetes `Secret`. + The default Secret resolver expects `accessKey`, `secretKey`, and optional + `sessionToken` keys. + properties: + group: + description: API group of the referenced + credential; empty selects the + core API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to + `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) + > 0)' + serviceName: + description: |- + AWS SigV4 signing service name, for example + `bedrock`, `bedrock-agentcore`, or `execute-api`). If unset, typed AWS + backends may provide this automatically. + maxLength: 256 + minLength: 1 + type: string + type: object + x-kubernetes-validations: + - message: secretRef and assumeRole are + mutually exclusive + rule: '!(has(self.secretRef) && has(self.assumeRole))' + key: + description: |- + Inline API key to use as the value of the `Authorization` header. + This option is the least secure; usage of a `Secret` is preferred. + maxLength: 2048 + type: string + location: + description: |- + Where API keys are inserted. Defaults to the `Authorization` header with + the `Bearer ` prefix. Applies to `key` and `secretRef`. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. + HTTP/2 pseudo-headers (names beginning + with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in + [header queryParameter cookie] must + be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + secretRef: + description: |- + Credential source for the API key, defaulting to a Kubernetes `Secret`. + By default, the value is read from the `Authorization` key; set + `secretRef.key` to override it. A `Bearer ` prefix is stripped only from + the default `Authorization` key. + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set + both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + type: object + x-kubernetes-validations: + - message: location may only be set for key + or secretRef auth + rule: 'has(self.location) ? has(self.key) + || has(self.secretRef) : true' + - message: exactly one of the fields in [key + secretRef aws] must be set + rule: (has(self.key)?1:0)+(has(self.secretRef)?1:0)+(has(self.aws)?1:0) + == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + region: + description: |- + AWS region where the guardrail is deployed, for example + `us-west-2`). + maxLength: 256 + minLength: 1 + type: string + version: + description: Version of the Guardrail policy to + use for the backend. + maxLength: 256 + minLength: 1 + type: string + required: + - identifier + - region + - version + type: object + googleModelArmor: + description: Google Model Armor settings for prompt + guarding. + properties: + action: + description: |- + Action controls whether flagged content is rejected or only observed. + `Reject` (the default) rejects flagged content; `Audit` records the + would-be rejection without blocking. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + location: + description: |- + Google Cloud location, for example `us-central1`. + Defaults to `us-central1` if not specified. + maxLength: 256 + minLength: 1 + type: string + policies: + description: Policies for communicating with Google + Model Armor. + properties: + auth: + description: Settings for authenticating to + Google Model Armor. + properties: + gcp: + description: |- + Google authentication method for Model Armor. Use `gcp: {}` for default + Google credential discovery. + properties: + audience: + description: |- + Explicit `aud` value for the ID token. Only + valid with `IdToken` type. If not set, the `aud` is automatically + derived from the backend hostname. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for ADC-compatible Google credential JSON, defaulting to + a Kubernetes `Secret`. By default, the value is read from + `credentials.json`; set `secretRef.key` to override it. When omitted, + ambient credentials are used. + properties: + group: + description: API group of the referenced + credential; empty selects the + core API group + type: string + key: + description: Key in the referenced + Secret. If omitted, a location-specific + default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to + `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) + > 0)' + type: + description: |- + The type of token to generate. To authenticate to GCP services, + generally an `AccessToken` is used. To authenticate to Cloud Run, an + `IdToken` is used. + enum: + - AccessToken + - IdToken + type: string + type: object + x-kubernetes-validations: + - message: audience is only valid with IdToken + rule: 'has(self.audience) ? self.type + == ''IdToken'' : true' + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [gcp] + must be set + rule: (has(self.gcp)?1:0) == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + projectId: + description: Google Cloud project ID. + maxLength: 256 + minLength: 1 + type: string + templateId: + description: Template ID for Google Model Armor. + maxLength: 256 + minLength: 1 + type: string + required: + - projectId + - templateId + type: object + openAIModeration: + description: |- + Passes prompt data through the OpenAI Moderations + endpoint. + See https://developers.openai.com/api/reference/resources/moderations for more information. + properties: + action: + description: |- + Action controls whether flagged content is rejected or only observed. + `Reject` (the default) rejects flagged content; `Audit` records the + would-be rejection without blocking. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + model: + description: |- + Moderation model to use. For example, + `omni-moderation`. + type: string + policies: + description: Policies for communicating with OpenAI. + properties: + auth: + description: Settings for authenticating to + OpenAI. + properties: + key: + description: |- + Inline key to use as the value of the + `Authorization` header. This option is the least secure; usage of a + `Secret` is preferred. + maxLength: 2048 + type: string + location: + description: |- + Where backend credentials are inserted. Defaults to the `Authorization` + header with the `Bearer ` prefix. Applies to `key` and `secretRef`. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. + HTTP/2 pseudo-headers (names beginning + with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in + [header queryParameter cookie] must + be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + secretRef: + description: |- + Credential source for the authorization value, defaulting to a Kubernetes + `Secret`. By default, the value is read from the `Authorization` key; set + `secretRef.key` to override it. A `Bearer ` prefix is stripped only from + the default `Authorization` key. + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set + both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [key + secretRef] must be set + rule: (has(self.key)?1:0)+(has(self.secretRef)?1:0) + == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + type: object + regex: + description: Regular expression (regex) matching for + prompt guards and data masking. + properties: + action: + description: |- + The action to take if a regex pattern is matched in a request or response. + The action applies to request and response matches alike. Note that + `Mask` is not applied to streamed responses: matched content in a + streamed response is passed through unmodified. + Defaults to `Mask`. + enum: + - Audit + - Mask + - Reject + type: string + builtins: + description: |- + Built-in regex patterns to match against the request or response. + Matches and built-ins are additive. + items: + description: |- + Built-in regex patterns for specific types of strings in prompts. + For example, if you specify `CreditCard`, any credit card numbers + in the request or response are matched. + enum: + - CaSin + - CreditCard + - Email + - PhoneNumber + - Ssn + type: string + type: array + matches: + description: |- + Regex patterns to match against the request or response. + Matches and built-ins are additive. + items: + maxLength: 1024 + minLength: 1 + type: string + type: array + type: object + response: + description: |- + Custom response message to return to the client. If not specified, defaults to + `The request was rejected due to inappropriate content`. + properties: + message: + description: |- + Custom response message to return to the client. If not specified, defaults to + `The request was rejected due to inappropriate content`. + type: string + statusCode: + description: Status code to return to the client. + Defaults to 403. + format: int32 + maximum: 599 + minimum: 200 + type: integer + type: object + scope: + description: |- + Which parts of the request this guard inspects. When unset, defaults to + `SystemPrompt` and `Messages`. Tool call inputs and outputs are not + inspected unless `ToolInput`/`ToolOutput` are listed explicitly. + In APIs that send tool arguments as opaque JSON, such as Completions, the + arguments are masked as a single string, meaning a prompt guard has the + potential to rewrite the arguments into invalid JSON. + items: + description: Which category of request content a prompt + guard inspects. + enum: + - Messages + - SystemPrompt + - ToolInput + - ToolOutput + type: string + maxItems: 4 + minItems: 1 + type: array + x-kubernetes-list-type: set + webhook: + description: Webhook that receives requests for prompt + guarding. + properties: + action: + description: |- + Action controls whether the webhook's verdict is enforced or only observed. + `Reject` (the default) enforces it; `Audit` records the would-be action + without blocking or masking. + enum: + - Audit + - Reject + type: string + backendRef: + description: |- + Webhook server to reach. + + Supported types: Service and Backend. + properties: + group: + default: "" + description: Group of the referent, for example + `gateway.networking.k8s.io`. Empty selects + the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind of the + referent, for example `Service`. Defaults + to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. Defaults + to the local namespace. A cross-namespace + reference requires a ReferenceGrant in the + referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. Required + when the referent is a Kubernetes `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == + ''Service'') ? has(self.port) : true' + failureMode: + description: |- + Behavior when the webhook guardrail is unavailable + or returns an error. `FailOpen` allows the request to continue. + `FailClosed` (default) rejects the request. + enum: + - FailClosed + - FailOpen + type: string + forwardHeaderMatches: + description: |- + HTTP header matches used to select the headers to forward to the webhook. + Request headers are used when forwarding requests and response headers + are used when forwarding responses. + By default, no headers are forwarded. + items: + description: |- + HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request + headers. + properties: + name: + description: Name of the HTTP header to match, + case-insensitive. When names are equivalent, + only the first matching entry is used + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: 'How to match against the header + value: `Exact` (default) or `RegularExpression`. + The regex dialect is implementation-specific' + enum: + - Exact + - RegularExpression + type: string + value: + description: Value of the HTTP header to match + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + headers: + additionalProperties: + description: A Common Expression Language (CEL) + expression. + maxLength: 16384 + minLength: 1 + type: string + description: CEL-computed headers to include in + webhook requests. + maxProperties: 64 + type: object + required: + - backendRef + type: object + type: object + x-kubernetes-validations: + - message: 'scope: only regex and bedrockGuardrails guards + support a non-default scope; other guard kinds always + inspect the default (SystemPrompt + Messages)' + rule: '!has(self.scope) || has(self.regex) || has(self.bedrockGuardrails) + || (self.scope.size() == 2 && ''SystemPrompt'' in self.scope + && ''Messages'' in self.scope)' + - message: exactly one of the fields in [regex webhook openAIModeration + bedrockGuardrails googleModelArmor] must be set + rule: (has(self.regex)?1:0)+(has(self.webhook)?1:0)+(has(self.openAIModeration)?1:0)+(has(self.bedrockGuardrails)?1:0)+(has(self.googleModelArmor)?1:0) + == 1 + maxItems: 8 + minItems: 1 + type: array + response: + description: Prompt guards to apply to responses returned + by the LLM provider. + items: + description: Prompt guards to apply to responses returned + by the LLM provider. + properties: + bedrockGuardrails: + description: |- + AWS Bedrock Guardrails settings for prompt + guarding. + properties: + action: + description: |- + Action controls whether the guardrail's verdict is enforced or only + observed. `Reject` (the default) enforces the guardrail: a blocked + assessment rejects the request/response and an anonymized assessment masks + the matched content. `Audit` runs the guardrail in observe mode: it is + invoked and its assessment recorded (metrics + structured log), but the + request/response is never blocked or masked. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + identifier: + description: Identifier of the Guardrail policy + to use for the backend. + maxLength: 256 + minLength: 1 + type: string + policies: + description: Policies for communicating with AWS + Bedrock Guardrails. + properties: + auth: + description: Settings for authenticating to + AWS Bedrock Guardrails. + properties: + aws: + description: |- + AWS authentication method for Bedrock Guardrails. Use `aws: {}` for + default AWS SDK credential discovery. + properties: + assumeRole: + description: |- + AWS STS AssumeRole settings to use before signing backend requests. + Ambient AWS credentials are used as the source credentials for STS. + properties: + externalId: + description: ExternalID is set when + the role's trust policy requires + sts:ExternalId. + maxLength: 1224 + minLength: 2 + pattern: ^[\w+=,.@:/-]+$ + type: string + roleArn: + description: AWS IAM role ARN to + assume. + minLength: 1 + pattern: ^arn:aws[a-z-]*:iam::[0-9]{12}:role/.+$ + type: string + sessionName: + description: |- + SessionName is a custom session name (RoleSessionName) for CloudTrail and + Cost & Usage Report attribution. If unset, AWS generates a random name. + pattern: ^[\w+=,.@-]{2,64}$ + type: string + sessionNameExpression: + description: |- + SessionNameExpression is a CEL expression evaluated against each request + to produce the session name (RoleSessionName), for example `jwt.sub` or + `request.headers["x-team"]`. If the expression does not produce a valid + session name at request time, the request is rejected. + maxLength: 16384 + minLength: 1 + type: string + tags: + description: |- + Session tags passed to STS AssumeRole for cost attribution in the AWS Cost + & Usage Report, once activated. STS allows at most 50 per role session. + items: + description: |- + AwsSessionTag is an AWS STS session tag passed to AssumeRole for cost + attribution. Exactly one of value and expression must be set. + properties: + expression: + description: |- + CEL expression evaluated against each request to produce the tag value, + for example `jwt.sub` or `request.headers["x-app"]`. Requests with invalid + tag values are rejected. + maxLength: 16384 + minLength: 1 + type: string + key: + description: Key is the tag + key. + maxLength: 128 + minLength: 1 + type: string + value: + description: Value is a static + tag value. + maxLength: 256 + type: string + required: + - key + type: object + x-kubernetes-validations: + - message: exactly one of value + or expression must be set + rule: has(self.value) != has(self.expression) + maxItems: 50 + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + required: + - roleArn + type: object + x-kubernetes-validations: + - message: at most one of the fields + in [sessionName sessionNameExpression] + may be set + rule: (has(self.sessionName)?1:0)+(has(self.sessionNameExpression)?1:0) + <= 1 + region: + description: |- + AWS SigV4 signing region, for example `us-east-1`. Set this when the + target AWS service is in a different region than the gateway. If unset, + typed AWS backends may provide this automatically; otherwise the ambient + AWS region is used. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for AWS credentials, defaulting to a Kubernetes `Secret`. + The default Secret resolver expects `accessKey`, `secretKey`, and optional + `sessionToken` keys. + properties: + group: + description: API group of the referenced + credential; empty selects the + core API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to + `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) + > 0)' + serviceName: + description: |- + AWS SigV4 signing service name, for example + `bedrock`, `bedrock-agentcore`, or `execute-api`). If unset, typed AWS + backends may provide this automatically. + maxLength: 256 + minLength: 1 + type: string + type: object + x-kubernetes-validations: + - message: secretRef and assumeRole are + mutually exclusive + rule: '!(has(self.secretRef) && has(self.assumeRole))' + key: + description: |- + Inline API key to use as the value of the `Authorization` header. + This option is the least secure; usage of a `Secret` is preferred. + maxLength: 2048 + type: string + location: + description: |- + Where API keys are inserted. Defaults to the `Authorization` header with + the `Bearer ` prefix. Applies to `key` and `secretRef`. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. + HTTP/2 pseudo-headers (names beginning + with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in + [header queryParameter cookie] must + be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + secretRef: + description: |- + Credential source for the API key, defaulting to a Kubernetes `Secret`. + By default, the value is read from the `Authorization` key; set + `secretRef.key` to override it. A `Bearer ` prefix is stripped only from + the default `Authorization` key. + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set + both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + type: object + x-kubernetes-validations: + - message: location may only be set for key + or secretRef auth + rule: 'has(self.location) ? has(self.key) + || has(self.secretRef) : true' + - message: exactly one of the fields in [key + secretRef aws] must be set + rule: (has(self.key)?1:0)+(has(self.secretRef)?1:0)+(has(self.aws)?1:0) + == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + region: + description: |- + AWS region where the guardrail is deployed, for example + `us-west-2`). + maxLength: 256 + minLength: 1 + type: string + version: + description: Version of the Guardrail policy to + use for the backend. + maxLength: 256 + minLength: 1 + type: string + required: + - identifier + - region + - version + type: object + googleModelArmor: + description: Google Model Armor settings for prompt + guarding. + properties: + action: + description: |- + Action controls whether flagged content is rejected or only observed. + `Reject` (the default) rejects flagged content; `Audit` records the + would-be rejection without blocking. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + location: + description: |- + Google Cloud location, for example `us-central1`. + Defaults to `us-central1` if not specified. + maxLength: 256 + minLength: 1 + type: string + policies: + description: Policies for communicating with Google + Model Armor. + properties: + auth: + description: Settings for authenticating to + Google Model Armor. + properties: + gcp: + description: |- + Google authentication method for Model Armor. Use `gcp: {}` for default + Google credential discovery. + properties: + audience: + description: |- + Explicit `aud` value for the ID token. Only + valid with `IdToken` type. If not set, the `aud` is automatically + derived from the backend hostname. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for ADC-compatible Google credential JSON, defaulting to + a Kubernetes `Secret`. By default, the value is read from + `credentials.json`; set `secretRef.key` to override it. When omitted, + ambient credentials are used. + properties: + group: + description: API group of the referenced + credential; empty selects the + core API group + type: string + key: + description: Key in the referenced + Secret. If omitted, a location-specific + default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to + `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) + > 0)' + type: + description: |- + The type of token to generate. To authenticate to GCP services, + generally an `AccessToken` is used. To authenticate to Cloud Run, an + `IdToken` is used. + enum: + - AccessToken + - IdToken + type: string + type: object + x-kubernetes-validations: + - message: audience is only valid with IdToken + rule: 'has(self.audience) ? self.type + == ''IdToken'' : true' + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [gcp] + must be set + rule: (has(self.gcp)?1:0) == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + projectId: + description: Google Cloud project ID. + maxLength: 256 + minLength: 1 + type: string + templateId: + description: Template ID for Google Model Armor. + maxLength: 256 + minLength: 1 + type: string + required: + - projectId + - templateId + type: object + regex: + description: Regular expression (regex) matching for + prompt guards and data masking. + properties: + action: + description: |- + The action to take if a regex pattern is matched in a request or response. + The action applies to request and response matches alike. Note that + `Mask` is not applied to streamed responses: matched content in a + streamed response is passed through unmodified. + Defaults to `Mask`. + enum: + - Audit + - Mask + - Reject + type: string + builtins: + description: |- + Built-in regex patterns to match against the request or response. + Matches and built-ins are additive. + items: + description: |- + Built-in regex patterns for specific types of strings in prompts. + For example, if you specify `CreditCard`, any credit card numbers + in the request or response are matched. + enum: + - CaSin + - CreditCard + - Email + - PhoneNumber + - Ssn + type: string + type: array + matches: + description: |- + Regex patterns to match against the request or response. + Matches and built-ins are additive. + items: + maxLength: 1024 + minLength: 1 + type: string + type: array + type: object + response: + description: |- + Custom response message to return to the client. If not specified, defaults to + `The response was rejected due to inappropriate content`. + properties: + message: + description: |- + Custom response message to return to the client. If not specified, defaults to + `The request was rejected due to inappropriate content`. + type: string + statusCode: + description: Status code to return to the client. + Defaults to 403. + format: int32 + maximum: 599 + minimum: 200 + type: integer + type: object + webhook: + description: Webhook that receives responses for prompt + guarding. + properties: + action: + description: |- + Action controls whether the webhook's verdict is enforced or only observed. + `Reject` (the default) enforces it; `Audit` records the would-be action + without blocking or masking. + enum: + - Audit + - Reject + type: string + backendRef: + description: |- + Webhook server to reach. + + Supported types: Service and Backend. + properties: + group: + default: "" + description: Group of the referent, for example + `gateway.networking.k8s.io`. Empty selects + the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind of the + referent, for example `Service`. Defaults + to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. Defaults + to the local namespace. A cross-namespace + reference requires a ReferenceGrant in the + referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. Required + when the referent is a Kubernetes `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == + ''Service'') ? has(self.port) : true' + failureMode: + description: |- + Behavior when the webhook guardrail is unavailable + or returns an error. `FailOpen` allows the request to continue. + `FailClosed` (default) rejects the request. + enum: + - FailClosed + - FailOpen + type: string + forwardHeaderMatches: + description: |- + HTTP header matches used to select the headers to forward to the webhook. + Request headers are used when forwarding requests and response headers + are used when forwarding responses. + By default, no headers are forwarded. + items: + description: |- + HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request + headers. + properties: + name: + description: Name of the HTTP header to match, + case-insensitive. When names are equivalent, + only the first matching entry is used + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: 'How to match against the header + value: `Exact` (default) or `RegularExpression`. + The regex dialect is implementation-specific' + enum: + - Exact + - RegularExpression + type: string + value: + description: Value of the HTTP header to match + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + headers: + additionalProperties: + description: A Common Expression Language (CEL) + expression. + maxLength: 16384 + minLength: 1 + type: string + description: CEL-computed headers to include in + webhook requests. + maxProperties: 64 + type: object + required: + - backendRef + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [regex webhook bedrockGuardrails + googleModelArmor] must be set + rule: (has(self.regex)?1:0)+(has(self.webhook)?1:0)+(has(self.bedrockGuardrails)?1:0)+(has(self.googleModelArmor)?1:0) + == 1 + maxItems: 8 + minItems: 1 + type: array + streaming: + description: |- + Apply prompt guards to streaming responses and realtime websocket messages. + Defaults to disabled to preserve streaming throughput unless explicitly enabled. + enum: + - Enabled + type: string + type: object + x-kubernetes-validations: + - message: at least one of the fields in [request response] must + be set + rule: '[has(self.request),has(self.response)].filter(x,x==true).size() + >= 1' + tls: + description: TLS settings for connections to this model provider. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced CA certificate source. + Omitted defaults to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client identity and + trust roots (`Inline` default, or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group and + kind + rule: '(!has(self.group) || size(self.group) == 0) ? (!has(self.kind) + || size(self.kind) == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs may not + be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) : true' + - message: insecureSkipVerify and verifySubjectAltNames may not + be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not be combined with mtlsCertificateRef, + caCertificateRefs, or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) && !has(self.caCertificateRefs) + && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + transformations: + description: CEL transformations applied to fields in the provider + request body. + items: + description: |- + Maps a request JSON field to a CEL expression. + The expression is evaluated against the current request body and its result + is assigned to the configured field. + properties: + expression: + description: CEL expression used to compute the field value. + maxLength: 16384 + minLength: 1 + type: string + field: + description: Name of the field to set. + maxLength: 256 + minLength: 1 + type: string + required: + - expression + - field + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - field + x-kubernetes-list-type: map + tunnel: + description: Proxy tunnel used to reach this model provider. + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind of the referent, + for example `Service`. Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. Defaults to the + local namespace. A cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. Required when the + referent is a Kubernetes `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') + ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef url] must + be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) == 1 + - message: url must not include a path for backend tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth authorization finalTransformations + headers health promptGuard tls transformations tunnel] must be + set + rule: '[has(self.auth),has(self.authorization),has(self.finalTransformations),has(self.headers),has(self.health),has(self.promptGuard),has(self.tls),has(self.transformations),has(self.tunnel)].filter(x,x==true).size() + >= 1' + provider: + description: |- + Provider serving this concrete model. Provider-specific configuration is + set by the corresponding field below when needed. + enum: + - Anthropic + - Azure + - Baseten + - Bedrock + - Cerebras + - Cohere + - Custom + - Deepinfra + - Deepseek + - Fireworks + - Gemini + - Groq + - Huggingface + - Meta + - Mistral + - Ollama + - OpenAI + - Openrouter + - TogetherAI + - VertexAI + - XAI + type: string + vertexai: + description: Provider-specific settings for Vertex AI. + properties: + projectId: + description: The ID of the Google Cloud Project that you use for + the Vertex AI. + maxLength: 64 + minLength: 1 + type: string + region: + description: |- + The location of the Google Cloud Project that you use for the Vertex AI. + Special values: `global` uses the global endpoint, while `us` and `eu` use restricted + multi-region endpoints. Other values are treated as regional locations. + Defaults to `global` if not specified. + maxLength: 64 + minLength: 1 + type: string + required: + - projectId + type: object + virtualModel: + description: Request-time routing among concrete AgentgatewayModel + resources. + properties: + conditional: + description: Ordered condition-based model selection. + properties: + targets: + description: |- + Concrete model targets evaluated in order. The first matching condition is + selected. One final target may omit when to act as the fallback. + items: + properties: + model: + description: |- + Concrete model name selected through the referenced model. It is required + when modelRef points to a wildcard match.model. When omitted, the referenced + model's exact effective match.model is used. + maxLength: 1024 + minLength: 1 + type: string + modelRef: + description: Same-namespace AgentgatewayModel resource + selected by this target. + properties: + name: + default: "" + description: Name of the referent + type: string + type: object + x-kubernetes-map-type: atomic + when: + description: |- + CEL expression that must evaluate to true for this target to be selected. + Omit only on the final fallback target. + maxLength: 16384 + minLength: 1 + type: string + required: + - modelRef + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-validations: + - message: conditional targets without when must be last + rule: self.filter(e, !has(e.when)).size() <= 1 && (!self.exists(e, + !has(e.when)) || !has(self[size(self) - 1].when)) + required: + - targets + type: object + failover: + description: Priority-based model selection with failover between + priority groups. + properties: + targets: + description: Concrete model targets grouped by priority. Lower + values are preferred. + items: + properties: + model: + description: |- + Concrete model name selected through the referenced model. It is required + when modelRef points to a wildcard match.model. When omitted, the referenced + model's exact effective match.model is used. + maxLength: 1024 + minLength: 1 + type: string + modelRef: + description: Same-namespace AgentgatewayModel resource + selected by this target. + properties: + name: + default: "" + description: Name of the referent + type: string + type: object + x-kubernetes-map-type: atomic + priority: + description: |- + Priority of this target. Lower values are preferred. Targets at the same + priority are selected using a score that considers health and latency. The + next priority is used only when every target at this priority is degraded. + Configure policies.health on concrete target models to customize + degradation and eviction behavior. + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - modelRef + - priority + type: object + maxItems: 64 + minItems: 1 + type: array + required: + - targets + type: object + weighted: + description: Weight-based model selection. + properties: + targets: + description: Concrete model targets and their relative weights. + items: + properties: + model: + description: |- + Concrete model name selected through the referenced model. It is required + when modelRef points to a wildcard match.model. When omitted, the referenced + model's exact effective match.model is used. + maxLength: 1024 + minLength: 1 + type: string + modelRef: + description: Same-namespace AgentgatewayModel resource + selected by this target. + properties: + name: + default: "" + description: Name of the referent + type: string + type: object + x-kubernetes-map-type: atomic + weight: + description: Relative traffic weight. Defaults to 1. + format: int32 + maximum: 1000000 + minimum: 1 + type: integer + required: + - modelRef + type: object + maxItems: 64 + minItems: 1 + type: array + required: + - targets + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [weighted failover conditional] + must be set + rule: (has(self.weighted)?1:0)+(has(self.failover)?1:0)+(has(self.conditional)?1:0) + == 1 + visibility: + description: |- + Controls whether clients can request this model directly. Internal models + can only be selected by virtual models. Defaults to Public. + enum: + - Internal + - Public + type: string + required: + - parentRefs + type: object + x-kubernetes-validations: + - message: baseURL requires provider + rule: has(self.provider) || !has(self.baseURL) + - message: policies cannot be used with virtualModel + rule: '!has(self.virtualModel) || !has(self.policies)' + - message: virtual models must be public + rule: '!has(self.virtualModel) || !has(self.visibility) || self.visibility + != ''Internal''' + - message: virtual model match.model must be an exact name + rule: '!has(self.virtualModel) || !has(self.match) || !has(self.match.model) + || !self.match.model.contains(''*'')' + - message: ollama requires baseURL + rule: '!has(self.provider) || self.provider != ''Ollama'' || has(self.baseURL)' + - message: baseURL must be an absolute http or https URL with a host + rule: '!has(self.baseURL) || (isURL(self.baseURL) && (url(self.baseURL).getScheme() + == ''http'' || url(self.baseURL).getScheme() == ''https'') && url(self.baseURL).getHostname() + != "")' + - message: baseURL cannot target localhost, loopback, or link-local addresses + rule: '!has(self.baseURL) || !self.baseURL.matches("(?i)^https?://(localhost|[^/]+\\.localhost)(:[0-9]+)?(/|$)")' + - message: baseURL cannot target localhost, loopback, or link-local addresses + rule: '!has(self.baseURL) || !self.baseURL.matches("^https?://127(\\.[0-9]{1,3}){0,3}(:[0-9]+)?(/|$)")' + - message: baseURL cannot target localhost, loopback, or link-local addresses + rule: '!has(self.baseURL) || !self.baseURL.matches("^https?://169\\.254\\.[0-9]{1,3}\\.[0-9]{1,3}(:[0-9]+)?(/|$)")' + - message: baseURL cannot target localhost, loopback, or link-local addresses + rule: '!has(self.baseURL) || !self.baseURL.matches("(?i)^https?://\\[(::1|fe[89ab][0-9a-f:]*)\\](:[0-9]+)?(/|$)")' + - message: azure must be set if and only if provider is Azure + rule: has(self.azure) == (has(self.provider) && self.provider == 'Azure') + - message: vertexai must be set if and only if provider is VertexAI + rule: has(self.vertexai) == (has(self.provider) && self.provider == + 'VertexAI') + - message: bedrock must be set if and only if provider is Bedrock + rule: has(self.bedrock) == (has(self.provider) && self.provider == 'Bedrock') + - message: custom must be set if and only if provider is Custom + rule: has(self.custom) == (has(self.provider) && self.provider == 'Custom') + - message: exactly one of the fields in [provider virtualModel] must be + set + rule: (has(self.provider)?1:0)+(has(self.virtualModel)?1:0) == 1 + status: + description: Current model attachment status. + properties: + parents: + description: Status for each Gateway parent to which this model is + attached. + items: + description: |- + RouteParentStatus describes the status of a route with respect to an + associated Parent. + properties: + conditions: + description: |- + Conditions describes the status of the route with respect to the Gateway. + Note that the route's availability is also subject to the Gateway's own + status conditions and listener status. + + If the Route's ParentRef specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller has sufficient access, + then that Gateway's controller MUST set the "Accepted" condition on the + Route, to indicate whether the route has been accepted or rejected by the + Gateway, and why. + + A Route MUST be considered "Accepted" if at least one of the Route's + rules is implemented by the Gateway. + + There are a number of cases where the "Accepted" condition may not be set + due to lack of controller visibility, that includes when: + + * The Route refers to a nonexistent parent. + * The Route is of a type that the controller does not support. + * The Route is in a namespace to which the controller does not have access. + + + + Notes for implementors: + + Conditions are a listType `map`, which means that they function like a + map with a key of the `type` field _in the k8s apiserver_. + + This means that implementations must obey some rules when updating this + section. + + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + * Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. + * Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. + * Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. + * If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + + + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: |- + ControllerName is a domain/path string that indicates the name of the + controller that wrote this status. This corresponds with the + controllerName field on GatewayClass. + + Example: "example.net/gateway-controller". + + The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are + valid Kubernetes names + (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + + Controllers MUST populate this field when writing status. Controllers should ensure that + entries to status populated with their ControllerName are cleaned up when they are no + longer necessary. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: |- + ParentRef corresponds with a ParentRef in the spec that this + RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: 'Namespace of the referent. Defaults to the + Route''s local namespace. Cross-namespace references must + be explicitly allowed, for example via ReferenceGrant. + Support: Core' + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: 'Port this Route targets on the parent, interpreted + per parent kind (for example a Gateway listener port or + Service port). Support: Extended' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: 'Name of a section within the target resource, + for example a Gateway Listener name or Service port name. + Empty references the entire resource. Support: Core' + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - conditions + - controllerName + - parentRef + type: object + maxItems: 16 + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/test/crd/inference-extension/httproute.yaml b/test/crd/inference-extension/httproute.yaml index 82e8cbfc3..dd7cd7ab6 100644 --- a/test/crd/inference-extension/httproute.yaml +++ b/test/crd/inference-extension/httproute.yaml @@ -68,6 +68,8 @@ spec: items: type: object properties: + name: + type: string matches: type: array items: diff --git a/test/e2e-rbac/crds/README.md b/test/e2e-rbac/crds/README.md index cb711e9b4..de9c2a06a 100644 --- a/test/e2e-rbac/crds/README.md +++ b/test/e2e-rbac/crds/README.md @@ -1,9 +1,10 @@ # CRDs for `make test-e2e-rbac` with RBAC_E2E_GATEWAY_API=true. -These two files are symlinks into the vendored upstream CRDs +These three files are symlinks into the vendored upstream CRDs (`test/crd/inference-extension/README.md` records their provenance). The -AgentGateway plane needs the InferencePool and AgentgatewayPolicy kinds, which -the upstream Gateway API standard bundle does not ship. +AgentGateway plane needs the InferencePool, AgentgatewayPolicy and +AgentgatewayModel kinds, which the upstream Gateway API standard bundle does +not ship. The vendored directory also contains hand-made Gateway and HTTPRoute stubs. Those must NOT be installed next to `RBAC_E2E_GATEWAY_API=true`: the script diff --git a/test/e2e-rbac/crds/agentgatewaymodel.yaml b/test/e2e-rbac/crds/agentgatewaymodel.yaml new file mode 120000 index 000000000..d25b603f9 --- /dev/null +++ b/test/e2e-rbac/crds/agentgatewaymodel.yaml @@ -0,0 +1 @@ +../../crd/inference-extension/agentgatewaymodel.yaml \ No newline at end of file diff --git a/test/e2e-rbac/fixtures/50-agentgateway.yaml b/test/e2e-rbac/fixtures/50-agentgateway.yaml index 81baabf92..6f0aad816 100644 --- a/test/e2e-rbac/fixtures/50-agentgateway.yaml +++ b/test/e2e-rbac/fixtures/50-agentgateway.yaml @@ -58,3 +58,24 @@ spec: route: backends: - local +--- +# The same backend through agentGatewayRef.routing: Model. This exercises +# agentgatewaymodels create/update, and the lists every Model-mode reconcile +# runs to sweep the other mode's objects and the models of removed backends. +# Deleting it exercises owner-reference garbage collection of the route and +# the model, as for the router above. +apiVersion: inference.llmkube.dev/v1alpha1 +kind: ModelRouter +metadata: + name: rbac-agw-model-router +spec: + dataPlane: AgentGateway + agentGatewayRef: + name: rbac-agw-gateway + routing: Model + sectionName: http + backends: + - name: local + tier: local + inferenceServiceRef: + name: rbac-agw-cpu