diff --git a/api/v1alpha1/modelrouter_types.go b/api/v1alpha1/modelrouter_types.go index 2bc055c81..f6fe52263 100644 --- a/api/v1alpha1/modelrouter_types.go +++ b/api/v1alpha1/modelrouter_types.go @@ -803,19 +803,63 @@ type MCPServerSpec struct { Enabled bool `json:"enabled,omitempty"` } +// AgentGatewayRouting selects what a dataPlane: AgentGateway ModelRouter +// compiles to. +type AgentGatewayRouting string + +const ( + AgentGatewayRoutingInferencePool AgentGatewayRouting = "InferencePool" + AgentGatewayRoutingModel AgentGatewayRouting = "Model" +) + // AgentGatewayReference identifies the pre-installed agentgateway data plane a // dataPlane: AgentGateway ModelRouter compiles onto. The Gateway is the // gateway.networking.k8s.io Gateway whose GatewayClass is agentgateway; the // EndpointPicker is the Endpoint Picker (EPP) Service that selects a model // server endpoint per request for each generated InferencePool. The Gateway, // the Gateway API Inference Extension CRDs, and a running EPP are documented -// prerequisites that LLMKube does not install or own. +// prerequisites that LLMKube does not install or own. With routing: Model only +// the Gateway and agentgateway v1.6.0 or later are. // +kubebuilder:validation:XValidation:rule="has(self.endpointPicker) == has(self.endpointPickerPort)",message="endpointPicker and endpointPickerPort must be set together" +// +kubebuilder:validation:XValidation:rule="!(has(self.routing) && self.routing == 'Model') || !has(self.endpointPicker)",message="endpointPicker and endpointPickerPort apply only to routing: InferencePool" +// +kubebuilder:validation:XValidation:rule="!(has(self.routing) && self.routing == 'Model') || has(self.sectionName) || has(self.hostnames)",message="routing: Model requires sectionName or hostnames: its HTTPRoute matches every path, so it must not take over a whole Gateway" +// +kubebuilder:validation:XValidation:rule="(has(self.routing) && self.routing == 'Model') || (!has(self.sectionName) && !has(self.hostnames))",message="sectionName and hostnames apply only to routing: Model" type AgentGatewayReference struct { // Name is the agentgateway Gateway's name. // +kubebuilder:validation:Required Name string `json:"name"` + // Routing selects the compile target. "InferencePool" (the default) is the + // Gateway API Inference Extension path described on spec.agentGatewayRef. + // "Model" uses agentgateway's model router (v1.6.0+): one AgentgatewayModel + // per backend, matched on its displayName (else name), behind one + // catch-all HTTPRoute. It needs no Inference Extension CRDs or Endpoint + // Picker. spec.rules, spec.defaultRoute, external backends, resolution: + // endpoint, and per-backend endpoint pickers are refused in this mode. + // +kubebuilder:validation:Enum=InferencePool;Model + // +kubebuilder:default=InferencePool + // +optional + Routing AgentGatewayRouting `json:"routing,omitempty"` + + // SectionName names the Gateway listener the generated HTTPRoute attaches + // to. routing: Model only, where it or hostnames is required, because the + // route takes every path on its listeners. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +optional + SectionName string `json:"sectionName,omitempty"` + + // Hostnames limit the generated HTTPRoute to these hosts. routing: Model + // only, where it or sectionName is required. + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=253 + // +kubebuilder:validation:items:Pattern=`^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +optional + Hostnames []string `json:"hostnames,omitempty"` + // Namespace is the Gateway's namespace. Empty means the ModelRouter's own // namespace. A different namespace is supported: the generated // model-extraction AgentgatewayPolicy then lives in the Gateway's diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 977b6048f..6c2266b54 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -54,6 +54,11 @@ func (in *AccelerationStatus) DeepCopy() *AccelerationStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AgentGatewayReference) DeepCopyInto(out *AgentGatewayReference) { *out = *in + if in.Hostnames != nil { + in, out := &in.Hostnames, &out.Hostnames + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentGatewayReference. @@ -1508,7 +1513,7 @@ func (in *ModelRouterSpec) DeepCopyInto(out *ModelRouterSpec) { if in.AgentGatewayRef != nil { in, out := &in.AgentGatewayRef, &out.AgentGatewayRef *out = new(AgentGatewayReference) - **out = **in + (*in).DeepCopyInto(*out) } if in.Backends != nil { in, out := &in.Backends, &out.Backends diff --git a/charts/llmkube/templates/clusterrole.yaml b/charts/llmkube/templates/clusterrole.yaml index dbf79836c..d783cc8b9 100644 --- a/charts/llmkube/templates/clusterrole.yaml +++ b/charts/llmkube/templates/clusterrole.yaml @@ -225,6 +225,7 @@ rules: - apiGroups: - agentgateway.dev resources: + - agentgatewaymodels - agentgatewaypolicies verbs: - create diff --git a/charts/llmkube/templates/crds/modelrouters.yaml b/charts/llmkube/templates/crds/modelrouters.yaml index 8141a6bf3..dddb7c524 100644 --- a/charts/llmkube/templates/crds/modelrouters.yaml +++ b/charts/llmkube/templates/crds/modelrouters.yaml @@ -110,6 +110,18 @@ spec: maximum: 65535 minimum: 1 type: integer + hostnames: + description: |- + Hostnames limit the generated HTTPRoute to these hosts. routing: Model + only, where it or sectionName is required. + items: + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + minItems: 1 + type: array name: description: Name is the agentgateway Gateway's name. type: string @@ -124,12 +136,46 @@ spec: EndpointPicker Service all live in this ModelRouter's namespace regardless. type: string + routing: + default: InferencePool + description: |- + Routing selects the compile target. "InferencePool" (the default) is the + Gateway API Inference Extension path described on spec.agentGatewayRef. + "Model" uses agentgateway's model router (v1.6.0+): one AgentgatewayModel + per backend, matched on its displayName (else name), behind one + catch-all HTTPRoute. It needs no Inference Extension CRDs or Endpoint + Picker. spec.rules, spec.defaultRoute, external backends, resolution: + endpoint, and per-backend endpoint pickers are refused in this mode. + enum: + - InferencePool + - Model + type: string + sectionName: + description: |- + SectionName names the Gateway listener the generated HTTPRoute attaches + to. routing: Model only, where it or hostnames is required, because the + route takes every path on its listeners. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string required: - name type: object x-kubernetes-validations: - message: endpointPicker and endpointPickerPort must be set together rule: has(self.endpointPicker) == has(self.endpointPickerPort) + - message: 'endpointPicker and endpointPickerPort apply only to routing: + InferencePool' + rule: '!(has(self.routing) && self.routing == ''Model'') || !has(self.endpointPicker)' + - message: 'routing: Model requires sectionName or hostnames: its + HTTPRoute matches every path, so it must not take over a whole + Gateway' + rule: '!(has(self.routing) && self.routing == ''Model'') || has(self.sectionName) + || has(self.hostnames)' + - message: 'sectionName and hostnames apply only to routing: Model' + rule: (has(self.routing) && self.routing == 'Model') || (!has(self.sectionName) + && !has(self.hostnames)) backends: description: |- Backends are the candidate destinations the router can dispatch to. diff --git a/config/crd/bases/inference.llmkube.dev_modelrouters.yaml b/config/crd/bases/inference.llmkube.dev_modelrouters.yaml index a1b0c1371..13075e27f 100644 --- a/config/crd/bases/inference.llmkube.dev_modelrouters.yaml +++ b/config/crd/bases/inference.llmkube.dev_modelrouters.yaml @@ -106,6 +106,18 @@ spec: maximum: 65535 minimum: 1 type: integer + hostnames: + description: |- + Hostnames limit the generated HTTPRoute to these hosts. routing: Model + only, where it or sectionName is required. + items: + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + minItems: 1 + type: array name: description: Name is the agentgateway Gateway's name. type: string @@ -120,12 +132,46 @@ spec: EndpointPicker Service all live in this ModelRouter's namespace regardless. type: string + routing: + default: InferencePool + description: |- + Routing selects the compile target. "InferencePool" (the default) is the + Gateway API Inference Extension path described on spec.agentGatewayRef. + "Model" uses agentgateway's model router (v1.6.0+): one AgentgatewayModel + per backend, matched on its displayName (else name), behind one + catch-all HTTPRoute. It needs no Inference Extension CRDs or Endpoint + Picker. spec.rules, spec.defaultRoute, external backends, resolution: + endpoint, and per-backend endpoint pickers are refused in this mode. + enum: + - InferencePool + - Model + type: string + sectionName: + description: |- + SectionName names the Gateway listener the generated HTTPRoute attaches + to. routing: Model only, where it or hostnames is required, because the + route takes every path on its listeners. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string required: - name type: object x-kubernetes-validations: - message: endpointPicker and endpointPickerPort must be set together rule: has(self.endpointPicker) == has(self.endpointPickerPort) + - message: 'endpointPicker and endpointPickerPort apply only to routing: + InferencePool' + rule: '!(has(self.routing) && self.routing == ''Model'') || !has(self.endpointPicker)' + - message: 'routing: Model requires sectionName or hostnames: its + HTTPRoute matches every path, so it must not take over a whole + Gateway' + rule: '!(has(self.routing) && self.routing == ''Model'') || has(self.sectionName) + || has(self.hostnames)' + - message: 'sectionName and hostnames apply only to routing: Model' + rule: (has(self.routing) && self.routing == 'Model') || (!has(self.sectionName) + && !has(self.hostnames)) backends: description: |- Backends are the candidate destinations the router can dispatch to. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 67edd30a3..708e5f54e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -81,6 +81,7 @@ rules: - apiGroups: - agentgateway.dev resources: + - agentgatewaymodels - agentgatewaypolicies verbs: - create diff --git a/docs/site/concepts/model-router.md b/docs/site/concepts/model-router.md index 0fcc5371b..68959c47a 100644 --- a/docs/site/concepts/model-router.md +++ b/docs/site/concepts/model-router.md @@ -347,6 +347,65 @@ Each `InferencePool` is named after its `InferenceService`, and a router never a `primary-fallback` routes each rule to its first healthy backend. When that backend's `InferenceService` has no ready replica, the next reconcile moves the rule to the next backend. Failover follows replica readiness, not individual request errors: a primary that is up but failing requests keeps its traffic. +## Model routing on dataPlane: AgentGateway + +`agentGatewayRef.routing: Model` compiles the router onto agentgateway's own model router instead of the Inference Extension. It needs agentgateway v1.6.0 or later, which ships the `AgentgatewayModel` CRD, and a Gateway. It needs no Gateway API Inference Extension CRDs, no Endpoint Picker, and no Gateway-wide policy, which suits single-replica `InferenceService`s reached through their Service. + +```yaml +apiVersion: inference.llmkube.dev/v1alpha1 +kind: ModelRouter +metadata: + name: llm-router +spec: + dataPlane: AgentGateway + agentGatewayRef: + name: ai-gateway + routing: Model + sectionName: llm + backends: + - name: chat + displayName: chat-large + inferenceServiceRef: + name: chat-vllm + - name: chat-small + inferenceServiceRef: + name: chat-llama + - name: reranker + inferenceServiceRef: + name: reranker +``` + +`chat-vllm` must start vLLM with `--served-model-name=chat-large` in its `extraArgs`, since LLMKube does not set that flag. + +The router generates, all in its own namespace and owned by it: + +- one `HTTPRoute` named after the router, attached to the Gateway, with a single rule named `models` at `/` whose only backend is `AgentgatewayModel` `*`. The rule sits at `/` because agentgateway strips the rule's path prefix before it classifies the endpoint: under `/v1`, `/v1/embeddings` would parse as a chat completion; +- one `AgentgatewayModel` per backend, named `-` (cut and hashed past 63 characters), attached to that rule. Its `match.model` is the backend's `displayName`, else its `name`, so served names such as `Qwen3.5-2B` or `org/model` work. It reaches the `InferenceService`'s Service through a custom provider, and the request keeps its model name, so the served name must be one the runtime answers to: llama.cpp ignores it, and vLLM needs a matching `--served-model-name` in `extraArgs`. + +The provider and API formats come from the `InferenceService`: + +| Runtime | `spec.mode` (or inferred) | `providerOverride` | Formats | +|---|---|---|---| +| `llamacpp` (or unset) | chat | `llamacpp` | `Completions` | +| `vllm` | chat | `vllm` | `Completions`, `Messages`, `AnthropicTokenCount`, `Responses` | +| `llamacpp`, `vllm` | embedding | as above | `Embeddings` | +| `llamacpp`, `vllm` | rerank | as above | `Rerank` | + +agentgateway then routes each request on the body's `model`, answers `GET /v1/models` itself, and returns a JSON 404 `model_not_found` for a model no backend serves. + +A rule at `/` takes every path on the listeners it attaches to, so this mode requires `agentGatewayRef.sectionName` (one listener), `agentGatewayRef.hostnames`, or both. Your own routes still share the listener: a rule with a longer match, such as a `PathPrefix` `/v1` with a header match, outranks the router's `/`. Use one `routing: Model` router per listener and hostname. Two routers' `/` rules tie, Gateway API gives the tie to the older route, and the newer router's models get no traffic. + +The router reports `AgentGatewayReady=False` and generates nothing when: + +- the `AgentgatewayModel` CRD is not installed (`AgentgatewayModelCRDNotInstalled`); +- the spec has `rules`, a `defaultRoute`, an `external` backend, `resolution: endpoint`, a backend `endpointPicker`, a served name containing `*`, or two backends serving one name (`UnsupportedInModelRouting`; the webhook rejects these at apply time). Fallback between backends and external backends need an `AgentgatewayBackend`, which this mode does not generate yet; +- a backend's `InferenceService` runs a runtime other than `llamacpp` or `vllm`, or runs vLLM without a `--served-model-name` matching the backend's served name (`UnsupportedRuntimeInModelRouting`); +- a generated name is held by an object LLMKube did not create (`GeneratedResourceNameTaken`); delete or rename that object. + +When you replace hand-written `AgentgatewayModel`s with a router, remove the hand-written ones for the same model names on that listener, so each name has one model. + +Removing a backend deletes its model. Switching `routing` in either direction deletes the other mode's objects, including the `InferencePool` mode's extraction policy in the Gateway's namespace. + ## Composition with LiteLLM LLMKube does not replace [LiteLLM](https://github.com/BerriAI/litellm). For organizations already running a LiteLLM proxy as their cloud-provider abstraction, point a `ModelRouter` external backend at it: diff --git a/internal/controller/gateway_resources_agentgateway.go b/internal/controller/gateway_resources_agentgateway.go index 7863fea60..c837ff484 100644 --- a/internal/controller/gateway_resources_agentgateway.go +++ b/internal/controller/gateway_resources_agentgateway.go @@ -209,15 +209,6 @@ func newAgentGatewayHTTPRoute( rules []agentGatewayRuleResource, backends []agentGatewayBackendResource, ) *unstructured.Unstructured { - parentRef := map[string]interface{}{ - metadataNameField: ref.Name, - "kind": "Gateway", - "group": gatewayBackendRefGroupAPI, - } - if ref.Namespace != "" { - parentRef["namespace"] = ref.Namespace - } - compiledRules := make([]interface{}, 0, len(rules)) poolFor := make(map[string]string, len(backends)) for _, b := range backends { @@ -235,12 +226,29 @@ func newAgentGatewayHTTPRoute( // leaving-AgentGateway sweep can find it (mirrors the extraction policy). u.SetLabels(map[string]string{agentGatewayRouterUIDLabel: string(mr.UID)}) u.Object["spec"] = map[string]interface{}{ - "parentRefs": []interface{}{parentRef}, + "parentRefs": []interface{}{agentGatewayRouteParentRef(ref)}, "rules": compiledRules, } return u } +// agentGatewayRouteParentRef only carries a sectionName with routing: Model, +// where the CRD allows it, so the InferencePool route renders unchanged. +func agentGatewayRouteParentRef(ref *inferencev1alpha1.AgentGatewayReference) map[string]interface{} { + parentRef := map[string]interface{}{ + metadataNameField: ref.Name, + "kind": "Gateway", + "group": gatewayBackendRefGroupAPI, + } + if ref.Namespace != "" { + parentRef["namespace"] = ref.Namespace + } + if ref.SectionName != "" { + parentRef["sectionName"] = ref.SectionName + } + return parentRef +} + // compileAgentGatewayRouteRule builds one HTTPRoute rule: the matches block (a // match per model name, ANDed with header matches; a header-only match when the // rule declares no models, e.g. a catch-all) and the backendRefs block. diff --git a/internal/controller/modelrouter_agentgateway_controller.go b/internal/controller/modelrouter_agentgateway_controller.go index b9b0ec12e..a289cdbf4 100644 --- a/internal/controller/modelrouter_agentgateway_controller.go +++ b/internal/controller/modelrouter_agentgateway_controller.go @@ -174,10 +174,15 @@ type ModelRouterAgentGatewayReconciler struct { // InferencePool kind and agentgateway.dev's AgentgatewayPolicy kind. detectorOnce sync.Once detector *crdDetector + + // modelDetector gates routing: Model on the AgentgatewayModel CRD alone. + modelDetectorOnce sync.Once + modelDetector *crdDetector } // +kubebuilder:rbac:groups=inference.networking.k8s.io,resources=inferencepools,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=agentgateway.dev,resources=agentgatewaypolicies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=agentgateway.dev,resources=agentgatewaymodels,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch @@ -224,7 +229,7 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c // status.agentGateway block (set exactly when generation succeeded), or a // TRUE AgentGatewayReady condition. A FALSE condition is not evidence: // refusals set it without generating anything, and counting it would make - // every refusal reconcile re-run the three Lists below - the cost this + // every refusal reconcile re-run the four Lists below - the cost this // gate exists to remove. readyCond := apimeta.FindStatusCondition(mr.Status.Conditions, ModelRouterAgentGatewayConditionReady) usedAgentGateway := hasFinalizer || mr.Status.AgentGateway != nil || @@ -240,10 +245,11 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{}, err } if deleting || leaving { - // Pools and routes are owner-referenced, but owner-ref GC only - // fires when the router object goes away: a router that leaves - // AgentGateway mode must stop attaching to the Gateway now. - if err := r.cleanupGeneratedPoolsAndRoutes(ctx, mr); err != nil { + // Pools, routes and models are owner-referenced, but owner-ref GC + // only fires when the router object goes away: a router that + // leaves AgentGateway mode must stop attaching to the Gateway now. + if err := r.deleteGeneratedObjects(ctx, mr, nil, + inferencePoolGVK(), gatewayHTTPRouteGVK(), agentGatewayModelGVK()); err != nil { return ctrl.Result{}, err } } @@ -309,7 +315,8 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c // kind is meant, rather than having to read the detector; #1725). A // transient discovery error (not a missing kind) is returned so we requeue // rather than disabling. - present, missing, err := r.requiredCRDsMissing(log) + modelRouting := agentGatewayModelRouting(mr) + present, missing, err := r.requiredCRDsMissing(log, modelRouting) if err != nil { return ctrl.Result{}, err } @@ -318,6 +325,11 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c for _, gvk := range missing { names = append(names, fmt.Sprintf("%s (%s/%s)", gvk.Kind, gvk.Group, gvk.Version)) } + if modelRouting { + return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonModelCRDMissing, + "agentGatewayRef.routing: Model needs agentgateway v1.6.0 or later, and its AgentgatewayModel CRD is not "+ + "installed; agentgateway exposure is disabled; missing: "+strings.Join(names, ", ")) + } return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonCRDsMissing, "Gateway API Inference Extension and agentgateway CRDs are not installed; agentgateway exposure is disabled; missing: "+ strings.Join(names, ", ")) @@ -373,50 +385,53 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil } - // Fail loud when the AgentgatewayPolicy this router would generate already - // exists and LLMKube did not create it. Runs BEFORE the conflict check so - // a foreign policy that both holds our name and conflicts reports the more - // specific name-taken reason, and before any generation so a name-taken - // router yields no partial resources. The pools and the route are checked - // the same way inside reconcileAgentGatewayResources, once their names are - // known from the resolved backends. - if err := r.checkGeneratedNameAvailable(ctx, mr, newAgentGatewayModelExtractionPolicy(mr, mr.Spec.AgentGatewayRef)); err != nil { - var taken *agentGatewayNameTakenError - if errors.As(err, &taken) { - // Whether the name is free depends on objects outside the router's - // spec; requeue so deleting or renaming the foreign object takes - // effect without an unrelated event. - if serr := r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonResourceNameTaken, err.Error()); serr != nil { - return ctrl.Result{}, serr + // routing: Model writes no extraction policy, so these checks don't apply. + if !modelRouting { + // Fail loud when the AgentgatewayPolicy this router would generate already + // exists and LLMKube did not create it. Runs BEFORE the conflict check so + // a foreign policy that both holds our name and conflicts reports the more + // specific name-taken reason, and before any generation so a name-taken + // router yields no partial resources. The pools and the route are checked + // the same way inside reconcileAgentGatewayResources, once their names are + // known from the resolved backends. + if err := r.checkGeneratedNameAvailable(ctx, mr, newAgentGatewayModelExtractionPolicy(mr, mr.Spec.AgentGatewayRef)); err != nil { + var taken *agentGatewayNameTakenError + if errors.As(err, &taken) { + // Whether the name is free depends on objects outside the router's + // spec; requeue so deleting or renaming the foreign object takes + // effect without an unrelated event. + if serr := r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonResourceNameTaken, err.Error()); serr != nil { + return ctrl.Result{}, serr + } + return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil } - return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil + return ctrl.Result{}, err } - return ctrl.Result{}, err - } - // Fail loud when the target Gateway already carries a PreRouting - // transformation policy this router does not own. agentgateway merges - // equal-specificity policies oldest-wins per whole field and never - // combines list entries (and no condition reports the dropped field), so a - // second transformation on the same Gateway either silently disables the - // generated model extraction or is silently disabled by it. Refuse before - // any generation so a conflicted router yields no partial resources. - conflicts, err := r.agentGatewayPreRoutingConflicts(ctx, mr, gateway.GetLabels()) - if err != nil { - return ctrl.Result{}, err - } - if len(conflicts) > 0 { - if err := r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonPolicyConflict, - fmt.Sprintf("Gateway %s/%s already carries PreRouting transformation policy(ies) that LLMKube does not own: %s. "+ - "agentgateway merges same-specificity policies oldest-wins per field without combining list entries, "+ - "so the generated model extraction could be silently dropped, or silently shadow the existing policy. "+ - "Remove or scope the other policy", agentGatewayPolicyNamespace(mr), mr.Spec.AgentGatewayRef.Name, - strings.Join(conflicts, ", "))); err != nil { + // Fail loud when the target Gateway already carries a PreRouting + // transformation policy this router does not own. agentgateway merges + // equal-specificity policies oldest-wins per whole field and never + // combines list entries (and no condition reports the dropped field), so a + // second transformation on the same Gateway either silently disables the + // generated model extraction or is silently disabled by it. Refuse before + // any generation so a conflicted router yields no partial resources. + conflicts, err := r.agentGatewayPreRoutingConflicts(ctx, mr, gateway.GetLabels()) + if err != nil { return ctrl.Result{}, err } - // Recheck on the conflict cadence so the condition clears by itself - // when the foreign policy is deleted. - return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil + if len(conflicts) > 0 { + if err := r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonPolicyConflict, + fmt.Sprintf("Gateway %s/%s already carries PreRouting transformation policy(ies) that LLMKube does not own: %s. "+ + "agentgateway merges same-specificity policies oldest-wins per field without combining list entries, "+ + "so the generated model extraction could be silently dropped, or silently shadow the existing policy. "+ + "Remove or scope the other policy", agentGatewayPolicyNamespace(mr), mr.Spec.AgentGatewayRef.Name, + strings.Join(conflicts, ", "))); err != nil { + return ctrl.Result{}, err + } + // Recheck on the conflict cadence so the condition clears by itself + // when the foreign policy is deleted. + return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil + } } // Fail-loud on matches the agentgateway data plane cannot express. We @@ -424,7 +439,10 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c // user expects. The agentgateway plane matches on the OpenAI "model" field // (via the extracted agentGatewayHTTPModelHeader) and on headers (via // HTTPRoute header matches); everything the Gateway plane rejects is - // rejected here too. + // rejected here too. routing: Model has its own boundary, checked first. + if msg := agentGatewayModelRoutingMessage(mr); msg != "" { + return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonUnsupportedModelRouting, msg) + } if msg := agentGatewayUnsupportedMatchMessage(mr); msg != "" { return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonUnsupported, msg) } @@ -480,7 +498,12 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, reason, msg) } - ejected, err := r.reconcileAgentGatewayResources(ctx, mr) + var ejected []string + if modelRouting { + err = r.reconcileAgentGatewayModels(ctx, mr) + } else { + ejected, err = r.reconcileAgentGatewayResources(ctx, mr) + } if err != nil { var taken *agentGatewayNameTakenError if errors.As(err, &taken) { @@ -492,6 +515,11 @@ func (r *ModelRouterAgentGatewayReconciler) Reconcile(ctx context.Context, req c } return ctrl.Result{RequeueAfter: agentGatewayConflictRequeueAfter}, nil } + // No requeue: the InferenceService watch re-reconciles on a fix. + var refusal *agentGatewayRefusalError + if errors.As(err, &refusal) { + return ctrl.Result{}, r.setAgentGatewayNotReady(ctx, mr, refusal.reason, refusal.message) + } _ = r.setAgentGatewayNotReady(ctx, mr, modelRouterAgentGatewayReasonReconcile, err.Error()) return ctrl.Result{}, err } @@ -577,6 +605,10 @@ func (r *ModelRouterAgentGatewayReconciler) reconcileAgentGatewayResources( return nil, fmt.Errorf("%s/%s: %w", obj.GetKind(), obj.GetName(), err) } } + // Remove what routing: Model left behind. + if err := r.deleteGeneratedObjects(ctx, mr, nil, agentGatewayModelGVK()); err != nil { + return nil, err + } return ejected, nil } @@ -595,27 +627,9 @@ func (r *ModelRouterAgentGatewayReconciler) resolveAgentGatewayBackends( pickers, _, _ := agentGatewayResolvePoolPickers(mr) resolved := make([]agentGatewayBackendResource, 0, len(mr.Spec.Backends)) for _, b := range mr.Spec.Backends { - if b.External != nil { - return nil, fmt.Errorf("backend %q is External; dataPlane: AgentGateway routes to an InferencePool "+ - "that selects model-server pods, so off-cluster backends cannot be attached", b.Name) - } - if b.InferenceServiceRef == nil { - return nil, fmt.Errorf("backend %q has no inferenceServiceRef", b.Name) - } - - isvc := &inferencev1alpha1.InferenceService{} - key := types.NamespacedName{Name: b.InferenceServiceRef.Name, Namespace: mr.Namespace} - if err := r.Get(ctx, key, isvc); err != nil { - if apierrors.IsNotFound(err) { - return nil, fmt.Errorf("backend %q references InferenceService %q which does not exist in namespace %q", - b.Name, b.InferenceServiceRef.Name, mr.Namespace) - } - return nil, fmt.Errorf("backend %q: looking up InferenceService %q: %w", b.Name, b.InferenceServiceRef.Name, err) - } - - port := int64(8080) - if isvc.Spec.Endpoint != nil && isvc.Spec.Endpoint.Port > 0 { - port = int64(isvc.Spec.Endpoint.Port) + isvc, err := r.backendInferenceService(ctx, mr, b) + if err != nil { + return nil, err } resolved = append(resolved, agentGatewayBackendResource{ Name: b.Name, @@ -623,7 +637,7 @@ func (r *ModelRouterAgentGatewayReconciler) resolveAgentGatewayBackends( // InferenceService name so one pool serves one model's pods. PoolName: sanitizeDNSName(isvc.Name), FQDN: fmt.Sprintf("%s.%s.svc.cluster.local", sanitizeDNSName(isvc.Name), isvc.Namespace), - Port: port, + Port: gatewayServicePort(isvc), Picker: pickers[sanitizeDNSName(isvc.Name)], // A backend is healthy iff its InferenceService has at least one ready // replica. An unhealthy backend is ejected from the route's @@ -635,6 +649,33 @@ func (r *ModelRouterAgentGatewayReconciler) resolveAgentGatewayBackends( return resolved, nil } +// backendInferenceService fetches a backend's InferenceService. External +// backends and missing InferenceServices are errors. +func (r *ModelRouterAgentGatewayReconciler) backendInferenceService( + ctx context.Context, + mr *inferencev1alpha1.ModelRouter, + b inferencev1alpha1.RouterBackend, +) (*inferencev1alpha1.InferenceService, error) { + if b.External != nil { + return nil, fmt.Errorf("backend %q is External; dataPlane: AgentGateway routes to an InferencePool "+ + "that selects model-server pods, so off-cluster backends cannot be attached", b.Name) + } + if b.InferenceServiceRef == nil { + return nil, fmt.Errorf("backend %q has no inferenceServiceRef", b.Name) + } + + isvc := &inferencev1alpha1.InferenceService{} + key := types.NamespacedName{Name: b.InferenceServiceRef.Name, Namespace: mr.Namespace} + if err := r.Get(ctx, key, isvc); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf("backend %q references InferenceService %q which does not exist in namespace %q", + b.Name, b.InferenceServiceRef.Name, mr.Namespace) + } + return nil, fmt.Errorf("backend %q: looking up InferenceService %q: %w", b.Name, b.InferenceServiceRef.Name, err) + } + return isvc, nil +} + // applyAgentGatewayResource creates-or-updates desired. Objects in the // ModelRouter's namespace (pools, route, same-namespace extraction policy) are // controller-referenced to the ModelRouter so owner-ref GC removes them. The @@ -695,7 +736,13 @@ func (r *ModelRouterAgentGatewayReconciler) applyAgentGatewayResource( // requiredCRDsMissing reports whether the CRDs this slice needs are registered, // and, when they are not, exactly which GVKs are missing (so the disabled // condition can name them). Delegates to the shared crdDetector. -func (r *ModelRouterAgentGatewayReconciler) requiredCRDsMissing(log logr.Logger) (bool, []schema.GroupVersionKind, error) { +func (r *ModelRouterAgentGatewayReconciler) requiredCRDsMissing(log logr.Logger, modelRouting bool) (bool, []schema.GroupVersionKind, error) { + if modelRouting { + r.modelDetectorOnce.Do(func() { + r.modelDetector = newCRDDetector("agentgateway-model", []schema.GroupVersionKind{agentGatewayModelGVK()}) + }) + return r.modelDetector.PresentMissing(r.Client, log) + } r.detectorOnce.Do(func() { r.detector = newCRDDetector("agentgateway", modelRouterAgentGatewayGVKs()) }) @@ -756,6 +803,7 @@ func (r *ModelRouterAgentGatewayReconciler) agentGatewayAllowedRoutesViolation( mr *inferencev1alpha1.ModelRouter, ) (string, error) { policyNS := agentGatewayPolicyNamespace(mr) + sectionName := mr.Spec.AgentGatewayRef.SectionName listeners, _, _ := unstructured.NestedSlice(gateway.Object, "spec", "listeners") nsLabels := map[string]string{} @@ -805,6 +853,13 @@ func (r *ModelRouterAgentGatewayReconciler) agentGatewayAllowedRoutesViolation( kinds, _, _ := unstructured.NestedSlice(listener, "allowedRoutes", "kinds") summary = append(summary, describeListener(from, proto, kinds)) + // A sectionName route attaches to that listener only. + if sectionName != "" { + if name, _, _ := unstructured.NestedString(listener, metadataNameField); name != sectionName { + continue + } + } + // An HTTPRoute attaches only to HTTP/HTTPS listeners, and only to // kinds the listener lists: an unset kinds list defaults by protocol // (HTTP/HTTPS with no kinds admits HTTPRoute), while a non-empty list @@ -862,10 +917,17 @@ func (r *ModelRouterAgentGatewayReconciler) agentGatewayAllowedRoutesViolation( } } } - return fmt.Sprintf("Gateway %s/%s has no listener that admits namespace %q for an HTTPRoute (listeners: %s); "+ - "the generated HTTPRoute would be refused (NotAllowedByListeners), and the model-extraction AgentgatewayPolicy "+ - "must not be written without that consent", - policyNS, mr.Spec.AgentGatewayRef.Name, mr.Namespace, strings.Join(summary, ", ")), nil + listener := "no listener" + if sectionName != "" { + listener = fmt.Sprintf("no listener named %q", sectionName) + } + msg := fmt.Sprintf("Gateway %s/%s has %s that admits namespace %q for an HTTPRoute (listeners: %s); "+ + "the generated HTTPRoute would be refused (NotAllowedByListeners)", + policyNS, mr.Spec.AgentGatewayRef.Name, listener, mr.Namespace, strings.Join(summary, ", ")) + if agentGatewayModelRouting(mr) { + return msg, nil + } + return msg + ", and the model-extraction AgentgatewayPolicy must not be written without that consent", nil } // labelSelectorFromUnstructured converts an unstructured LabelSelector @@ -1104,21 +1166,24 @@ func (r *ModelRouterAgentGatewayReconciler) cleanupExtractionPolicies( return nil } -// cleanupGeneratedPoolsAndRoutes deletes the InferencePools and HTTPRoute this -// router generated, matched by the router-UID label in the router's namespace. -// It runs when the router leaves AgentGateway mode (dataPlane change or ref -// removal) and on deletion: owner-ref GC only fires when the router object -// goes away, but a router that leaves the mode must stop attaching to the -// Gateway immediately. The sweep is label-based, not owner-ref-based, because -// the Gateway plane also owner-references objects in the router's namespace; -// the UID label is what makes it mode-precise. A missing CRD (NoMatch, or the -// cached-mapping 404 after CRD removal) means the API server removed the CRs -// with it, so there is nothing to clean. -func (r *ModelRouterAgentGatewayReconciler) cleanupGeneratedPoolsAndRoutes( +// deleteGeneratedObjects deletes the objects of each given kind this router +// generated, matched by the router-UID label in the router's namespace, except +// the names in keep. It runs when the router leaves AgentGateway mode +// (dataPlane change or ref removal) and on deletion: owner-ref GC only fires +// when the router object goes away, but a router that leaves the mode must +// stop attaching to the Gateway immediately. Each routing mode also runs it to +// clear the other mode's kinds. The sweep is label-based, not owner-ref-based, +// because the Gateway plane also owner-references objects in the router's +// namespace; the UID label is what makes it mode-precise. A missing CRD +// (NoMatch, or the cached-mapping 404 after CRD removal) means the API server +// removed the CRs with it, so there is nothing to clean. +func (r *ModelRouterAgentGatewayReconciler) deleteGeneratedObjects( ctx context.Context, mr *inferencev1alpha1.ModelRouter, + keep map[string]bool, + gvks ...schema.GroupVersionKind, ) error { - for _, gvk := range []schema.GroupVersionKind{inferencePoolGVK(), gatewayHTTPRouteGVK()} { + for _, gvk := range gvks { list := &unstructured.UnstructuredList{} list.SetGroupVersionKind(gvk) if err := r.List(ctx, list, client.InNamespace(mr.Namespace), @@ -1130,6 +1195,9 @@ func (r *ModelRouterAgentGatewayReconciler) cleanupGeneratedPoolsAndRoutes( } for i := range list.Items { obj := &list.Items[i] + if keep[obj.GetName()] { + continue + } if err := r.Delete(ctx, obj); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("delete %s %s/%s: %w", gvk.Kind, obj.GetNamespace(), obj.GetName(), err) } @@ -1167,7 +1235,7 @@ func (r *ModelRouterAgentGatewayReconciler) setAgentGatewayReady( ) error { patch := client.MergeFrom(mr.DeepCopy()) mr.Status.AgentGateway = &inferencev1alpha1.AgentGatewayStatus{ - PoolReady: true, + PoolReady: !agentGatewayModelRouting(mr), Endpoint: agentGatewayEndpointAddress(mr.Spec.AgentGatewayRef), } message := agentGatewayReadyMessage(mr) @@ -1184,8 +1252,12 @@ func (r *ModelRouterAgentGatewayReconciler) setAgentGatewayReady( } // agentGatewayReadyMessage builds the success condition message: the rule count -// and the resolved endpoint. +// (with routing: Model, the model count) and the resolved endpoint. func agentGatewayReadyMessage(mr *inferencev1alpha1.ModelRouter) string { + if agentGatewayModelRouting(mr) { + return fmt.Sprintf("compiled %d model(s) onto agentgateway %s", len(mr.Spec.Backends), + agentGatewayEndpointAddress(mr.Spec.AgentGatewayRef)) + } return fmt.Sprintf("compiled %d rule(s) onto agentgateway %s", len(mr.Spec.Rules), agentGatewayEndpointAddress(mr.Spec.AgentGatewayRef)) } @@ -1337,12 +1409,8 @@ func compileAgentGatewayRules(mr *inferencev1alpha1.ModelRouter) ([]agentGateway // defaultRoute catch-all. First-match ordering mirrors the proxy. if mr.Spec.DefaultRouteStrategy == inferencev1alpha1.DefaultRouteStrategyBackendNameMatch { for _, b := range mr.Spec.Backends { - modelID := b.Name - if b.DisplayName != "" { - modelID = b.DisplayName - } rules = append(rules, agentGatewayRuleResource{ - Models: []string{modelID}, + Models: []string{agentGatewayServedName(b)}, BackendRefs: []agentGatewayBackendRef{{Name: b.Name}}, }) } diff --git a/internal/controller/modelrouter_agentgateway_model.go b/internal/controller/modelrouter_agentgateway_model.go new file mode 100644 index 000000000..5488e1c68 --- /dev/null +++ b/internal/controller/modelrouter_agentgateway_model.go @@ -0,0 +1,351 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + inferencev1alpha1 "github.com/defilantech/llmkube/api/v1alpha1" +) + +// routing: Model compiles a router to agentgateway's own model router +// (v1.6.0+): one HTTPRoute whose single rule hands every request to +// AgentgatewayModel "*", and one AgentgatewayModel per backend. agentgateway +// matches the body's model, and answers GET /v1/models and unknown models +// itself. + +const ( + agentGatewayModelKind = "AgentgatewayModel" + + // Models name the route's rule as sectionName: agentgateway lets a + // parentRef omit it only while the route has exactly one rule. + agentGatewayModelRuleName = "models" + + // A DNS-1123 label, so a generated name also fits a label value. + agentGatewayModelNameMaxLength = 63 + + // The AgentgatewayModel CRD's match.model maxLength. + agentGatewayModelMatchMaxLength = 1024 + + modelRouterAgentGatewayReasonModelCRDMissing = "AgentgatewayModelCRDNotInstalled" + modelRouterAgentGatewayReasonUnsupportedModelRouting = "UnsupportedInModelRouting" + modelRouterAgentGatewayReasonUnsupportedRuntime = "UnsupportedRuntimeInModelRouting" +) + +func agentGatewayModelGVK() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: agentGatewayPolicyGroup, Version: agentGatewayPolicyVersion, Kind: agentGatewayModelKind} +} + +// agentGatewayModelRouting reports routing: Model; unset means InferencePool. +func agentGatewayModelRouting(mr *inferencev1alpha1.ModelRouter) bool { + ref := mr.Spec.AgentGatewayRef + return ref != nil && ref.Routing == inferencev1alpha1.AgentGatewayRoutingModel +} + +func agentGatewayServedName(b inferencev1alpha1.RouterBackend) string { + if b.DisplayName != "" { + return b.DisplayName + } + return b.Name +} + +// agentGatewayModelName derives the object name from router and backend, not +// from the served name, which need not be a valid name ("org/model"). Long or +// dash-ending names are cut and hashed. Collisions ("a-b"+"c" vs "a"+"b-c", or +// routers "a.b" and "a-b") are refused as GeneratedResourceNameTaken. +func agentGatewayModelName(mr *inferencev1alpha1.ModelRouter, backend string) string { + name := sanitizeDNSName(mr.Name) + "-" + backend + if len(name) <= agentGatewayModelNameMaxLength && !strings.HasSuffix(name, "-") { + return name + } + sum := sha256.Sum256([]byte(mr.Name + "/" + backend)) + tag := "-" + hex.EncodeToString(sum[:])[:8] + keep := min(len(name), agentGatewayModelNameMaxLength-len(tag)) + return strings.TrimRight(name[:keep], "-") + tag +} + +// agentGatewayModelProvider maps an InferenceService's runtime and serving mode +// to agentgateway's provider identity and API formats. ok is false for other +// runtimes: a guessed format would fail every request. +func agentGatewayModelProvider(isvc *inferencev1alpha1.InferenceService) (provider string, formats []string, ok bool) { + var chat []string + switch runtimeNameLabel(isvc) { + case "llamacpp": + provider, chat = "llamacpp", []string{"Completions"} + case RuntimeVLLM: + // vLLM serves Messages and Responses itself. + provider, chat = "vllm", []string{"Completions", "Messages", "AnthropicTokenCount", "Responses"} + default: + return "", nil, false + } + switch resolveServingMode(isvc) { + case servingModeEmbedding: + return provider, []string{"Embeddings"}, true + case servingModeRerank: + return provider, []string{"Rerank"}, true + } + return provider, chat, true +} + +// agentGatewayModelRoutingMessage returns why a routing: Model spec cannot +// compile, or "". The reconciler and the webhook share it. +func agentGatewayModelRoutingMessage(mr *inferencev1alpha1.ModelRouter) string { + if !agentGatewayModelRouting(mr) { + return "" + } + if len(mr.Spec.Rules) > 0 { + return fmt.Sprintf("rule %q does not compile with agentGatewayRef.routing: Model: agentgateway's model router "+ + "sends each request to the backend whose displayName (else name) equals its model field. Remove "+ + "spec.rules, or use routing: InferencePool", mr.Spec.Rules[0].Name) + } + if mr.Spec.DefaultRoute != "" { + return "spec.defaultRoute does not compile with agentGatewayRef.routing: Model: agentgateway answers an " + + "unknown model with 404 model_not_found. Remove it, or use routing: InferencePool" + } + servedBy := make(map[string]string, len(mr.Spec.Backends)) + for _, b := range mr.Spec.Backends { + switch { + case b.External != nil: + return fmt.Sprintf("backend %q is external; agentGatewayRef.routing: Model compiles only "+ + "inferenceServiceRef backends", b.Name) + case b.Resolution == inferencev1alpha1.RouterBackendResolutionEndpoint: + return fmt.Sprintf("backend %q sets resolution: endpoint; agentGatewayRef.routing: Model reaches the "+ + "InferenceService through its Service (resolution: service)", b.Name) + case b.EndpointPicker != nil: + return fmt.Sprintf("backend %q sets endpointPicker, which applies only to agentGatewayRef.routing: "+ + "InferencePool", b.Name) + } + served := agentGatewayServedName(b) + if len(served) > agentGatewayModelMatchMaxLength { + return fmt.Sprintf("backend %q serves a model name of %d characters; agentgateway's match.model accepts at most %d", + b.Name, len(served), agentGatewayModelMatchMaxLength) + } + if strings.Contains(served, "*") { + return fmt.Sprintf("backend %q serves model %q; agentgateway reads '*' in a model name as a wildcard", b.Name, served) + } + if other, taken := servedBy[served]; taken { + return fmt.Sprintf("backends %q and %q both serve model %q; agentgateway's model router needs one "+ + "backend per model name", other, b.Name, served) + } + servedBy[served] = b.Name + } + return "" +} + +// agentGatewayRefusalError is a refusal found during generation, reported as a +// condition rather than a reconcile error. +type agentGatewayRefusalError struct { + reason, message string +} + +func (e *agentGatewayRefusalError) Error() string { return e.message } + +type agentGatewayModelBackend struct { + ObjectName string + ServedName string + Service string + Port int64 + Provider string + Formats []string +} + +// newAgentGatewayModelRoute builds the router's HTTPRoute. The rule sits at "/" +// because agentgateway strips the rule's path prefix before it classifies the +// endpoint: under /v1, an embeddings request would parse as a completion. It +// reuses the InferencePool route's name, so a mode switch rewrites one route. +func newAgentGatewayModelRoute(mr *inferencev1alpha1.ModelRouter) *unstructured.Unstructured { + ref := mr.Spec.AgentGatewayRef + spec := map[string]interface{}{ + "parentRefs": []interface{}{agentGatewayRouteParentRef(ref)}, + "rules": []interface{}{ + map[string]interface{}{ + metadataNameField: agentGatewayModelRuleName, + "backendRefs": []interface{}{ + map[string]interface{}{ + metadataNameField: "*", + "kind": agentGatewayModelKind, + "group": agentGatewayPolicyGroup, + }, + }, + }, + }, + } + if len(ref.Hostnames) > 0 { + spec["hostnames"] = toInterfaceSlice(ref.Hostnames) + } + + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gatewayHTTPRouteGVK()) + u.SetName(modelRouterAgentGatewayResourceName(mr)) + u.SetNamespace(mr.Namespace) + u.SetLabels(map[string]string{agentGatewayRouterUIDLabel: string(mr.UID)}) + u.Object["spec"] = spec + return u +} + +// newAgentGatewayModel sets group and kind where the CRD would default them, +// so an unchanged spec causes no update. +func newAgentGatewayModel(mr *inferencev1alpha1.ModelRouter, route string, b agentGatewayModelBackend) *unstructured.Unstructured { + formats := make([]interface{}, 0, len(b.Formats)) + for _, f := range b.Formats { + formats = append(formats, map[string]interface{}{"type": f}) + } + + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(agentGatewayModelGVK()) + u.SetName(b.ObjectName) + u.SetNamespace(mr.Namespace) + u.SetLabels(map[string]string{agentGatewayRouterUIDLabel: string(mr.UID)}) + u.Object["spec"] = map[string]interface{}{ + "parentRefs": []interface{}{ + map[string]interface{}{ + metadataNameField: route, + "kind": "HTTPRoute", + "group": gatewayBackendRefGroupAPI, + "sectionName": agentGatewayModelRuleName, + }, + }, + "match": map[string]interface{}{"model": b.ServedName}, + "provider": "Custom", + "custom": map[string]interface{}{ + "backendRef": map[string]interface{}{ + metadataNameField: b.Service, + "port": b.Port, + }, + "providerOverride": b.Provider, + "formats": formats, + }, + } + return u +} + +// vllmServesModel reports whether vLLM answers to model. The request keeps its +// model name, and LLMKube starts vLLM without --served-model-name (#1988), so +// the name must come from the user's args. vLLM sees spec.args only with a +// command override (setCommandAndArgs), otherwise just spec.extraArgs. +func vllmServesModel(isvc *inferencev1alpha1.InferenceService, model string) bool { + args := isvc.Spec.ExtraArgs + if len(isvc.Spec.Command) > 0 { + args = append(append([]string{}, isvc.Spec.Args...), args...) + } + for i, a := range args { + if v, ok := strings.CutPrefix(a, "--served-model-name="); ok && v == model { + return true + } + if a != "--served-model-name" { + continue + } + // vLLM takes one or more names after the flag. + for _, v := range args[i+1:] { + if strings.HasPrefix(v, "-") { + break + } + if v == model { + return true + } + } + } + return false +} + +// resolveAgentGatewayModelBackends returns an agentGatewayRefusalError for an +// unmapped runtime. +func (r *ModelRouterAgentGatewayReconciler) resolveAgentGatewayModelBackends( + ctx context.Context, + mr *inferencev1alpha1.ModelRouter, +) ([]agentGatewayModelBackend, error) { + resolved := make([]agentGatewayModelBackend, 0, len(mr.Spec.Backends)) + for _, b := range mr.Spec.Backends { + isvc, err := r.backendInferenceService(ctx, mr, b) + if err != nil { + return nil, err + } + provider, formats, ok := agentGatewayModelProvider(isvc) + if !ok { + return nil, &agentGatewayRefusalError{ + reason: modelRouterAgentGatewayReasonUnsupportedRuntime, + message: fmt.Sprintf("backend %q: InferenceService %q runs runtime %q, which agentGatewayRef.routing: "+ + "Model has no agentgateway provider mapping for (supported: llamacpp, vllm); use routing: "+ + "InferencePool for it", b.Name, isvc.Name, runtimeNameLabel(isvc)), + } + } + served := agentGatewayServedName(b) + if runtimeNameLabel(isvc) == RuntimeVLLM && !vllmServesModel(isvc, served) { + return nil, &agentGatewayRefusalError{ + reason: modelRouterAgentGatewayReasonUnsupportedRuntime, + message: fmt.Sprintf("backend %q: vLLM InferenceService %q does not serve model %q, so every request "+ + "would get a 404; add \"--served-model-name=%s\" to its extraArgs", b.Name, isvc.Name, served, served), + } + } + resolved = append(resolved, agentGatewayModelBackend{ + ObjectName: agentGatewayModelName(mr, b.Name), + ServedName: served, + Service: sanitizeDNSName(isvc.Name), + Port: gatewayServicePort(isvc), + Provider: provider, + Formats: formats, + }) + } + return resolved, nil +} + +// reconcileAgentGatewayModels applies the models and route, then deletes the +// models of removed backends and whatever routing: InferencePool left behind. +// Deleting last means a mode switch never points the route at deleted pools. +func (r *ModelRouterAgentGatewayReconciler) reconcileAgentGatewayModels(ctx context.Context, mr *inferencev1alpha1.ModelRouter) error { + backends, err := r.resolveAgentGatewayModelBackends(ctx, mr) + if err != nil { + return err + } + + // Models first, so a rejected model leaves the previous route in place. + route := newAgentGatewayModelRoute(mr) + desired := make([]*unstructured.Unstructured, 0, len(backends)+1) + keep := make(map[string]bool, len(backends)) + for _, b := range backends { + desired = append(desired, newAgentGatewayModel(mr, route.GetName(), b)) + keep[b.ObjectName] = true + } + desired = append(desired, route) + + // Refuse before any write if a generated name belongs to someone else. + for _, obj := range desired { + if err := r.checkGeneratedNameAvailable(ctx, mr, obj); err != nil { + return err + } + } + + for _, obj := range desired { + if err := r.applyAgentGatewayResource(ctx, mr, obj); err != nil { + return fmt.Errorf("%s/%s: %w", obj.GetKind(), obj.GetName(), err) + } + } + if err := r.deleteGeneratedObjects(ctx, mr, keep, agentGatewayModelGVK()); err != nil { + return err + } + if err := r.deleteGeneratedObjects(ctx, mr, nil, inferencePoolGVK()); err != nil { + return err + } + return r.withdrawAgentGatewayPolicy(ctx, mr) +} diff --git a/internal/controller/modelrouter_agentgateway_model_test.go b/internal/controller/modelrouter_agentgateway_model_test.go new file mode 100644 index 000000000..8b9c87424 --- /dev/null +++ b/internal/controller/modelrouter_agentgateway_model_test.go @@ -0,0 +1,784 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "reflect" + "strings" + "testing" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + inferencev1alpha1 "github.com/defilantech/llmkube/api/v1alpha1" +) + +func TestAgentGatewayModelProvider(t *testing.T) { + chatVLLM := []string{"Completions", "Messages", "AnthropicTokenCount", "Responses"} + tests := []struct { + name string + spec inferencev1alpha1.InferenceServiceSpec + wantProvider string + wantFormats []string + wantOK bool + }{ + {"unset runtime is llama.cpp chat", inferencev1alpha1.InferenceServiceSpec{}, "llamacpp", []string{"Completions"}, true}, + {"llamacpp chat", inferencev1alpha1.InferenceServiceSpec{Runtime: "llamacpp"}, "llamacpp", []string{"Completions"}, true}, + {"llamacpp embedding mode", inferencev1alpha1.InferenceServiceSpec{Mode: "embedding"}, "llamacpp", []string{"Embeddings"}, true}, + {"llamacpp rerank inferred from --reranking", inferencev1alpha1.InferenceServiceSpec{ + ExtraArgs: []string{"--reranking", "--embedding"}, + }, "llamacpp", []string{"Rerank"}, true}, + {"llamacpp embedding inferred from endpoint path", inferencev1alpha1.InferenceServiceSpec{ + Endpoint: &inferencev1alpha1.EndpointSpec{Path: "/v1/embeddings"}, + }, "llamacpp", []string{"Embeddings"}, true}, + {"vllm chat", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeVLLM}, "vllm", chatVLLM, true}, + {"vllm embedding", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeVLLM, Mode: "embedding"}, "vllm", []string{"Embeddings"}, true}, + {"vllm rerank", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeVLLM, Mode: "rerank"}, "vllm", []string{"Rerank"}, true}, + {"sglang is unmapped", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeSGLANG}, "", nil, false}, + {"tgi is unmapped", inferencev1alpha1.InferenceServiceSpec{Runtime: "tgi"}, "", nil, false}, + {"llamacpp-router is unmapped", inferencev1alpha1.InferenceServiceSpec{Runtime: RuntimeLlamaCppRouter}, "", nil, false}, + {"generic is unmapped", inferencev1alpha1.InferenceServiceSpec{Runtime: "generic"}, "", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider, formats, ok := agentGatewayModelProvider(&inferencev1alpha1.InferenceService{Spec: tt.spec}) + if ok != tt.wantOK || provider != tt.wantProvider || !reflect.DeepEqual(formats, tt.wantFormats) { + t.Errorf("agentGatewayModelProvider = (%q, %v, %v), want (%q, %v, %v)", + provider, formats, ok, tt.wantProvider, tt.wantFormats, tt.wantOK) + } + }) + } +} + +func TestVLLMServesModel(t *testing.T) { + tests := []struct { + name string + args []string + want bool + }{ + {"no flag", nil, false}, + {"inline", []string{"--served-model-name=chat"}, true}, + {"inline, other name", []string{"--served-model-name=other"}, false}, + {"one of several names", []string{"--served-model-name", "other", "chat", "--port", "8000"}, true}, + {"name after the next flag", []string{"--served-model-name", "other", "--port", "chat"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + isvc := &inferencev1alpha1.InferenceService{Spec: inferencev1alpha1.InferenceServiceSpec{ExtraArgs: tt.args}} + if got := vllmServesModel(isvc, "chat"); got != tt.want { + t.Errorf("vllmServesModel(%v) = %v, want %v", tt.args, got, tt.want) + } + }) + } + + // spec.args reaches vLLM only with a command override (setCommandAndArgs). + flag := []string{"--served-model-name=chat"} + for _, tt := range []struct { + name string + command []string + want bool + }{ + {"args without command", nil, false}, + {"args with command", []string{"vllm", "serve"}, true}, + } { + t.Run(tt.name, func(t *testing.T) { + isvc := &inferencev1alpha1.InferenceService{Spec: inferencev1alpha1.InferenceServiceSpec{Command: tt.command, Args: flag}} + if got := vllmServesModel(isvc, "chat"); got != tt.want { + t.Errorf("vllmServesModel(command %v, args %v) = %v, want %v", tt.command, flag, got, tt.want) + } + }) + } +} + +func TestAgentGatewayModelName(t *testing.T) { + router := func(name string) *inferencev1alpha1.ModelRouter { + return &inferencev1alpha1.ModelRouter{ObjectMeta: metav1.ObjectMeta{Name: name}} + } + long := strings.Repeat("r", 60) + + tests := []struct { + name, router, backend, want string + }{ + {"short names join", "llm-router", "chat-llama", "llm-router-chat-llama"}, + {"dots in the router name are sanitized", "ai.models", "chat-llama", "ai-models-chat-llama"}, + {"a long pair is cut and hashed", long, "chat-llama", ""}, + {"a trailing dash is hashed away", "llm-router", "chat-", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := agentGatewayModelName(router(tt.router), tt.backend) + if errs := validation.IsDNS1123Label(got); len(errs) > 0 { + t.Fatalf("agentGatewayModelName(%q, %q) = %q, not a DNS-1123 label: %v", tt.router, tt.backend, got, errs) + } + if tt.want != "" && got != tt.want { + t.Errorf("agentGatewayModelName(%q, %q) = %q, want %q", tt.router, tt.backend, got, tt.want) + } + if again := agentGatewayModelName(router(tt.router), tt.backend); again != got { + t.Errorf("name is not deterministic: %q then %q", got, again) + } + }) + } + + // Two routers in one namespace serving the same backend name get two + // objects; so do two long pairs that only differ past the cut. + if a, b := agentGatewayModelName(router("team-a"), "chat-llama"), agentGatewayModelName(router("team-b"), "chat-llama"); a == b { + t.Errorf("two routers derived the same model name %q", a) + } + if a, b := agentGatewayModelName(router(long), "backend-one"), agentGatewayModelName(router(long), "backend-two"); a == b { + t.Errorf("two long pairs that differ past the cut derived the same name %q", a) + } + // "chat" and "chat-" are distinct backend names and must stay distinct. + if a, b := agentGatewayModelName(router("llm-router"), "chat"), agentGatewayModelName(router("llm-router"), "chat-"); a == b { + t.Errorf("backends chat and chat- derived the same name %q", a) + } +} + +// modelRoutingRouter attaches to ai-gateway's http listener, one backend per +// InferenceService. +func modelRoutingRouter(ns, name string, isvcs ...string) *inferencev1alpha1.ModelRouter { + mr := &inferencev1alpha1.ModelRouter{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: inferencev1alpha1.ModelRouterSpec{ + DataPlane: inferencev1alpha1.ModelRouterDataPlaneAgentGateway, + AgentGatewayRef: &inferencev1alpha1.AgentGatewayReference{ + Name: "ai-gateway", + Routing: inferencev1alpha1.AgentGatewayRoutingModel, + SectionName: "http", + }, + }, + } + for _, isvc := range isvcs { + mr.Spec.Backends = append(mr.Spec.Backends, + inferencev1alpha1.RouterBackend{Name: isvc, InferenceServiceRef: corev1LocalRef(isvc)}) + } + return mr +} + +func TestAgentGatewayModelRoutingMessage(t *testing.T) { + tests := []struct { + name string + mutate func(*inferencev1alpha1.ModelRouter) + want string + }{ + {"clean", func(*inferencev1alpha1.ModelRouter) {}, ""}, + {"rules", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Rules = []inferencev1alpha1.RouterRule{{ + Name: "by-name", + Match: &inferencev1alpha1.RuleMatch{Models: []string{"a"}}, + Route: inferencev1alpha1.RuleRoute{Backends: []string{"a"}}, + }} + }, `rule "by-name"`}, + {"defaultRoute", func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.DefaultRoute = "a" }, "spec.defaultRoute"}, + {"external backend", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Backends = append(mr.Spec.Backends, inferencev1alpha1.RouterBackend{ + Name: "cloud", External: &inferencev1alpha1.ExternalProvider{Provider: "openai", Model: "gpt-5"}, + }) + }, `backend "cloud" is external`}, + {"resolution endpoint", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Backends[0].Resolution = inferencev1alpha1.RouterBackendResolutionEndpoint + }, "resolution: endpoint"}, + {"backend endpointPicker", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Backends[0].EndpointPicker = &inferencev1alpha1.RouterEndpointPicker{ + Service: &inferencev1alpha1.EndpointPickerService{Name: "epp", Port: 9002}, + } + }, "sets endpointPicker"}, + {"wildcard served name", func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.Backends[0].DisplayName = "chat-*" }, "wildcard"}, + {"served name over match.model's limit", func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Backends[0].DisplayName = strings.Repeat("m", agentGatewayModelMatchMaxLength+1) + }, "at most 1024"}, + {"duplicate served name", func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.Backends[1].DisplayName = "a" }, `both serve model "a"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mr := modelRoutingRouter("ns", "router", "a", "b") + tt.mutate(mr) + got := agentGatewayModelRoutingMessage(mr) + if tt.want == "" && got != "" { + t.Fatalf("clean router refused: %s", got) + } + if !strings.Contains(got, tt.want) { + t.Errorf("agentGatewayModelRoutingMessage = %q, want it to contain %q", got, tt.want) + } + }) + } +} + +func TestModelRouterValidator_AgentGatewayModelRouting(t *testing.T) { + v := &ModelRouterValidator{} + + if _, err := v.ValidateCreate(context.Background(), modelRoutingRouter("ns", "router", "a", "b")); err != nil { + t.Errorf("a routing: Model router without pickers must be admitted: %v", err) + } + + pool := modelRoutingRouter("ns", "router", "a", "b") + pool.Spec.AgentGatewayRef.Routing = inferencev1alpha1.AgentGatewayRoutingInferencePool + pool.Spec.AgentGatewayRef.SectionName = "" + if _, err := v.ValidateCreate(context.Background(), pool); err == nil || !strings.Contains(err.Error(), "Endpoint Picker") { + t.Errorf("routing: InferencePool without a picker: got %v, want the EndpointPickerRequired message", err) + } + + withRule := modelRoutingRouter("ns", "router", "a") + withRule.Spec.Rules = []inferencev1alpha1.RouterRule{{ + Name: "r", + Match: &inferencev1alpha1.RuleMatch{Models: []string{"a"}}, + Route: inferencev1alpha1.RuleRoute{Backends: []string{"a"}}, + }} + if _, err := v.ValidateCreate(context.Background(), withRule); err == nil || !strings.Contains(err.Error(), "routing: Model") { + t.Errorf("routing: Model with rules: got %v, want the routing: Model refusal", err) + } +} + +// An empty want means the router must be admitted. +func TestModelRouterAgentGateway_ModelRoutingCEL(t *testing.T) { + c, _, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + tests := []struct { + name, want string + mutate func(*inferencev1alpha1.AgentGatewayReference) + }{ + {"sectionName", "", func(*inferencev1alpha1.AgentGatewayReference) {}}, + {"hostnames only", "", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.SectionName = "" + ref.Hostnames = []string{"llm.example.com"} + }}, + {"no listener and no hostnames", "requires sectionName or hostnames", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.SectionName = "" + }}, + {"router-wide picker", "apply only to routing: InferencePool", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.EndpointPicker, ref.EndpointPickerPort = "epp", 9002 + }}, + {"sectionName on InferencePool", "apply only to routing: Model", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.Routing = inferencev1alpha1.AgentGatewayRoutingInferencePool + }}, + {"hostnames on InferencePool", "apply only to routing: Model", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.Routing, ref.SectionName = inferencev1alpha1.AgentGatewayRoutingInferencePool, "" + ref.Hostnames = []string{"llm.example.com"} + }}, + {"sectionName that is not a Gateway API section name", "spec.agentGatewayRef.sectionName", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.SectionName = "HTTP" + }}, + {"hostname that is not a Gateway API hostname", "spec.agentGatewayRef.hostnames[0]", func(ref *inferencev1alpha1.AgentGatewayReference) { + ref.Hostnames = []string{"LLM.example.com"} + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mr := modelRoutingRouter(ns, "", "a") + mr.GenerateName = "cel-" + tt.mutate(mr.Spec.AgentGatewayRef) + err := c.Create(context.Background(), mr) + if tt.want == "" { + if err != nil { + t.Errorf("create must be admitted: %v", err) + } + } else if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Errorf("create: got %v, want an error containing %q", err, tt.want) + } + }) + } +} + +// A router written before routing existed compiles exactly as before. +func TestModelRouterAgentGateway_InferencePoolStaysDefault(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "legacy-cuda") + mr := sameNamespaceRouterFixture(ns, "legacy", "legacy-cuda") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("a router without routing must be admitted: %v", err) + } + if got := mr.Spec.AgentGatewayRef.Routing; got != inferencev1alpha1.AgentGatewayRoutingInferencePool { + t.Errorf("routing defaulted to %q, want InferencePool", got) + } + reconcileAgentGatewayRouter(t, newModelRouterAgentGatewayReconciler(t, cfg), mr) + + getUnstructuredInNamespace(t, c, inferencePoolGVK(), ns, "legacy-cuda") + getPolicyInNamespace(t, c, ns, "legacy") + route := getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "legacy") + parentRefs, _, _ := unstructured.NestedSlice(route.Object, "spec", "parentRefs") + wantParents := []interface{}{map[string]interface{}{ + "name": "ai-gateway", "kind": "Gateway", "group": "gateway.networking.k8s.io", + }} + if !reflect.DeepEqual(parentRefs, wantParents) { + t.Errorf("route parentRefs = %v, want %v", parentRefs, wantParents) + } + if _, has, _ := unstructured.NestedSlice(route.Object, "spec", "hostnames"); has { + t.Errorf("route grew hostnames: %v", route.Object["spec"]) + } + if models := listGeneratedNames(t, c, mr, agentGatewayModelGVK()); len(models) != 0 { + t.Errorf("routing: InferencePool generated AgentgatewayModels %v", models) + } +} + +func makeModelISvc(t *testing.T, c client.Client, ns, name, rt, mode string, port int32, extraArgs ...string) { + t.Helper() + isvc := &inferencev1alpha1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: inferencev1alpha1.InferenceServiceSpec{ + ModelRef: name, + Runtime: rt, + Mode: mode, + Endpoint: &inferencev1alpha1.EndpointSpec{Port: port}, + ExtraArgs: extraArgs, + }, + } + if err := c.Create(context.Background(), isvc); err != nil { + t.Fatalf("create isvc %s: %v", name, err) + } + setISvcReadyReplicasInNamespace(t, c, ns, name, 1) +} + +func agentGatewayReadyCondition(t *testing.T, c client.Client, mr *inferencev1alpha1.ModelRouter) (*metav1.Condition, *inferencev1alpha1.ModelRouter) { + t.Helper() + fresh := &inferencev1alpha1.ModelRouter{} + if err := c.Get(context.Background(), types.NamespacedName{Name: mr.Name, Namespace: mr.Namespace}, fresh); err != nil { + t.Fatalf("get modelrouter: %v", err) + } + return apimeta.FindStatusCondition(fresh.Status.Conditions, ModelRouterAgentGatewayConditionReady), fresh +} + +func listGeneratedNames(t *testing.T, c client.Client, mr *inferencev1alpha1.ModelRouter, gvk schema.GroupVersionKind) []string { + t.Helper() + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk) + if err := c.List(context.Background(), list, client.InNamespace(mr.Namespace), + client.MatchingLabels{agentGatewayRouterUIDLabel: string(mr.UID)}); err != nil { + t.Fatalf("list %s: %v", gvk.Kind, err) + } + names := make([]string, 0, len(list.Items)) + for _, item := range list.Items { + names = append(names, item.GetName()) + } + return names +} + +// One model per runtime and format, validated by the upstream CRD. +func TestModelRouterAgentGateway_ModelRoutingCompiles(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeModelISvc(t, c, ns, "chat-vllm", RuntimeVLLM, "", 8000, "--served-model-name=chat-large") + makeBackendISvcInNamespace(t, c, ns, "chat-llama") + makeModelISvc(t, c, ns, "reranker", "llamacpp", "rerank", 8080) + makeModelISvc(t, c, ns, "embedder", "", "embedding", 8080) + + mr := modelRoutingRouter(ns, "llm-router", "chat-vllm", "chat-llama", "reranker", "embedder") + mr.Spec.Backends[0].DisplayName = "chat-large" + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + + r := newModelRouterAgentGatewayReconciler(t, cfg) + reconcileAgentGatewayRouter(t, r, mr) + + cond, fresh := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Status != metav1.ConditionTrue || !strings.Contains(cond.Message, "compiled 4 model(s)") { + t.Fatalf("expected AgentGatewayReady=True compiling 4 models, got %+v", cond) + } + if fresh.Status.AgentGateway == nil || fresh.Status.AgentGateway.PoolReady { + t.Errorf("status.agentGateway = %+v, want set with poolReady false (no pool in routing: Model)", fresh.Status.AgentGateway) + } + + route := getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "llm-router") + assertOwnedByRouter(t, route, fresh) + wantRoute := map[string]interface{}{ + "parentRefs": []interface{}{map[string]interface{}{ + "name": "ai-gateway", "kind": "Gateway", "group": "gateway.networking.k8s.io", "sectionName": "http", + }}, + "rules": []interface{}{map[string]interface{}{ + "name": "models", + "backendRefs": []interface{}{map[string]interface{}{ + "name": "*", "kind": "AgentgatewayModel", "group": "agentgateway.dev", + }}, + }}, + } + if !reflect.DeepEqual(route.Object["spec"], wantRoute) { + t.Errorf("route spec = %#v\nwant %#v", route.Object["spec"], wantRoute) + } + + model := func(served, service string, port int64, provider string, formats ...string) map[string]interface{} { + fs := make([]interface{}, 0, len(formats)) + for _, f := range formats { + fs = append(fs, map[string]interface{}{"type": f}) + } + return map[string]interface{}{ + "parentRefs": []interface{}{map[string]interface{}{ + "name": "llm-router", "kind": "HTTPRoute", "group": "gateway.networking.k8s.io", "sectionName": "models", + }}, + "match": map[string]interface{}{"model": served}, + "provider": "Custom", + "custom": map[string]interface{}{ + "backendRef": map[string]interface{}{"name": service, "port": port}, + "providerOverride": provider, + "formats": fs, + }, + } + } + want := map[string]map[string]interface{}{ + "llm-router-chat-vllm": model("chat-large", "chat-vllm", 8000, "vllm", + "Completions", "Messages", "AnthropicTokenCount", "Responses"), + "llm-router-chat-llama": model("chat-llama", "chat-llama", 8080, "llamacpp", "Completions"), + "llm-router-reranker": model("reranker", "reranker", 8080, "llamacpp", "Rerank"), + "llm-router-embedder": model("embedder", "embedder", 8080, "llamacpp", "Embeddings"), + } + for name, spec := range want { + got := getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, name) + assertOwnedByRouter(t, got, fresh) + if label := got.GetLabels()[agentGatewayRouterUIDLabel]; label != string(fresh.UID) { + t.Errorf("%s uid label = %q, want %q", name, label, fresh.UID) + } + if !reflect.DeepEqual(got.Object["spec"], spec) { + t.Errorf("%s spec = %#v\nwant %#v", name, got.Object["spec"], spec) + } + } + + if pools := listGeneratedNames(t, c, fresh, inferencePoolGVK()); len(pools) != 0 { + t.Errorf("routing: Model generated InferencePools %v", pools) + } + assertNotExistsInNamespace(t, c, agentGatewayPolicyGVK(), ns, "llm-router") +} + +// Served names that are not object names, from two routers in one namespace. +func TestModelRouterAgentGateway_ModelRoutingServedNames(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "chat-llama") + makeModelISvc(t, c, ns, "org-model", RuntimeVLLM, "", 8000, "--served-model-name", "org/model") + + r := newModelRouterAgentGatewayReconciler(t, cfg) + for _, name := range []string{"team-a", "team-b"} { + mr := modelRoutingRouter(ns, name, "chat-llama", "org-model") + mr.Spec.Backends[0].DisplayName = "Qwen3.5-2B" + mr.Spec.Backends[1].DisplayName = "org/model" + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter %s: %v", name, err) + } + reconcileAgentGatewayRouter(t, r, mr) + if cond, _ := agentGatewayReadyCondition(t, c, mr); cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("router %s: expected AgentGatewayReady=True, got %+v", name, cond) + } + + for backend, served := range map[string]string{"chat-llama": "Qwen3.5-2B", "org-model": "org/model"} { + m := getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, name+"-"+backend) + if got, _, _ := unstructured.NestedString(m.Object, "spec", "match", "model"); got != served { + t.Errorf("%s match.model = %q, want %q", m.GetName(), got, served) + } + } + } +} + +func TestModelRouterAgentGateway_ModelRoutingHostnames(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "chat-llama") + mr := modelRoutingRouter(ns, "by-host", "chat-llama") + mr.Spec.AgentGatewayRef.SectionName = "" + mr.Spec.AgentGatewayRef.Hostnames = []string{"llm.example.com", "*.llm.example.com"} + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, newModelRouterAgentGatewayReconciler(t, cfg), mr) + + route := getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "by-host") + hostnames, _, _ := unstructured.NestedStringSlice(route.Object, "spec", "hostnames") + if want := []string{"llm.example.com", "*.llm.example.com"}; !reflect.DeepEqual(hostnames, want) { + t.Errorf("route hostnames = %v, want %v", hostnames, want) + } + parentRefs, _, _ := unstructured.NestedSlice(route.Object, "spec", "parentRefs") + if len(parentRefs) != 1 || parentRefs[0].(map[string]interface{})["sectionName"] != nil { + t.Errorf("route parentRefs = %v, want one parentRef without sectionName", parentRefs) + } +} + +func TestModelRouterAgentGateway_ModelRoutingPrunesRemovedBackend(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "keep") + makeBackendISvcInNamespace(t, c, ns, "drop") + + mr := modelRoutingRouter(ns, "prune", "keep", "drop") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + r := newModelRouterAgentGatewayReconciler(t, cfg) + reconcileAgentGatewayRouter(t, r, mr) + getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, "prune-drop") + + _, fresh := agentGatewayReadyCondition(t, c, mr) + fresh.Spec.Backends = fresh.Spec.Backends[:1] + if err := c.Update(context.Background(), fresh); err != nil { + t.Fatalf("remove backend: %v", err) + } + reconcileAgentGatewayRouter(t, r, fresh) + + if got := listGeneratedNames(t, c, fresh, agentGatewayModelGVK()); !reflect.DeepEqual(got, []string{"prune-keep"}) { + t.Errorf("models after removing backend drop = %v, want [prune-keep]", got) + } +} + +// InferencePool -> Model -> InferencePool -> Model -> Proxy, with a +// cross-namespace Gateway so the extraction policy carries a finalizer. +func TestModelRouterAgentGateway_ModelRoutingSwitchesCleanUp(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + gwNS := newTestNamespace(t, c) + ctx := context.Background() + + makeGateway(t, c, gwNS, "ai-gateway", nil, []interface{}{gatewayListenerAllowingAll()}) + makeBackendISvcInNamespace(t, c, ns, "switch-cuda") + + mr := crossNamespaceRouterFixture(ns, "switcher", gwNS, "switch-cuda") + if err := c.Create(ctx, mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + r := newModelRouterAgentGatewayReconciler(t, cfg) + reconcileAgentGatewayRouter(t, r, mr) + policyName := ns + "-switcher" + getPolicyInNamespace(t, c, gwNS, policyName) + getUnstructuredInNamespace(t, c, inferencePoolGVK(), ns, "switch-cuda") + + update := func(mutate func(*inferencev1alpha1.ModelRouter)) *inferencev1alpha1.ModelRouter { + t.Helper() + _, fresh := agentGatewayReadyCondition(t, c, mr) + mutate(fresh) + if err := c.Update(ctx, fresh); err != nil { + t.Fatalf("update modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, r, fresh) + return fresh + } + + // InferencePool -> Model. + toModel := func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.Rules = nil + mr.Spec.AgentGatewayRef.Routing = inferencev1alpha1.AgentGatewayRoutingModel + mr.Spec.AgentGatewayRef.SectionName = "http" + mr.Spec.AgentGatewayRef.EndpointPicker, mr.Spec.AgentGatewayRef.EndpointPickerPort = "", 0 + } + update(toModel) + cond, fresh := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("after switching to routing: Model: expected AgentGatewayReady=True, got %+v", cond) + } + assertNoPolicyInNamespace(t, c, gwNS, policyName) + assertNotExistsInNamespace(t, c, inferencePoolGVK(), ns, "switch-cuda") + if controllerutil.ContainsFinalizer(fresh, modelRouterAgentGatewayFinalizer) { + t.Errorf("routing: Model kept the cross-namespace policy finalizer") + } + getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, "switcher-switch-cuda") + route := getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "switcher") + if rules, _, _ := unstructured.NestedSlice(route.Object, "spec", "rules"); len(rules) != 1 || + rules[0].(map[string]interface{})["name"] != agentGatewayModelRuleName { + t.Errorf("route was not rewritten to the model router rule: %v", rules) + } + + // Model -> InferencePool. + update(func(mr *inferencev1alpha1.ModelRouter) { + back := crossNamespaceRouterFixture(ns, "switcher", gwNS, "switch-cuda") + mr.Spec.AgentGatewayRef = back.Spec.AgentGatewayRef + mr.Spec.Rules = back.Spec.Rules + }) + if cond, _ := agentGatewayReadyCondition(t, c, mr); cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("after switching back to InferencePool: expected AgentGatewayReady=True, got %+v", cond) + } + assertNotExistsInNamespace(t, c, agentGatewayModelGVK(), ns, "switcher-switch-cuda") + getPolicyInNamespace(t, c, gwNS, policyName) + getUnstructuredInNamespace(t, c, inferencePoolGVK(), ns, "switch-cuda") + + // Model -> Proxy: leaving the plane sweeps the route and the models. + update(toModel) + getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), ns, "switcher-switch-cuda") + update(func(mr *inferencev1alpha1.ModelRouter) { + mr.Spec.DataPlane = inferencev1alpha1.ModelRouterDataPlaneProxy + }) + assertNotExistsInNamespace(t, c, agentGatewayModelGVK(), ns, "switcher-switch-cuda") + assertNotExistsInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "switcher") +} + +func TestModelRouterAgentGateway_ModelRoutingRefusesUnownedModel(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "chat-llama") + + handmade := &unstructured.Unstructured{} + handmade.SetGroupVersionKind(agentGatewayModelGVK()) + handmade.SetName("llm-router-chat-llama") + handmade.SetNamespace(ns) + handmade.Object["spec"] = map[string]interface{}{ + "parentRefs": []interface{}{map[string]interface{}{"kind": "HTTPRoute", "name": "hand-written"}}, + "provider": "Custom", + "custom": map[string]interface{}{ + "backendRef": map[string]interface{}{"name": "chat-llama", "port": int64(8080)}, + "formats": []interface{}{map[string]interface{}{"type": "Completions"}}, + }, + } + if err := c.Create(context.Background(), handmade); err != nil { + t.Fatalf("create handmade model: %v", err) + } + + mr := modelRoutingRouter(ns, "llm-router", "chat-llama") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + r := newModelRouterAgentGatewayReconciler(t, cfg) + res, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: mr.Name, Namespace: ns}}) + if err != nil { + t.Fatalf("reconcile must refuse, not error: %v", err) + } + if res.RequeueAfter != agentGatewayConflictRequeueAfter { + t.Errorf("requeue after = %v, want %v (the name-taken recheck)", res.RequeueAfter, agentGatewayConflictRequeueAfter) + } + cond, _ := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Reason != modelRouterAgentGatewayReasonResourceNameTaken || + !strings.Contains(cond.Message, "AgentgatewayModel "+ns+"/llm-router-chat-llama") { + t.Fatalf("expected AgentGatewayReady=False/%s naming the model, got %+v", modelRouterAgentGatewayReasonResourceNameTaken, cond) + } + assertForeignObjectUntouched(t, c, handmade) + assertNotExistsInNamespace(t, c, gatewayHTTPRouteGVK(), ns, "llm-router") +} + +func TestModelRouterAgentGateway_ModelRoutingRefusals(t *testing.T) { + c, cfg, stop := startAgentGatewayTestEnv(t, true) + defer stop() + ns := newTestNamespace(t, c) + + makeGateway(t, c, ns, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, ns, "llama") + makeModelISvc(t, c, ns, "sglang", RuntimeSGLANG, "", 8080) + makeModelISvc(t, c, ns, "bare-vllm", RuntimeVLLM, "", 8000) + r := newModelRouterAgentGatewayReconciler(t, cfg) + + tests := []struct { + name, isvc, reason, message string + mutate func(*inferencev1alpha1.ModelRouter) + }{ + {"default-route", "llama", modelRouterAgentGatewayReasonUnsupportedModelRouting, "spec.defaultRoute", + func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.DefaultRoute = "llama" }}, + {"unmapped-runtime", "sglang", modelRouterAgentGatewayReasonUnsupportedRuntime, `runtime "sglang"`, + func(*inferencev1alpha1.ModelRouter) {}}, + {"vllm-without-served-name", "bare-vllm", modelRouterAgentGatewayReasonUnsupportedRuntime, + `"--served-model-name=bare-vllm"`, func(*inferencev1alpha1.ModelRouter) {}}, + {"unknown-listener", "llama", modelRouterAgentGatewayReasonNotAllowed, `no listener named "models"`, + func(mr *inferencev1alpha1.ModelRouter) { mr.Spec.AgentGatewayRef.SectionName = "models" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mr := modelRoutingRouter(ns, "refused-"+tt.name, tt.isvc) + tt.mutate(mr) + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, r, mr) + cond, _ := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != tt.reason || !strings.Contains(cond.Message, tt.message) { + t.Fatalf("expected AgentGatewayReady=False/%s containing %q, got %+v", tt.reason, tt.message, cond) + } + assertNotExistsInNamespace(t, c, gatewayHTTPRouteGVK(), ns, mr.Name) + assertNotExistsInNamespace(t, c, agentGatewayModelGVK(), ns, mr.Name+"-"+tt.isvc) + }) + } +} + +// routing: Model needs the AgentgatewayModel CRD and nothing else. +func TestModelRouterAgentGateway_ModelRoutingCRDGate(t *testing.T) { + t.Run("without the AgentgatewayModel CRD", func(t *testing.T) { + // The gate runs before the Gateway lookup, so the base-only env needs + // no Gateway. + c, cfg, stop := startAgentGatewayTestEnv(t, false) + defer stop() + ns := newTestNamespace(t, c) + makeBackendISvcInNamespace(t, c, ns, "gate-llama") + mr := modelRoutingRouter(ns, "gate", "gate-llama") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, newModelRouterAgentGatewayReconciler(t, cfg), mr) + cond, _ := agentGatewayReadyCondition(t, c, mr) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != modelRouterAgentGatewayReasonModelCRDMissing || + !strings.Contains(cond.Message, "AgentgatewayModel (agentgateway.dev/v1alpha1)") { + t.Errorf("expected AgentGatewayReady=False/%s naming the kind, got %+v", modelRouterAgentGatewayReasonModelCRDMissing, cond) + } + }) + + t.Run("with only the AgentgatewayModel CRD", func(t *testing.T) { + // A RESTMapper that knows only this package's types, the Gateway API + // kinds the reconcile touches, and AgentgatewayModel: no InferencePool + // or AgentgatewayPolicy. + s := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(s); err != nil { + t.Fatalf("scheme: %v", err) + } + if err := inferencev1alpha1.AddToScheme(s); err != nil { + t.Fatalf("scheme: %v", err) + } + mapper := apimeta.NewDefaultRESTMapper(nil) + for gvk := range s.AllKnownTypes() { + mapper.Add(gvk, apimeta.RESTScopeNamespace) + } + for _, gvk := range []schema.GroupVersionKind{ + {Group: "gateway.networking.k8s.io", Version: "v1", Kind: "Gateway"}, gatewayHTTPRouteGVK(), agentGatewayModelGVK(), + } { + mapper.Add(gvk, apimeta.RESTScopeNamespace) + } + c := fake.NewClientBuilder().WithScheme(s).WithRESTMapper(mapper). + WithStatusSubresource(&inferencev1alpha1.ModelRouter{}, &inferencev1alpha1.InferenceService{}).Build() + + makeGateway(t, c, testNS, "ai-gateway", nil, nil) + makeBackendISvcInNamespace(t, c, testNS, "gate-llama") + mr := modelRoutingRouter(testNS, "gate", "gate-llama") + if err := c.Create(context.Background(), mr); err != nil { + t.Fatalf("create modelrouter: %v", err) + } + reconcileAgentGatewayRouter(t, &ModelRouterAgentGatewayReconciler{Client: c, Scheme: s}, mr) + if cond, _ := agentGatewayReadyCondition(t, c, mr); cond == nil || cond.Status != metav1.ConditionTrue { + t.Errorf("expected AgentGatewayReady=True, got %+v", cond) + } + getUnstructuredInNamespace(t, c, agentGatewayModelGVK(), testNS, "gate-gate-llama") + getUnstructuredInNamespace(t, c, gatewayHTTPRouteGVK(), testNS, "gate") + }) +} diff --git a/internal/controller/modelrouter_agentgateway_picker.go b/internal/controller/modelrouter_agentgateway_picker.go index df2522425..eab9b6acb 100644 --- a/internal/controller/modelrouter_agentgateway_picker.go +++ b/internal/controller/modelrouter_agentgateway_picker.go @@ -37,8 +37,12 @@ type agentGatewayPicker struct { // every pool resolves exactly one picker and no picker serves two pools. A // non-empty reason names the refusal; the reconciler and the webhook both call // this, so admission and reconcile cannot diverge. External backends are not -// counted: the reconciler refuses them separately. +// counted: the reconciler refuses them separately. routing: Model compiles no +// pool, so it resolves none. func agentGatewayResolvePoolPickers(mr *inferencev1alpha1.ModelRouter) (map[string]agentGatewayPicker, string, string) { + if agentGatewayModelRouting(mr) { + return nil, "", "" + } var def *agentGatewayPicker if ref := mr.Spec.AgentGatewayRef; ref != nil && ref.EndpointPicker != "" { def = &agentGatewayPicker{Name: ref.EndpointPicker, Port: ref.EndpointPickerPort} diff --git a/internal/controller/modelrouter_agentgateway_rbac_test.go b/internal/controller/modelrouter_agentgateway_rbac_test.go index b0c8750ab..9128dd0e6 100644 --- a/internal/controller/modelrouter_agentgateway_rbac_test.go +++ b/internal/controller/modelrouter_agentgateway_rbac_test.go @@ -43,9 +43,10 @@ func TestModelRouterAgentGateway_RBACCoversClientVerbs(t *testing.T) { // resource: the guard chain Gets the referenced Gateway and, for // allowedRoutes Selector matching, the router's namespace; backend // resolution Gets InferenceServices; the apply creates/updates pools, the - // extraction policy, and the HTTPRoute; the conflict detector Lists - // policies; the deletion, leaving, and consent-withdrawal sweeps List and - // Delete policies, pools, and routes; the pre-flight name check Gets each + // extraction policy, the HTTPRoute, and (routing: Model) the + // AgentgatewayModels; the conflict detector Lists policies; the deletion, + // leaving, consent-withdrawal, and mode-switch sweeps and the stale-model + // prune List and Delete policies, pools, routes, and models; the pre-flight name check Gets each // generated object; status and finalizer writes update and patch the // router; the Recorder emits events. // @@ -66,6 +67,7 @@ func TestModelRouterAgentGateway_RBACCoversClientVerbs(t *testing.T) { {"inference.networking.k8s.io/inferencepools", []string{"get", "list", "create", "update", "patch", "delete"}}, {"gateway.networking.k8s.io/httproutes", []string{"get", "list", "create", "update", "patch", "delete"}}, {"agentgateway.dev/agentgatewaypolicies", []string{"get", "list", "create", "update", "patch", "delete"}}, + {"agentgateway.dev/agentgatewaymodels", []string{"get", "list", "create", "update", "patch", "delete"}}, {"gateway.networking.k8s.io/gateways", []string{"get"}}, {"/namespaces", []string{"get", "list", "watch"}}, {"/events", []string{"create", "patch"}}, diff --git a/internal/controller/modelrouter_agentgateway_test.go b/internal/controller/modelrouter_agentgateway_test.go index 393bd61b9..04def84c8 100644 --- a/internal/controller/modelrouter_agentgateway_test.go +++ b/internal/controller/modelrouter_agentgateway_test.go @@ -2462,8 +2462,8 @@ func newCountingModelRouterAgentGatewayReconciler(t *testing.T, cfg *rest.Config } // TestModelRouterAgentGateway_SweepOnlyForUsedRouters pins the evidence gate -// on the leave-AgentGateway sweep: the sweep runs three uncached Lists -// (cluster-wide policies, pools, routes), so a router that never used +// on the leave-AgentGateway sweep: the sweep runs four uncached Lists +// (cluster-wide policies, pools, routes, models), so a router that never used // AgentGateway mode must make none, and a router that switches away must be // swept exactly once - after the sweep clears the plane's status, later // reconciles find no evidence and skip it. @@ -2523,14 +2523,14 @@ func TestModelRouterAgentGateway_SweepOnlyForUsedRouters(t *testing.T) { } // The switch reconcile sweeps: one List each for policies - // (cluster-wide), pools, and routes. + // (cluster-wide), pools, routes, and models. if _, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{Name: mr.Name, Namespace: ns}, }); err != nil { t.Fatalf("reconcile after switch: %v", err) } - if got := counter.lists - before; got != 3 { - t.Errorf("the switch reconcile made %d List calls, want exactly 3 (one per swept kind)", got) + if got := counter.lists - before; got != 4 { + t.Errorf("the switch reconcile made %d List calls, want exactly 4 (one per swept kind)", got) } assertAllGoneLike(t, c, ns, mr.Name, "sweep1-cuda") @@ -2543,7 +2543,7 @@ func TestModelRouterAgentGateway_SweepOnlyForUsedRouters(t *testing.T) { t.Fatalf("reconcile %d after sweep: %v", i+1, err) } } - if got := counter.lists - before - 3; got != 0 { + if got := counter.lists - before - 4; got != 0 { t.Errorf("later reconciles made %d List calls after the sweep, want 0", got) } }) @@ -2568,14 +2568,14 @@ func TestModelRouterAgentGateway_SweepOnlyForUsedRouters(t *testing.T) { } // The ref-removal reconcile sweeps: one List each for policies - // (cluster-wide), pools, and routes. + // (cluster-wide), pools, routes, and models. if _, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{Name: mr.Name, Namespace: ns}, }); err != nil { t.Fatalf("reconcile after ref removal: %v", err) } - if got := counter.lists - before; got != 3 { - t.Errorf("the ref-removal reconcile made %d List calls, want exactly 3 (one per swept kind)", got) + if got := counter.lists - before; got != 4 { + t.Errorf("the ref-removal reconcile made %d List calls, want exactly 4 (one per swept kind)", got) } assertAllGoneLike(t, c, ns, mr.Name, "sweep2-cuda") @@ -2588,7 +2588,7 @@ func TestModelRouterAgentGateway_SweepOnlyForUsedRouters(t *testing.T) { t.Fatalf("reconcile %d after sweep: %v", i+1, err) } } - if got := counter.lists - before - 3; got != 0 { + if got := counter.lists - before - 4; got != 0 { t.Errorf("later reconciles made %d List calls after the sweep, want 0", got) } refless := &inferencev1alpha1.ModelRouter{} diff --git a/internal/controller/modelrouter_webhook.go b/internal/controller/modelrouter_webhook.go index d04f000e0..b2dd4035d 100644 --- a/internal/controller/modelrouter_webhook.go +++ b/internal/controller/modelrouter_webhook.go @@ -184,6 +184,9 @@ func agentGatewayModeViolations(mr *inferencev1alpha1.ModelRouter, specPath *fie if _, reason, msg := agentGatewayResolvePoolPickers(mr); reason != "" { errs = append(errs, field.Invalid(specPath.Child("backends"), nil, msg)) } + if msg := agentGatewayModelRoutingMessage(mr); msg != "" { + errs = append(errs, field.Invalid(specPath, nil, msg)) + } if msg := agentGatewayUnsupportedMatchMessage(mr); msg != "" { errs = append(errs, field.Invalid(specPath.Child("rules"), nil, msg)) } diff --git a/test/crd/inference-extension/README.md b/test/crd/inference-extension/README.md index e1470f51a..c9c6018ef 100644 --- a/test/crd/inference-extension/README.md +++ b/test/crd/inference-extension/README.md @@ -9,12 +9,15 @@ schemas on purpose: the generated InferencePool spec must pass server-side validation (a flat `selector` map was once emitted here and only the real CRD's `matchLabels` requirement catches it), and the AgentgatewayPolicy CRD's CEL rules (PreRouting targets a Gateway/ListenerSet, same-namespace targetRefs) are -why the generated extraction policy lives in the Gateway's namespace. +why the generated extraction policy lives in the Gateway's namespace. The +AgentgatewayModel CRD's CEL rules (a custom provider's backendRef kind, a +match.model wildcard shape) check what agentGatewayRef.routing: Model generates. | File | Source | Pinned | License | |--------------------------|----------------------------------------------------------------------|----------|------------| | inferencepool.yaml | kubernetes-sigs/gateway-api-inference-extension config/crd/bases | v1.6.2 | Apache-2.0 | | agentgatewaypolicy.yaml | agentgateway/agentgateway agentgateway-crds chart template | v1.6.0 | Apache-2.0 | +| agentgatewaymodel.yaml | agentgateway/agentgateway agentgateway-crds chart template | v1.6.0 | Apache-2.0 | | gateway.yaml | Hand-made minimal stub (Gateway API v1 Gateway; permissive spec) | - | - | | httproute.yaml | Hand-made minimal stub (Gateway API v1 HTTPRoute) | - | - | diff --git a/test/crd/inference-extension/agentgatewaymodel.yaml b/test/crd/inference-extension/agentgatewaymodel.yaml new file mode 100644 index 000000000..2833859e7 --- /dev/null +++ b/test/crd/inference-extension/agentgatewaymodel.yaml @@ -0,0 +1,5646 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + labels: + app: agentgateway + app.kubernetes.io/name: agentgateway + name: agentgatewaymodels.agentgateway.dev +spec: + group: agentgateway.dev + names: + categories: + - agentgateway + kind: AgentgatewayModel + listKind: AgentgatewayModelList + plural: agentgatewaymodels + shortNames: + - agmodel + singular: agentgatewaymodel + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Model name matched from client requests + jsonPath: .spec.match.model + name: Model Match + type: string + - description: The age of the model. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Desired model configuration. + properties: + azure: + description: Provider-specific settings for Azure AI. + properties: + apiVersion: + description: |- + The version of the Azure OpenAI API to use. + If unset, defaults to `v1`. + maxLength: 64 + minLength: 1 + type: string + projectName: + description: |- + The Foundry project name, required when `resourceType` is `Foundry`. + Used to construct paths: /api/projects/{projectName}/openai/v1/... + maxLength: 256 + minLength: 1 + type: string + resourceName: + description: |- + The Azure resource name used to construct the endpoint host. + For OpenAI: {resourceName}.openai.azure.com + For Foundry: {resourceName}.services.ai.azure.com + Note: when the Azure portal "Foundry legacy" template was used, the + generated resource name may end in "-resource" (e.g. "myproject-resource"); + that suffix is part of the resource name as the user configured it, not + part of the hostname suffix agentgateway should append. + maxLength: 256 + minLength: 1 + type: string + resourceType: + description: The type of Azure endpoint. Determines the host suffix. + enum: + - Foundry + - OpenAI + type: string + required: + - resourceName + - resourceType + type: object + x-kubernetes-validations: + - message: projectName is required when resourceType is Foundry + rule: self.resourceType != 'Foundry' || has(self.projectName) + baseURL: + description: |- + BaseURL overrides the provider address and base path prefix. It must use the + http or https scheme. Backend policies may override the default TLS + configuration. Query parameters, fragments, and user info are not supported. + The URL path is the upstream base path and defaults to / when omitted. + Provider-specific endpoint paths are appended to this base path. + For example, `https://api.openai.com/v1` sends completions to `/v1/chat/completions`, + while `https://api.openai.com` sends them to `/chat/completions`. + format: uri + maxLength: 1024 + minLength: 1 + type: string + bedrock: + description: Provider-specific settings for Amazon Bedrock. + properties: + endpointPreference: + description: |- + EndpointPreference selects which Bedrock API surface to prefer. + Defaults to `RuntimePreferred`, preferring runtime over mantle. + Decides which endpoint to pick mainly based on the catalog tags + `mantle` and `runtime`. + enum: + - MantleOnly + - MantlePreferred + - RuntimeOnly + - RuntimePreferred + type: string + guardrail: + description: |- + Guardrail policy to use for the backend. See + . + If not specified, the AWS Guardrail policy will not be used. + properties: + identifier: + description: Identifier of the Guardrail policy to use for + the backend. + maxLength: 256 + minLength: 1 + type: string + version: + description: Version of the Guardrail policy to use for the + backend. + maxLength: 256 + minLength: 1 + type: string + required: + - identifier + - version + type: object + region: + description: |- + AWS region to use for the backend. + Defaults to `us-east-1` if not specified. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9-]+$ + type: string + type: object + x-kubernetes-validations: + - message: Bedrock guardrails cannot be used with MantlePreferred + or MantleOnly + rule: '!has(self.guardrail) || !has(self.endpointPreference) || + !(self.endpointPreference in [''MantlePreferred'', ''MantleOnly''])' + custom: + description: Provider-specific settings for a custom provider. + properties: + backendRef: + description: |- + Kubernetes backend that serves this provider. + `backendRef` may target only a namespace-local Service or InferencePool. + If unset, host and port must be set on the parent provider. + properties: + group: + description: |- + API group of the referenced resource. For example, `gateway.networking.k8s.io`. + Defaults to the empty string, which identifies the core API group. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: |- + Kind of the referenced resource. For example, `Service`. + Defaults to "Service" when not specified. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name of the referenced resource. + maxLength: 253 + minLength: 1 + type: string + port: + description: |- + Destination port number to use for this resource. + Required when the referenced resource is a Kubernetes Service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '((!has(self.group) || size(self.group) == 0) && (!has(self.kind) + || self.kind == ''Service'')) ? has(self.port) : true' + formats: + description: Provider-native API formats this provider supports. + items: + description: Provider-native LLM API format settings. + properties: + path: + description: |- + Default upstream path override for this format. + If unset, agentgateway uses the default path for the format. + maxLength: 1024 + minLength: 1 + type: string + type: + description: Provider-native API format. + enum: + - AnthropicTokenCount + - Completions + - Embeddings + - Messages + - Realtime + - Rerank + - Responses + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: path must start with / + rule: '!has(self.path) || self.path.startsWith(''/'')' + maxItems: 6 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + providerOverride: + description: |- + Provider identity used for cost-catalog lookup and telemetry. + Defaults to "custom" when unset. + maxLength: 256 + minLength: 1 + type: string + required: + - formats + type: object + x-kubernetes-validations: + - message: custom provider backendRef may target only Service or InferencePool + rule: '!has(self.backendRef) || (((!has(self.backendRef.group) || + self.backendRef.group == "") && (!has(self.backendRef.kind) || + self.backendRef.kind == ''Service'')) || (has(self.backendRef.group) + && self.backendRef.group == ''inference.networking.k8s.io'' && + has(self.backendRef.kind) && self.backendRef.kind == ''InferencePool''))' + match: + description: Conditions for selecting this model from client requests. + properties: + model: + description: |- + Model name matched against client requests. It may be exact, a suffix + wildcard such as `gpt-*`, a prefix wildcard such as `*-latest`, or `*`. + When omitted, the model matches metadata.name exactly. + maxLength: 1024 + minLength: 1 + type: string + x-kubernetes-validations: + - message: model wildcards must be '*', a suffix like 'gpt-*', + or a prefix like '*-latest' + rule: '!self.contains(''*'') || (self.indexOf(''*'') == self.lastIndexOf(''*'') + && (self.indexOf(''*'') == 0 || self.indexOf(''*'') == size(self) + - 1))' + type: object + parentRefs: + description: |- + Parent resources to which this model attaches. Supported parent kinds are + Gateway, ListenerSet, and HTTPRoute. + + A Gateway or ListenerSet parent attaches the model directly to its + listeners. An HTTPRoute parent attaches the model to the referenced rule; + sectionName selects a named rule, or the HTTPRoute must contain exactly + one rule when sectionName is omitted. The selected rule must use exactly + one AgentgatewayModel backend with name "*". If the rule has path matches, + they must use PathPrefix matching. + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: 'Namespace of the referent. Defaults to the Route''s + local namespace. Cross-namespace references must be explicitly + allowed, for example via ReferenceGrant. Support: Core' + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: 'Port this Route targets on the parent, interpreted + per parent kind (for example a Gateway listener port or Service + port). Support: Extended' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: 'Name of a section within the target resource, + for example a Gateway Listener name or Service port name. + Empty references the entire resource. Support: Core' + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 16 + minItems: 1 + type: array + policies: + description: Policies applied to this concrete model. + properties: + auth: + description: Credentials used to authenticate requests to this + model provider. + properties: + aws: + description: |- + Explicit AWS authentication method for the model provider. When omitted, + default AWS SDK credential discovery is used. + properties: + assumeRole: + description: |- + AWS STS AssumeRole settings to use before signing backend requests. + Ambient AWS credentials are used as the source credentials for STS. + properties: + externalId: + description: ExternalID is set when the role's trust + policy requires sts:ExternalId. + maxLength: 1224 + minLength: 2 + pattern: ^[\w+=,.@:/-]+$ + type: string + roleArn: + description: AWS IAM role ARN to assume. + minLength: 1 + pattern: ^arn:aws[a-z-]*:iam::[0-9]{12}:role/.+$ + type: string + sessionName: + description: |- + SessionName is a custom session name (RoleSessionName) for CloudTrail and + Cost & Usage Report attribution. If unset, AWS generates a random name. + pattern: ^[\w+=,.@-]{2,64}$ + type: string + sessionNameExpression: + description: |- + SessionNameExpression is a CEL expression evaluated against each request + to produce the session name (RoleSessionName), for example `jwt.sub` or + `request.headers["x-team"]`. If the expression does not produce a valid + session name at request time, the request is rejected. + maxLength: 16384 + minLength: 1 + type: string + tags: + description: |- + Session tags passed to STS AssumeRole for cost attribution in the AWS Cost + & Usage Report, once activated. STS allows at most 50 per role session. + items: + description: |- + AwsSessionTag is an AWS STS session tag passed to AssumeRole for cost + attribution. Exactly one of value and expression must be set. + properties: + expression: + description: |- + CEL expression evaluated against each request to produce the tag value, + for example `jwt.sub` or `request.headers["x-app"]`. Requests with invalid + tag values are rejected. + maxLength: 16384 + minLength: 1 + type: string + key: + description: Key is the tag key. + maxLength: 128 + minLength: 1 + type: string + value: + description: Value is a static tag value. + maxLength: 256 + type: string + required: + - key + type: object + x-kubernetes-validations: + - message: exactly one of value or expression must + be set + rule: has(self.value) != has(self.expression) + maxItems: 50 + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + required: + - roleArn + type: object + x-kubernetes-validations: + - message: at most one of the fields in [sessionName sessionNameExpression] + may be set + rule: (has(self.sessionName)?1:0)+(has(self.sessionNameExpression)?1:0) + <= 1 + region: + description: |- + AWS SigV4 signing region, for example `us-east-1`. Set this when the + target AWS service is in a different region than the gateway. If unset, + typed AWS backends may provide this automatically; otherwise the ambient + AWS region is used. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for AWS credentials, defaulting to a Kubernetes `Secret`. + The default Secret resolver expects `accessKey`, `secretKey`, and optional + `sessionToken` keys. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == 0) ? + (!has(self.kind) || size(self.kind) == 0 || self.kind + == ''Secret'') : (has(self.kind) && size(self.kind) + > 0)' + serviceName: + description: |- + AWS SigV4 signing service name, for example + `bedrock`, `bedrock-agentcore`, or `execute-api`). If unset, typed AWS + backends may provide this automatically. + maxLength: 256 + minLength: 1 + type: string + type: object + x-kubernetes-validations: + - message: secretRef and assumeRole are mutually exclusive + rule: '!(has(self.secretRef) && has(self.assumeRole))' + azure: + description: Azure authentication method for the model provider. + properties: + managedIdentity: + description: |- + Managed identity authentication settings. Leave this object empty to use + the system-assigned identity. To use a user-assigned identity, set one of + `clientId`, `objectId`, or `resourceId`. + properties: + clientId: + description: Client ID of the user-assigned managed + identity. + type: string + objectId: + description: Object ID of the user-assigned managed + identity. + type: string + resourceId: + description: Resource ID of the user-assigned managed + identity. + type: string + type: object + x-kubernetes-validations: + - message: at most one of the fields in [clientId objectId + resourceId] may be set + rule: (has(self.clientId)?1:0)+(has(self.objectId)?1:0)+(has(self.resourceId)?1:0) + <= 1 + scopes: + description: |- + Scopes requested for the Azure access token. When omitted, the scope is + inferred from the backend hostname. Managed Identity supports exactly one + scope. + items: + type: string + maxItems: 64 + minItems: 1 + type: array + secretRef: + description: |- + Credential source for Azure credentials, defaulting to a Kubernetes + `Secret`. The default Secret resolver expects `clientID`, `tenantID`, and + `clientSecret` keys. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == 0) ? + (!has(self.kind) || size(self.kind) == 0 || self.kind + == ''Secret'') : (has(self.kind) && size(self.kind) + > 0)' + workloadIdentity: + description: |- + Workload identity authentication settings. Uses the federated token and + Azure env vars projected into the data plane pod. Recommended on AKS with + Workload Identity enabled. + type: object + type: object + x-kubernetes-validations: + - message: managedIdentity supports exactly one scope + rule: '!has(self.managedIdentity) || !has(self.scopes) || + self.scopes.size() == 1' + - message: at most one of the fields in [secretRef managedIdentity + workloadIdentity] may be set + rule: (has(self.secretRef)?1:0)+(has(self.managedIdentity)?1:0)+(has(self.workloadIdentity)?1:0) + <= 1 + credentials: + description: |- + Credentials is a list of additional credentials to inject on the backend + request. Each entry resolves a Secret key and writes its value to the + entry's location. `credentials` is independent of the primary + `key`/`secretRef`/`passthrough` mechanism and may be set on its own or + alongside it. + items: + description: |- + BackendAuthCredential specifies one additional credential to inject on the + backend request. + properties: + location: + description: Where the credential is inserted on the + backend request. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 + pseudo-headers (names beginning with `:`) + are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + secretRef: + description: |- + SecretRef references a Kubernetes Secret holding the credential value, and + optionally overrides the key read from it. Defaults to `Authorization`, + matching the key convention used by the top-level `secretRef`. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. If omitted, + a location-specific default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; + empty defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == 0) + ? (!has(self.kind) || size(self.kind) == 0 || self.kind + == ''Secret'') : (has(self.kind) && size(self.kind) + > 0)' + required: + - location + - secretRef + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + gcp: + description: |- + Google authentication method for the model provider. When omitted, + default Google credential discovery is used. + properties: + audience: + description: |- + Explicit `aud` value for the ID token. Only + valid with `IdToken` type. If not set, the `aud` is automatically + derived from the backend hostname. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for ADC-compatible Google credential JSON, defaulting to + a Kubernetes `Secret`. By default, the value is read from + `credentials.json`; set `secretRef.key` to override it. When omitted, + ambient credentials are used. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. If omitted, + a location-specific default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == 0) ? + (!has(self.kind) || size(self.kind) == 0 || self.kind + == ''Secret'') : (has(self.kind) && size(self.kind) + > 0)' + type: + description: |- + The type of token to generate. To authenticate to GCP services, + generally an `AccessToken` is used. To authenticate to Cloud Run, an + `IdToken` is used. + enum: + - AccessToken + - IdToken + type: string + type: object + x-kubernetes-validations: + - message: audience is only valid with IdToken + rule: 'has(self.audience) ? self.type == ''IdToken'' : true' + key: + description: |- + Inline key to use as the value of the `Authorization` header. This option + is the least secure; usage of a `Secret` is preferred. + maxLength: 2048 + type: string + location: + description: |- + Where backend credentials are inserted. If omitted, credentials are + written to the `Authorization` header with the `Bearer ` prefix. This + applies to `key`, `secretRef`, and `passthrough`. Entries in + `credentials` carry their own location. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 pseudo-headers + (names beginning with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + oauthTokenExchange: + description: |- + OAuth 2.0 token exchange (RFC 8693) / jwt-bearer (RFC 7523) + authentication. + properties: + actorToken: + description: RFC 8693 delegation actor token. TokenExchange + grant only. + properties: + mayAct: + description: may_act claim validation mode. When omitted, + may_act is not enforced. + enum: + - Required + type: string + source: + description: Where to read the actor token. Actor + tokens have no default source. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + expression: + description: CEL expression that extracts the + credential from the request. + maxLength: 16384 + minLength: 1 + type: string + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 + pseudo-headers (names beginning with `:`) + are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie expression] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0)+(has(self.expression)?1:0) + == 1 + tokenType: + description: |- + OAuth token type. Empty defaults to AccessToken. Custom absolute URI values + are supported for actor tokens. + type: string + required: + - source + type: object + x-kubernetes-validations: + - message: mayAct Required requires tokenType Jwt + rule: '!has(self.mayAct) || self.mayAct != ''Required'' + || (has(self.tokenType) && self.tokenType == ''Jwt'')' + additionalParams: + additionalProperties: + description: A Common Expression Language (CEL) expression. + maxLength: 16384 + minLength: 1 + type: string + description: Extra form params; values are CEL expressions + over the incoming request. + maxProperties: 64 + type: object + audiences: + description: Audiences sent to the token endpoint. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 64 + minItems: 1 + type: array + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind of the referent, + for example `Service`. Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. Defaults to + the local namespace. A cross-namespace reference + requires a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. Required when + the referent is a Kubernetes `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') + ? has(self.port) : true' + cache: + description: Response cache configuration. + properties: + inMemory: + properties: + defaultTtl: + description: TTL used when the token endpoint + omits expires_in. Default 300s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: defaultTtl must be at least 1 second + rule: duration(self) >= duration('1s') + maxEntries: + description: Default 8192; 0 disables the cache. + format: int32 + type: integer + type: object + type: object + clientAuth: + description: Client authentication for the token endpoint. + When unset, none is sent. + properties: + clientId: + description: Client ID sent to the token endpoint. + minLength: 1 + type: string + method: + description: Client authentication method. Defaults + to ClientSecretBasic. + enum: + - ClientSecretBasic + - ClientSecretPost + - PrivateKeyJwt + type: string + privateKeyJwt: + description: Client assertion settings. Required when + method is PrivateKeyJwt. + properties: + alg: + description: JWS signing algorithm. Defaults to + RS256. + enum: + - ES256 + - ES384 + - PS256 + - RS256 + - RS384 + - RS512 + type: string + assertionAudience: + description: Audience for the client assertion, + typically the token endpoint URL. + minLength: 1 + type: string + certificateHeader: + description: JWS certificate header. Required + when certificateRef is set. + enum: + - x5c + - x5t#S256 + type: string + certificateRef: + description: |- + PEM-encoded X.509 certificate chain, leaf first, for certificateHeader. The + leaf public key should match signingKeyRef; a mismatch only logs a warning + but the token endpoint will reject the assertions. Required when + certificateHeader is set. The key defaults to `certificate`. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; + empty defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both + group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + kid: + description: Optional JWS key ID header. + type: string + signingKeyRef: + description: PEM-encoded RSA or EC private key; + key defaults to `signingKey`. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; + empty defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both + group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + required: + - assertionAudience + - signingKeyRef + type: object + x-kubernetes-validations: + - message: certificateRef and certificateHeader must + be set together + rule: has(self.certificateRef) == has(self.certificateHeader) + secretRef: + description: |- + Secret providing the `clientSecret` key by default; override via + `secretRef.key`. When omitted, client_id is sent without a secret, which + is only valid with ClientSecretPost. + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + key: + description: Key in the referenced Secret. If + omitted, a location-specific default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; + empty defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group + and kind + rule: '(!has(self.group) || size(self.group) == + 0) ? (!has(self.kind) || size(self.kind) == 0 + || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + required: + - clientId + type: object + x-kubernetes-validations: + - message: privateKeyJwt settings require method PrivateKeyJwt + rule: has(self.privateKeyJwt) == (has(self.method) && + self.method == 'PrivateKeyJwt') + - message: secretRef is not valid with method PrivateKeyJwt + rule: '!has(self.secretRef) || !has(self.method) || + self.method != ''PrivateKeyJwt''' + - message: clientAuth without secretRef requires method + ClientSecretPost or PrivateKeyJwt + rule: has(self.secretRef) || has(self.privateKeyJwt) + || (has(self.method) && self.method == 'ClientSecretPost') + grantType: + description: RFC followed by the request. Defaults to + TokenExchange (RFC 8693). + enum: + - JwtBearer + - TokenExchange + type: string + location: + description: |- + Where the exchanged token is written to the backend request. + Defaults to Authorization: Bearer. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 pseudo-headers + (names beginning with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + path: + description: Token endpoint path; defaults to "/". Must + start with "/". + pattern: ^/ + type: string + requestedTokenType: + description: |- + RFC 8693 requested_token_type. Unlike subject/actor token types, only the + built-in values may be requested; custom URIs are not supported here. + enum: + - AccessToken + - Jwt + - IdToken + - IdJag + type: string + resources: + description: Resources sent to the token endpoint. + items: + type: string + maxItems: 64 + minItems: 1 + type: array + scopes: + description: Scopes sent to the token endpoint. + items: + type: string + maxItems: 64 + minItems: 1 + type: array + subjectToken: + description: |- + Subject token / assertion source and type. Defaults to Authorization Bearer, AccessToken. + The token type may be a built-in value or a custom absolute URI for providers + that support custom token exchange profiles. + properties: + source: + description: Where to read the token. CEL `expression` + variant is permitted. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + expression: + description: CEL expression that extracts the + credential from the request. + maxLength: 16384 + minLength: 1 + type: string + header: + properties: + name: + description: Name of an HTTP header. HTTP/2 + pseudo-headers (names beginning with `:`) + are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [header queryParameter + cookie expression] must be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0)+(has(self.expression)?1:0) + == 1 + tokenType: + description: |- + OAuth token type. Empty defaults to AccessToken. Custom absolute URI values + are supported for subject tokens. + type: string + type: object + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: actorToken is only valid with TokenExchange grantType + rule: '!(has(self.actorToken) && has(self.grantType) && + self.grantType == ''JwtBearer'')' + - message: requestedTokenType is only valid with TokenExchange + grantType + rule: '!(has(self.requestedTokenType) && has(self.grantType) + && self.grantType == ''JwtBearer'')' + - message: requestedTokenType IdJag is only supported by crossAppAccess + rule: '!has(self.requestedTokenType) || self.requestedTokenType + != ''IdJag''' + - message: path may not be set when url is set + rule: 'has(self.url) ? !has(self.path) : true' + - message: exactly one of the fields in [backendRef url] must + be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) == + 1 + passthrough: + description: |- + Reuses a client token already validated by another policy. Those policies + may strip client credentials; passthrough adds the original token back to + the backend request. Without client auth policies, this has no effect. + type: object + secretRef: + description: |- + Credential source for the authorization value, defaulting to a Kubernetes + `Secret`. By default, the value is read from the `Authorization` key; set + `secretRef.key` to override it. A `Bearer ` prefix is stripped only from + the default `Authorization` key. + properties: + group: + description: API group of the referenced credential; empty + selects the core API group + type: string + key: + description: Key in the referenced Secret. If omitted, + a location-specific default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group and + kind + rule: '(!has(self.group) || size(self.group) == 0) ? (!has(self.kind) + || size(self.kind) == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) > 0)' + type: object + x-kubernetes-validations: + - message: must specify credentials, or at most one of key/secretRef/passthrough/aws/azure/gcp/oauthTokenExchange + (credentials may be combined with a primary auth kind) + rule: has(self.credentials) || has(self.key) || has(self.secretRef) + || has(self.passthrough) || has(self.aws) || has(self.azure) + || has(self.gcp) || has(self.oauthTokenExchange) + - message: location may only be set for key, secretRef, or passthrough + auth + rule: 'has(self.location) ? has(self.key) || has(self.secretRef) + || has(self.passthrough) : true' + - message: at most one of the fields in [key secretRef passthrough + aws azure gcp oauthTokenExchange] may be set + rule: (has(self.key)?1:0)+(has(self.secretRef)?1:0)+(has(self.passthrough)?1:0)+(has(self.aws)?1:0)+(has(self.azure)?1:0)+(has(self.gcp)?1:0)+(has(self.oauthTokenExchange)?1:0) + <= 1 + authorization: + description: Authorization rules that clients must satisfy to + use this model. + properties: + action: + description: |- + The effect of this rule when it matches. + If unspecified, defaults to `Allow`. + `Require` rules are cumulative: all require rules must match. + enum: + - Allow + - Deny + - Require + type: string + policy: + description: |- + The authorization rule to evaluate. + + * `Allow`: any matching allow rule allows the request. + * `Require`: every require rule must match for the request to be allowed. + * `Deny`: any matching deny rule denies the request. + + `Deny` is not recommended because expression failures fail to deny; prefer + `Allow` or `Require`. If used, design expressions defensively against evaluation errors. + + If at least one `Allow` rule is configured, requests are denied unless at + least one allow rule matches. + properties: + matchExpressions: + description: CEL expressions that must all evaluate to + true for the rule to match. + items: + description: A Common Expression Language (CEL) expression. + maxLength: 16384 + minLength: 1 + type: string + maxItems: 256 + minItems: 1 + type: array + required: + - matchExpressions + type: object + required: + - policy + type: object + finalTransformations: + description: |- + CEL transformations applied to fields in the provider request body. + After conversion from one provider format to another, these transformations are applied to the converted request body. + items: + description: |- + Maps a request JSON field to a CEL expression. + The expression is evaluated against the current request body and its result + is assigned to the configured field. + properties: + expression: + description: CEL expression used to compute the field value. + maxLength: 16384 + minLength: 1 + type: string + field: + description: Name of the field to set. + maxLength: 256 + minLength: 1 + type: string + required: + - expression + - field + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - field + x-kubernetes-list-type: map + headers: + description: Request and response header changes applied to provider + traffic. + properties: + request: + description: Header changes to apply before forwarding a request. + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name + and value as defined by RFC 7230. + properties: + name: + description: Name of the HTTP header, case-insensitive + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value for the HTTP header + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name + and value as defined by RFC 7230. + properties: + name: + description: Name of the HTTP header, case-insensitive + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value for the HTTP header + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + response: + description: Header changes to apply before returning a response. + properties: + add: + description: |- + Add adds the given header(s) (name, value) to the request + before the action. It appends to any existing values associated + with the header name. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + add: + - name: "my-header" + value: "bar,baz" + + Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz + items: + description: HTTPHeader represents an HTTP Header name + and value as defined by RFC 7230. + properties: + name: + description: Name of the HTTP header, case-insensitive + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value for the HTTP header + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: |- + Remove the given header(s) from the HTTP request before the action. The + value of Remove is a list of HTTP header names. Note that the header + names are case-insensitive (see + https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + + Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz + + Config: + remove: ["my-header1", "my-header3"] + + Output: + GET /foo HTTP/1.1 + my-header2: bar + items: + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + set: + description: |- + Set overwrites the request with the given header (name, value) + before the action. + + Input: + GET /foo HTTP/1.1 + my-header: foo + + Config: + set: + - name: "my-header" + value: "bar" + + Output: + GET /foo HTTP/1.1 + my-header: bar + items: + description: HTTPHeader represents an HTTP Header name + and value as defined by RFC 7230. + properties: + name: + description: Name of the HTTP header, case-insensitive + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value for the HTTP header + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + x-kubernetes-validations: + - message: at least one of the fields in [request response] must + be set + rule: '[has(self.request),has(self.response)].filter(x,x==true).size() + >= 1' + health: + description: Health checking and eviction behavior for this model + provider. + properties: + eviction: + description: Settings for evicting unhealthy backends. + properties: + consecutiveFailures: + description: |- + Number of consecutive unhealthy responses required before the backend is evicted. + For example, a value of 5 means the backend must receive 5 unhealthy responses in a row before being evicted. + When both consecutiveFailures and healthThreshold are set, the backend is evicted when either condition is met. + When neither is set, a single unhealthy response can trigger eviction. + format: int32 + minimum: 0 + type: integer + duration: + description: |- + Base time a backend should be evicted after being marked unhealthy. + Subsequent evictions use multiplicative backoff (duration * times_evicted). + If all endpoints are evicted, the load balancer falls back to returning evicted endpoints + rather than failing entirely. + If unset, defaults to `3s`. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: duration must be at least 1 second + rule: duration(self) >= duration('1s') + healthThreshold: + description: |- + EWMA health score threshold, from 0 to 100. When set, a backend is evicted + only if its computed health drops below this value after an unhealthy + response (e.g. 50 evicts when EWMA health falls below 50%). Unlike + consecutiveFailures, this sliding-window average lets a single success delay + eviction. If both are set, either condition evicts; if neither, a single + unhealthy response evicts. + format: int32 + maximum: 100 + minimum: 0 + type: integer + restoreHealth: + description: |- + Health score from 0 to 100 assigned to a backend when it returns from eviction. + For gradual recovery, set below 100; for full recovery immediately, set 100. + If unset, the backend resumes with the health it had when evicted. + format: int32 + maximum: 100 + minimum: 0 + type: integer + type: object + unhealthyCondition: + description: |- + CEL expression that determines whether a response indicates an unhealthy backend. + When the expression evaluates to true, the backend is considered unhealthy and may be evicted. + + For example, to evict on 5xx responses: `response.code >= 500`. + + When unset, any 5xx response, or a connection failure, is treated as unhealthy. + This default lowers the backend's health score but does not trigger eviction on its own. + maxLength: 16384 + minLength: 1 + type: string + type: object + promptGuard: + description: Guardrails for requests and responses sent to this + model provider. + properties: + request: + description: Prompt guards to apply to requests sent by the + client. + items: + description: Prompt guards to apply to requests sent by + the client. + properties: + bedrockGuardrails: + description: |- + AWS Bedrock Guardrails settings for prompt + guarding. + properties: + action: + description: |- + Action controls whether the guardrail's verdict is enforced or only + observed. `Reject` (the default) enforces the guardrail: a blocked + assessment rejects the request/response and an anonymized assessment masks + the matched content. `Audit` runs the guardrail in observe mode: it is + invoked and its assessment recorded (metrics + structured log), but the + request/response is never blocked or masked. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + identifier: + description: Identifier of the Guardrail policy + to use for the backend. + maxLength: 256 + minLength: 1 + type: string + policies: + description: Policies for communicating with AWS + Bedrock Guardrails. + properties: + auth: + description: Settings for authenticating to + AWS Bedrock Guardrails. + properties: + aws: + description: |- + AWS authentication method for Bedrock Guardrails. Use `aws: {}` for + default AWS SDK credential discovery. + properties: + assumeRole: + description: |- + AWS STS AssumeRole settings to use before signing backend requests. + Ambient AWS credentials are used as the source credentials for STS. + properties: + externalId: + description: ExternalID is set when + the role's trust policy requires + sts:ExternalId. + maxLength: 1224 + minLength: 2 + pattern: ^[\w+=,.@:/-]+$ + type: string + roleArn: + description: AWS IAM role ARN to + assume. + minLength: 1 + pattern: ^arn:aws[a-z-]*:iam::[0-9]{12}:role/.+$ + type: string + sessionName: + description: |- + SessionName is a custom session name (RoleSessionName) for CloudTrail and + Cost & Usage Report attribution. If unset, AWS generates a random name. + pattern: ^[\w+=,.@-]{2,64}$ + type: string + sessionNameExpression: + description: |- + SessionNameExpression is a CEL expression evaluated against each request + to produce the session name (RoleSessionName), for example `jwt.sub` or + `request.headers["x-team"]`. If the expression does not produce a valid + session name at request time, the request is rejected. + maxLength: 16384 + minLength: 1 + type: string + tags: + description: |- + Session tags passed to STS AssumeRole for cost attribution in the AWS Cost + & Usage Report, once activated. STS allows at most 50 per role session. + items: + description: |- + AwsSessionTag is an AWS STS session tag passed to AssumeRole for cost + attribution. Exactly one of value and expression must be set. + properties: + expression: + description: |- + CEL expression evaluated against each request to produce the tag value, + for example `jwt.sub` or `request.headers["x-app"]`. Requests with invalid + tag values are rejected. + maxLength: 16384 + minLength: 1 + type: string + key: + description: Key is the tag + key. + maxLength: 128 + minLength: 1 + type: string + value: + description: Value is a static + tag value. + maxLength: 256 + type: string + required: + - key + type: object + x-kubernetes-validations: + - message: exactly one of value + or expression must be set + rule: has(self.value) != has(self.expression) + maxItems: 50 + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + required: + - roleArn + type: object + x-kubernetes-validations: + - message: at most one of the fields + in [sessionName sessionNameExpression] + may be set + rule: (has(self.sessionName)?1:0)+(has(self.sessionNameExpression)?1:0) + <= 1 + region: + description: |- + AWS SigV4 signing region, for example `us-east-1`. Set this when the + target AWS service is in a different region than the gateway. If unset, + typed AWS backends may provide this automatically; otherwise the ambient + AWS region is used. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for AWS credentials, defaulting to a Kubernetes `Secret`. + The default Secret resolver expects `accessKey`, `secretKey`, and optional + `sessionToken` keys. + properties: + group: + description: API group of the referenced + credential; empty selects the + core API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to + `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) + > 0)' + serviceName: + description: |- + AWS SigV4 signing service name, for example + `bedrock`, `bedrock-agentcore`, or `execute-api`). If unset, typed AWS + backends may provide this automatically. + maxLength: 256 + minLength: 1 + type: string + type: object + x-kubernetes-validations: + - message: secretRef and assumeRole are + mutually exclusive + rule: '!(has(self.secretRef) && has(self.assumeRole))' + key: + description: |- + Inline API key to use as the value of the `Authorization` header. + This option is the least secure; usage of a `Secret` is preferred. + maxLength: 2048 + type: string + location: + description: |- + Where API keys are inserted. Defaults to the `Authorization` header with + the `Bearer ` prefix. Applies to `key` and `secretRef`. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. + HTTP/2 pseudo-headers (names beginning + with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in + [header queryParameter cookie] must + be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + secretRef: + description: |- + Credential source for the API key, defaulting to a Kubernetes `Secret`. + By default, the value is read from the `Authorization` key; set + `secretRef.key` to override it. A `Bearer ` prefix is stripped only from + the default `Authorization` key. + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set + both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + type: object + x-kubernetes-validations: + - message: location may only be set for key + or secretRef auth + rule: 'has(self.location) ? has(self.key) + || has(self.secretRef) : true' + - message: exactly one of the fields in [key + secretRef aws] must be set + rule: (has(self.key)?1:0)+(has(self.secretRef)?1:0)+(has(self.aws)?1:0) + == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + region: + description: |- + AWS region where the guardrail is deployed, for example + `us-west-2`). + maxLength: 256 + minLength: 1 + type: string + version: + description: Version of the Guardrail policy to + use for the backend. + maxLength: 256 + minLength: 1 + type: string + required: + - identifier + - region + - version + type: object + googleModelArmor: + description: Google Model Armor settings for prompt + guarding. + properties: + action: + description: |- + Action controls whether flagged content is rejected or only observed. + `Reject` (the default) rejects flagged content; `Audit` records the + would-be rejection without blocking. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + location: + description: |- + Google Cloud location, for example `us-central1`. + Defaults to `us-central1` if not specified. + maxLength: 256 + minLength: 1 + type: string + policies: + description: Policies for communicating with Google + Model Armor. + properties: + auth: + description: Settings for authenticating to + Google Model Armor. + properties: + gcp: + description: |- + Google authentication method for Model Armor. Use `gcp: {}` for default + Google credential discovery. + properties: + audience: + description: |- + Explicit `aud` value for the ID token. Only + valid with `IdToken` type. If not set, the `aud` is automatically + derived from the backend hostname. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for ADC-compatible Google credential JSON, defaulting to + a Kubernetes `Secret`. By default, the value is read from + `credentials.json`; set `secretRef.key` to override it. When omitted, + ambient credentials are used. + properties: + group: + description: API group of the referenced + credential; empty selects the + core API group + type: string + key: + description: Key in the referenced + Secret. If omitted, a location-specific + default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to + `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) + > 0)' + type: + description: |- + The type of token to generate. To authenticate to GCP services, + generally an `AccessToken` is used. To authenticate to Cloud Run, an + `IdToken` is used. + enum: + - AccessToken + - IdToken + type: string + type: object + x-kubernetes-validations: + - message: audience is only valid with IdToken + rule: 'has(self.audience) ? self.type + == ''IdToken'' : true' + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [gcp] + must be set + rule: (has(self.gcp)?1:0) == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + projectId: + description: Google Cloud project ID. + maxLength: 256 + minLength: 1 + type: string + templateId: + description: Template ID for Google Model Armor. + maxLength: 256 + minLength: 1 + type: string + required: + - projectId + - templateId + type: object + openAIModeration: + description: |- + Passes prompt data through the OpenAI Moderations + endpoint. + See https://developers.openai.com/api/reference/resources/moderations for more information. + properties: + action: + description: |- + Action controls whether flagged content is rejected or only observed. + `Reject` (the default) rejects flagged content; `Audit` records the + would-be rejection without blocking. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + model: + description: |- + Moderation model to use. For example, + `omni-moderation`. + type: string + policies: + description: Policies for communicating with OpenAI. + properties: + auth: + description: Settings for authenticating to + OpenAI. + properties: + key: + description: |- + Inline key to use as the value of the + `Authorization` header. This option is the least secure; usage of a + `Secret` is preferred. + maxLength: 2048 + type: string + location: + description: |- + Where backend credentials are inserted. Defaults to the `Authorization` + header with the `Bearer ` prefix. Applies to `key` and `secretRef`. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. + HTTP/2 pseudo-headers (names beginning + with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in + [header queryParameter cookie] must + be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + secretRef: + description: |- + Credential source for the authorization value, defaulting to a Kubernetes + `Secret`. By default, the value is read from the `Authorization` key; set + `secretRef.key` to override it. A `Bearer ` prefix is stripped only from + the default `Authorization` key. + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set + both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [key + secretRef] must be set + rule: (has(self.key)?1:0)+(has(self.secretRef)?1:0) + == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + type: object + regex: + description: Regular expression (regex) matching for + prompt guards and data masking. + properties: + action: + description: |- + The action to take if a regex pattern is matched in a request or response. + The action applies to request and response matches alike. Note that + `Mask` is not applied to streamed responses: matched content in a + streamed response is passed through unmodified. + Defaults to `Mask`. + enum: + - Audit + - Mask + - Reject + type: string + builtins: + description: |- + Built-in regex patterns to match against the request or response. + Matches and built-ins are additive. + items: + description: |- + Built-in regex patterns for specific types of strings in prompts. + For example, if you specify `CreditCard`, any credit card numbers + in the request or response are matched. + enum: + - CaSin + - CreditCard + - Email + - PhoneNumber + - Ssn + type: string + type: array + matches: + description: |- + Regex patterns to match against the request or response. + Matches and built-ins are additive. + items: + maxLength: 1024 + minLength: 1 + type: string + type: array + type: object + response: + description: |- + Custom response message to return to the client. If not specified, defaults to + `The request was rejected due to inappropriate content`. + properties: + message: + description: |- + Custom response message to return to the client. If not specified, defaults to + `The request was rejected due to inappropriate content`. + type: string + statusCode: + description: Status code to return to the client. + Defaults to 403. + format: int32 + maximum: 599 + minimum: 200 + type: integer + type: object + scope: + description: |- + Which parts of the request this guard inspects. When unset, defaults to + `SystemPrompt` and `Messages`. Tool call inputs and outputs are not + inspected unless `ToolInput`/`ToolOutput` are listed explicitly. + In APIs that send tool arguments as opaque JSON, such as Completions, the + arguments are masked as a single string, meaning a prompt guard has the + potential to rewrite the arguments into invalid JSON. + items: + description: Which category of request content a prompt + guard inspects. + enum: + - Messages + - SystemPrompt + - ToolInput + - ToolOutput + type: string + maxItems: 4 + minItems: 1 + type: array + x-kubernetes-list-type: set + webhook: + description: Webhook that receives requests for prompt + guarding. + properties: + action: + description: |- + Action controls whether the webhook's verdict is enforced or only observed. + `Reject` (the default) enforces it; `Audit` records the would-be action + without blocking or masking. + enum: + - Audit + - Reject + type: string + backendRef: + description: |- + Webhook server to reach. + + Supported types: Service and Backend. + properties: + group: + default: "" + description: Group of the referent, for example + `gateway.networking.k8s.io`. Empty selects + the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind of the + referent, for example `Service`. Defaults + to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. Defaults + to the local namespace. A cross-namespace + reference requires a ReferenceGrant in the + referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. Required + when the referent is a Kubernetes `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == + ''Service'') ? has(self.port) : true' + failureMode: + description: |- + Behavior when the webhook guardrail is unavailable + or returns an error. `FailOpen` allows the request to continue. + `FailClosed` (default) rejects the request. + enum: + - FailClosed + - FailOpen + type: string + forwardHeaderMatches: + description: |- + HTTP header matches used to select the headers to forward to the webhook. + Request headers are used when forwarding requests and response headers + are used when forwarding responses. + By default, no headers are forwarded. + items: + description: |- + HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request + headers. + properties: + name: + description: Name of the HTTP header to match, + case-insensitive. When names are equivalent, + only the first matching entry is used + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: 'How to match against the header + value: `Exact` (default) or `RegularExpression`. + The regex dialect is implementation-specific' + enum: + - Exact + - RegularExpression + type: string + value: + description: Value of the HTTP header to match + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + headers: + additionalProperties: + description: A Common Expression Language (CEL) + expression. + maxLength: 16384 + minLength: 1 + type: string + description: CEL-computed headers to include in + webhook requests. + maxProperties: 64 + type: object + required: + - backendRef + type: object + type: object + x-kubernetes-validations: + - message: 'scope: only regex and bedrockGuardrails guards + support a non-default scope; other guard kinds always + inspect the default (SystemPrompt + Messages)' + rule: '!has(self.scope) || has(self.regex) || has(self.bedrockGuardrails) + || (self.scope.size() == 2 && ''SystemPrompt'' in self.scope + && ''Messages'' in self.scope)' + - message: exactly one of the fields in [regex webhook openAIModeration + bedrockGuardrails googleModelArmor] must be set + rule: (has(self.regex)?1:0)+(has(self.webhook)?1:0)+(has(self.openAIModeration)?1:0)+(has(self.bedrockGuardrails)?1:0)+(has(self.googleModelArmor)?1:0) + == 1 + maxItems: 8 + minItems: 1 + type: array + response: + description: Prompt guards to apply to responses returned + by the LLM provider. + items: + description: Prompt guards to apply to responses returned + by the LLM provider. + properties: + bedrockGuardrails: + description: |- + AWS Bedrock Guardrails settings for prompt + guarding. + properties: + action: + description: |- + Action controls whether the guardrail's verdict is enforced or only + observed. `Reject` (the default) enforces the guardrail: a blocked + assessment rejects the request/response and an anonymized assessment masks + the matched content. `Audit` runs the guardrail in observe mode: it is + invoked and its assessment recorded (metrics + structured log), but the + request/response is never blocked or masked. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + identifier: + description: Identifier of the Guardrail policy + to use for the backend. + maxLength: 256 + minLength: 1 + type: string + policies: + description: Policies for communicating with AWS + Bedrock Guardrails. + properties: + auth: + description: Settings for authenticating to + AWS Bedrock Guardrails. + properties: + aws: + description: |- + AWS authentication method for Bedrock Guardrails. Use `aws: {}` for + default AWS SDK credential discovery. + properties: + assumeRole: + description: |- + AWS STS AssumeRole settings to use before signing backend requests. + Ambient AWS credentials are used as the source credentials for STS. + properties: + externalId: + description: ExternalID is set when + the role's trust policy requires + sts:ExternalId. + maxLength: 1224 + minLength: 2 + pattern: ^[\w+=,.@:/-]+$ + type: string + roleArn: + description: AWS IAM role ARN to + assume. + minLength: 1 + pattern: ^arn:aws[a-z-]*:iam::[0-9]{12}:role/.+$ + type: string + sessionName: + description: |- + SessionName is a custom session name (RoleSessionName) for CloudTrail and + Cost & Usage Report attribution. If unset, AWS generates a random name. + pattern: ^[\w+=,.@-]{2,64}$ + type: string + sessionNameExpression: + description: |- + SessionNameExpression is a CEL expression evaluated against each request + to produce the session name (RoleSessionName), for example `jwt.sub` or + `request.headers["x-team"]`. If the expression does not produce a valid + session name at request time, the request is rejected. + maxLength: 16384 + minLength: 1 + type: string + tags: + description: |- + Session tags passed to STS AssumeRole for cost attribution in the AWS Cost + & Usage Report, once activated. STS allows at most 50 per role session. + items: + description: |- + AwsSessionTag is an AWS STS session tag passed to AssumeRole for cost + attribution. Exactly one of value and expression must be set. + properties: + expression: + description: |- + CEL expression evaluated against each request to produce the tag value, + for example `jwt.sub` or `request.headers["x-app"]`. Requests with invalid + tag values are rejected. + maxLength: 16384 + minLength: 1 + type: string + key: + description: Key is the tag + key. + maxLength: 128 + minLength: 1 + type: string + value: + description: Value is a static + tag value. + maxLength: 256 + type: string + required: + - key + type: object + x-kubernetes-validations: + - message: exactly one of value + or expression must be set + rule: has(self.value) != has(self.expression) + maxItems: 50 + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + required: + - roleArn + type: object + x-kubernetes-validations: + - message: at most one of the fields + in [sessionName sessionNameExpression] + may be set + rule: (has(self.sessionName)?1:0)+(has(self.sessionNameExpression)?1:0) + <= 1 + region: + description: |- + AWS SigV4 signing region, for example `us-east-1`. Set this when the + target AWS service is in a different region than the gateway. If unset, + typed AWS backends may provide this automatically; otherwise the ambient + AWS region is used. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for AWS credentials, defaulting to a Kubernetes `Secret`. + The default Secret resolver expects `accessKey`, `secretKey`, and optional + `sessionToken` keys. + properties: + group: + description: API group of the referenced + credential; empty selects the + core API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to + `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) + > 0)' + serviceName: + description: |- + AWS SigV4 signing service name, for example + `bedrock`, `bedrock-agentcore`, or `execute-api`). If unset, typed AWS + backends may provide this automatically. + maxLength: 256 + minLength: 1 + type: string + type: object + x-kubernetes-validations: + - message: secretRef and assumeRole are + mutually exclusive + rule: '!(has(self.secretRef) && has(self.assumeRole))' + key: + description: |- + Inline API key to use as the value of the `Authorization` header. + This option is the least secure; usage of a `Secret` is preferred. + maxLength: 2048 + type: string + location: + description: |- + Where API keys are inserted. Defaults to the `Authorization` header with + the `Bearer ` prefix. Applies to `key` and `secretRef`. + properties: + cookie: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + header: + properties: + name: + description: Name of an HTTP header. + HTTP/2 pseudo-headers (names beginning + with `:`) are not supported + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + prefix: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + queryParameter: + properties: + name: + maxLength: 256 + minLength: 1 + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in + [header queryParameter cookie] must + be set + rule: (has(self.header)?1:0)+(has(self.queryParameter)?1:0)+(has(self.cookie)?1:0) + == 1 + secretRef: + description: |- + Credential source for the API key, defaulting to a Kubernetes `Secret`. + By default, the value is read from the `Authorization` key; set + `secretRef.key` to override it. A `Bearer ` prefix is stripped only from + the default `Authorization` key. + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + key: + description: Key in the referenced Secret. + If omitted, a location-specific default + is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set + both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : (has(self.kind) + && size(self.kind) > 0)' + type: object + x-kubernetes-validations: + - message: location may only be set for key + or secretRef auth + rule: 'has(self.location) ? has(self.key) + || has(self.secretRef) : true' + - message: exactly one of the fields in [key + secretRef aws] must be set + rule: (has(self.key)?1:0)+(has(self.secretRef)?1:0)+(has(self.aws)?1:0) + == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + region: + description: |- + AWS region where the guardrail is deployed, for example + `us-west-2`). + maxLength: 256 + minLength: 1 + type: string + version: + description: Version of the Guardrail policy to + use for the backend. + maxLength: 256 + minLength: 1 + type: string + required: + - identifier + - region + - version + type: object + googleModelArmor: + description: Google Model Armor settings for prompt + guarding. + properties: + action: + description: |- + Action controls whether flagged content is rejected or only observed. + `Reject` (the default) rejects flagged content; `Audit` records the + would-be rejection without blocking. + enum: + - Audit + - Reject + type: string + failureMode: + description: |- + Behavior when the provider is unavailable or returns an error. + `FailOpen` allows the request to continue; `FailClosed` (default) rejects it. + enum: + - FailClosed + - FailOpen + type: string + location: + description: |- + Google Cloud location, for example `us-central1`. + Defaults to `us-central1` if not specified. + maxLength: 256 + minLength: 1 + type: string + policies: + description: Policies for communicating with Google + Model Armor. + properties: + auth: + description: Settings for authenticating to + Google Model Armor. + properties: + gcp: + description: |- + Google authentication method for Model Armor. Use `gcp: {}` for default + Google credential discovery. + properties: + audience: + description: |- + Explicit `aud` value for the ID token. Only + valid with `IdToken` type. If not set, the `aud` is automatically + derived from the backend hostname. + maxLength: 256 + minLength: 1 + type: string + secretRef: + description: |- + Credential source for ADC-compatible Google credential JSON, defaulting to + a Kubernetes `Secret`. By default, the value is read from + `credentials.json`; set `secretRef.key` to override it. When omitted, + ambient credentials are used. + properties: + group: + description: API group of the referenced + credential; empty selects the + core API group + type: string + key: + description: Key in the referenced + Secret. If omitted, a location-specific + default is used + maxLength: 253 + minLength: 1 + type: string + kind: + description: Kind of the referenced + credential; empty defaults to + `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) + > 0)' + type: + description: |- + The type of token to generate. To authenticate to GCP services, + generally an `AccessToken` is used. To authenticate to Cloud Run, an + `IdToken` is used. + enum: + - AccessToken + - IdToken + type: string + type: object + x-kubernetes-validations: + - message: audience is only valid with IdToken + rule: 'has(self.audience) ? self.type + == ''IdToken'' : true' + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [gcp] + must be set + rule: (has(self.gcp)?1:0) == 1 + http: + description: Settings for managing HTTP requests + to the backend + properties: + requestTimeout: + description: Deadline for receiving a response + from the backend. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: requestTimeout must be at least + 1ms + rule: duration(self) >= duration('1ms') + version: + description: |- + HTTP protocol version for backend connections. If unset, it is inferred: + `Service` appProtocol, `HTTP2` for gRPC, the original protocol for + plaintext HTTP, or `HTTP1` for HTTPS because clients often upgrade HTTPS + to HTTP/2 even when the backend does not support it. + enum: + - HTTP1 + - HTTP2 + type: string + type: object + tcp: + description: Settings for managing TCP connections + to the backend + properties: + connectTimeout: + description: |- + Deadline for establishing a connection to + the destination. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: connectTimeout must be at least + 100ms + rule: duration(self) >= duration('100ms') + keepalive: + description: |- + Settings for enabling TCP keepalives on the + connection. + properties: + interval: + description: |- + Time between keepalive probes. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: interval must be at least + 1 second + rule: duration(self) >= duration('1s') + retries: + description: |- + Maximum number of keepalive probes to send before dropping the connection. + If unset, this defaults to 9. + format: int32 + maximum: 64 + minimum: 1 + type: integer + time: + description: |- + Time a connection needs to be idle before keepalive probes start being sent. + If unset, this defaults to 180s. + maxLength: 32 + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + x-kubernetes-validations: + - message: time must be at least 1 second + rule: duration(self) >= duration('1s') + type: object + type: object + tls: + description: |- + Settings for managing TLS connections to the backend + + When set, TLS is originated to the backend using the system trusted CA + certificates, and SNI is inferred from the destination. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced + CA certificate source. Omitted defaults + to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced + CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client + identity and trust roots (`Inline` default, + or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced + credential; empty selects the core + API group + type: string + kind: + description: Kind of the referenced + credential; empty defaults to `Secret` + type: string + name: + description: Name of the referenced + credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must + set both group and kind + rule: '(!has(self.group) || size(self.group) + == 0) ? (!has(self.kind) || size(self.kind) + == 0 || self.kind == ''Secret'') : + (has(self.kind) && size(self.kind) + > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs + may not be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) + : true' + - message: insecureSkipVerify and verifySubjectAltNames + may not be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not + be combined with mtlsCertificateRef, caCertificateRefs, + or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) + && !has(self.caCertificateRefs) && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + tunnel: + description: Settings for managing tunnel connections + to the backend, like `HTTPS_PROXY` + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, + for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind + of the referent, for example `Service`. + Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. + Defaults to the local namespace. A + cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. + Required when the referent is a Kubernetes + `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef + url] must be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) + == 1 + - message: url must not include a path for backend + tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth http + tcp tls tunnel] must be set + rule: '[has(self.auth),has(self.http),has(self.tcp),has(self.tls),has(self.tunnel)].filter(x,x==true).size() + >= 1' + projectId: + description: Google Cloud project ID. + maxLength: 256 + minLength: 1 + type: string + templateId: + description: Template ID for Google Model Armor. + maxLength: 256 + minLength: 1 + type: string + required: + - projectId + - templateId + type: object + regex: + description: Regular expression (regex) matching for + prompt guards and data masking. + properties: + action: + description: |- + The action to take if a regex pattern is matched in a request or response. + The action applies to request and response matches alike. Note that + `Mask` is not applied to streamed responses: matched content in a + streamed response is passed through unmodified. + Defaults to `Mask`. + enum: + - Audit + - Mask + - Reject + type: string + builtins: + description: |- + Built-in regex patterns to match against the request or response. + Matches and built-ins are additive. + items: + description: |- + Built-in regex patterns for specific types of strings in prompts. + For example, if you specify `CreditCard`, any credit card numbers + in the request or response are matched. + enum: + - CaSin + - CreditCard + - Email + - PhoneNumber + - Ssn + type: string + type: array + matches: + description: |- + Regex patterns to match against the request or response. + Matches and built-ins are additive. + items: + maxLength: 1024 + minLength: 1 + type: string + type: array + type: object + response: + description: |- + Custom response message to return to the client. If not specified, defaults to + `The response was rejected due to inappropriate content`. + properties: + message: + description: |- + Custom response message to return to the client. If not specified, defaults to + `The request was rejected due to inappropriate content`. + type: string + statusCode: + description: Status code to return to the client. + Defaults to 403. + format: int32 + maximum: 599 + minimum: 200 + type: integer + type: object + webhook: + description: Webhook that receives responses for prompt + guarding. + properties: + action: + description: |- + Action controls whether the webhook's verdict is enforced or only observed. + `Reject` (the default) enforces it; `Audit` records the would-be action + without blocking or masking. + enum: + - Audit + - Reject + type: string + backendRef: + description: |- + Webhook server to reach. + + Supported types: Service and Backend. + properties: + group: + default: "" + description: Group of the referent, for example + `gateway.networking.k8s.io`. Empty selects + the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind of the + referent, for example `Service`. Defaults + to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. Defaults + to the local namespace. A cross-namespace + reference requires a ReferenceGrant in the + referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. Required + when the referent is a Kubernetes `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == + ''Service'') ? has(self.port) : true' + failureMode: + description: |- + Behavior when the webhook guardrail is unavailable + or returns an error. `FailOpen` allows the request to continue. + `FailClosed` (default) rejects the request. + enum: + - FailClosed + - FailOpen + type: string + forwardHeaderMatches: + description: |- + HTTP header matches used to select the headers to forward to the webhook. + Request headers are used when forwarding requests and response headers + are used when forwarding responses. + By default, no headers are forwarded. + items: + description: |- + HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request + headers. + properties: + name: + description: Name of the HTTP header to match, + case-insensitive. When names are equivalent, + only the first matching entry is used + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: 'How to match against the header + value: `Exact` (default) or `RegularExpression`. + The regex dialect is implementation-specific' + enum: + - Exact + - RegularExpression + type: string + value: + description: Value of the HTTP header to match + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + headers: + additionalProperties: + description: A Common Expression Language (CEL) + expression. + maxLength: 16384 + minLength: 1 + type: string + description: CEL-computed headers to include in + webhook requests. + maxProperties: 64 + type: object + required: + - backendRef + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [regex webhook bedrockGuardrails + googleModelArmor] must be set + rule: (has(self.regex)?1:0)+(has(self.webhook)?1:0)+(has(self.bedrockGuardrails)?1:0)+(has(self.googleModelArmor)?1:0) + == 1 + maxItems: 8 + minItems: 1 + type: array + streaming: + description: |- + Apply prompt guards to streaming responses and realtime websocket messages. + Defaults to disabled to preserve streaming throughput unless explicitly enabled. + enum: + - Enabled + type: string + type: object + x-kubernetes-validations: + - message: at least one of the fields in [request response] must + be set + rule: '[has(self.request),has(self.response)].filter(x,x==true).size() + >= 1' + tls: + description: TLS settings for connections to this model provider. + properties: + alpnProtocols: + description: |- + Application-Layer Protocol Negotiation (`ALPN`) + value to use in the TLS handshake. + + If not present, defaults to `["h2", "http/1.1"]`. + items: + maxLength: 64 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + caCertificateRefs: + description: |- + CA certificate source to use to verify the server certificate. Omitted kind + and `ConfigMap` select a ConfigMap; `Secret` selects a Secret. The bundle is + read from the `ca.crt` key unless `key` names a different one. If unset, the + system's trusted certificates are used. + items: + description: |- + LocalCACertificateRef references a same-namespace CA certificate source. + An omitted kind defaults to ConfigMap, and an omitted key to `ca.crt`. + properties: + key: + description: |- + Key within the referenced source holding the PEM-encoded CA bundle. + Omitted defaults to `ca.crt`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$ + type: string + kind: + description: Kind of the referenced CA certificate source. + Omitted defaults to ConfigMap. + enum: + - ConfigMap + - Secret + type: string + name: + description: Name of the referenced CA certificate source. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + certificateSource: + description: Source for the gateway's client identity and + trust roots (`Inline` default, or `SPIFFE`). + enum: + - Inline + - SPIFFE + type: string + insecureSkipVerify: + description: |- + Originates TLS but skips verification of the backend's certificate + WARNING: insecure; only use if the risks are understood + + Modes: + * `All` disables all TLS verification + * `Hostname` trusts the CA certificate but ignores hostname/SAN mismatches. + Still insecure; prefer `verifySubjectAltNames` where possible. + enum: + - All + - Hostname + type: string + keyExchangeGroups: + description: |- + Ordered list of key exchange groups for a TLS connection. + For example: `X25519_MLKEM768,X25519`. + items: + enum: + - P-256 + - P-384 + - X25519 + - X25519_MLKEM768 + type: string + type: array + mtlsCertificateRef: + description: |- + Enables mutual TLS to the backend using `tls.key` and `tls.crt` from the + referenced credential source (defaulting to a Kubernetes `Secret`). An + optional `ca.cert`, if present, verifies the server certificate, but + `caCertificateRefs` takes priority. If unspecified, no client certificate + is used. + items: + description: |- + References a same-namespace credential + Set only `name` for a Kubernetes Secret + properties: + group: + description: API group of the referenced credential; + empty selects the core API group + type: string + kind: + description: Kind of the referenced credential; empty + defaults to `Secret` + type: string + name: + description: Name of the referenced credential + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: custom credential refs must set both group and + kind + rule: '(!has(self.group) || size(self.group) == 0) ? (!has(self.kind) + || size(self.kind) == 0 || self.kind == ''Secret'') + : (has(self.kind) && size(self.kind) > 0)' + maxItems: 1 + type: array + x-kubernetes-list-type: atomic + sni: + description: |- + Server Name Indicator (`SNI`) to use in the TLS + handshake. If unset, the `SNI` is automatically set based on the + destination hostname. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + verifySubjectAltNames: + description: |- + Subject Alternative Names (`SAN`) + to verify in the server certificate. + If not present, the destination hostname is automatically used. + items: + maxLength: 256 + minLength: 1 + type: string + maxItems: 16 + minItems: 1 + type: array + type: object + x-kubernetes-validations: + - message: insecureSkipVerify All and caCertificateRefs may not + be set together + rule: 'has(self.insecureSkipVerify) && self.insecureSkipVerify + == ''All'' ? !has(self.caCertificateRefs) : true' + - message: insecureSkipVerify and verifySubjectAltNames may not + be set together + rule: 'has(self.insecureSkipVerify) ? !has(self.verifySubjectAltNames) + : true' + - message: certificateSource SPIFFE may not be combined with mtlsCertificateRef, + caCertificateRefs, or insecureSkipVerify + rule: '!has(self.certificateSource) || self.certificateSource + != ''SPIFFE'' || (!has(self.mtlsCertificateRef) && !has(self.caCertificateRefs) + && !has(self.insecureSkipVerify))' + - message: at most one of the fields in [verifySubjectAltNames + insecureSkipVerify] may be set + rule: (has(self.verifySubjectAltNames)?1:0)+(has(self.insecureSkipVerify)?1:0) + <= 1 + transformations: + description: CEL transformations applied to fields in the provider + request body. + items: + description: |- + Maps a request JSON field to a CEL expression. + The expression is evaluated against the current request body and its result + is assigned to the configured field. + properties: + expression: + description: CEL expression used to compute the field value. + maxLength: 16384 + minLength: 1 + type: string + field: + description: Name of the field to set. + maxLength: 256 + minLength: 1 + type: string + required: + - expression + - field + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - field + x-kubernetes-list-type: map + tunnel: + description: Proxy tunnel used to reach this model provider. + properties: + backendRef: + description: |- + `backendRef` selects a backend for this policy. + Mutually exclusive with `url`. + properties: + group: + default: "" + description: Group of the referent, for example `gateway.networking.k8s.io`. + Empty selects the core API group + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kubernetes resource kind of the referent, + for example `Service`. Defaults to `Service` + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the referent. Defaults to the + local namespace. A cross-namespace reference requires + a ReferenceGrant in the referent namespace + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Destination port number. Required when the + referent is a Kubernetes `Service` + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') + ? has(self.port) : true' + mode: + description: |- + How requests are sent through the proxy. + Defaults to `Auto`. + enum: + - Auto + - Connect + type: string + url: + description: |- + `url` directly specifies the HTTP(S) endpoint for this policy. + When the scheme is `https`, backend TLS is enabled automatically. + Mutually exclusive with `backendRef`. + URLs are opaque; referencing a Kubernetes service hostname like `hello.ns.svc.cluster.local` + will not apply Service policies or load balancing. + maxLength: 1024 + minLength: 1 + pattern: ^https?://[^/?#@]+(/[^?#]*)?$ + type: string + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [backendRef url] must + be set + rule: (has(self.backendRef)?1:0)+(has(self.url)?1:0) == 1 + - message: url must not include a path for backend tunnel + rule: '!has(self.url) || self.url.matches(''^https?://[^/?#]+$'')' + type: object + x-kubernetes-validations: + - message: at least one of the fields in [auth authorization finalTransformations + headers health promptGuard tls transformations tunnel] must be + set + rule: '[has(self.auth),has(self.authorization),has(self.finalTransformations),has(self.headers),has(self.health),has(self.promptGuard),has(self.tls),has(self.transformations),has(self.tunnel)].filter(x,x==true).size() + >= 1' + provider: + description: |- + Provider serving this concrete model. Provider-specific configuration is + set by the corresponding field below when needed. + enum: + - Anthropic + - Azure + - Baseten + - Bedrock + - Cerebras + - Cohere + - Custom + - Deepinfra + - Deepseek + - Fireworks + - Gemini + - Groq + - Huggingface + - Meta + - Mistral + - Ollama + - OpenAI + - Openrouter + - TogetherAI + - VertexAI + - XAI + type: string + vertexai: + description: Provider-specific settings for Vertex AI. + properties: + projectId: + description: The ID of the Google Cloud Project that you use for + the Vertex AI. + maxLength: 64 + minLength: 1 + type: string + region: + description: |- + The location of the Google Cloud Project that you use for the Vertex AI. + Special values: `global` uses the global endpoint, while `us` and `eu` use restricted + multi-region endpoints. Other values are treated as regional locations. + Defaults to `global` if not specified. + maxLength: 64 + minLength: 1 + type: string + required: + - projectId + type: object + virtualModel: + description: Request-time routing among concrete AgentgatewayModel + resources. + properties: + conditional: + description: Ordered condition-based model selection. + properties: + targets: + description: |- + Concrete model targets evaluated in order. The first matching condition is + selected. One final target may omit when to act as the fallback. + items: + properties: + model: + description: |- + Concrete model name selected through the referenced model. It is required + when modelRef points to a wildcard match.model. When omitted, the referenced + model's exact effective match.model is used. + maxLength: 1024 + minLength: 1 + type: string + modelRef: + description: Same-namespace AgentgatewayModel resource + selected by this target. + properties: + name: + default: "" + description: Name of the referent + type: string + type: object + x-kubernetes-map-type: atomic + when: + description: |- + CEL expression that must evaluate to true for this target to be selected. + Omit only on the final fallback target. + maxLength: 16384 + minLength: 1 + type: string + required: + - modelRef + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-validations: + - message: conditional targets without when must be last + rule: self.filter(e, !has(e.when)).size() <= 1 && (!self.exists(e, + !has(e.when)) || !has(self[size(self) - 1].when)) + required: + - targets + type: object + failover: + description: Priority-based model selection with failover between + priority groups. + properties: + targets: + description: Concrete model targets grouped by priority. Lower + values are preferred. + items: + properties: + model: + description: |- + Concrete model name selected through the referenced model. It is required + when modelRef points to a wildcard match.model. When omitted, the referenced + model's exact effective match.model is used. + maxLength: 1024 + minLength: 1 + type: string + modelRef: + description: Same-namespace AgentgatewayModel resource + selected by this target. + properties: + name: + default: "" + description: Name of the referent + type: string + type: object + x-kubernetes-map-type: atomic + priority: + description: |- + Priority of this target. Lower values are preferred. Targets at the same + priority are selected using a score that considers health and latency. The + next priority is used only when every target at this priority is degraded. + Configure policies.health on concrete target models to customize + degradation and eviction behavior. + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - modelRef + - priority + type: object + maxItems: 64 + minItems: 1 + type: array + required: + - targets + type: object + weighted: + description: Weight-based model selection. + properties: + targets: + description: Concrete model targets and their relative weights. + items: + properties: + model: + description: |- + Concrete model name selected through the referenced model. It is required + when modelRef points to a wildcard match.model. When omitted, the referenced + model's exact effective match.model is used. + maxLength: 1024 + minLength: 1 + type: string + modelRef: + description: Same-namespace AgentgatewayModel resource + selected by this target. + properties: + name: + default: "" + description: Name of the referent + type: string + type: object + x-kubernetes-map-type: atomic + weight: + description: Relative traffic weight. Defaults to 1. + format: int32 + maximum: 1000000 + minimum: 1 + type: integer + required: + - modelRef + type: object + maxItems: 64 + minItems: 1 + type: array + required: + - targets + type: object + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [weighted failover conditional] + must be set + rule: (has(self.weighted)?1:0)+(has(self.failover)?1:0)+(has(self.conditional)?1:0) + == 1 + visibility: + description: |- + Controls whether clients can request this model directly. Internal models + can only be selected by virtual models. Defaults to Public. + enum: + - Internal + - Public + type: string + required: + - parentRefs + type: object + x-kubernetes-validations: + - message: baseURL requires provider + rule: has(self.provider) || !has(self.baseURL) + - message: policies cannot be used with virtualModel + rule: '!has(self.virtualModel) || !has(self.policies)' + - message: virtual models must be public + rule: '!has(self.virtualModel) || !has(self.visibility) || self.visibility + != ''Internal''' + - message: virtual model match.model must be an exact name + rule: '!has(self.virtualModel) || !has(self.match) || !has(self.match.model) + || !self.match.model.contains(''*'')' + - message: ollama requires baseURL + rule: '!has(self.provider) || self.provider != ''Ollama'' || has(self.baseURL)' + - message: baseURL must be an absolute http or https URL with a host + rule: '!has(self.baseURL) || (isURL(self.baseURL) && (url(self.baseURL).getScheme() + == ''http'' || url(self.baseURL).getScheme() == ''https'') && url(self.baseURL).getHostname() + != "")' + - message: baseURL cannot target localhost, loopback, or link-local addresses + rule: '!has(self.baseURL) || !self.baseURL.matches("(?i)^https?://(localhost|[^/]+\\.localhost)(:[0-9]+)?(/|$)")' + - message: baseURL cannot target localhost, loopback, or link-local addresses + rule: '!has(self.baseURL) || !self.baseURL.matches("^https?://127(\\.[0-9]{1,3}){0,3}(:[0-9]+)?(/|$)")' + - message: baseURL cannot target localhost, loopback, or link-local addresses + rule: '!has(self.baseURL) || !self.baseURL.matches("^https?://169\\.254\\.[0-9]{1,3}\\.[0-9]{1,3}(:[0-9]+)?(/|$)")' + - message: baseURL cannot target localhost, loopback, or link-local addresses + rule: '!has(self.baseURL) || !self.baseURL.matches("(?i)^https?://\\[(::1|fe[89ab][0-9a-f:]*)\\](:[0-9]+)?(/|$)")' + - message: azure must be set if and only if provider is Azure + rule: has(self.azure) == (has(self.provider) && self.provider == 'Azure') + - message: vertexai must be set if and only if provider is VertexAI + rule: has(self.vertexai) == (has(self.provider) && self.provider == + 'VertexAI') + - message: bedrock must be set if and only if provider is Bedrock + rule: has(self.bedrock) == (has(self.provider) && self.provider == 'Bedrock') + - message: custom must be set if and only if provider is Custom + rule: has(self.custom) == (has(self.provider) && self.provider == 'Custom') + - message: exactly one of the fields in [provider virtualModel] must be + set + rule: (has(self.provider)?1:0)+(has(self.virtualModel)?1:0) == 1 + status: + description: Current model attachment status. + properties: + parents: + description: Status for each Gateway parent to which this model is + attached. + items: + description: |- + RouteParentStatus describes the status of a route with respect to an + associated Parent. + properties: + conditions: + description: |- + Conditions describes the status of the route with respect to the Gateway. + Note that the route's availability is also subject to the Gateway's own + status conditions and listener status. + + If the Route's ParentRef specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller has sufficient access, + then that Gateway's controller MUST set the "Accepted" condition on the + Route, to indicate whether the route has been accepted or rejected by the + Gateway, and why. + + A Route MUST be considered "Accepted" if at least one of the Route's + rules is implemented by the Gateway. + + There are a number of cases where the "Accepted" condition may not be set + due to lack of controller visibility, that includes when: + + * The Route refers to a nonexistent parent. + * The Route is of a type that the controller does not support. + * The Route is in a namespace to which the controller does not have access. + + + + Notes for implementors: + + Conditions are a listType `map`, which means that they function like a + map with a key of the `type` field _in the k8s apiserver_. + + This means that implementations must obey some rules when updating this + section. + + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + * Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. + * Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. + * Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. + * If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + + + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: |- + ControllerName is a domain/path string that indicates the name of the + controller that wrote this status. This corresponds with the + controllerName field on GatewayClass. + + Example: "example.net/gateway-controller". + + The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are + valid Kubernetes names + (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + + Controllers MUST populate this field when writing status. Controllers should ensure that + entries to status populated with their ControllerName are cleaned up when they are no + longer necessary. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: |- + ParentRef corresponds with a ParentRef in the spec that this + RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: 'Namespace of the referent. Defaults to the + Route''s local namespace. Cross-namespace references must + be explicitly allowed, for example via ReferenceGrant. + Support: Core' + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: 'Port this Route targets on the parent, interpreted + per parent kind (for example a Gateway listener port or + Service port). Support: Extended' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: 'Name of a section within the target resource, + for example a Gateway Listener name or Service port name. + Empty references the entire resource. Support: Core' + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - conditions + - controllerName + - parentRef + type: object + maxItems: 16 + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/test/crd/inference-extension/httproute.yaml b/test/crd/inference-extension/httproute.yaml index 82e8cbfc3..dd7cd7ab6 100644 --- a/test/crd/inference-extension/httproute.yaml +++ b/test/crd/inference-extension/httproute.yaml @@ -68,6 +68,8 @@ spec: items: type: object properties: + name: + type: string matches: type: array items: diff --git a/test/e2e-rbac/crds/README.md b/test/e2e-rbac/crds/README.md index cb711e9b4..de9c2a06a 100644 --- a/test/e2e-rbac/crds/README.md +++ b/test/e2e-rbac/crds/README.md @@ -1,9 +1,10 @@ # CRDs for `make test-e2e-rbac` with RBAC_E2E_GATEWAY_API=true. -These two files are symlinks into the vendored upstream CRDs +These three files are symlinks into the vendored upstream CRDs (`test/crd/inference-extension/README.md` records their provenance). The -AgentGateway plane needs the InferencePool and AgentgatewayPolicy kinds, which -the upstream Gateway API standard bundle does not ship. +AgentGateway plane needs the InferencePool, AgentgatewayPolicy and +AgentgatewayModel kinds, which the upstream Gateway API standard bundle does +not ship. The vendored directory also contains hand-made Gateway and HTTPRoute stubs. Those must NOT be installed next to `RBAC_E2E_GATEWAY_API=true`: the script diff --git a/test/e2e-rbac/crds/agentgatewaymodel.yaml b/test/e2e-rbac/crds/agentgatewaymodel.yaml new file mode 120000 index 000000000..d25b603f9 --- /dev/null +++ b/test/e2e-rbac/crds/agentgatewaymodel.yaml @@ -0,0 +1 @@ +../../crd/inference-extension/agentgatewaymodel.yaml \ No newline at end of file diff --git a/test/e2e-rbac/fixtures/50-agentgateway.yaml b/test/e2e-rbac/fixtures/50-agentgateway.yaml index 81baabf92..6f0aad816 100644 --- a/test/e2e-rbac/fixtures/50-agentgateway.yaml +++ b/test/e2e-rbac/fixtures/50-agentgateway.yaml @@ -58,3 +58,24 @@ spec: route: backends: - local +--- +# The same backend through agentGatewayRef.routing: Model. This exercises +# agentgatewaymodels create/update, and the lists every Model-mode reconcile +# runs to sweep the other mode's objects and the models of removed backends. +# Deleting it exercises owner-reference garbage collection of the route and +# the model, as for the router above. +apiVersion: inference.llmkube.dev/v1alpha1 +kind: ModelRouter +metadata: + name: rbac-agw-model-router +spec: + dataPlane: AgentGateway + agentGatewayRef: + name: rbac-agw-gateway + routing: Model + sectionName: http + backends: + - name: local + tier: local + inferenceServiceRef: + name: rbac-agw-cpu