From fccc9abb70c2bec72cfeeed454a126e99f02bc9b Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sun, 20 Sep 2026 08:22:01 +0800 Subject: [PATCH 01/18] feat: serve TLSRoute in Passthrough mode The stream proxy can now forward a TLS stream to the upstream untouched while still picking that upstream from the SNI, which it prereads from the ClientHello rather than learning from a handshake it performed itself (apache/apisix#13912). That is exactly what Gateway API asks of a listener in Passthrough mode, and until now the only thing the stream subsystem could not do - routing by SNI implied terminating. A Passthrough listener therefore behaved as Terminate: the translator never looked at tls.mode, so the gateway decrypted a stream the backend was supposed to own, and the handshake failed against a certificate the gateway does not have. - TLSRoute now reads the tls.mode of the listeners it attached to and sets tls_passthrough on the stream routes bound to a Passthrough port. Its controller populates tctx.Listeners for that, as the TCPRoute and UDPRoute ones already did. - Every matched listener on a port has to agree on the mode. A physical stream listen is either terminating or prereading, never both, so listeners that disagree fall back to terminating rather than to a guess. Within one Gateway such a port is already ProtocolConflict and attaches no routes at all. Two adjacent defects in the same code, both of which passthrough would have made visible: - every hostname produced its own StreamRoute under one name, so they all collapsed onto a single id and only the last survived. One StreamRoute now carries them all - sni for a single hostname, snis beyond that, never both, which APISIX rejects. - the StreamRoutes carried no server_port, so several listener ports fell onto one route and shared an id (the TCPRoute/UDPRoute side of this was #2802). TLSRoute now goes through the same per-port fan-out, gated by the same listener_port_match_mode. A TLSRoute with no hostnames used to produce no StreamRoute at all - attached, but unserved. It now falls back to the listener hostnames and, failing those, to the catch-all "*". Conformance: the four TLSRoute tests pinned to Passthrough are no longer skipped. Their Gateway listener is fixed at port 443, so the conformance data plane points its 443 service port at the stream tls_passthrough listen instead of the HTTP ssl listen - one port cannot serve both, and the HTTPRoute tests that would want HTTP-over-TLS there are skipped for unrelated SAN reasons. E2E adds a Passthrough spec that verifies the served chain against the backend's own CA: the gateway holds no certificate for a Passthrough listener, so a chain that validates there can only have come from the backend. Requires ADC to carry the new fields (api7/adc#618). --- api/adc/types.go | 9 +- api/adc/zz_generated.deepcopy.go | 10 + docs/en/latest/concepts/gateway-api.md | 2 +- internal/adc/translator/l4route_test.go | 15 +- internal/adc/translator/tcproute.go | 34 ++-- internal/adc/translator/tlsroute.go | 94 ++++++++- internal/adc/translator/tlsroute_test.go | 223 +++++++++++++++++++++ internal/controller/tlsroute_controller.go | 9 + test/conformance/conformance_test.go | 12 -- test/conformance/suite_test.go | 7 + test/e2e/framework/manifests/apisix.yaml | 18 +- test/e2e/gatewayapi/tlsroute.go | 86 ++++++++ test/e2e/scaffold/apisix_deployer.go | 28 ++- test/e2e/scaffold/apisix_prewarm.go | 15 +- test/e2e/scaffold/deployer.go | 9 +- test/e2e/scaffold/scaffold.go | 71 ++++++- 16 files changed, 577 insertions(+), 65 deletions(-) create mode 100644 internal/adc/translator/tlsroute_test.go diff --git a/api/adc/types.go b/api/adc/types.go index f7adae1cd..df58231f1 100644 --- a/api/adc/types.go +++ b/api/adc/types.go @@ -170,7 +170,14 @@ type StreamRoute struct { RemoteAddr string `json:"remote_addr,omitempty"` ServerAddr string `json:"server_addr,omitempty"` ServerPort int32 `json:"server_port,omitempty"` - SNI string `json:"sni,omitempty"` + // SNI and SNIs are the singular and plural forms of the same match; APISIX + // rejects a stream route carrying both, so only one is ever set. + SNI string `json:"sni,omitempty"` + SNIs []string `json:"snis,omitempty"` + // TLSPassthrough forwards the TLS stream to the upstream untouched instead + // of terminating it on the gateway. APISIX only consults it on a stream + // listen configured with both tls and tls_passthrough. + TLSPassthrough *bool `json:"tls_passthrough,omitempty"` } // +k8s:deepcopy-gen=true diff --git a/api/adc/zz_generated.deepcopy.go b/api/adc/zz_generated.deepcopy.go index fc11631fc..6e413a2fb 100644 --- a/api/adc/zz_generated.deepcopy.go +++ b/api/adc/zz_generated.deepcopy.go @@ -567,6 +567,16 @@ func (in *StreamRoute) DeepCopyInto(out *StreamRoute) { *out = *in in.Metadata.DeepCopyInto(&out.Metadata) out.Plugins = in.Plugins.DeepCopy() + if in.SNIs != nil { + in, out := &in.SNIs, &out.SNIs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TLSPassthrough != nil { + in, out := &in.TLSPassthrough, &out.TLSPassthrough + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StreamRoute. diff --git a/docs/en/latest/concepts/gateway-api.md b/docs/en/latest/concepts/gateway-api.md index 27f003580..ca763ab3e 100644 --- a/docs/en/latest/concepts/gateway-api.md +++ b/docs/en/latest/concepts/gateway-api.md @@ -85,6 +85,6 @@ The fields below are specified in the Gateway API specification but are either p | `spec.listeners[].port` | Not supported* | The configuration is required but ignored. This is due to limitations in the data plane: it cannot dynamically open new ports. Since the Ingress Controller does not manage the data plane deployment, it cannot automatically update the configuration or restart the data plane to apply port changes. | | `spec.listeners[].tls.certificateRefs[].group` | Partially supported | Only `""` is supported; other group values cause validation failure. | | `spec.listeners[].tls.certificateRefs[].kind` | Partially supported | Only `Secret` is supported. | -| `spec.listeners[].tls.mode` | Partially supported | `Terminate` is implemented; `Passthrough` is effectively unsupported for Gateway listeners. | +| `spec.listeners[].tls.mode` | Partially supported | `Terminate` and `Passthrough` are both implemented. A `Passthrough` listener needs APISIX to be listening on that port with [`tls_passthrough`](https://apisix.apache.org/docs/apisix/stream-proxy/); the controller cannot open data plane ports itself. A single port cannot mix the two modes: listeners that disagree on `tls.mode` for one port are reported `Accepted=False` with `ProtocolConflict`. | | `spec.listeners[].tls.frontendValidation` | Partially supported | Enables downstream (client) mTLS. `caCertificateRefs` may reference a `ConfigMap` (Gateway API Core support) or a `Secret` (implementation-specific) holding the CA certificate under the `ca.crt` key; clients are then required to present a certificate signed by one of the referenced CAs. | | `spec.addresses` | Not supported | Controller does not read or act on `spec.addresses`. | diff --git a/internal/adc/translator/l4route_test.go b/internal/adc/translator/l4route_test.go index 62d0ca038..2bb9fc736 100644 --- a/internal/adc/translator/l4route_test.go +++ b/internal/adc/translator/l4route_test.go @@ -251,13 +251,20 @@ func TestTranslateTLSRouteWithL4RoutePolicy(t *testing.T) { require.NoError(t, err) require.Len(t, result.Services, 1) - // Verify stream routes are created per SNI hostname - if len(tt.hostnames) > 0 { - assert.Len(t, result.Services[0].StreamRoutes, len(tt.hostnames)) + // One stream route carries every hostname: the singular sni for a + // single one, the plural snis beyond that. APISIX rejects both at once. + require.Len(t, result.Services[0].StreamRoutes, 1) + switch len(tt.hostnames) { + case 1: + assert.Equal(t, tt.hostnames[0], result.Services[0].StreamRoutes[0].SNI) + assert.Empty(t, result.Services[0].StreamRoutes[0].SNIs) + default: + assert.Equal(t, tt.hostnames, result.Services[0].StreamRoutes[0].SNIs) + assert.Empty(t, result.Services[0].StreamRoutes[0].SNI) } // Plugins are attached at the stream_route level so the APISIX stream proxy - // applies them; with multiple SNIs each stream_route carries its own copy. + // applies them. require.NotEmpty(t, result.Services[0].StreamRoutes) plugins := result.Services[0].StreamRoutes[0].Plugins if tt.wantNoPlugins { diff --git a/internal/adc/translator/tcproute.go b/internal/adc/translator/tcproute.go index 7de4a67ec..e08e09ee7 100644 --- a/internal/adc/translator/tcproute.go +++ b/internal/adc/translator/tcproute.go @@ -54,6 +54,26 @@ func listenerPortSet(tctx *provider.TranslateContext) map[int32]struct{} { return portSet } +// l4StreamRoutePorts returns the listener ports to emit one StreamRoute for +// each, or the single sentinel 0 meaning one StreamRoute with no server_port +// match at all. See buildL4StreamRoutes for why the injection is opt-in. +func (t *Translator) l4StreamRoutePorts(tctx *provider.TranslateContext) []int32 { + var ports []int32 + if portSet := listenerPortSet(tctx); t.shouldInjectServerPortVars(tctx.HasExplicitListenerMatch, portSet) { + ports = make([]int32, 0, len(portSet)) + for port := range portSet { + ports = append(ports, port) + } + sort.Slice(ports, func(i, j int) bool { return ports[i] < ports[j] }) + } + if len(ports) == 0 { + // No server_port isolation: a single StreamRoute that matches all + // connections on the stream listener, as before. + return []int32{0} + } + return ports +} + // buildL4StreamRoutes builds the StreamRoutes for one L4 route rule. // // A StreamRoute without a server_port match matches every connection on any @@ -68,19 +88,7 @@ func listenerPortSet(tctx *provider.TranslateContext) map[int32]struct{} { // or more than one listener port). When it is not injected we keep the previous // single portless StreamRoute, preserving backward compatibility. func (t *Translator) buildL4StreamRoutes(tctx *provider.TranslateContext, namespace, name string, ruleIndex int, typ, routeKind string, labels map[string]string) []*adctypes.StreamRoute { - var ports []int32 - if portSet := listenerPortSet(tctx); t.shouldInjectServerPortVars(tctx.HasExplicitListenerMatch, portSet) { - ports = make([]int32, 0, len(portSet)) - for port := range portSet { - ports = append(ports, port) - } - sort.Slice(ports, func(i, j int) bool { return ports[i] < ports[j] }) - } - if len(ports) == 0 { - // No server_port isolation: a single StreamRoute that matches all - // connections on the stream listener, as before. - ports = []int32{0} - } + ports := t.l4StreamRoutePorts(tctx) streamRoutes := make([]*adctypes.StreamRoute, 0, len(ports)) for _, port := range ports { streamRoute := adctypes.NewDefaultStreamRoute() diff --git a/internal/adc/translator/tlsroute.go b/internal/adc/translator/tlsroute.go index 8d1fd0a6f..659a91664 100644 --- a/internal/adc/translator/tlsroute.go +++ b/internal/adc/translator/tlsroute.go @@ -20,6 +20,7 @@ package translator import ( "fmt" + "k8s.io/utils/ptr" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" adctypes "github.com/apache/apisix-ingress-controller/api/adc" @@ -34,10 +35,7 @@ func (t *Translator) TranslateTLSRoute(tctx *provider.TranslateContext, tlsRoute result := &TranslateResult{} rules := tlsRoute.Spec.Rules labels := label.GenLabel(tlsRoute) - hosts := make([]string, 0, len(tlsRoute.Spec.Hostnames)) - for _, hostname := range tlsRoute.Spec.Hostnames { - hosts = append(hosts, string(hostname)) - } + snis := tlsRouteSNIs(tctx, tlsRoute) for ruleIndex, rule := range rules { service := adctypes.NewDefaultService() service.Labels = labels @@ -143,16 +141,33 @@ func (t *Translator) TranslateTLSRoute(tctx *provider.TranslateContext, tlsRoute } } - for _, host := range hosts { + for _, port := range t.l4StreamRoutePorts(tctx) { streamRoute := adctypes.NewDefaultStreamRoute() - streamRouteName := adctypes.ComposeStreamRouteName(tlsRoute.Namespace, tlsRoute.Name, fmt.Sprintf("%d", ruleIndex), "TLS") + ruleKey := fmt.Sprintf("%d", ruleIndex) + if port != 0 { + // Include the port in the name key so multiple listeners produce + // distinct StreamRoute names/IDs instead of colliding. + ruleKey = fmt.Sprintf("%d-%d", ruleIndex, port) + streamRoute.ServerPort = port + } + streamRouteName := adctypes.ComposeStreamRouteName(tlsRoute.Namespace, tlsRoute.Name, ruleKey, "TLS") streamRoute.Name = streamRouteName streamRoute.ID = id.GenID(streamRouteName) - streamRoute.SNI = host + // A single SNI keeps using the singular form: it is what every + // APISIX version understands, and snis only earns its place once + // there is more than one to match. + if len(snis) == 1 { + streamRoute.SNI = snis[0] + } else { + streamRoute.SNIs = snis + } + if tlsPassthroughOnPort(tctx.Listeners, port) { + streamRoute.TLSPassthrough = ptr.To(true) + } streamRoute.Labels = labels // Attach L4RoutePolicy plugins at the stream_route level: the APISIX stream proxy - // applies plugins from the stream_route, not from the service. With multiple SNIs - // each stream_route carries its own copy of the plugins. + // applies plugins from the stream_route, not from the service. With multiple + // listener ports each stream_route carries its own copy of the plugins. streamRoute.Plugins = make(adctypes.Plugins) t.AttachL4RoutePolicyPlugins(tctx.L4RoutePolicies, tlsRoute.Namespace, tlsRoute.Name, "TLSRoute", streamRoute.Plugins, tctx.Secrets) service.StreamRoutes = append(service.StreamRoutes, streamRoute) @@ -162,3 +177,64 @@ func (t *Translator) TranslateTLSRoute(tctx *provider.TranslateContext, tlsRoute } return result, nil } + +// tlsRouteSNIs returns the SNIs the route's stream routes match on. +// +// A TLSRoute without hostnames matches everything its listeners accept, so it +// falls back to the matched listener hostnames and, when those carry none +// either, to the catch-all "*". Emitting nothing - which is what the per +// hostname loop used to do - left such a route attached but unserved. +func tlsRouteSNIs(tctx *provider.TranslateContext, tlsRoute *gatewayv1.TLSRoute) []string { + if len(tlsRoute.Spec.Hostnames) > 0 { + snis := make([]string, 0, len(tlsRoute.Spec.Hostnames)) + for _, hostname := range tlsRoute.Spec.Hostnames { + snis = append(snis, string(hostname)) + } + return snis + } + + snis := make([]string, 0, len(tctx.Listeners)) + seen := make(map[string]struct{}, len(tctx.Listeners)) + for _, listener := range tctx.Listeners { + if listener.Hostname == nil || *listener.Hostname == "" { + continue + } + hostname := string(*listener.Hostname) + if _, ok := seen[hostname]; ok { + continue + } + seen[hostname] = struct{}{} + snis = append(snis, hostname) + } + if len(snis) == 0 { + return []string{"*"} + } + return snis +} + +// tlsPassthroughOnPort reports whether the stream routes bound to port must +// forward the connection untouched instead of having the gateway terminate it. +// port 0 means the StreamRoute carries no server_port match, so every matched +// listener applies. +// +// Every matched TLS listener on the port has to agree. Within one Gateway a +// port carrying both modes is already reported ProtocolConflict and attaches +// no routes; across Gateways the combination is unrepresentable, since the +// physical stream listen has a single mode - so the terminating behaviour wins +// rather than a guess. +func tlsPassthroughOnPort(listeners []gatewayv1.Listener, port int32) bool { + matched := false + for _, listener := range listeners { + if listener.Protocol != gatewayv1.TLSProtocolType { + continue + } + if port != 0 && listener.Port != port { + continue + } + if listener.TLS == nil || listener.TLS.Mode == nil || *listener.TLS.Mode != gatewayv1.TLSModePassthrough { + return false + } + matched = true + } + return matched +} diff --git a/internal/adc/translator/tlsroute_test.go b/internal/adc/translator/tlsroute_test.go new file mode 100644 index 000000000..a3fd393dc --- /dev/null +++ b/internal/adc/translator/tlsroute_test.go @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translator + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/apache/apisix-ingress-controller/internal/controller/config" + "github.com/apache/apisix-ingress-controller/internal/provider" +) + +func tlsModeListener(name string, port int32, mode *gatewayv1.TLSModeType, hostname string) gatewayv1.Listener { + listener := gatewayv1.Listener{ + Name: gatewayv1.SectionName(name), + Protocol: gatewayv1.TLSProtocolType, + Port: port, + TLS: &gatewayv1.ListenerTLSConfig{Mode: mode}, + } + if hostname != "" { + listener.Hostname = ptr.To(gatewayv1.Hostname(hostname)) + } + return listener +} + +func translateTLSRoute(t *testing.T, listeners []gatewayv1.Listener, explicit bool, hostnames ...string) *TranslateResult { + t.Helper() + + // listener_port_match_mode defaults to off; auto is what the e2e and + // conformance runs use, and what makes the per-port fan-out observable. + translator := NewTranslator(logr.Discard(), config.ListenerPortMatchModeAuto) + tctx := provider.NewDefaultTranslateContext(context.Background()) + tctx.Listeners = listeners + tctx.HasExplicitListenerMatch = explicit + + specHostnames := make([]gatewayv1.Hostname, 0, len(hostnames)) + for _, hostname := range hostnames { + specHostnames = append(specHostnames, gatewayv1.Hostname(hostname)) + } + + result, err := translator.TranslateTLSRoute(tctx, &gatewayv1.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "my-tls", Namespace: "default"}, + Spec: gatewayv1.TLSRouteSpec{ + Hostnames: specHostnames, + Rules: []gatewayv1.TLSRouteRule{ + {BackendRefs: []gatewayv1.BackendRef{}}, + }, + }, + }) + require.NoError(t, err) + require.Len(t, result.Services, 1) + return result +} + +func TestTranslateTLSRouteTLSPassthrough(t *testing.T) { + terminate := gatewayv1.TLSModeTerminate + passthrough := gatewayv1.TLSModePassthrough + + for _, tc := range []struct { + name string + listeners []gatewayv1.Listener + explicit bool + want []*bool + }{ + { + name: "passthrough listener asks the data plane to pass the stream through", + listeners: []gatewayv1.Listener{tlsModeListener("tls", 9110, &passthrough, "")}, + explicit: true, + want: []*bool{ptr.To(true)}, + }, + { + name: "terminate listener leaves the flag off", + listeners: []gatewayv1.Listener{tlsModeListener("tls", 9110, &terminate, "")}, + explicit: true, + want: []*bool{nil}, + }, + { + // tls.mode is optional and defaults to Terminate. + name: "omitted mode is terminate", + listeners: []gatewayv1.Listener{tlsModeListener("tls", 9110, nil, "")}, + explicit: true, + want: []*bool{nil}, + }, + { + name: "each port carries the mode of its own listener", + listeners: []gatewayv1.Listener{ + tlsModeListener("terminate", 9110, &terminate, ""), + tlsModeListener("passthrough", 9120, &passthrough, ""), + }, + want: []*bool{nil, ptr.To(true)}, + }, + { + // A physical stream listen has one mode, so a port whose matched + // listeners disagree cannot be served both ways; terminating is + // what the gateway did before passthrough existed. + name: "listeners disagreeing on one port fall back to terminate", + listeners: []gatewayv1.Listener{ + tlsModeListener("a", 9110, &passthrough, ""), + tlsModeListener("b", 9110, &terminate, ""), + }, + explicit: true, + want: []*bool{nil}, + }, + { + // No listener port to pin to: the single portless StreamRoute takes + // the mode every matched listener agrees on. + name: "portless stream route still follows the listener mode", + listeners: []gatewayv1.Listener{tlsModeListener("tls", 9110, &passthrough, "")}, + want: []*bool{ptr.To(true)}, + }, + { + name: "no matched listener leaves the flag off", + listeners: nil, + want: []*bool{nil}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + result := translateTLSRoute(t, tc.listeners, tc.explicit, "example.com") + + streamRoutes := result.Services[0].StreamRoutes + require.Len(t, streamRoutes, len(tc.want)) + for i, want := range tc.want { + assert.Equal(t, want, streamRoutes[i].TLSPassthrough, "stream route %d", i) + } + }) + } +} + +func TestTranslateTLSRouteSNIs(t *testing.T) { + passthrough := gatewayv1.TLSModePassthrough + + t.Run("a single hostname uses the singular sni", func(t *testing.T) { + result := translateTLSRoute(t, nil, false, "example.com") + + streamRoutes := result.Services[0].StreamRoutes + require.Len(t, streamRoutes, 1) + assert.Equal(t, "example.com", streamRoutes[0].SNI) + assert.Empty(t, streamRoutes[0].SNIs) + }) + + t.Run("several hostnames share one stream route", func(t *testing.T) { + result := translateTLSRoute(t, nil, false, "a.example.com", "b.example.com") + + // Regression: every hostname used to produce its own StreamRoute under + // one name, so they collided on a single id and only the last survived. + streamRoutes := result.Services[0].StreamRoutes + require.Len(t, streamRoutes, 1) + assert.Equal(t, []string{"a.example.com", "b.example.com"}, streamRoutes[0].SNIs) + assert.Empty(t, streamRoutes[0].SNI) + }) + + t.Run("no hostname falls back to the listener hostnames", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("tls", 9110, &passthrough, "*.example.com"), + }, true) + + streamRoutes := result.Services[0].StreamRoutes + require.Len(t, streamRoutes, 1) + assert.Equal(t, "*.example.com", streamRoutes[0].SNI) + }) + + t.Run("no hostname anywhere falls back to the catch-all", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("tls", 9110, &passthrough, ""), + }, true) + + streamRoutes := result.Services[0].StreamRoutes + require.Len(t, streamRoutes, 1) + assert.Equal(t, "*", streamRoutes[0].SNI) + }) +} + +func TestTranslateTLSRouteServerPort(t *testing.T) { + passthrough := gatewayv1.TLSModePassthrough + + t.Run("each listener port gets its own stream route", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("a", 9110, &passthrough, ""), + tlsModeListener("b", 9120, &passthrough, ""), + }, false, "example.com") + + // Regression: without a server_port match both listeners' traffic fell + // onto one StreamRoute, and the two shared a name, hence an id. + streamRoutes := result.Services[0].StreamRoutes + require.Len(t, streamRoutes, 2) + assert.Equal(t, int32(9110), streamRoutes[0].ServerPort) + assert.Equal(t, int32(9120), streamRoutes[1].ServerPort) + assert.NotEqual(t, streamRoutes[0].ID, streamRoutes[1].ID) + assert.NotEqual(t, streamRoutes[0].Name, streamRoutes[1].Name) + }) + + t.Run("a single listener without explicit targeting stays portless", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("a", 9110, &passthrough, ""), + }, false, "example.com") + + streamRoutes := result.Services[0].StreamRoutes + require.Len(t, streamRoutes, 1) + assert.Zero(t, streamRoutes[0].ServerPort) + }) +} diff --git a/internal/controller/tlsroute_controller.go b/internal/controller/tlsroute_controller.go index 5bbd5a591..2d93ffc3e 100644 --- a/internal/controller/tlsroute_controller.go +++ b/internal/controller/tlsroute_controller.go @@ -317,6 +317,15 @@ func (r *TLSRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c acceptStatus.status = false acceptStatus.msg = err.Error() } + // Populate the matched listeners so the translator can derive the + // StreamRoute server_port and its tls.mode from the listener the route + // attaches to. + if len(gateway.Listeners) > 0 { + tctx.Listeners = appendListeners(tctx.Listeners, gateway.Listeners...) + } else if gateway.Listener != nil { + tctx.Listeners = appendListeners(tctx.Listeners, *gateway.Listener) + } + tctx.HasExplicitListenerMatch = tctx.HasExplicitListenerMatch || gateway.ExplicitListenerMatch } var backendRefErr error diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index cf606fc60..86b2df310 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -31,17 +31,6 @@ var skippedTestsForSSL = []string{ tests.HTTPRouteRedirectPortAndScheme.ShortName, } -// APISIX terminates TLS on its stream proxy and matches stream routes by SNI, -// which implements TLSRoute in Terminate mode (declared via the -// TLSRouteModeTerminate feature) but never forwards the encrypted stream -// untouched. Every test below pins its listener to mode: Passthrough. -var skippedTestsForTLSPassthrough = []string{ - tests.TLSRouteSimpleSameNamespace.ShortName, - tests.TLSRouteHostnameIntersection.ShortName, - tests.TLSRouteInvalidBackendRefNonexistent.ShortName, - tests.TLSRouteInvalidBackendRefUnknownKind.ShortName, -} - // Known gaps tracked for follow-up. These are genuine feature gaps rather than // architectural limits, so they are expected to shrink over time. var skippedTestsForKnownGaps = []string{ @@ -77,7 +66,6 @@ func TestGatewayAPIConformance(t *testing.T) { opts.CleanupBaseResources = true opts.GatewayClassName = gatewayClassName opts.SkipTests = append(opts.SkipTests, skippedTestsForSSL...) - opts.SkipTests = append(opts.SkipTests, skippedTestsForTLSPassthrough...) opts.SkipTests = append(opts.SkipTests, skippedTestsForKnownGaps...) opts.Implementation = conformancev1.Implementation{ Organization: "APISIX", diff --git a/test/conformance/suite_test.go b/test/conformance/suite_test.go index e7ab91b11..64783c31f 100644 --- a/test/conformance/suite_test.go +++ b/test/conformance/suite_test.go @@ -154,6 +154,13 @@ func TestMain(m *testing.M) { ServiceType: "LoadBalancer", ServiceHTTPPort: 80, ServiceHTTPSPort: 443, + // The TLSRoute tests pin their Gateway listener to port 443 in + // Passthrough mode and dial the Gateway address there, so 443 has to + // reach APISIX's stream tls_passthrough listen rather than its HTTP ssl + // listen. One port cannot serve both, and the HTTPRoute tests that would + // want HTTP-over-TLS on 443 are skipped for unrelated reasons + // (skippedTestsForSSL). + ServiceHTTPSTargetPort: 9120, }) svc := s.GetDataplaneService() diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index 302891948..d36074f59 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -49,6 +49,11 @@ data: - 9100 - addr: 9110 tls: true + # TLS passthrough: the SNI is read out of the prereaded ClientHello + # and the stream reaches the backend untouched. A listen is either + # terminating or prereading, never both, so this needs its own port. + - addr: 9120 + tls_passthrough: true udp: # UDP proxy port list - 9200 discovery: @@ -119,6 +124,9 @@ spec: - name: tls containerPort: 9110 protocol: TCP + - name: tls-passthrough + containerPort: 9120 + protocol: TCP volumeMounts: - name: config-writable mountPath: /usr/local/apisix/conf @@ -151,7 +159,11 @@ spec: - port: {{ .ServiceHTTPSPort }} name: https protocol: TCP - targetPort: 9443 + # Normally APISIX's HTTP ssl listen. The conformance run redirects it to + # the stream tls_passthrough listen instead: a Gateway API TLSRoute + # Passthrough listener is fixed at 443, and one port cannot serve both + # HTTP-over-TLS and an untouched stream. + targetPort: {{ .ServiceHTTPSTargetPort }} - port: 9180 name: admin protocol: TCP @@ -168,6 +180,10 @@ spec: port: 9110 protocol: TCP targetPort: 9110 + - name: tls-passthrough + port: 9120 + protocol: TCP + targetPort: 9120 selector: app.kubernetes.io/name: apisix type: {{ .ServiceType | default "NodePort" }} diff --git a/test/e2e/gatewayapi/tlsroute.go b/test/e2e/gatewayapi/tlsroute.go index 08d4e4a36..a6361ac82 100644 --- a/test/e2e/gatewayapi/tlsroute.go +++ b/test/e2e/gatewayapi/tlsroute.go @@ -20,10 +20,13 @@ package gatewayapi import ( "fmt" "net/http" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/utils/ptr" + "github.com/apache/apisix-ingress-controller/test/e2e/framework" "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" ) @@ -114,4 +117,87 @@ spec: }).Should(ContainSubstring("EOF"), "should get EOF after deleting TLSRoute") }) }) + + Context("TLSRoute Passthrough", func() { + // The certificate the e2e nginx serves, and the CA that signed it. + const backendSNI = "server.example.com" + + var passthroughGateway = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: tls-passthrough-gateway +spec: + gatewayClassName: %s + listeners: + - name: passthrough + protocol: TLS + # Must equal APISIX's physical stream_proxy listen configured with + # tls_passthrough (see the e2e apisix manifest): the controller runs with + # listener_port_match_mode=auto, so the port reaches the stream route as a + # server_port match and has to be one the data plane actually accepts on. + port: 9120 + hostname: server.example.com + tls: + mode: Passthrough + infrastructure: + parametersRef: + group: apisix.apache.org + kind: GatewayProxy + name: apisix-proxy-config +` + + var passthroughRoute = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: TLSRoute +metadata: + name: tls-passthrough-route +spec: + parentRefs: + - name: tls-passthrough-gateway + sectionName: passthrough + hostnames: ["server.example.com"] + rules: + - backendRefs: + - name: nginx + port: 443 +` + + BeforeEach(func() { + By("create GatewayProxy") + Expect(s.CreateResourceFromString(s.GetGatewayProxySpec())).NotTo(HaveOccurred(), "creating GatewayProxy") + + By("create GatewayClass") + Expect(s.CreateResourceFromString(s.GetGatewayClassYaml())).NotTo(HaveOccurred(), "creating GatewayClass") + + By("create Gateway with a Passthrough listener") + Expect(s.CreateResourceFromString(fmt.Sprintf(passthroughGateway, s.Namespace()))).NotTo(HaveOccurred(), "creating Gateway") + + By("deploy the TLS backend") + s.DeployNginx(framework.NginxOptions{ + Namespace: s.Namespace(), + Replicas: ptr.To(int32(1)), + }) + }) + + It("forwards the stream to the backend that owns the certificate", func() { + s.ResourceApplied("TLSRoute", "tls-passthrough-route", passthroughRoute, 1) + + // The client verifies the served chain against the backend's own CA. + // The gateway holds no certificate for this listener - Passthrough + // takes no certificateRefs - so a chain that validates here can only + // have come from nginx, which is what passthrough means. + s.RequestAssert(&scaffold.RequestAssert{ + Client: s.NewAPISIXClientWithTLSPassthrough(backendSNI, []byte(framework.TestCACert)), + Method: http.MethodGet, + Path: "/", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedBodyContains("Hello, World!"), + }, + Timeout: time.Minute * 3, + Interval: time.Second * 2, + }) + }) + }) }) diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index aa6d8fad3..9170e587f 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -43,6 +43,11 @@ type APISIXDeployOptions struct { ServiceType string ServiceHTTPPort int ServiceHTTPSPort int + // ServiceHTTPSTargetPort is the container port the HTTPS service port + // forwards to: APISIX's HTTP ssl listen (9443) by default, or its stream + // tls_passthrough listen (9120). See the manifest for why the two cannot + // share a port. + ServiceHTTPSTargetPort int ConfigProvider string Replicas *int @@ -175,11 +180,12 @@ func (s *APISIXDeployer) AfterEach() { func (s *APISIXDeployer) DeployDataplane(deployOpts DeployDataplaneOptions) { opts := APISIXDeployOptions{ - Namespace: s.namespace, - AdminKey: s.runtimeOpts.APISIXAdminAPIKey, - ServiceHTTPPort: 9080, - ServiceHTTPSPort: 9443, - Replicas: ptr.To(1), + Namespace: s.namespace, + AdminKey: s.runtimeOpts.APISIXAdminAPIKey, + ServiceHTTPPort: 9080, + ServiceHTTPSPort: 9443, + ServiceHTTPSTargetPort: 9443, + Replicas: ptr.To(1), } if deployOpts.Namespace != "" { @@ -194,6 +200,9 @@ func (s *APISIXDeployer) DeployDataplane(deployOpts DeployDataplaneOptions) { if deployOpts.ServiceHTTPSPort != 0 { opts.ServiceHTTPSPort = deployOpts.ServiceHTTPSPort } + if deployOpts.ServiceHTTPSTargetPort != 0 { + opts.ServiceHTTPSTargetPort = deployOpts.ServiceHTTPSTargetPort + } if deployOpts.AdminKey != "" { opts.AdminKey = deployOpts.AdminKey } @@ -410,10 +419,11 @@ func (s *APISIXDeployer) CreateAdditionalGatewayWithOptions(namePrefix string, o // Deploy dataplane for this additional gateway o := APISIXDeployOptions{ - Namespace: additionalNS, - AdminKey: adminKey, - ServiceHTTPPort: 9080, - ServiceHTTPSPort: 9443, + Namespace: additionalNS, + AdminKey: adminKey, + ServiceHTTPPort: 9080, + ServiceHTTPSPort: 9443, + ServiceHTTPSTargetPort: 9443, } if opts.Namespace != "" { o.Namespace = opts.Namespace diff --git a/test/e2e/scaffold/apisix_prewarm.go b/test/e2e/scaffold/apisix_prewarm.go index 030917365..7cce59dea 100644 --- a/test/e2e/scaffold/apisix_prewarm.go +++ b/test/e2e/scaffold/apisix_prewarm.go @@ -165,13 +165,14 @@ func provisionDataplane(t *bgTestingT, env *pooledEnv, _ Options) (*corev1.Servi } deployOpts := APISIXDeployOptions{ - Namespace: env.namespace, - AdminKey: env.adminKey, - ServiceName: serviceName, - ServiceHTTPPort: 9080, - ServiceHTTPSPort: 9443, - ConfigProvider: configProvider, - Replicas: ptr.To(1), + Namespace: env.namespace, + AdminKey: env.adminKey, + ServiceName: serviceName, + ServiceHTTPPort: 9080, + ServiceHTTPSPort: 9443, + ServiceHTTPSTargetPort: 9443, + ConfigProvider: configProvider, + Replicas: ptr.To(1), } buf := bytes.NewBuffer(nil) if err := framework.APISIXStandaloneTpl.Execute(buf, &deployOpts); err != nil { diff --git a/test/e2e/scaffold/deployer.go b/test/e2e/scaffold/deployer.go index d9ddb2f60..ad4127685 100644 --- a/test/e2e/scaffold/deployer.go +++ b/test/e2e/scaffold/deployer.go @@ -45,7 +45,10 @@ type DeployDataplaneOptions struct { SkipCreateTunnels bool ServiceHTTPPort int ServiceHTTPSPort int - Replicas *int - AdminKey string - ProviderType string + // ServiceHTTPSTargetPort redirects the HTTPS service port at the data + // plane's stream tls_passthrough listen instead of its HTTP ssl listen. + ServiceHTTPSTargetPort int + Replicas *int + AdminKey string + ProviderType string } diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 41ac85c74..6961481c8 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -20,6 +20,7 @@ package scaffold import ( "context" "crypto/tls" + "crypto/x509" "fmt" "net/http" "net/url" @@ -91,6 +92,9 @@ type Tunnels struct { TCP Tunnel HTTP2 Tunnel TLS Tunnel + // TLSPassthrough reaches the stream listen that prereads the SNI and + // forwards the connection without terminating it. + TLSPassthrough Tunnel } func (t *Tunnels) Close() { @@ -114,6 +118,10 @@ func (t *Tunnels) Close() { t.safeClose(t.TLS.Close) t.TLS = nil } + if t.TLSPassthrough != nil { + t.safeClose(t.TLSPassthrough.Close) + t.TLSPassthrough = nil + } } func (t *Tunnels) safeClose(close func()) { @@ -301,6 +309,45 @@ func (s *Scaffold) NewAPISIXClientWithTCPProxy() *httpexpect.Expect { }) } +// tlsPassthroughPortName is the data plane Service port that forwards to the +// APISIX stream listen configured with tls_passthrough. +const tlsPassthroughPortName = "tls-passthrough" + +// NewAPISIXClientWithTLSPassthrough dials the data plane's TLS passthrough +// stream listen with sni and verifies the served chain against caCert. +// +// Verifying is the point: the gateway holds no certificate for a passthrough +// listener, so a chain that validates against the backend's own CA can only +// have come from the backend itself. +func (s *Scaffold) NewAPISIXClientWithTLSPassthrough(sni string, caCert []byte) *httpexpect.Expect { + Expect(s.apisixTunnels.TLSPassthrough).NotTo(BeNil(), "tls passthrough tunnel") + + pool := x509.NewCertPool() + Expect(pool.AppendCertsFromPEM(caCert)).To(BeTrue(), "parsing CA certificate") + + u := url.URL{ + Scheme: apiv2.SchemeHTTPS, + Host: s.apisixTunnels.TLSPassthrough.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: pool, + ServerName: sni, + }, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(s.GinkgoT), + ), + }) +} + func (s *Scaffold) NewAPISIXClientWithTLSProxy(host string) *httpexpect.Expect { u := url.URL{ Scheme: apiv2.SchemeHTTPS, @@ -412,11 +459,12 @@ func (s *Scaffold) createDataplaneTunnels( serviceName string, ) (*Tunnels, error) { var ( - httpPort int - httpsPort int - tcpPort int - http2Port int - tlsPort int + httpPort int + httpsPort int + tcpPort int + http2Port int + tlsPort int + tlsPassthroughPort int ) for _, port := range svc.Spec.Ports { @@ -431,6 +479,8 @@ func (s *Scaffold) createDataplaneTunnels( http2Port = int(port.Port) case apiv2.SchemeTLS: tlsPort = int(port.Port) + case tlsPassthroughPortName: + tlsPassthroughPort = int(port.Port) } } @@ -474,6 +524,17 @@ func (s *Scaffold) createDataplaneTunnels( tunnels.HTTP2 = http2Tunnel } + // Absent on a gateway deployed from an older manifest revision; the specs + // that need it assert on the tunnel being there. + if tlsPassthroughPort != 0 { + tlsPassthroughTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, + 0, tlsPassthroughPort) + if err := tlsPassthroughTunnel.ForwardPortE(s.t); err != nil { + return nil, err + } + tunnels.TLSPassthrough = tlsPassthroughTunnel + } + return tunnels, nil } From c586e8622c9387eef7120065128cac29c99632a6 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sun, 20 Sep 2026 08:22:01 +0800 Subject: [PATCH 02/18] fix: intersect TLSRoute hostnames with the listener hostname A TLSRoute's hostnames become the SNIs its stream routes match on, and they were used verbatim. Gateway API defines the effective hostnames as the intersection with the listener hostname, so a route attached to a narrower listener served names that listener never accepted. On one shared stream listen that is not merely over-serving. Every Gateway resolves to the same data plane, so an over-broad SNI takes that name from the route whose listener does accept it: a route with "*.example.com" attached to an "abc.example.com" listener answered every *.example.com connection, including the ones a sibling Gateway's route was there to serve. HTTPRoute has narrowed its hostnames this way since it was written - filterHostnames plus getMinimumHostnameIntersection. Both now share intersectRouteHostnames, and the TLSRoute reconciler applies it exactly as the HTTPRoute one does, translating the narrowed copy and reporting NoMatchingListenerHostname when nothing intersects. The Gateway API conformance test TLSRouteHostnameIntersection is what surfaced this; with the fix its intersections pass. --- internal/controller/listener_utils_test.go | 92 ++++++++++++++++++++++ internal/controller/tlsroute_controller.go | 16 +++- internal/controller/utils.go | 60 +++++++++----- 3 files changed, 149 insertions(+), 19 deletions(-) diff --git a/internal/controller/listener_utils_test.go b/internal/controller/listener_utils_test.go index 9fa4cc9ab..3862b11ef 100644 --- a/internal/controller/listener_utils_test.go +++ b/internal/controller/listener_utils_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -115,3 +116,94 @@ func TestGetMinimumHostnameIntersectionUsesMatchedListeners(t *testing.T) { require.Equal(t, gatewayv1.Hostname("b.example.com"), getMinimumHostnameIntersection(whole, "b.example.com")) } + +// The Gateway API conformance case TLSRouteHostnameIntersection turns on this: +// four Gateways whose TLS listeners carry different hostnames all resolve to one +// physical stream listen, so a TLSRoute keeping its own hostname verbatim serves +// SNIs its listener never accepted and steals them from the route whose listener +// did. +func TestFilterTLSRouteHostnames(t *testing.T) { + exact := gatewayv1.Hostname("abc.example.com") + moreSpecificWildcard := gatewayv1.Hostname("*.example.com") + lessSpecificWildcard := gatewayv1.Hostname("*.com") + + tlsListener := func(name string, hostname *gatewayv1.Hostname) gatewayv1.Listener { + return gatewayv1.Listener{ + Name: gatewayv1.SectionName(name), + Protocol: gatewayv1.TLSProtocolType, + Port: 443, + Hostname: hostname, + } + } + route := func(hostnames ...gatewayv1.Hostname) *gatewayv1.TLSRoute { + return &gatewayv1.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: gatewayv1.TLSRouteSpec{Hostnames: hostnames}, + } + } + + for _, tc := range []struct { + name string + listener *gatewayv1.Hostname + hostnames []gatewayv1.Hostname + want []gatewayv1.Hostname + wantErr bool + }{ + { + name: "a wildcard route narrows to the exact listener hostname", + listener: &exact, + hostnames: []gatewayv1.Hostname{moreSpecificWildcard}, + want: []gatewayv1.Hostname{exact}, + }, + { + name: "a broader wildcard route narrows to the listener wildcard", + listener: &moreSpecificWildcard, + hostnames: []gatewayv1.Hostname{lessSpecificWildcard}, + want: []gatewayv1.Hostname{moreSpecificWildcard}, + }, + { + name: "an exact route under a listener wildcard keeps its own hostname", + listener: &moreSpecificWildcard, + hostnames: []gatewayv1.Hostname{exact}, + want: []gatewayv1.Hostname{exact}, + }, + { + name: "a listener without a hostname leaves the route alone", + listener: nil, + hostnames: []gatewayv1.Hostname{lessSpecificWildcard}, + want: []gatewayv1.Hostname{lessSpecificWildcard}, + }, + { + name: "a route without hostnames takes the listener hostname", + listener: &moreSpecificWildcard, + hostnames: nil, + want: []gatewayv1.Hostname{moreSpecificWildcard}, + }, + { + name: "a route without hostnames under a hostname-less listener matches anything", + listener: nil, + hostnames: nil, + want: nil, + }, + { + name: "no intersection is rejected", + listener: &exact, + hostnames: []gatewayv1.Hostname{gatewayv1.Hostname("other.example.net")}, + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + gateways := []RouteParentRefContext{ + {Listeners: []gatewayv1.Listener{tlsListener("tls", tc.listener)}}, + } + + filtered, err := filterTLSRouteHostnames(gateways, route(tc.hostnames...).DeepCopy()) + if tc.wantErr { + require.ErrorIs(t, err, ErrNoMatchingListenerHostname) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, filtered.Spec.Hostnames) + }) + } +} diff --git a/internal/controller/tlsroute_controller.go b/internal/controller/tlsroute_controller.go index 2d93ffc3e..955f7e542 100644 --- a/internal/controller/tlsroute_controller.go +++ b/internal/controller/tlsroute_controller.go @@ -349,6 +349,17 @@ func (r *TLSRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if r.supportsL4RoutePolicy { ProcessL4RoutePolicy(r.Client, r.Log, tctx, tr.Namespace, tr.Name, types.KindTLSRoute) } + + // The hostnames become the SNIs the stream routes match on, so they have to be + // narrowed to what the listeners accept first. Every Gateway shares one physical + // stream listen, so a route keeping its own broader hostname does not merely + // over-serve: it takes that name from the route whose listener does accept it. + filteredTLSRoute, hostnameErr := filterTLSRouteHostnames(gateways, tr.DeepCopy()) + if hostnameErr != nil { + acceptStatus.status = false + acceptStatus.msg = hostnameErr.Error() + } + tr.Status.Parents = make([]gatewayv1.RouteParentStatus, 0, len(gateways)) for _, gateway := range gateways { parentStatus := gatewayv1.RouteParentStatus{} @@ -377,8 +388,11 @@ func (r *TLSRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c }), }) UpdateStatus(r.Updater, r.Log, tctx) - if isRouteAccepted(gateways) { + if isRouteAccepted(gateways) && hostnameErr == nil { routeToUpdate := tr + if filteredTLSRoute != nil { + routeToUpdate = filteredTLSRoute + } if err := r.Provider.Update(ctx, tctx, routeToUpdate); err != nil { return ctrl.Result{}, err } diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 3b7e31fdc..b09058190 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -1409,30 +1409,54 @@ func FullTypeName(a any) string { // filterHostnames accepts a list of gateways and an HTTPRoute, and returns a copy of the HTTPRoute with only the hostnames that match the listener hostnames of the gateways. // If the HTTPRoute hostnames do not intersect with the listener hostnames of the gateways, it returns an ErrNoMatchingListenerHostname error. func filterHostnames(gateways []RouteParentRefContext, httpRoute *gatewayv1.HTTPRoute) (*gatewayv1.HTTPRoute, error) { - filteredHostnames := make([]gatewayv1.Hostname, 0) + hostnames, err := intersectRouteHostnames(gateways, httpRoute.Spec.Hostnames) + if err != nil { + return httpRoute, err + } + httpRoute.Spec.Hostnames = hostnames + return httpRoute, nil +} - // If the HTTPRoute does not specify hostnames, we use the union of the listener hostnames of all supported gateways - // If any supported listener does not specify a hostname, the HTTPRoute hostnames remain empty to match any hostname - if len(httpRoute.Spec.Hostnames) == 0 { +// filterTLSRouteHostnames is filterHostnames for a TLSRoute. Its hostnames become +// the SNIs the stream routes match on, so a route left carrying its own broader +// hostname would serve names the listener it attached to never accepted - and, on +// a single shared stream listen, would take them from the route that should have. +func filterTLSRouteHostnames(gateways []RouteParentRefContext, tlsRoute *gatewayv1.TLSRoute) (*gatewayv1.TLSRoute, error) { + hostnames, err := intersectRouteHostnames(gateways, tlsRoute.Spec.Hostnames) + if err != nil { + return tlsRoute, err + } + tlsRoute.Spec.Hostnames = hostnames + return tlsRoute, nil +} + +// intersectRouteHostnames narrows a route's hostnames to what the listeners it +// attached to actually accept. +// +// A route without hostnames takes the union of the listener hostnames, and stays +// empty - matching any hostname - as soon as one supported listener carries no +// hostname of its own. Otherwise every hostname is replaced by its smallest +// intersection with a listener hostname, and a route that intersects with none +// is ErrNoMatchingListenerHostname. +func intersectRouteHostnames(gateways []RouteParentRefContext, routeHostnames []gatewayv1.Hostname) ([]gatewayv1.Hostname, error) { + if len(routeHostnames) == 0 { hostnames, matchAnyHost := getUnionOfGatewayHostnames(gateways) if matchAnyHost { - return httpRoute, nil - } - filteredHostnames = hostnames - } else { - // If the HTTPRoute specifies hostnames, we need to find the intersection with the gateway listener hostnames - for _, hostname := range httpRoute.Spec.Hostnames { - if hostnameMatching := getMinimumHostnameIntersection(gateways, hostname); hostnameMatching != "" { - filteredHostnames = append(filteredHostnames, hostnameMatching) - } - } - if len(filteredHostnames) == 0 { - return httpRoute, ErrNoMatchingListenerHostname + return routeHostnames, nil } + return hostnames, nil } - httpRoute.Spec.Hostnames = filteredHostnames - return httpRoute, nil + filteredHostnames := make([]gatewayv1.Hostname, 0, len(routeHostnames)) + for _, hostname := range routeHostnames { + if hostnameMatching := getMinimumHostnameIntersection(gateways, hostname); hostnameMatching != "" { + filteredHostnames = append(filteredHostnames, hostnameMatching) + } + } + if len(filteredHostnames) == 0 { + return nil, ErrNoMatchingListenerHostname + } + return filteredHostnames, nil } // getUnionOfGatewayHostnames returns the union of the hostnames specified in all supported gateways From 2a425aff87b54a28d42ba4feef2d5b2ad4bcae94 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sun, 20 Sep 2026 08:22:01 +0800 Subject: [PATCH 03/18] test: skip TLSRouteHostnameIntersection as a known architectural gap One assertion in it cannot hold here, and not for want of translating correctly. The test stands four Gateways up on port 443 with different listener hostnames; every Gateway resolves to the one data plane address and the one physical stream listen, so their SNI namespaces are shared. The Gateway whose listener carries no hostname keeps its route's "*.com" verbatim - correctly, and its own subtest depends on it - which then also answers "non.matching.com" on the address of the Gateway that should have rejected that connection. Which Gateway a connection was addressed to is not on the wire, so nothing is left to discriminate on. This is the same limitation HTTPRouteMultipleGateways is already skipped for, and it sits beside it. Every other assertion in the test passes, including the hostname intersections themselves. --- test/conformance/conformance_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index 86b2df310..223a0a73c 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -58,6 +58,18 @@ var skippedTestsForKnownGaps = []string{ // A single HTTPRoute attached to several Gateways is not served from each // parent independently. tests.HTTPRouteMultipleGateways.ShortName, + + // The same limitation for TLSRoute, and not something the translator can + // fix. The test stands four Gateways up on port 443 with different listener + // hostnames; every Gateway resolves to the one data plane address and the one + // physical stream listen, so their SNI namespaces are shared. The Gateway + // whose listener carries no hostname keeps its route's "*.com" verbatim - + // correctly, and its own subtest depends on it - which then also answers + // "non.matching.com" on the address of the Gateway that should have rejected + // it. Which Gateway a connection was addressed to is not on the wire, so + // there is nothing left to discriminate on. Every other assertion in this + // test passes, including the hostname intersections themselves. + tests.TLSRouteHostnameIntersection.ShortName, } func TestGatewayAPIConformance(t *testing.T) { From 9b5b991b3b6824135be64f2ee3d83fac69839308 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sun, 20 Sep 2026 08:28:07 +0800 Subject: [PATCH 04/18] test: declare the TLSRoute gap on the API7 gateway The APISIX suite stops skipping the TLSRoute Passthrough tests in this series, because APISIX can serve them. The API7 gateway cannot: its stream_route schema carries `sni` only, under additionalProperties = false, and has no tls_passthrough - it predates apache/apisix#13912. A stream route carrying either `snis` or `tls_passthrough` is rejected outright. Those four tests were never skipped here and have been failing unnoticed behind continue-on-error. Skipping them with the reason recorded says what is actually missing, and they come back as soon as the gateway carries the fields. --- test/conformance/api7ee/conformance_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/conformance/api7ee/conformance_test.go b/test/conformance/api7ee/conformance_test.go index a9572a9f2..f9aade686 100644 --- a/test/conformance/api7ee/conformance_test.go +++ b/test/conformance/api7ee/conformance_test.go @@ -31,6 +31,22 @@ var skippedTestsForSSL = []string{ tests.HTTPRouteRedirectPortAndScheme.ShortName, } +// The API7 gateway's stream_route schema carries `sni` only, under +// additionalProperties = false, and has no tls_passthrough - it predates +// apache/apisix#13912. A stream route carrying either `snis` or +// `tls_passthrough` is rejected outright, so a TLSRoute needs its own gateway +// support before these can run here. The APISIX provider runs them already; see +// test/conformance/conformance_test.go. +var skippedTestsForGatewaySchema = []string{ + // Pinned to mode: Passthrough, which the gateway cannot serve. + tests.TLSRouteSimpleSameNamespace.ShortName, + tests.TLSRouteInvalidBackendRefNonexistent.ShortName, + tests.TLSRouteInvalidBackendRefUnknownKind.ShortName, + // Passthrough as well, and its routes carry several hostnames, which the + // translator emits as `snis`. + tests.TLSRouteHostnameIntersection.ShortName, +} + // TODO: HTTPRoute hostname intersection and listener hostname matching func TestGatewayAPIConformance(t *testing.T) { @@ -39,6 +55,7 @@ func TestGatewayAPIConformance(t *testing.T) { opts.CleanupBaseResources = true opts.GatewayClassName = gatewayClassName opts.SkipTests = append(opts.SkipTests, skippedTestsForSSL...) + opts.SkipTests = append(opts.SkipTests, skippedTestsForGatewaySchema...) opts.Implementation = conformancev1.Implementation{ Organization: "APISIX", Project: "apisix-ingress-controller", From f224f783d08c93333c590134c18cc0fe9e60741d Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sun, 20 Sep 2026 09:08:05 +0800 Subject: [PATCH 05/18] ci: run the APISIX conformance job against adc:dev The four TLSRoute tests this series stops skipping fail on this job, and not in traffic - the routes never reach the data plane: Accepted condition set to Status False with Reason SyncFailed HTTP 400 {"code":"unrecognized_keys","keys":["tls_passthrough"], "path":["services",0,"stream_routes",0]} ADC learned `snis` and `tls_passthrough` in api7/adc#618, which no release carries yet. `kind-load-adc-image` pulls `adc:$(ADC_VERSION)` and retags it as `:dev`, so the Makefile default 0.29.0 is what ran. Both e2e workflows already set `ADC_VERSION: dev` at the workflow level; this job had it commented out, and in the "Build images" step env, where it could never have reached `kind-load-adc-image` anyway. Declared the same way as the siblings instead. --- .github/workflows/apisix-conformance-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/apisix-conformance-test.yml b/.github/workflows/apisix-conformance-test.yml index dbff2fba8..cd9bd03d9 100644 --- a/.github/workflows/apisix-conformance-test.yml +++ b/.github/workflows/apisix-conformance-test.yml @@ -32,6 +32,9 @@ concurrency: permissions: pull-requests: write +env: + ADC_VERSION: dev + jobs: conformance-test: env: @@ -70,7 +73,6 @@ jobs: ARCH: amd64 ENABLE_PROXY: "false" BASE_IMAGE_TAG: "debug" - # ADC_VERSION: "dev" run: | echo "building images..." make build-image From 6f9f3cae895b0da91f608521d3da86f27964723b Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sun, 20 Sep 2026 09:11:11 +0800 Subject: [PATCH 06/18] test: skip the Passthrough e2e spec in API7EE mode The API7 gateway's stream_route schema carries `sni` only, under additionalProperties = false, and has no tls_passthrough, so the route the spec applies never reaches the data plane. The control plane rejects it first: PUT /apisix/admin/stream_routes/... 400 can not create a Stream Route to the HTTP Service Same gap the api7ee conformance suite now declares, so the spec declares it the same way rather than failing the job. The APISIX providers keep running it. --- test/e2e/gatewayapi/tlsroute.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/e2e/gatewayapi/tlsroute.go b/test/e2e/gatewayapi/tlsroute.go index a6361ac82..5aa5352f1 100644 --- a/test/e2e/gatewayapi/tlsroute.go +++ b/test/e2e/gatewayapi/tlsroute.go @@ -181,6 +181,11 @@ spec: }) It("forwards the stream to the backend that owns the certificate", func() { + if s.Deployer.Name() == framework.ProviderTypeAPI7EE { + // The API7 gateway's stream_route schema has no tls_passthrough and + // refuses unknown keys, so the route never reaches the data plane. + Skip("skipping test in API7EE mode") + } s.ResourceApplied("TLSRoute", "tls-passthrough-route", passthroughRoute, 1) // The client verifies the served chain against the backend's own CA. From 47ddb1661e0b51c4fb600e364bd8f558cb577c66 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 08:45:42 +0800 Subject: [PATCH 07/18] fix: verify the TLS passthrough tunnel with a handshake, and rebuild it Backport of the same fix in apache/apisix-ingress-controller#2882, where this spec failed on every attempt across both providers until it landed. The tunnel was forwarded alongside the HTTP ones when the data plane was deployed, and handed to the spec as-is. Two problems with that. The forward can die mid-flight - inside the pod's netns: an error occurred forwarding -> 9120: ... read tcp4 127.0.0.1:...->127.0.0.1:9120: read: connection reset by peer after which the local end is simply gone, and every request reports dial tcp [::1]:: connect: connection refused which points at the wrong layer: a reset rather than ECONNREFUSED means the data plane listened and hung up, and the stream route synced for it is correct. What makes the far end reset is not established, and this does not guess at it. And checking the forward with a bare TCP dial proves nothing, because kubectl port-forward accepts locally first and only then tries to proxy. So the tunnel is now proven the way the spec uses it - a real handshake for the SNI - and an attempt that cannot complete one closes the forward and builds another. It is also forwarded on demand: createDataplaneTunnels runs for every spec in the suite, so one consumer was costing ~118 forwards. tlsPassthroughTunnel is byte-identical to the upstream version; the HTTP/2 tunnel this fork carries is kept alongside. --- test/e2e/scaffold/scaffold.go | 89 ++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 6961481c8..54f6bc28a 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "net" "net/http" "net/url" "os" @@ -319,15 +320,72 @@ const tlsPassthroughPortName = "tls-passthrough" // Verifying is the point: the gateway holds no certificate for a passthrough // listener, so a chain that validates against the backend's own CA can only // have come from the backend itself. -func (s *Scaffold) NewAPISIXClientWithTLSPassthrough(sni string, caCert []byte) *httpexpect.Expect { - Expect(s.apisixTunnels.TLSPassthrough).NotTo(BeNil(), "tls passthrough tunnel") +// tlsPassthroughTunnel forwards the data plane's tls_passthrough stream port and +// hands back a tunnel that has been proven to carry a handshake end to end. +// +// Two things make this port different from the others the scaffold forwards. +// +// A tls_passthrough listen prereads the ClientHello to pick a stream route, so a +// connection that carries no TLS gets reset. kubectl port-forward treats that +// reset as fatal and exits, taking the local listener with it - every later dial +// then gets "connection refused" rather than anything describing the cause. So +// the tunnel is verified with a real handshake for sni, never a bare TCP dial, +// and a failed attempt rebuilds it rather than reusing a forwarder that may +// already be gone. +// +// It is also forwarded on demand rather than alongside the HTTP tunnels when the +// data plane is deployed: APISIX opens its stream listens later than its HTTP +// ones, and only this one spec dials this port, so the rest of the suite would +// pay for a forward it never uses. +func (s *Scaffold) tlsPassthroughTunnel(sni string, pool *x509.CertPool) Tunnel { + svc := s.dataplaneService + Expect(svc).NotTo(BeNil(), "data plane service") + + var port int + for _, p := range svc.Spec.Ports { + if p.Name == tlsPassthroughPortName { + port = int(p.Port) + } + } + Expect(port).NotTo(BeZero(), "data plane service has no %s port", tlsPassthroughPortName) + + kubectlOpts := k8s.NewKubectlOptions("", "", svc.Namespace) + var tunnel *k8s.Tunnel + Eventually(func() error { + t := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, svc.Name, 0, port) + if err := t.ForwardPortE(s.t); err != nil { + return err + } + // ForwardPortE returns before the forward has proxied anything, so the + // handshake is what decides whether this tunnel works. + conn, err := tls.DialWithDialer( + &net.Dialer{Timeout: 10 * time.Second}, + "tcp", t.Endpoint(), + &tls.Config{ServerName: sni, RootCAs: pool}, + ) + if err != nil { + t.Close() + return err + } + _ = conn.Close() + tunnel = t + return nil + }).WithTimeout(time.Minute*2).WithPolling(time.Second*2). + Should(Succeed(), "forwarding the tls passthrough port") + + s.apisixTunnels.TLSPassthrough = tunnel + return tunnel +} +func (s *Scaffold) NewAPISIXClientWithTLSPassthrough(sni string, caCert []byte) *httpexpect.Expect { pool := x509.NewCertPool() Expect(pool.AppendCertsFromPEM(caCert)).To(BeTrue(), "parsing CA certificate") + tunnel := s.tlsPassthroughTunnel(sni, pool) + u := url.URL{ Scheme: apiv2.SchemeHTTPS, - Host: s.apisixTunnels.TLSPassthrough.Endpoint(), + Host: tunnel.Endpoint(), } return httpexpect.WithConfig(httpexpect.Config{ BaseURL: u.String(), @@ -459,12 +517,11 @@ func (s *Scaffold) createDataplaneTunnels( serviceName string, ) (*Tunnels, error) { var ( - httpPort int - httpsPort int - tcpPort int - http2Port int - tlsPort int - tlsPassthroughPort int + httpPort int + httpsPort int + tcpPort int + http2Port int + tlsPort int ) for _, port := range svc.Spec.Ports { @@ -479,8 +536,6 @@ func (s *Scaffold) createDataplaneTunnels( http2Port = int(port.Port) case apiv2.SchemeTLS: tlsPort = int(port.Port) - case tlsPassthroughPortName: - tlsPassthroughPort = int(port.Port) } } @@ -524,16 +579,8 @@ func (s *Scaffold) createDataplaneTunnels( tunnels.HTTP2 = http2Tunnel } - // Absent on a gateway deployed from an older manifest revision; the specs - // that need it assert on the tunnel being there. - if tlsPassthroughPort != 0 { - tlsPassthroughTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, - 0, tlsPassthroughPort) - if err := tlsPassthroughTunnel.ForwardPortE(s.t); err != nil { - return nil, err - } - tunnels.TLSPassthrough = tlsPassthroughTunnel - } + // The TLS passthrough tunnel is deliberately not forwarded here. See + // tlsPassthroughTunnel. return tunnels, nil } From f5325f92bf14eaa0baeef71af6ec09af67792341 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 09:05:10 +0800 Subject: [PATCH 08/18] fix: address review findings on the hostname intersection and the e2e scaffold Three findings from the PR review, all confirmed in the code. **A rejected parentRef widened the hostname intersection.** ParseRouteParentRefs returns a context per parentRef whether or not it was accepted, and a rejected one carries no matched listener - `Listener` nil, `Listeners` empty, `ListenerName` empty - so listenersForGatewayContext falls through to its last branch: ```go return gateway.Gateway.Spec.Listeners ``` every listener on that Gateway. A hostname only that Gateway accepts therefore survived intersectRouteHostnames and, for a TLSRoute, became an SNI served through the parentRef that *was* accepted. Only accepted contexts may widen the set now. This is pre-existing behaviour of filterHostnames for HTTPRoute, which this series extended to TLSRoute by sharing the helper; a test covers both the union and the minimum-intersection branch. **The passthrough tunnel ignored the configured kubeconfig.** It built fresh `NewKubectlOptions("", "", ns)` while every other dataplane tunnel is forwarded with the scaffold's own options, so a run that sets a kubeconfig or context would have targeted the ambient cluster instead. **An additional gateway dropped a ServiceHTTPSTargetPort override.** CreateAdditionalGatewayWithOptions copies the HTTP and HTTPS port overrides but not the target port, so an additional gateway stayed on 9443 and could not be pointed at the tls_passthrough listen. --- internal/controller/listener_utils_test.go | 50 ++++++++++++++++++++++ internal/controller/utils.go | 29 +++++++++++++ test/e2e/scaffold/apisix_deployer.go | 3 ++ test/e2e/scaffold/scaffold.go | 9 +++- 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/internal/controller/listener_utils_test.go b/internal/controller/listener_utils_test.go index 3862b11ef..ab1127d8b 100644 --- a/internal/controller/listener_utils_test.go +++ b/internal/controller/listener_utils_test.go @@ -207,3 +207,53 @@ func TestFilterTLSRouteHostnames(t *testing.T) { }) } } + +// A rejected parentRef carries no matched listener, so listenersForGatewayContext +// falls back to every listener on its Gateway. Without filtering, a hostname only +// that Gateway accepts survives the intersection and the TLS translator serves it +// as an SNI through the parent that was accepted. +func TestIntersectRouteHostnamesIgnoresRejectedParents(t *testing.T) { + acceptedHost := gatewayv1.Hostname("a.example.com") + rejectedHost := gatewayv1.Hostname("b.example.com") + + gatewayWith := func(hostname gatewayv1.Hostname) *gatewayv1.Gateway { + // The protocol has to be one isListenerHostnameEffective accepts, or + // the union branch skips the listener and reads as "no hostnames". + l := listener("tls", 443, string(hostname)) + l.Protocol = gatewayv1.TLSProtocolType + return &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: string(hostname), Namespace: "default"}, + Spec: gatewayv1.GatewaySpec{Listeners: []gatewayv1.Listener{l}}, + } + } + condition := func(status metav1.ConditionStatus) []metav1.Condition { + return []metav1.Condition{{ + Type: string(gatewayv1.RouteConditionAccepted), + Status: status, + }} + } + + accepted := gatewayWith(acceptedHost) + rejected := gatewayWith(rejectedHost) + gateways := []RouteParentRefContext{ + { + Gateway: accepted, + Listeners: accepted.Spec.Listeners, + Conditions: condition(metav1.ConditionTrue), + }, + { + // No matched listener, as ParseRouteParentRefs leaves it. + Gateway: rejected, + Conditions: condition(metav1.ConditionFalse), + }, + } + + got, err := intersectRouteHostnames(gateways, []gatewayv1.Hostname{acceptedHost, rejectedHost}) + require.NoError(t, err) + require.Equal(t, []gatewayv1.Hostname{acceptedHost}, got) + + // A route without hostnames takes the union, which must not be widened either. + got, err = intersectRouteHostnames(gateways, nil) + require.NoError(t, err) + require.Equal(t, []gatewayv1.Hostname{acceptedHost}, got) +} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 6cd9a14c2..538a028e7 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -1439,6 +1439,8 @@ func filterTLSRouteHostnames(gateways []RouteParentRefContext, tlsRoute *gateway // intersection with a listener hostname, and a route that intersects with none // is ErrNoMatchingListenerHostname. func intersectRouteHostnames(gateways []RouteParentRefContext, routeHostnames []gatewayv1.Hostname) ([]gatewayv1.Hostname, error) { + gateways = acceptedParentRefs(gateways) + if len(routeHostnames) == 0 { hostnames, matchAnyHost := getUnionOfGatewayHostnames(gateways) if matchAnyHost { @@ -1561,6 +1563,33 @@ func isListenerHostnameEffective(listener gatewayv1.Listener) bool { listener.Protocol == gatewayv1.TLSProtocolType } +// acceptedParentRefs keeps only the contexts whose parentRef was accepted. +// +// ParseRouteParentRefs returns a context per parentRef, accepted or not, and a +// rejected one carries no matched listener at all - so listenersForGatewayContext +// falls back to every listener on its Gateway. A hostname that only such a +// Gateway accepts would then survive the intersection and be served through the +// parentRef that was accepted. Only an accepted parent may widen the set. +// +// When nothing was accepted the contexts are returned unchanged: the route is +// not translated in that case, and narrowing here would only change which +// reason its status reports. +func acceptedParentRefs(gateways []RouteParentRefContext) []RouteParentRefContext { + accepted := make([]RouteParentRefContext, 0, len(gateways)) + for _, gateway := range gateways { + for _, condition := range gateway.Conditions { + if condition.Type == string(gatewayv1.RouteConditionAccepted) && condition.Status == metav1.ConditionTrue { + accepted = append(accepted, gateway) + break + } + } + } + if len(accepted) == 0 { + return gateways + } + return accepted +} + func isRouteAccepted(gateways []RouteParentRefContext) bool { for _, gateway := range gateways { for _, condition := range gateway.Conditions { diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index 0c66af8e8..a08b38851 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -439,6 +439,9 @@ func (s *APISIXDeployer) CreateAdditionalGatewayWithOptions(namePrefix string, o if opts.ServiceHTTPSPort != 0 { o.ServiceHTTPSPort = opts.ServiceHTTPSPort } + if opts.ServiceHTTPSTargetPort != 0 { + o.ServiceHTTPSTargetPort = opts.ServiceHTTPSTargetPort + } if opts.ProviderType != "" { if opts.ProviderType == framework.ProviderTypeAPISIX { o.ConfigProvider = framework.ConfigProviderTypeEtcd diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index bdf546cff..cbf675571 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -353,10 +353,15 @@ func (s *Scaffold) tlsPassthroughTunnel(sni string, pool *x509.CertPool) Tunnel } Expect(port).NotTo(BeZero(), "data plane service has no %s port", tlsPassthroughPortName) - kubectlOpts := k8s.NewKubectlOptions("", "", svc.Namespace) + // Copy the scaffold's options rather than building fresh ones: a run that + // configures a kubeconfig or context would otherwise be ignored here while + // every other dataplane tunnel honours it. + kubectlOpts := *s.kubectlOptions + kubectlOpts.Namespace = svc.Namespace + var tunnel *k8s.Tunnel Eventually(func() error { - t := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, svc.Name, 0, port) + t := k8s.NewTunnel(&kubectlOpts, k8s.ResourceTypeService, svc.Name, 0, port) if err := t.ForwardPortE(s.t); err != nil { return err } From e1147b1488eb0b98d0714045eb5a712909374381 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 09:05:10 +0800 Subject: [PATCH 09/18] chore: require an ADC release carrying snis and tls_passthrough ADC 0.29.0 rejects both fields outright, so a multi-host or Passthrough TLSRoute fails to sync on the default path: HTTP 400 {"code":"unrecognized_keys","keys":["tls_passthrough"], "path":["services",0,"stream_routes",0]} CI was unaffected because both e2e workflows and now the conformance one set ADC_VERSION=dev, but anything using the Makefile default - a local `make e2e-test`, or a deployment built from it - hits this. 0.31.0 is the first release containing api7/adc commit 63a09371 (feat(core): support snis and tls_passthrough configurations for stream route); verified with `gh api repos/api7/adc/compare/63a09371...v0.31.0` reporting behind_by 0. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 73cf92849..260793e42 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ KIND_NAME ?= apisix-ingress-cluster KIND_NODE_IMAGE ?= kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f DASHBOARD_VERSION ?= dev -ADC_VERSION ?= 0.29.0 +ADC_VERSION ?= 0.31.0 DIR := $(shell pwd) From 303d630688fdfd015ee43165e3b0765baa600b0d Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 09:09:46 +0800 Subject: [PATCH 10/18] test: make the rejected-parent test self-contained So that this test reads the same in both repositories: the two keep these hostname tests in differently named files with different local helpers, and the test body no longer depends on either. --- internal/controller/listener_utils_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/controller/listener_utils_test.go b/internal/controller/listener_utils_test.go index ab1127d8b..5f3740ce5 100644 --- a/internal/controller/listener_utils_test.go +++ b/internal/controller/listener_utils_test.go @@ -217,13 +217,19 @@ func TestIntersectRouteHostnamesIgnoresRejectedParents(t *testing.T) { rejectedHost := gatewayv1.Hostname("b.example.com") gatewayWith := func(hostname gatewayv1.Hostname) *gatewayv1.Gateway { - // The protocol has to be one isListenerHostnameEffective accepts, or - // the union branch skips the listener and reads as "no hostnames". - l := listener("tls", 443, string(hostname)) - l.Protocol = gatewayv1.TLSProtocolType + h := hostname return &gatewayv1.Gateway{ ObjectMeta: metav1.ObjectMeta{Name: string(hostname), Namespace: "default"}, - Spec: gatewayv1.GatewaySpec{Listeners: []gatewayv1.Listener{l}}, + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{{ + Name: "tls", + Port: 443, + // Has to be a protocol isListenerHostnameEffective accepts, or + // the union branch skips the listener and reads as "no hostnames". + Protocol: gatewayv1.TLSProtocolType, + Hostname: &h, + }}, + }, } } condition := func(status metav1.ConditionStatus) []metav1.Condition { From c9a30d8f8599314c2b0bce8814570b5702db20d6 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 09:32:18 +0800 Subject: [PATCH 11/18] Revert "test: declare the TLSRoute gap on the API7 gateway" The gap does not exist. api7/api7-ee-3-gateway#2173 forward-ported TLS passthrough on 2026-09-07, 29 seconds after the upstream apache/apisix#13912 it mirrors, so the gateway has carried `snis`, `tls_passthrough` and the mutual-exclusion constraint on stream_route since ten days before this series was opened: $ git log -1 --format='%ci %s' 62bca1d2 2026-09-07 14:52:11 +0800 feat(stream): support TLS passthrough on the stream proxy (#2173) I read the schema off a checkout that was not on master and concluded the fields were missing, without checking the ref. The api7ee failure I attached to that conclusion was a different thing entirely - `can not create a Stream Route to the HTTP Service`, about the service type, and a retry in the same run succeeded - so it was never evidence for a missing field. Both skips are therefore removed, and the Passthrough e2e spec is now identical to the upstream one. TLSRouteHostnameIntersection stays skipped for the reason the APISIX suite already records: every Gateway resolves to one data plane address and one physical stream listen, so their SNI namespaces are shared. --- test/conformance/api7ee/conformance_test.go | 23 ++++++++------------- test/e2e/gatewayapi/tlsroute.go | 5 ----- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/test/conformance/api7ee/conformance_test.go b/test/conformance/api7ee/conformance_test.go index f9aade686..bf92e1ceb 100644 --- a/test/conformance/api7ee/conformance_test.go +++ b/test/conformance/api7ee/conformance_test.go @@ -31,19 +31,14 @@ var skippedTestsForSSL = []string{ tests.HTTPRouteRedirectPortAndScheme.ShortName, } -// The API7 gateway's stream_route schema carries `sni` only, under -// additionalProperties = false, and has no tls_passthrough - it predates -// apache/apisix#13912. A stream route carrying either `snis` or -// `tls_passthrough` is rejected outright, so a TLSRoute needs its own gateway -// support before these can run here. The APISIX provider runs them already; see -// test/conformance/conformance_test.go. -var skippedTestsForGatewaySchema = []string{ - // Pinned to mode: Passthrough, which the gateway cannot serve. - tests.TLSRouteSimpleSameNamespace.ShortName, - tests.TLSRouteInvalidBackendRefNonexistent.ShortName, - tests.TLSRouteInvalidBackendRefUnknownKind.ShortName, - // Passthrough as well, and its routes carry several hostnames, which the - // translator emits as `snis`. +// The TLSRoute tests all pin their listener to mode: Passthrough, which the +// API7 gateway serves since api7/api7-ee-3-gateway#2173. One of them cannot +// pass here for the same reason it cannot on the APISIX provider, recorded in +// test/conformance/conformance_test.go: every Gateway resolves to one data +// plane address and one physical stream listen, so their SNI namespaces are +// shared and a name no listener on the addressed Gateway accepts is still +// answered by a sibling Gateway's route. +var skippedTestsForKnownGaps = []string{ tests.TLSRouteHostnameIntersection.ShortName, } @@ -55,7 +50,7 @@ func TestGatewayAPIConformance(t *testing.T) { opts.CleanupBaseResources = true opts.GatewayClassName = gatewayClassName opts.SkipTests = append(opts.SkipTests, skippedTestsForSSL...) - opts.SkipTests = append(opts.SkipTests, skippedTestsForGatewaySchema...) + opts.SkipTests = append(opts.SkipTests, skippedTestsForKnownGaps...) opts.Implementation = conformancev1.Implementation{ Organization: "APISIX", Project: "apisix-ingress-controller", diff --git a/test/e2e/gatewayapi/tlsroute.go b/test/e2e/gatewayapi/tlsroute.go index 5aa5352f1..a6361ac82 100644 --- a/test/e2e/gatewayapi/tlsroute.go +++ b/test/e2e/gatewayapi/tlsroute.go @@ -181,11 +181,6 @@ spec: }) It("forwards the stream to the backend that owns the certificate", func() { - if s.Deployer.Name() == framework.ProviderTypeAPI7EE { - // The API7 gateway's stream_route schema has no tls_passthrough and - // refuses unknown keys, so the route never reaches the data plane. - Skip("skipping test in API7EE mode") - } s.ResourceApplied("TLSRoute", "tls-passthrough-route", passthroughRoute, 1) // The client verifies the served chain against the backend's own CA. From b473b478946af78da3873a8e936c52832d54d0b7 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 10:15:06 +0800 Subject: [PATCH 12/18] fix: configure the passthrough listen on the API7 data plane too The api7ee e2e leg failed in tlsPassthroughTunnel: data plane service has no tls-passthrough port The gateway has carried TLS passthrough since api7/api7-ee-3-gateway#2173, but only manifests/apisix.yaml was given the listen; manifests/dp.yaml, which the API7 data plane is deployed from, never got it. So the capability was there and the test environment had no port to reach it on. Added the same three pieces this series added to the APISIX manifest - the stream_proxy listen, the container port and the Service port - so the spec exercises passthrough on this provider rather than being skipped for it. Also reworded the TLSPassthrough doc comment, which said APISIX "only consults it on a stream listen configured with both tls and tls_passthrough". That is accurate - stream_tls_route_phase, the only reader of the field, is the preread phase of a mixed listen - but it reads as though both have to be set for passthrough to work at all, which a review took it to mean. --- api/adc/types.go | 8 ++++++-- test/e2e/framework/manifests/dp.yaml | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/api/adc/types.go b/api/adc/types.go index 904a200ee..bae1a3736 100644 --- a/api/adc/types.go +++ b/api/adc/types.go @@ -186,8 +186,12 @@ type StreamRoute struct { SNI string `json:"sni,omitempty"` SNIs []string `json:"snis,omitempty"` // TLSPassthrough forwards the TLS stream to the upstream untouched instead - // of terminating it on the gateway. APISIX only consults it on a stream - // listen configured with both tls and tls_passthrough. + // of terminating it on the gateway. + // + // A listen carrying tls_passthrough alone passes every connection through + // and never reads this; a listen carrying tls alone always terminates. It + // decides only on a listen configured with both, where the matched stream + // route picks between the terminating and the prereading server. TLSPassthrough *bool `json:"tls_passthrough,omitempty"` } diff --git a/test/e2e/framework/manifests/dp.yaml b/test/e2e/framework/manifests/dp.yaml index 7608c9de6..96071b028 100644 --- a/test/e2e/framework/manifests/dp.yaml +++ b/test/e2e/framework/manifests/dp.yaml @@ -36,6 +36,11 @@ data: - 9100 - addr: 9110 tls: true + # TLS passthrough: the SNI is read out of the prereaded ClientHello + # and the stream reaches the backend untouched. A listen is either + # terminating or prereading, never both, so this needs its own port. + - addr: 9120 + tls_passthrough: true udp: - 9200 nginx_config: @@ -227,6 +232,9 @@ spec: - containerPort: 9110 name: tls protocol: TCP + - containerPort: 9120 + name: tls-passthrough + protocol: TCP readinessProbe: failureThreshold: 10 initialDelaySeconds: 3 @@ -295,6 +303,10 @@ spec: port: 9110 protocol: TCP targetPort: 9110 + - name: tls-passthrough + port: 9120 + protocol: TCP + targetPort: 9120 selector: app.kubernetes.io/instance: api7ee3 app.kubernetes.io/name: apisix From 5f628b8f04c8184aec2905196777e1aa705c9177 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 10:18:26 +0800 Subject: [PATCH 13/18] fix: stop de-duplicating aggregated listeners A review found that `appendListeners` keyed the aggregate by name and port, and that this slice spans every Gateway the route attaches to - so the conventional "tls" listener on 443 that two Gateways both declare collapsed to whichever parentRef happened to be processed first, taking its hostname and its tls.mode with it. This series reads both of those per port, so the order decided what the translator saw. Upstream had already fixed it, by dropping de-duplication altogether: ```go // appendListeners appends listeners without de-duplication. // Route translation aggregates listeners across multiple Gateways, and listener // names are only unique within a single Gateway. func appendListeners(target []gatewayv1.Listener, source ...gatewayv1.Listener) []gatewayv1.Listener { return append(target, source...) } ``` Taken verbatim, along with its table test, rather than inventing a wider key here - duplicates are harmless downstream (`listenerPortSet` is a set, `tlsRouteSNIs` de-duplicates the hostname fallback) and this keeps the two repositories identical. --- internal/controller/listener_utils_test.go | 47 +++++++++++++++++----- internal/controller/utils.go | 26 +++--------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/internal/controller/listener_utils_test.go b/internal/controller/listener_utils_test.go index 5f3740ce5..96e034df8 100644 --- a/internal/controller/listener_utils_test.go +++ b/internal/controller/listener_utils_test.go @@ -20,6 +20,7 @@ package controller import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -36,19 +37,43 @@ func listener(name string, port gatewayv1.PortNumber, hostname string) gatewayv1 } return l } +func TestAppendListeners(t *testing.T) { + listenerA := gatewayv1.Listener{Name: "a", Port: 80} + listenerB := gatewayv1.Listener{Name: "b", Port: 81} + listenerA2 := gatewayv1.Listener{Name: "a", Port: 82} + listenerA3 := gatewayv1.Listener{Name: "a", Port: 80} -func TestAppendListenersKeepsSameNameOnDifferentPorts(t *testing.T) { - // Listener names are unique per Gateway only, and this slice spans every - // Gateway a route attaches to. - got := appendListeners(nil, - listener("http", 80, ""), - listener("http", 8080, ""), - listener("http", 80, ""), - ) + tests := []struct { + name string + target []gatewayv1.Listener + source []gatewayv1.Listener + expected []gatewayv1.Listener + }{ + { + name: "empty target, add listeners", + target: nil, + source: []gatewayv1.Listener{listenerA, listenerB}, + expected: []gatewayv1.Listener{listenerA, listenerB}, + }, + { + name: "preserves same listener names from different gateways", + target: []gatewayv1.Listener{listenerA}, + source: []gatewayv1.Listener{listenerA, listenerB}, + expected: []gatewayv1.Listener{listenerA, listenerA, listenerB}, + }, + { + name: "preserves all listeners when names collide", + target: []gatewayv1.Listener{listenerA}, + source: []gatewayv1.Listener{listenerB, listenerA2, listenerA3}, + expected: []gatewayv1.Listener{listenerA, listenerB, listenerA2, listenerA3}, + }, + } - require.Len(t, got, 2) - require.Equal(t, gatewayv1.PortNumber(80), got[0].Port) - require.Equal(t, gatewayv1.PortNumber(8080), got[1].Port) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, appendListeners(tt.target, tt.source...)) + }) + } } func TestListenersForGatewayContext(t *testing.T) { diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 538a028e7..30886921e 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -1504,27 +1504,11 @@ func listenersForGatewayContext(gateway RouteParentRefContext) []gatewayv1.Liste return gateway.Gateway.Spec.Listeners } -// appendListeners appends listeners to the slice, avoiding duplicates by port+name. -func appendListeners(existing []gatewayv1.Listener, toAdd ...gatewayv1.Listener) []gatewayv1.Listener { - // Listener names are only unique within a Gateway, while this slice is - // aggregated across every Gateway the route attaches to, so the name alone - // would collapse distinct listeners such as two "http" on different ports. - type listenerKey struct { - name string - port gatewayv1.PortNumber - } - seen := make(map[listenerKey]struct{}, len(existing)) - for _, l := range existing { - seen[listenerKey{name: string(l.Name), port: l.Port}] = struct{}{} - } - for _, l := range toAdd { - key := listenerKey{name: string(l.Name), port: l.Port} - if _, ok := seen[key]; !ok { - existing = append(existing, l) - seen[key] = struct{}{} - } - } - return existing +// appendListeners appends listeners without de-duplication. +// Route translation aggregates listeners across multiple Gateways, and listener +// names are only unique within a single Gateway. +func appendListeners(target []gatewayv1.Listener, source ...gatewayv1.Listener) []gatewayv1.Listener { + return append(target, source...) } // getMinimumHostnameIntersection returns the smallest intersection hostname From 3364c833c69adb8971bb62a6fa2862c46d18f172 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 10:59:14 +0800 Subject: [PATCH 14/18] fix: close the previous passthrough tunnel before replacing it tlsPassthroughTunnel forwards a new port every call and overwrites the stored handle. Tunnels.Close only knows about the one currently stored, so a spec that built a second client - one per SNI, say - left the first kubectl port-forward running until the process exited. --- test/e2e/scaffold/scaffold.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index cbf675571..580d65520 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -382,6 +382,12 @@ func (s *Scaffold) tlsPassthroughTunnel(sni string, pool *x509.CertPool) Tunnel }).WithTimeout(time.Minute*2).WithPolling(time.Second*2). Should(Succeed(), "forwarding the tls passthrough port") + // Tunnels.Close only knows about the tunnel currently stored, so a spec + // that builds a second client would otherwise leave the first + // kubectl port-forward running until the process exits. + if previous := s.apisixTunnels.TLSPassthrough; previous != nil { + previous.Close() + } s.apisixTunnels.TLSPassthrough = tunnel return tunnel } From 19bab29c70563875fa7770efe7840889493d2f1c Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Sat, 10 Oct 2026 11:10:54 +0800 Subject: [PATCH 15/18] fix: point the api7ee conformance run at the passthrough listen The api7ee conformance suite failed every TLSRoute test, with the data plane refusing the handshake: tcp.go:99: client could not connect: remote error: tls: internal error The TLSRoute tests pin their Gateway listener to port 443 and dial the Gateway address there, so 443 has to reach the stream tls_passthrough listen. The APISIX suite redirects it with ServiceHTTPSTargetPort: 9120; this one still had 443 going to the HTTP ssl listen, which terminates and holds no certificate for the SNI the tests use - hence the alert from the server side rather than any routing failure. The option did not exist on this path: API7DeployOptions had no target port and dp.yaml hard-coded targetPort: 9443. Both now carry it, defaulting to 9443 so nothing else changes, and the conformance suite sets 9120. TLSRouteTerminateSimpleSameNamespace is skipped here for the reason the APISIX suite already records - it needs a standalone Gateway with no GatewayProxy to reach Accepted=True - which holds on this provider too, and a 443 pointed at a prereading listen could not serve it in any case. These tests failed on master as well (five of them; this branch already skipped one), so this is not a regression being fixed but a gap the series exposes by running them. --- test/conformance/api7ee/conformance_test.go | 6 ++++++ test/conformance/api7ee/suite_test.go | 5 +++++ test/e2e/framework/api7_gateway.go | 9 ++++++++- test/e2e/framework/manifests/dp.yaml | 6 +++++- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/test/conformance/api7ee/conformance_test.go b/test/conformance/api7ee/conformance_test.go index bf92e1ceb..3645d8a79 100644 --- a/test/conformance/api7ee/conformance_test.go +++ b/test/conformance/api7ee/conformance_test.go @@ -40,6 +40,12 @@ var skippedTestsForSSL = []string{ // answered by a sibling Gateway's route. var skippedTestsForKnownGaps = []string{ tests.TLSRouteHostnameIntersection.ShortName, + // Terminate mode itself is covered by TLSRouteListenerTerminateSupportedKinds + // and by the e2e TLSRoute suite. This provisional test additionally requires a + // standalone Gateway with no GatewayProxy attached to reach Accepted=True, and + // a stream proxy listening on the port it picks; neither holds here, so the + // Gateway is rejected with "gateway proxy not found" before any traffic flows. + tests.TLSRouteTerminateSimpleSameNamespace.ShortName, } // TODO: HTTPRoute hostname intersection and listener hostname matching diff --git a/test/conformance/api7ee/suite_test.go b/test/conformance/api7ee/suite_test.go index 2812d1af4..0d303b802 100644 --- a/test/conformance/api7ee/suite_test.go +++ b/test/conformance/api7ee/suite_test.go @@ -151,6 +151,11 @@ func TestMain(m *testing.M) { ServiceType: "LoadBalancer", ServiceHTTPPort: 80, ServiceHTTPSPort: 443, + // The TLSRoute tests pin their Gateway listener to port 443 in + // Passthrough mode and dial the Gateway address there, so 443 has to + // reach the stream tls_passthrough listen rather than the HTTP ssl + // listen. One port cannot serve both. + ServiceHTTPSTargetPort: 9120, }) if len(svc.Status.LoadBalancer.Ingress) == 0 { diff --git a/test/e2e/framework/api7_gateway.go b/test/e2e/framework/api7_gateway.go index fc5ccdb12..7662e7d45 100644 --- a/test/e2e/framework/api7_gateway.go +++ b/test/e2e/framework/api7_gateway.go @@ -59,7 +59,10 @@ type API7DeployOptions struct { ServiceType string ServiceHTTPPort int ServiceHTTPSPort int - Replicas *int + // ServiceHTTPSTargetPort redirects the HTTPS service port at the data + // plane's stream tls_passthrough listen instead of its HTTP ssl listen. + ServiceHTTPSTargetPort int + Replicas *int } func (f *Framework) DeployGateway(opts *API7DeployOptions) *corev1.Service { @@ -75,6 +78,10 @@ func (f *Framework) DeployGateway(opts *API7DeployOptions) *corev1.Service { opts.ServiceHTTPSPort = 443 } + if opts.ServiceHTTPSTargetPort == 0 { + opts.ServiceHTTPSTargetPort = 9443 + } + dpCert := f.GetDataplaneCertificates(opts.GatewayGroupID) f.applySSLSecret(opts.Namespace, diff --git a/test/e2e/framework/manifests/dp.yaml b/test/e2e/framework/manifests/dp.yaml index 96071b028..ee431b79a 100644 --- a/test/e2e/framework/manifests/dp.yaml +++ b/test/e2e/framework/manifests/dp.yaml @@ -287,7 +287,11 @@ spec: - name: https port: {{ .ServiceHTTPSPort }} protocol: TCP - targetPort: 9443 + # Normally the HTTP ssl listen. The conformance run redirects it to the + # stream tls_passthrough listen instead: a Gateway API TLSRoute Passthrough + # listener is fixed at 443, and one port cannot serve both HTTP-over-TLS + # and an untouched stream. + targetPort: {{ .ServiceHTTPSTargetPort }} - name: control-api port: 9090 protocol: TCP From 0d44e2c7e0b50b6b6b682307ef6ebeedc0243d7f Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Mon, 12 Oct 2026 10:48:16 +0800 Subject: [PATCH 16/18] fix: derive TLSRoute SNIs from the listeners of each port A TLSRoute's SNIs were computed once for the whole route and handed to every per-port stream route. A port then answered hostnames only another port's listener had accepted, and a port whose listener carries no hostname lost its catch-all as soon as a listener elsewhere named one - the fallback skipped hostname-less listeners and only reached "*" when no listener had a hostname at all. Each stream route now derives its SNIs from the listeners on its own port. Narrowing a route hostname against the listeners also kept only the first intersection, because the lookup returned a single "smallest intersection". A route whose wildcard covers two same-port listeners was emitted for one of them and the other's traffic was dropped. Every intersection is now collected and de-duplicated, which fixes the same gap for HTTPRoute. The hostname matchers move to internal/controller/utils so the translator can share them instead of re-implementing wildcard matching, and the Gateway API doc records that the ProtocolConflict check for mixed tls.mode is scoped to one Gateway. --- docs/en/latest/concepts/gateway-api.md | 2 +- internal/adc/translator/tlsroute.go | 84 ++++++++----- internal/adc/translator/tlsroute_test.go | 82 +++++++++++++ internal/controller/listener_utils_test.go | 78 +++++++++++-- internal/controller/utils.go | 116 +++--------------- internal/controller/utils/hostname.go | 130 +++++++++++++++++++++ 6 files changed, 353 insertions(+), 139 deletions(-) create mode 100644 internal/controller/utils/hostname.go diff --git a/docs/en/latest/concepts/gateway-api.md b/docs/en/latest/concepts/gateway-api.md index ca763ab3e..580436aca 100644 --- a/docs/en/latest/concepts/gateway-api.md +++ b/docs/en/latest/concepts/gateway-api.md @@ -85,6 +85,6 @@ The fields below are specified in the Gateway API specification but are either p | `spec.listeners[].port` | Not supported* | The configuration is required but ignored. This is due to limitations in the data plane: it cannot dynamically open new ports. Since the Ingress Controller does not manage the data plane deployment, it cannot automatically update the configuration or restart the data plane to apply port changes. | | `spec.listeners[].tls.certificateRefs[].group` | Partially supported | Only `""` is supported; other group values cause validation failure. | | `spec.listeners[].tls.certificateRefs[].kind` | Partially supported | Only `Secret` is supported. | -| `spec.listeners[].tls.mode` | Partially supported | `Terminate` and `Passthrough` are both implemented. A `Passthrough` listener needs APISIX to be listening on that port with [`tls_passthrough`](https://apisix.apache.org/docs/apisix/stream-proxy/); the controller cannot open data plane ports itself. A single port cannot mix the two modes: listeners that disagree on `tls.mode` for one port are reported `Accepted=False` with `ProtocolConflict`. | +| `spec.listeners[].tls.mode` | Partially supported | `Terminate` and `Passthrough` are both implemented. A `Passthrough` listener needs APISIX to be listening on that port with [`tls_passthrough`](https://apisix.apache.org/docs/apisix/stream-proxy/); the controller cannot open data plane ports itself. A single port cannot mix the two modes. Within one Gateway, listeners that disagree on `tls.mode` for one port are reported `Accepted=False` with `ProtocolConflict` and carry no routes. Two Gateways that disagree about the same port are each valid on their own, so no conflict is reported there; the routes on that port fall back to `Terminate`, because a physical stream listen has a single mode. | | `spec.listeners[].tls.frontendValidation` | Partially supported | Enables downstream (client) mTLS. `caCertificateRefs` may reference a `ConfigMap` (Gateway API Core support) or a `Secret` (implementation-specific) holding the CA certificate under the `ca.crt` key; clients are then required to present a certificate signed by one of the referenced CAs. | | `spec.addresses` | Not supported | Controller does not read or act on `spec.addresses`. | diff --git a/internal/adc/translator/tlsroute.go b/internal/adc/translator/tlsroute.go index d192d9651..4810dd1e3 100644 --- a/internal/adc/translator/tlsroute.go +++ b/internal/adc/translator/tlsroute.go @@ -26,6 +26,7 @@ import ( adctypes "github.com/apache/apisix-ingress-controller/api/adc" apiv2 "github.com/apache/apisix-ingress-controller/api/v2" "github.com/apache/apisix-ingress-controller/internal/controller/label" + cutils "github.com/apache/apisix-ingress-controller/internal/controller/utils" "github.com/apache/apisix-ingress-controller/internal/id" "github.com/apache/apisix-ingress-controller/internal/provider" "github.com/apache/apisix-ingress-controller/internal/types" @@ -35,7 +36,6 @@ func (t *Translator) TranslateTLSRoute(tctx *provider.TranslateContext, tlsRoute result := &TranslateResult{} rules := tlsRoute.Spec.Rules labels := label.GenLabel(tlsRoute) - snis := tlsRouteSNIs(tctx, tlsRoute) for ruleIndex, rule := range rules { service := adctypes.NewDefaultService() service.Labels = labels @@ -141,14 +141,22 @@ func (t *Translator) TranslateTLSRoute(tctx *provider.TranslateContext, tlsRoute } } - // One set of stream routes per rule carrying every SNI, not one set per - // hostname: buildL4StreamRoutes names them by rule and port only, so a per - // hostname loop produced N sets sharing one ID and only the last survived. + // One set of stream routes per rule, each carrying the SNIs of its own + // port, rather than one set per hostname: buildL4StreamRoutes names them by + // rule and port only, so a per hostname loop produced N sets sharing one ID + // and only the last survived. streamRoutes, err := t.buildL4StreamRoutes(tctx, tlsRoute.Namespace, tlsRoute.Name, ruleIndex, "TLS", "TLSRoute", labels) if err != nil { return nil, err } for _, streamRoute := range streamRoutes { + snis := tlsRouteSNIsForPort(tctx.Listeners, tlsRoute.Spec.Hostnames, streamRoute.ServerPort) + if len(snis) == 0 { + // Nothing for the stream route to match on. The listeners a route + // attaches to already intersect its hostnames, so this is a guard + // against emitting a route that would answer every SNI instead. + continue + } // A single SNI keeps using the singular form: it is what every // APISIX version understands, and snis only earns its place once // there is more than one to match. @@ -160,44 +168,66 @@ func (t *Translator) TranslateTLSRoute(tctx *provider.TranslateContext, tlsRoute if tlsPassthroughOnPort(tctx.Listeners, streamRoute.ServerPort) { streamRoute.TLSPassthrough = ptr.To(true) } + service.StreamRoutes = append(service.StreamRoutes, streamRoute) } - service.StreamRoutes = append(service.StreamRoutes, streamRoutes...) result.Services = append(result.Services, service) } return result, nil } -// tlsRouteSNIs returns the SNIs the route's stream routes match on. +// tlsRouteSNIsForPort returns the SNIs the stream route bound to port matches on. +// +// The set is derived from the listeners on that port alone. Handing every port +// the route-wide union instead would let a port answer a hostname its own +// listener never accepted, and would cost a port its catch-all as soon as a +// listener on another port named a hostname. port 0 means the stream route +// carries no server_port match, so every matched listener applies. // // A TLSRoute without hostnames matches everything its listeners accept, so it -// falls back to the matched listener hostnames and, when those carry none -// either, to the catch-all "*". Emitting nothing - which is what the per -// hostname loop used to do - left such a route attached but unserved. -func tlsRouteSNIs(tctx *provider.TranslateContext, tlsRoute *gatewayv1.TLSRoute) []string { - if len(tlsRoute.Spec.Hostnames) > 0 { - snis := make([]string, 0, len(tlsRoute.Spec.Hostnames)) - for _, hostname := range tlsRoute.Spec.Hostnames { - snis = append(snis, string(hostname)) +// falls back to the hostnames of the listeners on the port and, when one of them +// carries no hostname either, to the catch-all "*". Emitting nothing - which is +// what the per hostname loop used to do - left such a route attached but +// unserved. +func tlsRouteSNIsForPort(listeners []gatewayv1.Listener, routeHostnames []gatewayv1.Hostname, port int32) []string { + matched := make([]gatewayv1.Listener, 0, len(listeners)) + for _, listener := range listeners { + if listener.Protocol != gatewayv1.TLSProtocolType { + continue + } + if port != 0 && listener.Port != port { + continue } - return snis + matched = append(matched, listener) } - snis := make([]string, 0, len(tctx.Listeners)) - seen := make(map[string]struct{}, len(tctx.Listeners)) - for _, listener := range tctx.Listeners { - if listener.Hostname == nil || *listener.Hostname == "" { - continue + // No listener to narrow against: keep what the route itself asks for. + if len(matched) == 0 { + if len(routeHostnames) == 0 { + return []string{"*"} } - hostname := string(*listener.Hostname) - if _, ok := seen[hostname]; ok { - continue + return hostnamesToSNIs(routeHostnames) + } + + if len(routeHostnames) == 0 { + hostnames, matchAnyHost := cutils.UnionHostnames(matched) + if matchAnyHost { + return []string{"*"} } - seen[hostname] = struct{}{} - snis = append(snis, hostname) + return hostnamesToSNIs(cutils.DedupHostnames(hostnames)) } - if len(snis) == 0 { - return []string{"*"} + + snis := make([]gatewayv1.Hostname, 0, len(routeHostnames)) + for _, hostname := range routeHostnames { + snis = append(snis, cutils.NarrowHostname(matched, hostname)...) + } + return hostnamesToSNIs(cutils.DedupHostnames(snis)) +} + +func hostnamesToSNIs(hostnames []gatewayv1.Hostname) []string { + snis := make([]string, 0, len(hostnames)) + for _, hostname := range hostnames { + snis = append(snis, string(hostname)) } return snis } diff --git a/internal/adc/translator/tlsroute_test.go b/internal/adc/translator/tlsroute_test.go index b8e496e16..b207da8b2 100644 --- a/internal/adc/translator/tlsroute_test.go +++ b/internal/adc/translator/tlsroute_test.go @@ -191,3 +191,85 @@ func TestTranslateTLSRouteSNIs(t *testing.T) { assert.Equal(t, "*", streamRoutes[0].SNI) }) } + +func TestTranslateTLSRouteSNIsPerPort(t *testing.T) { + passthrough := gatewayv1.TLSModePassthrough + + snisByPort := func(result *TranslateResult) map[int32][]string { + byPort := make(map[int32][]string) + for _, streamRoute := range result.Services[0].StreamRoutes { + snis := streamRoute.SNIs + if streamRoute.SNI != "" { + snis = []string{streamRoute.SNI} + } + byPort[streamRoute.ServerPort] = snis + } + return byPort + } + + // The route-wide union went to every port, so each port also answered the + // hostname only the other port's listener accepted. + t.Run("a hostname-less route takes the hostnames of its own port", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("a", 9110, &passthrough, "a.example.com"), + tlsModeListener("b", 9120, &passthrough, "b.example.com"), + }, true) + + assert.Equal(t, map[int32][]string{ + 9110: {"a.example.com"}, + 9120: {"b.example.com"}, + }, snisByPort(result)) + }) + + // A listener carrying no hostname accepts every name. Collecting hostnames + // route-wide cost that port its catch-all as soon as a listener on another + // port named a hostname. + t.Run("a port whose listener names no hostname keeps the catch-all", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("named", 9110, &passthrough, "a.example.com"), + tlsModeListener("any", 9120, &passthrough, ""), + }, true) + + assert.Equal(t, map[int32][]string{ + 9110: {"a.example.com"}, + 9120: {"*"}, + }, snisByPort(result)) + }) + + t.Run("route hostnames are narrowed to the listeners of each port", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("a", 9110, &passthrough, "a.example.com"), + tlsModeListener("b", 9120, &passthrough, "b.example.com"), + }, true, "*.example.com") + + assert.Equal(t, map[int32][]string{ + 9110: {"a.example.com"}, + 9120: {"b.example.com"}, + }, snisByPort(result)) + }) + + // Both listeners on the port are covered by the route's wildcard, so the + // route answers both names rather than only the first intersection. + t.Run("one port with several listeners keeps every intersection", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("a", 9110, &passthrough, "a.example.com"), + tlsModeListener("b", 9110, &passthrough, "b.example.com"), + }, true, "*.example.com") + + assert.Equal(t, map[int32][]string{ + 9110: {"a.example.com", "b.example.com"}, + }, snisByPort(result)) + }) + + // A listener hostname that covers the route hostname accepts it whole, so + // the route keeps its own name instead of the listener's. + t.Run("a wildcard listener keeps the route hostname", func(t *testing.T) { + result := translateTLSRoute(t, []gatewayv1.Listener{ + tlsModeListener("wildcard", 9110, &passthrough, "*.example.com"), + }, true, "a.example.com") + + assert.Equal(t, map[int32][]string{ + 9110: {"a.example.com"}, + }, snisByPort(result)) + }) +} diff --git a/internal/controller/listener_utils_test.go b/internal/controller/listener_utils_test.go index 96e034df8..b852a94dd 100644 --- a/internal/controller/listener_utils_test.go +++ b/internal/controller/listener_utils_test.go @@ -24,12 +24,15 @@ import ( "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + cutils "github.com/apache/apisix-ingress-controller/internal/controller/utils" ) func listener(name string, port gatewayv1.PortNumber, hostname string) gatewayv1.Listener { l := gatewayv1.Listener{ - Name: gatewayv1.SectionName(name), - Port: port, + Name: gatewayv1.SectionName(name), + Protocol: gatewayv1.HTTPProtocolType, + Port: port, } if hostname != "" { h := gatewayv1.Hostname(hostname) @@ -115,11 +118,11 @@ func TestListenersForGatewayContext(t *testing.T) { }) } -// TestGetMinimumHostnameIntersectionUsesMatchedListeners pins the intersection to -// the listeners the parentRef actually selected. A parentRef selecting by port -// leaves ListenerName empty, so filtering by name alone would let a route keep a +// TestNarrowHostnameUsesMatchedListeners pins the intersection to the listeners +// the parentRef actually selected. A parentRef selecting by port leaves +// ListenerName empty, so filtering by name alone would let a route keep a // hostname that only a listener on a different port serves. -func TestGetMinimumHostnameIntersectionUsesMatchedListeners(t *testing.T) { +func TestNarrowHostnameUsesMatchedListeners(t *testing.T) { gateway := &gatewayv1.Gateway{} gateway.Spec.Listeners = []gatewayv1.Listener{ listener("http-a", 80, "a.example.com"), @@ -130,16 +133,16 @@ func TestGetMinimumHostnameIntersectionUsesMatchedListeners(t *testing.T) { Listeners: []gatewayv1.Listener{gateway.Spec.Listeners[0]}, }} - require.Equal(t, gatewayv1.Hostname("a.example.com"), - getMinimumHostnameIntersection(portSelected, "a.example.com")) - require.Empty(t, getMinimumHostnameIntersection(portSelected, "b.example.com"), + require.Equal(t, []gatewayv1.Hostname{"a.example.com"}, + cutils.NarrowHostname(hostnameEffectiveListeners(portSelected), "a.example.com")) + require.Empty(t, cutils.NarrowHostname(hostnameEffectiveListeners(portSelected), "b.example.com"), "a hostname served only by the untargeted listener must not survive") // Without a matched listener the whole spec is still considered, so both // hostnames intersect as before. whole := []RouteParentRefContext{{Gateway: gateway}} - require.Equal(t, gatewayv1.Hostname("b.example.com"), - getMinimumHostnameIntersection(whole, "b.example.com")) + require.Equal(t, []gatewayv1.Hostname{"b.example.com"}, + cutils.NarrowHostname(hostnameEffectiveListeners(whole), "b.example.com")) } // The Gateway API conformance case TLSRouteHostnameIntersection turns on this: @@ -288,3 +291,56 @@ func TestIntersectRouteHostnamesIgnoresRejectedParents(t *testing.T) { require.NoError(t, err) require.Equal(t, []gatewayv1.Hostname{acceptedHost}, got) } + +// A route hostname that covers several listener hostnames intersects with every +// one of them. Keeping only the first left the names the other listeners accept +// unserved. +func TestIntersectRouteHostnamesKeepsEveryIntersection(t *testing.T) { + aHostname := gatewayv1.Hostname("a.example.com") + bHostname := gatewayv1.Hostname("b.example.com") + gateways := []RouteParentRefContext{ + { + Listeners: []gatewayv1.Listener{ + {Name: "a", Protocol: gatewayv1.TLSProtocolType, Port: gatewayv1.PortNumber(443), Hostname: &aHostname}, + {Name: "b", Protocol: gatewayv1.TLSProtocolType, Port: gatewayv1.PortNumber(443), Hostname: &bHostname}, + }, + Conditions: []metav1.Condition{ + { + Type: string(gatewayv1.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }, + }, + }, + } + + got, err := intersectRouteHostnames(gateways, []gatewayv1.Hostname{"*.example.com"}) + require.NoError(t, err) + assert.Equal(t, []gatewayv1.Hostname{aHostname, bHostname}, got) +} + +// The same hostname contributed by two listeners is one hostname: listeners are +// aggregated across Gateways without de-duplication. +func TestIntersectRouteHostnamesDedups(t *testing.T) { + hostname := gatewayv1.Hostname("a.example.com") + listener := gatewayv1.Listener{ + Name: "a", Protocol: gatewayv1.TLSProtocolType, Port: gatewayv1.PortNumber(443), Hostname: &hostname, + } + accepted := []metav1.Condition{ + { + Type: string(gatewayv1.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }, + } + gateways := []RouteParentRefContext{ + {Listeners: []gatewayv1.Listener{listener}, Conditions: accepted}, + {Listeners: []gatewayv1.Listener{listener}, Conditions: accepted}, + } + + got, err := intersectRouteHostnames(gateways, []gatewayv1.Hostname{"*.example.com"}) + require.NoError(t, err) + assert.Equal(t, []gatewayv1.Hostname{hostname}, got) + + got, err = intersectRouteHostnames(gateways, nil) + require.NoError(t, err) + assert.Equal(t, []gatewayv1.Hostname{hostname}, got) +} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 30886921e..732a7414b 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -51,6 +51,7 @@ import ( apiv2 "github.com/apache/apisix-ingress-controller/api/v2" "github.com/apache/apisix-ingress-controller/internal/controller/config" "github.com/apache/apisix-ingress-controller/internal/controller/indexer" + cutils "github.com/apache/apisix-ingress-controller/internal/controller/utils" "github.com/apache/apisix-ingress-controller/internal/provider" sslutils "github.com/apache/apisix-ingress-controller/internal/ssl" "github.com/apache/apisix-ingress-controller/internal/types" @@ -725,7 +726,7 @@ func listenerHostnameIntersectWithRouteHostnames(listener gatewayv1.Listener, ho // iterate over all the hostnames and check that at least one intersect with the listener hostname for _, hostname := range hostnames { - if HostnamesIntersect(string(*listener.Hostname), string(hostname)) { + if cutils.HostnamesIntersect(string(*listener.Hostname), string(hostname)) { return true } } @@ -733,57 +734,6 @@ func listenerHostnameIntersectWithRouteHostnames(listener gatewayv1.Listener, ho return false } -func HostnamesIntersect(a, b string) bool { - return HostnamesMatch(a, b) || HostnamesMatch(b, a) -} - -// HostnamesMatch checks that the hostnameB matches the hostnameA. HostnameA is treated as mask -// to be checked against the hostnameB. -func HostnamesMatch(hostnameA, hostnameB string) bool { - // the hostnames are in the form of "foo.bar.com"; split them - // in a slice of substrings - hostnameALabels := strings.Split(hostnameA, ".") - hostnameBLabels := strings.Split(hostnameB, ".") - - var a, b int - var wildcard bool - - // iterate over the parts of both the hostnames - for a, b = 0, 0; a < len(hostnameALabels) && b < len(hostnameBLabels); a, b = a+1, b+1 { - var matchFound bool - - // if the current part of B is a wildcard, we need to find the first - // A part that matches with the following B part - if wildcard { - for ; b < len(hostnameBLabels); b++ { - if hostnameALabels[a] == hostnameBLabels[b] { - matchFound = true - break - } - } - } - - // if no match was found, the hostnames don't match - if wildcard && !matchFound { - return false - } - - // check if at least on of the current parts are a wildcard; if so, continue - if hostnameALabels[a] == "*" { - wildcard = true - continue - } - // reset the wildcard variables - wildcard = false - - // if the current a part is different from the b part, the hostnames are incompatible - if hostnameALabels[a] != hostnameBLabels[b] { - return false - } - } - return len(hostnameBLabels)-b == len(hostnameALabels)-a -} - func routeMatchesListenerAllowedRoutes( ctx context.Context, mgrc client.Client, @@ -1435,51 +1385,45 @@ func filterTLSRouteHostnames(gateways []RouteParentRefContext, tlsRoute *gateway // // A route without hostnames takes the union of the listener hostnames, and stays // empty - matching any hostname - as soon as one supported listener carries no -// hostname of its own. Otherwise every hostname is replaced by its smallest -// intersection with a listener hostname, and a route that intersects with none -// is ErrNoMatchingListenerHostname. +// hostname of its own. Otherwise every hostname is replaced by its intersections +// with the listener hostnames, and a route that intersects with none is +// ErrNoMatchingListenerHostname. func intersectRouteHostnames(gateways []RouteParentRefContext, routeHostnames []gatewayv1.Hostname) ([]gatewayv1.Hostname, error) { - gateways = acceptedParentRefs(gateways) + listeners := hostnameEffectiveListeners(acceptedParentRefs(gateways)) if len(routeHostnames) == 0 { - hostnames, matchAnyHost := getUnionOfGatewayHostnames(gateways) + hostnames, matchAnyHost := cutils.UnionHostnames(listeners) if matchAnyHost { return routeHostnames, nil } - return hostnames, nil + return cutils.DedupHostnames(hostnames), nil } filteredHostnames := make([]gatewayv1.Hostname, 0, len(routeHostnames)) for _, hostname := range routeHostnames { - if hostnameMatching := getMinimumHostnameIntersection(gateways, hostname); hostnameMatching != "" { - filteredHostnames = append(filteredHostnames, hostnameMatching) - } + filteredHostnames = append(filteredHostnames, cutils.NarrowHostname(listeners, hostname)...) } + filteredHostnames = cutils.DedupHostnames(filteredHostnames) if len(filteredHostnames) == 0 { return nil, ErrNoMatchingListenerHostname } return filteredHostnames, nil } -// getUnionOfGatewayHostnames returns the union of the hostnames specified in all supported gateways -// The second return value indicates whether any listener can match any hostname -func getUnionOfGatewayHostnames(gateways []RouteParentRefContext) ([]gatewayv1.Hostname, bool) { - hostnames := make([]gatewayv1.Hostname, 0) - +// hostnameEffectiveListeners flattens the listeners the route attached to, +// keeping only those a hostname can be matched against. +func hostnameEffectiveListeners(gateways []RouteParentRefContext) []gatewayv1.Listener { + listeners := make([]gatewayv1.Listener, 0, len(gateways)) for _, gateway := range gateways { for _, listener := range listenersForGatewayContext(gateway) { // Only consider listeners that can effectively configure hostnames (HTTP, HTTPS, or TLS). if !isListenerHostnameEffective(listener) { continue } - if listener.Hostname == nil { - return nil, true - } - hostnames = append(hostnames, *listener.Hostname) + listeners = append(listeners, listener) } } - - return hostnames, false + return listeners } // listenersForGatewayContext returns the listeners relevant for hostname resolution. @@ -1511,34 +1455,6 @@ func appendListeners(target []gatewayv1.Listener, source ...gatewayv1.Listener) return append(target, source...) } -// getMinimumHostnameIntersection returns the smallest intersection hostname -// - If the listener hostname is empty, return the HTTPRoute hostname -// - If the listener hostname is a wildcard of the HTTPRoute hostname, return the HTTPRoute hostname -// - If the HTTPRoute hostname is a wildcard of the listener hostname, return the listener hostname -// - If the HTTPRoute hostname and listener hostname are the same, return it -// - If none of the above, return an empty string -func getMinimumHostnameIntersection(gateways []RouteParentRefContext, hostname gatewayv1.Hostname) gatewayv1.Hostname { - for _, gateway := range gateways { - // Only the listeners the parentRef actually matched: a parentRef selecting - // by port alone leaves ListenerName empty, and intersecting against the - // whole spec would keep a hostname served only by a listener on another - // port. - for _, listener := range listenersForGatewayContext(gateway) { - if listener.Hostname == nil || *listener.Hostname == "" { - return hostname - } - if HostnamesMatch(string(*listener.Hostname), string(hostname)) { - return hostname - } - if HostnamesMatch(string(hostname), string(*listener.Hostname)) { - return *listener.Hostname - } - } - } - - return "" -} - // isListenerHostnameEffective checks if a listener can specify a hostname to match the hostname in the request // Basically, check if the listener uses HTTP, HTTPS, or TLS protocol func isListenerHostnameEffective(listener gatewayv1.Listener) bool { diff --git a/internal/controller/utils/hostname.go b/internal/controller/utils/hostname.go new file mode 100644 index 000000000..18b7bd4f3 --- /dev/null +++ b/internal/controller/utils/hostname.go @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "strings" + + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func HostnamesIntersect(a, b string) bool { + return HostnamesMatch(a, b) || HostnamesMatch(b, a) +} + +// HostnamesMatch checks that the hostnameB matches the hostnameA. HostnameA is treated as mask +// to be checked against the hostnameB. +func HostnamesMatch(hostnameA, hostnameB string) bool { + // the hostnames are in the form of "foo.bar.com"; split them + // in a slice of substrings + hostnameALabels := strings.Split(hostnameA, ".") + hostnameBLabels := strings.Split(hostnameB, ".") + + var a, b int + var wildcard bool + + // iterate over the parts of both the hostnames + for a, b = 0, 0; a < len(hostnameALabels) && b < len(hostnameBLabels); a, b = a+1, b+1 { + var matchFound bool + + // if the current part of B is a wildcard, we need to find the first + // A part that matches with the following B part + if wildcard { + for ; b < len(hostnameBLabels); b++ { + if hostnameALabels[a] == hostnameBLabels[b] { + matchFound = true + break + } + } + } + + // if no match was found, the hostnames don't match + if wildcard && !matchFound { + return false + } + + // check if at least on of the current parts are a wildcard; if so, continue + if hostnameALabels[a] == "*" { + wildcard = true + continue + } + // reset the wildcard variables + wildcard = false + + // if the current a part is different from the b part, the hostnames are incompatible + if hostnameALabels[a] != hostnameBLabels[b] { + return false + } + } + return len(hostnameBLabels)-b == len(hostnameALabels)-a +} + +// NarrowHostname returns every intersection of routeHostname with the hostnames +// of the given listeners. +// +// A listener carrying no hostname, or one whose hostname is a mask covering +// routeHostname, accepts routeHostname whole and is then the only intersection. +// Otherwise each listener hostname that routeHostname covers is an intersection +// of its own: keeping only the first - which is what looking up a single +// "smallest intersection" does - silently drops the traffic the remaining +// listeners accept. +// +// An empty result means the listeners accept none of routeHostname. +func NarrowHostname(listeners []gatewayv1.Listener, routeHostname gatewayv1.Hostname) []gatewayv1.Hostname { + narrowed := make([]gatewayv1.Hostname, 0, len(listeners)) + for _, listener := range listeners { + if listener.Hostname == nil || *listener.Hostname == "" || + HostnamesMatch(string(*listener.Hostname), string(routeHostname)) { + return []gatewayv1.Hostname{routeHostname} + } + if HostnamesMatch(string(routeHostname), string(*listener.Hostname)) { + narrowed = append(narrowed, *listener.Hostname) + } + } + return narrowed +} + +// UnionHostnames returns the union of the listener hostnames. The second return +// value reports that a listener carries no hostname of its own and therefore +// accepts every hostname, which makes the union meaningless. +func UnionHostnames(listeners []gatewayv1.Listener) ([]gatewayv1.Hostname, bool) { + hostnames := make([]gatewayv1.Hostname, 0, len(listeners)) + for _, listener := range listeners { + if listener.Hostname == nil || *listener.Hostname == "" { + return nil, true + } + hostnames = append(hostnames, *listener.Hostname) + } + return hostnames, false +} + +// DedupHostnames drops repeated hostnames, keeping the first occurrence. +// Listeners are aggregated across Gateways, so the same hostname can be +// contributed more than once. +func DedupHostnames(hostnames []gatewayv1.Hostname) []gatewayv1.Hostname { + deduped := make([]gatewayv1.Hostname, 0, len(hostnames)) + seen := make(map[gatewayv1.Hostname]struct{}, len(hostnames)) + for _, hostname := range hostnames { + if _, ok := seen[hostname]; ok { + continue + } + seen[hostname] = struct{}{} + deduped = append(deduped, hostname) + } + return deduped +} From 34399bc7366247eacba22be8d0f91a5131c9128d Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Mon, 12 Oct 2026 11:31:47 +0800 Subject: [PATCH 17/18] docs: narrow the cross-Gateway tls.mode guarantee The ProtocolConflict check is per Gateway and tlsPassthroughOnPort sees only the listeners the route itself matched, so the fallback to Terminate covers one route attached to disagreeing parents - not two routes attached to one disagreeing Gateway each. Those are translated for their own listener's mode, and only one of the two is the mode the stream listen was configured with. The controller does not own that listen, so it can neither detect nor resolve the conflict; say so instead of promising a fallback that does not happen. --- docs/en/latest/concepts/gateway-api.md | 2 +- internal/adc/translator/tlsroute.go | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/en/latest/concepts/gateway-api.md b/docs/en/latest/concepts/gateway-api.md index 580436aca..c78db3df6 100644 --- a/docs/en/latest/concepts/gateway-api.md +++ b/docs/en/latest/concepts/gateway-api.md @@ -85,6 +85,6 @@ The fields below are specified in the Gateway API specification but are either p | `spec.listeners[].port` | Not supported* | The configuration is required but ignored. This is due to limitations in the data plane: it cannot dynamically open new ports. Since the Ingress Controller does not manage the data plane deployment, it cannot automatically update the configuration or restart the data plane to apply port changes. | | `spec.listeners[].tls.certificateRefs[].group` | Partially supported | Only `""` is supported; other group values cause validation failure. | | `spec.listeners[].tls.certificateRefs[].kind` | Partially supported | Only `Secret` is supported. | -| `spec.listeners[].tls.mode` | Partially supported | `Terminate` and `Passthrough` are both implemented. A `Passthrough` listener needs APISIX to be listening on that port with [`tls_passthrough`](https://apisix.apache.org/docs/apisix/stream-proxy/); the controller cannot open data plane ports itself. A single port cannot mix the two modes. Within one Gateway, listeners that disagree on `tls.mode` for one port are reported `Accepted=False` with `ProtocolConflict` and carry no routes. Two Gateways that disagree about the same port are each valid on their own, so no conflict is reported there; the routes on that port fall back to `Terminate`, because a physical stream listen has a single mode. | +| `spec.listeners[].tls.mode` | Partially supported | `Terminate` and `Passthrough` are both implemented. A `Passthrough` listener needs APISIX to be listening on that port with [`tls_passthrough`](https://apisix.apache.org/docs/apisix/stream-proxy/); the controller cannot open data plane ports itself. A single port cannot mix the two modes. Within one Gateway, listeners that disagree on `tls.mode` for one port are reported `Accepted=False` with `ProtocolConflict` and carry no routes, and a route attached to two Gateways that disagree about the same port falls back to `Terminate`, because a physical stream listen has a single mode. Two Gateways that disagree are each valid on their own, so when different routes attach to each of them no conflict is reported and every route is translated for the mode its own listener declares — only one of which the listen can honour. The controller does not configure the stream listen, so it cannot detect or resolve that: declare one `tls.mode` per port across Gateways that share a data plane. | | `spec.listeners[].tls.frontendValidation` | Partially supported | Enables downstream (client) mTLS. `caCertificateRefs` may reference a `ConfigMap` (Gateway API Core support) or a `Secret` (implementation-specific) holding the CA certificate under the `ca.crt` key; clients are then required to present a certificate signed by one of the referenced CAs. | | `spec.addresses` | Not supported | Controller does not read or act on `spec.addresses`. | diff --git a/internal/adc/translator/tlsroute.go b/internal/adc/translator/tlsroute.go index 4810dd1e3..7d584f51c 100644 --- a/internal/adc/translator/tlsroute.go +++ b/internal/adc/translator/tlsroute.go @@ -237,11 +237,16 @@ func hostnamesToSNIs(hostnames []gatewayv1.Hostname) []string { // port 0 means the StreamRoute carries no server_port match, so every matched // listener applies. // -// Every matched TLS listener on the port has to agree. Within one Gateway a -// port carrying both modes is already reported ProtocolConflict and attaches -// no routes; across Gateways the combination is unrepresentable, since the -// physical stream listen has a single mode - so the terminating behaviour wins -// rather than a guess. +// Every TLS listener this route matched on the port has to agree. Within one +// Gateway a port carrying both modes is already reported ProtocolConflict and +// attaches no routes; a route attached to two Gateways that disagree falls back +// to terminating rather than to a guess, since the physical stream listen has a +// single mode. +// +// Two *different* routes attached to two disagreeing Gateways are each +// translated for the mode their own listener declares, and only one of those is +// the mode the listen was configured with. The controller does not configure the +// stream listen, so it can neither detect that nor resolve it. func tlsPassthroughOnPort(listeners []gatewayv1.Listener, port int32) bool { matched := false for _, listener := range listeners { From 5c7cb20339eaec38513a01fda0ffdb6110dc1d88 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Tue, 13 Oct 2026 08:55:57 +0800 Subject: [PATCH 18/18] test: rebuild the TLS passthrough forward on a failed dial A single handshake check is not enough: APISIX standalone workers apply the pushed config independently, so a later connection can still be reset by a worker that has not loaded the stream route yet. The reset kills kubectl port-forward and every retry then hits "connection refused". Verify the tunnel with a handshake on each dial and rebuild it on failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/e2e/scaffold/scaffold.go | 180 +++++++++++++++++++++++----------- 1 file changed, 122 insertions(+), 58 deletions(-) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 580d65520..20f19289d 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -27,6 +27,7 @@ import ( "net/url" "os" "strings" + "sync" "time" "github.com/api7/gopkg/pkg/log" @@ -324,24 +325,66 @@ const tlsPassthroughPortName = "tls-passthrough" // Verifying is the point: the gateway holds no certificate for a passthrough // listener, so a chain that validates against the backend's own CA can only // have come from the backend itself. -// tlsPassthroughTunnel forwards the data plane's tls_passthrough stream port and -// hands back a tunnel that has been proven to carry a handshake end to end. +func (s *Scaffold) NewAPISIXClientWithTLSPassthrough(sni string, caCert []byte) *httpexpect.Expect { + pool := x509.NewCertPool() + Expect(pool.AppendCertsFromPEM(caCert)).To(BeTrue(), "parsing CA certificate") + + forward := s.newTLSPassthroughForward(&tls.Config{ServerName: sni, RootCAs: pool}) + + u := url.URL{ + Scheme: apiv2.SchemeHTTPS, + Host: forward.endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{ + // Every request dials afresh, so a forwarder that died since the + // last one is noticed and replaced rather than reused. + DisableKeepAlives: true, + DialTLSContext: forward.dialTLSContext, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(s.GinkgoT), + ), + }) +} + +// tlsPassthroughForward owns the port-forward to the data plane's +// tls_passthrough stream port and replaces it whenever it stops working. // // Two things make this port different from the others the scaffold forwards. // -// A tls_passthrough listen prereads the ClientHello to pick a stream route, so a -// connection that carries no TLS gets reset. kubectl port-forward treats that -// reset as fatal and exits, taking the local listener with it - every later dial -// then gets "connection refused" rather than anything describing the cause. So -// the tunnel is verified with a real handshake for sni, never a bare TCP dial, -// and a failed attempt rebuilds it rather than reusing a forwarder that may -// already be gone. +// A tls_passthrough listen prereads the ClientHello to pick a stream route, and +// resets a connection no route matches. The port-forward treats that reset as +// fatal and exits, taking the local listener with it - every later dial then +// gets "connection refused" rather than anything describing the cause. A +// handshake that succeeded once does not rule this out: the data plane workers +// load a new stream route independently, so the next connection can still land +// on one that has not. So the forward is verified with a real handshake for the +// SNI, never a bare TCP dial, and any failed dial rebuilds it instead of +// leaving every later retry to hit a forwarder that is already gone. // // It is also forwarded on demand rather than alongside the HTTP tunnels when the // data plane is deployed: APISIX opens its stream listens later than its HTTP // ones, and only this one spec dials this port, so the rest of the suite would // pay for a forward it never uses. -func (s *Scaffold) tlsPassthroughTunnel(sni string, pool *x509.CertPool) Tunnel { +type tlsPassthroughForward struct { + s *Scaffold + opts k8s.KubectlOptions + service string + port int + tlsConfig *tls.Config + + mu sync.Mutex + tunnel *k8s.Tunnel +} + +func (s *Scaffold) newTLSPassthroughForward(tlsConfig *tls.Config) *tlsPassthroughForward { svc := s.dataplaneService Expect(svc).NotTo(BeNil(), "data plane service") @@ -356,69 +399,90 @@ func (s *Scaffold) tlsPassthroughTunnel(sni string, pool *x509.CertPool) Tunnel // Copy the scaffold's options rather than building fresh ones: a run that // configures a kubeconfig or context would otherwise be ignored here while // every other dataplane tunnel honours it. - kubectlOpts := *s.kubectlOptions - kubectlOpts.Namespace = svc.Namespace + f := &tlsPassthroughForward{ + s: s, + opts: *s.kubectlOptions, + service: svc.Name, + port: port, + tlsConfig: tlsConfig, + } + f.opts.Namespace = svc.Namespace - var tunnel *k8s.Tunnel Eventually(func() error { - t := k8s.NewTunnel(&kubectlOpts, k8s.ResourceTypeService, svc.Name, 0, port) - if err := t.ForwardPortE(s.t); err != nil { - return err - } - // ForwardPortE returns before the forward has proxied anything, so the - // handshake is what decides whether this tunnel works. - conn, err := tls.DialWithDialer( - &net.Dialer{Timeout: 10 * time.Second}, - "tcp", t.Endpoint(), - &tls.Config{ServerName: sni, RootCAs: pool}, - ) + t, err := f.open() if err != nil { - t.Close() return err } - _ = conn.Close() - tunnel = t + f.replace(t) return nil }).WithTimeout(time.Minute*2).WithPolling(time.Second*2). Should(Succeed(), "forwarding the tls passthrough port") + return f +} - // Tunnels.Close only knows about the tunnel currently stored, so a spec - // that builds a second client would otherwise leave the first - // kubectl port-forward running until the process exits. - if previous := s.apisixTunnels.TLSPassthrough; previous != nil { +// open forwards the port and proves the forward with a handshake, since +// ForwardPortE returns before the forward has proxied anything. +func (f *tlsPassthroughForward) open() (*k8s.Tunnel, error) { + t := k8s.NewTunnel(&f.opts, k8s.ResourceTypeService, f.service, 0, f.port) + if err := t.ForwardPortE(f.s.t); err != nil { + return nil, err + } + conn, err := f.handshake(context.Background(), t.Endpoint()) + if err != nil { + t.Close() + return nil, err + } + _ = conn.Close() + return t, nil +} + +func (f *tlsPassthroughForward) handshake(ctx context.Context, endpoint string) (*tls.Conn, error) { + d := &tls.Dialer{ + NetDialer: &net.Dialer{Timeout: 10 * time.Second}, + Config: f.tlsConfig.Clone(), + } + conn, err := d.DialContext(ctx, "tcp", endpoint) + if err != nil { + return nil, err + } + return conn.(*tls.Conn), nil +} + +// replace installs t as the current forward and closes the one it supersedes. +// Tunnels.Close only knows about the tunnel stored there, so a superseded +// forward would otherwise keep running until the process exits. +func (f *tlsPassthroughForward) replace(t *k8s.Tunnel) { + if f.tunnel != nil { + f.tunnel.Close() + } else if previous := f.s.apisixTunnels.TLSPassthrough; previous != nil { previous.Close() } - s.apisixTunnels.TLSPassthrough = tunnel - return tunnel + f.tunnel = t + f.s.apisixTunnels.TLSPassthrough = t } -func (s *Scaffold) NewAPISIXClientWithTLSPassthrough(sni string, caCert []byte) *httpexpect.Expect { - pool := x509.NewCertPool() - Expect(pool.AppendCertsFromPEM(caCert)).To(BeTrue(), "parsing CA certificate") +func (f *tlsPassthroughForward) endpoint() string { + f.mu.Lock() + defer f.mu.Unlock() + return f.tunnel.Endpoint() +} - tunnel := s.tlsPassthroughTunnel(sni, pool) +// dialTLSContext ignores addr: the request URL names whichever local port the +// forward had when the client was built, and that port changes on a rebuild. +func (f *tlsPassthroughForward) dialTLSContext(ctx context.Context, _, _ string) (net.Conn, error) { + f.mu.Lock() + defer f.mu.Unlock() - u := url.URL{ - Scheme: apiv2.SchemeHTTPS, - Host: tunnel.Endpoint(), + conn, err := f.handshake(ctx, f.tunnel.Endpoint()) + if err == nil { + return conn, nil } - return httpexpect.WithConfig(httpexpect.Config{ - BaseURL: u.String(), - Client: &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: pool, - ServerName: sni, - }, - }, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - }, - Reporter: httpexpect.NewAssertReporter( - httpexpect.NewAssertReporter(s.GinkgoT), - ), - }) + t, rebuildErr := f.open() + if rebuildErr != nil { + return nil, fmt.Errorf("%w; rebuilding the tls passthrough forward: %v", err, rebuildErr) + } + f.replace(t) + return f.handshake(ctx, t.Endpoint()) } func (s *Scaffold) NewAPISIXClientWithTLSProxy(host string) *httpexpect.Expect { @@ -595,7 +659,7 @@ func (s *Scaffold) createDataplaneTunnels( } // The TLS passthrough tunnel is deliberately not forwarded here. See - // tlsPassthroughTunnel. + // tlsPassthroughForward. return tunnels, nil }