From f2ec1b5e834ba09cc5eb6ef410c70d5b9c3e9d4a Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Sun, 18 Oct 2026 17:34:28 +0200 Subject: [PATCH 01/10] feat(controlplane): serve gRPC, gRPC-Web and HTTP from one port Implements the control plane part of the single-endpoint design (docs/specs/issue-3628-single-endpoint.md, refs #3628). The control plane now accepts every connection on its current gRPC port. A small connection classifier (pkg/httpgrpcmux) looks at the start of each new connection without consuming it and gives it to the gRPC server or to the HTTP server that runs today, so the middleware chain, the authentication and the gRPC-Web wrapper do not change. A connection whose first request carries a native gRPC content type goes to the gRPC server; HTTP/1.1, HTTP/2 without gRPC and gRPC-Web go to the HTTP server, which now also speaks plain HTTP/2 (h2c). A client that stalls waiting for the server to speak first is unblocked with an empty SETTINGS frame after a short grace period, whose acknowledgement the classifier consumes. When TLS is configured, the listener terminates it before the classifier and offers HTTP/2 and HTTP/1.1 through ALPN, so one certificate covers all protocols. The HTTP-only port stays on by default for the deprecation period, behind the new server.legacy_http_listener flag, and logs a deprecation warning at startup. The Helm chart needs no change and existing CLI configurations keep working. Assisted-by: Claude Code Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: ses_eb07a1f40ffedmc6fUbJZvStdv --- app/controlplane/cmd/main.go | 37 +- app/controlplane/cmd/main_test.go | 57 + app/controlplane/cmd/wire_gen.go | 46 +- .../conf/controlplane/config/v1/conf.pb.go | 66 +- .../conf/controlplane/config/v1/conf.proto | 12 + app/controlplane/internal/server/apimux.go | 111 ++ .../internal/server/apimux_test.go | 393 +++++++ app/controlplane/internal/server/grpc.go | 99 +- app/controlplane/internal/server/grpc_test.go | 18 +- app/controlplane/internal/server/http.go | 61 +- app/controlplane/internal/server/otel_test.go | 14 +- app/controlplane/internal/server/server.go | 1 + go.mod | 2 +- pkg/httpgrpcmux/mux.go | 989 ++++++++++++++++++ pkg/httpgrpcmux/mux_test.go | 938 +++++++++++++++++ 15 files changed, 2777 insertions(+), 67 deletions(-) create mode 100644 app/controlplane/internal/server/apimux.go create mode 100644 app/controlplane/internal/server/apimux_test.go create mode 100644 pkg/httpgrpcmux/mux.go create mode 100644 pkg/httpgrpcmux/mux_test.go diff --git a/app/controlplane/cmd/main.go b/app/controlplane/cmd/main.go index bb83fa78f..344f35bd7 100644 --- a/app/controlplane/cmd/main.go +++ b/app/controlplane/cmd/main.go @@ -17,7 +17,9 @@ package main import ( "context" + "errors" "math/rand" + "net" "os" "time" @@ -32,6 +34,7 @@ import ( "github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1" "github.com/chainloop-dev/chainloop/pkg/credentials" "github.com/chainloop-dev/chainloop/pkg/credentials/manager" + "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" "github.com/chainloop-dev/chainloop/pkg/natsconn" "github.com/chainloop-dev/chainloop/pkg/servicelogger" @@ -61,11 +64,41 @@ func init() { flag.StringVar(&flagconf, "conf", "../configs", "config path, eg: -conf config.yaml") } -func newApp(logger log.Logger, gs *grpc.Server, hs *http.Server, ms *server.HTTPMetricsServer, profilerSvc *server.HTTPProfilerServer, +// cleanShutdownServer tolerates the shutdown race with the mux: kratos starts +// and stops servers concurrently, so the mux may close its virtual listeners +// before the gRPC/HTTP servers flag their own shutdown. Their Start then +// returns net.ErrClosed even though the shutdown is orderly — kratos would +// surface it and main would panic on every SIGTERM the mux wins. The race is +// only tolerated when the mux is actually stopping: a fatal accept-loop +// failure surfaces as an error, so the process restarts instead of serving a +// dead single port. +type cleanShutdownServer struct { + transport.Server + mux *httpgrpcmux.Mux +} + +func (s *cleanShutdownServer) Start(ctx context.Context) error { + err := s.Server.Start(ctx) + if errors.Is(err, net.ErrClosed) && s.mux.Stopping() { + return nil + } + return err +} + +func newApp(logger log.Logger, am *httpgrpcmux.Mux, gs []*grpc.Server, hs []*http.Server, ms *server.HTTPMetricsServer, profilerSvc *server.HTTPProfilerServer, expirer *biz.WorkflowRunExpirerUseCase, plugins sdk.AvailablePlugins, userAccessSyncer *biz.UserAccessSyncerUseCase, casBackendChecker *biz.CASBackendChecker, apiTokenStaleRevoker *biz.APITokenStaleRevoker, loginHandoffs *biz.LoginHandoffUseCase, cfg *conf.Bootstrap) *app { - servers := []transport.Server{gs, hs, ms} + servers := make([]transport.Server, 0, len(gs)+len(hs)+3) + // The mux owns the single-port listener; kratos starts servers concurrently, + // so no ordering is required — the virtual listeners work before Start. + servers = append(servers, am, ms) + for _, g := range gs { + servers = append(servers, &cleanShutdownServer{Server: g, mux: am}) + } + for _, h := range hs { + servers = append(servers, &cleanShutdownServer{Server: h, mux: am}) + } if cfg.EnableProfiler { servers = append(servers, profilerSvc) } diff --git a/app/controlplane/cmd/main_test.go b/app/controlplane/cmd/main_test.go index 243ca7105..5b4b09ffa 100644 --- a/app/controlplane/cmd/main_test.go +++ b/app/controlplane/cmd/main_test.go @@ -15,15 +15,20 @@ package main import ( + "context" "fmt" + "net" "os" "path/filepath" "testing" "time" conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1" + "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" "github.com/go-kratos/kratos/v2/config" "github.com/go-kratos/kratos/v2/config/file" + kratosgrpc "github.com/go-kratos/kratos/v2/transport/grpc" + kratoshttp "github.com/go-kratos/kratos/v2/transport/http" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/durationpb" @@ -216,3 +221,55 @@ func loadBootstrap(t *testing.T, path string) (*conf.Bootstrap, error) { return &bootstrap, nil } + +// TestServerToleratesMuxShutdown proves the shutdown-race fix: when the mux +// stops first, the servers' Start returns net.ErrClosed — an orderly +// shutdown that must not surface as an error (kratos would propagate it and +// main would panic on SIGTERM). +func TestServerToleratesMuxShutdown(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + m := httpgrpcmux.New(l, nil) + require.NoError(t, m.Start(context.Background())) + + // A kratos HTTP server on the mux's HTTP side, and a kratos gRPC server + // on its gRPC side: both fed by virtual listeners. + hs := kratoshttp.NewServer(kratoshttp.Listener(m.HTTP())) + gs := kratosgrpc.NewServer(kratosgrpc.Listener(m.GRPC())) + + errs := make(chan error, 2) + go func() { errs <- (&cleanShutdownServer{Server: hs, mux: m}).Start(context.Background()) }() + go func() { errs <- (&cleanShutdownServer{Server: gs, mux: m}).Start(context.Background()) }() + + // Give the servers time to enter their accept loops, then close the mux + // first — exactly the SIGTERM ordering that used to panic. + time.Sleep(100 * time.Millisecond) + m.Close() + + for i := 0; i < 2; i++ { + select { + case err := <-errs: + require.NoError(t, err, "an orderly mux shutdown must not surface as a server error") + case <-time.After(5 * time.Second): + require.Fail(t, "server Start did not return after the mux stopped") + } + } +} + +// TestCleanShutdownServerPropagatesRealErrors proves the decorator only +// tolerates the mux shutdown: a real failure (bind conflict) surfaces as an +// error, so the process still crashes on genuine startup problems. +func TestCleanShutdownServerPropagatesRealErrors(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer l.Close() + + decorated := &cleanShutdownServer{ + Server: kratoshttp.NewServer(kratoshttp.Address(l.Addr().String())), + mux: httpgrpcmux.New(l, nil), + } + err = decorated.Start(context.Background()) + require.Error(t, err) + assert.NotErrorIs(t, err, net.ErrClosed) +} diff --git a/app/controlplane/cmd/wire_gen.go b/app/controlplane/cmd/wire_gen.go index 9da2a1733..74a6afb14 100644 --- a/app/controlplane/cmd/wire_gen.go +++ b/app/controlplane/cmd/wire_gen.go @@ -35,20 +35,28 @@ import ( // Injectors from wire.go: func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWriter credentials.ReaderWriter, logger log.Logger, availablePlugins sdk.AvailablePlugins) (*app, func(), error) { + confServer := bootstrap.Server + mux, cleanup, err := server.NewAPIMux(confServer, logger) + if err != nil { + return nil, nil, err + } config := authzConfig() casbinEnforcer, err := authz.NewCasbinEnforcer(config) if err != nil { + cleanup() return nil, nil, err } confData := bootstrap.Data data_Database := confData.Database databaseConfig := newDataConf(data_Database) - tracerProvider, cleanup, err := server.NewTracerProvider(bootstrap, logger) + tracerProvider, cleanup2, err := server.NewTracerProvider(bootstrap, logger) if err != nil { + cleanup() return nil, nil, err } - dataData, cleanup2, err := data.NewData(databaseConfig, tracerProvider, logger) + dataData, cleanup3, err := data.NewData(databaseConfig, tracerProvider, logger) if err != nil { + cleanup2() cleanup() return nil, nil, err } @@ -65,14 +73,16 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr casServerDefaultOpts := newCASServerOptions(bootstrap_CASServer) bootstrap_NatsServer := bootstrap.NatsServer natsconnConfig := newNatsConfig(bootstrap_NatsServer) - reloadableConnection, cleanup3, err := natsconn.New(natsconnConfig, logger) + reloadableConnection, cleanup4, err := natsconn.New(natsconnConfig, logger) if err != nil { + cleanup3() cleanup2() cleanup() return nil, nil, err } auditLogPublisher, err := auditor.NewAuditLogPublisher(contextContext, reloadableConnection, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -81,6 +91,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr auditorUseCase := biz.NewAuditorUseCase(auditLogPublisher, logger) casBackendUseCase, err := biz.NewCASBackendUseCase(casBackendRepo, readerWriter, providers, casServerDefaultOpts, auditorUseCase, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -117,6 +128,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr auth := bootstrap.Auth casCredentialsUseCase, err := biz.NewCASCredentialsUseCase(auth) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -127,6 +139,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr referrerRepo := data.NewReferrerRepo(dataData, workflowRepo, logger) referrerUseCase, err := biz.NewReferrerUseCase(referrerRepo, workflowRepo, membershipUseCase, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -135,6 +148,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr apiTokenJWTConfig := newJWTConfig(auth) apiTokenUseCase, err := biz.NewAPITokenUseCase(apiTokenRepo, apiTokenJWTConfig, authzUseCase, organizationUseCase, auditorUseCase, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -146,6 +160,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr v4 := newPolicyProviderConfig(v3) registry, err := policies.NewRegistry(logger, v4...) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -155,6 +170,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr workflowUseCase := biz.NewWorkflowUsecase(workflowRepo, projectsRepo, workflowContractUseCase, auditorUseCase, membershipUseCase, organizationRepo, logger) cache, err := newMembershipsCache(logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -162,6 +178,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } cacheCache, err := newClaimsCache(logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -170,6 +187,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr orgInvitationRepo := data.NewOrgInvitation(dataData, logger) orgInvitationUseCase, err := biz.NewOrgInvitationUseCase(orgInvitationRepo, membershipRepo, userRepo, auditorUseCase, groupRepo, projectsRepo, authzUseCase, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -183,6 +201,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr loginHandoffUseCase := biz.NewLoginHandoffUseCase(loginHandoffRepo, logger) authService, err := service.NewAuthService(userUseCase, organizationUseCase, membershipUseCase, orgInvitationUseCase, loginHandoffUseCase, auth, bootstrap, auditorUseCase, v5...) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -191,6 +210,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr workflowRunRepo := data.NewWorkflowRunRepo(dataData, logger) signingUseCase, err := biz.NewChainloopSigningUseCase(bootstrap, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -198,6 +218,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } attestationbundleCache, err := attestationbundle.New(contextContext, reloadableConnection, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -219,6 +240,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } workflowRunUseCase, err := biz.NewWorkflowRunUseCase(workflowRunUseCaseOpts) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -228,6 +250,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr projectVersionUseCase := biz.NewProjectVersionUseCase(projectVersionRepo, auditorUseCase, logger) policyevalbundleCache, err := policyevalbundle.New(contextContext, reloadableConnection, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -253,6 +276,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr orgMetricsRepo := data.NewOrgMetricsRepo(dataData, logger) orgMetricsUseCase, err := biz.NewOrgMetricsUseCase(orgMetricsRepo, organizationRepo, workflowUseCase, logger) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -291,6 +315,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr casBackendService := service.NewCASBackendService(casBackendUseCase, providers, v5...) casRedirectService, err := service.NewCASRedirectService(casMappingUseCase, casCredentialsUseCase, bootstrap_CASServer, v5...) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -302,6 +327,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr attestationStateRepo := data.NewAttestationStateRepo(dataData, logger) attestationStateUseCase, err := biz.NewAttestationStateUseCase(attestationStateRepo, workflowRunRepo) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -319,11 +345,11 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr prometheusService := service.NewPrometheusService(organizationUseCase, prometheusUseCase, v5...) groupService := service.NewGroupService(groupUseCase, v5...) projectService := service.NewProjectService(v5...) - confServer := bootstrap.Server federatedAuthentication := bootstrap.FederatedAuthentication operationAuthorizationProvider := bootstrap.OperationAuthorizationProvider validator, err := newProtoValidator() if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -366,6 +392,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr ProjectSvc: projectService, Logger: logger, ServerConfig: confServer, + APIMux: mux, AuthConfig: auth, FederatedConfig: federatedAuthentication, OperationAuthConfig: operationAuthorizationProvider, @@ -373,15 +400,17 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr Credentials: readerWriter, Validator: validator, } - grpcServer, err := server.NewGRPCServer(opts) + v7, err := server.NewGRPCServer(opts) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() return nil, nil, err } - httpServer, err := server.NewHTTPServer(opts, grpcServer) + v8, err := server.NewHTTPServer(opts, v7) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -389,6 +418,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } httpMetricsServer, err := server.NewHTTPMetricsServer(opts) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -396,6 +426,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } httpProfilerServer, err := server.NewHTTPProfilerServer(opts) if err != nil { + cleanup4() cleanup3() cleanup2() cleanup() @@ -405,8 +436,9 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr distributedLock := data.NewPostgresLock(dataData, logger) casBackendChecker := biz.NewCASBackendChecker(logger, casBackendRepo, casBackendUseCase, distributedLock) apiTokenStaleRevoker := biz.NewAPITokenStaleRevoker(organizationRepo, apiTokenRepo, apiTokenUseCase, logger) - mainApp := newApp(logger, grpcServer, httpServer, httpMetricsServer, httpProfilerServer, workflowRunExpirerUseCase, availablePlugins, userAccessSyncerUseCase, casBackendChecker, apiTokenStaleRevoker, loginHandoffUseCase, bootstrap) + mainApp := newApp(logger, mux, v7, v8, httpMetricsServer, httpProfilerServer, workflowRunExpirerUseCase, availablePlugins, userAccessSyncerUseCase, casBackendChecker, apiTokenStaleRevoker, loginHandoffUseCase, bootstrap) return mainApp, func() { + cleanup4() cleanup3() cleanup2() cleanup() diff --git a/app/controlplane/internal/conf/controlplane/config/v1/conf.pb.go b/app/controlplane/internal/conf/controlplane/config/v1/conf.pb.go index 16c083973..43cf11e54 100644 --- a/app/controlplane/internal/conf/controlplane/config/v1/conf.pb.go +++ b/app/controlplane/internal/conf/controlplane/config/v1/conf.pb.go @@ -627,9 +627,15 @@ type Server struct { Http *Server_HTTP `protobuf:"bytes,1,opt,name=http,proto3" json:"http,omitempty"` Grpc *Server_GRPC `protobuf:"bytes,2,opt,name=grpc,proto3" json:"grpc,omitempty"` // HTTPMetrics defines the HTTP server that exposes prometheus metrics - HttpMetrics *Server_HTTP `protobuf:"bytes,3,opt,name=http_metrics,json=httpMetrics,proto3" json:"http_metrics,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + HttpMetrics *Server_HTTP `protobuf:"bytes,3,opt,name=http_metrics,json=httpMetrics,proto3" json:"http_metrics,omitempty"` + // Legacy gRPC-only listener that serves native gRPC on its own port, next + // to the single port that already serves gRPC, gRPC-Web and HTTP. It exists + // for the deprecation period of docs/specs/issue-3628-single-endpoint.md. + // Defaults to true when unset. Set it to false to listen on the single port + // only, plus the internal metrics and profiler ports. + LegacyGrpcListener *bool `protobuf:"varint,4,opt,name=legacy_grpc_listener,json=legacyGrpcListener,proto3,oneof" json:"legacy_grpc_listener,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Server) Reset() { @@ -683,6 +689,13 @@ func (x *Server) GetHttpMetrics() *Server_HTTP { return nil } +func (x *Server) GetLegacyGrpcListener() bool { + if x != nil && x.LegacyGrpcListener != nil { + return *x.LegacyGrpcListener + } + return false +} + type Data struct { state protoimpl.MessageState `protogen:"open.v1"` Database *Data_Database `protobuf:"bytes,1,opt,name=database,proto3" json:"database,omitempty"` @@ -1363,8 +1376,14 @@ type Server_HTTP struct { Addr string `protobuf:"bytes,2,opt,name=addr,proto3" json:"addr,omitempty"` // In the form of [scheme]://[host] i.e https://instance.chainloop.dev // Optional - ExternalUrl string `protobuf:"bytes,4,opt,name=external_url,json=externalUrl,proto3" json:"external_url,omitempty"` - Timeout *durationpb.Duration `protobuf:"bytes,5,opt,name=timeout,proto3" json:"timeout,omitempty"` + ExternalUrl string `protobuf:"bytes,4,opt,name=external_url,json=externalUrl,proto3" json:"external_url,omitempty"` + Timeout *durationpb.Duration `protobuf:"bytes,5,opt,name=timeout,proto3" json:"timeout,omitempty"` + // TLS for the single port, which serves gRPC, gRPC-Web and HTTP: the + // listener negotiates HTTP/2 and HTTP/1.1 through ALPN. A partial + // configuration (only one of certificate/private_key) is a startup + // error. Note: server.http_metrics reuses this message type but does + // not support TLS. + TlsConfig *Server_TLS `protobuf:"bytes,6,opt,name=tls_config,json=tlsConfig,proto3" json:"tls_config,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1427,6 +1446,13 @@ func (x *Server_HTTP) GetTimeout() *durationpb.Duration { return nil } +func (x *Server_HTTP) GetTlsConfig() *Server_TLS { + if x != nil { + return x.TlsConfig + } + return nil +} + type Server_TLS struct { state protoimpl.MessageState `protogen:"open.v1"` // path to certificate and private key @@ -1941,16 +1967,19 @@ const file_controlplane_config_v1_conf_proto_rawDesc = "" + "\rname.dns-1123\x12:must contain only lowercase letters, numbers, and hyphens.\x1a/this.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')R\x04name\x12\x18\n" + "\adefault\x18\x02 \x01(\bR\adefault\x12\x16\n" + "\x04host\x18\x03 \x01(\tB\x02\x18\x01R\x04host\x12\x10\n" + - "\x03url\x18\x04 \x01(\tR\x03url\"\xfe\x04\n" + + "\x03url\x18\x04 \x01(\tR\x03url\"\x91\x06\n" + "\x06Server\x127\n" + "\x04http\x18\x01 \x01(\v2#.controlplane.config.v1.Server.HTTPR\x04http\x127\n" + "\x04grpc\x18\x02 \x01(\v2#.controlplane.config.v1.Server.GRPCR\x04grpc\x12F\n" + - "\fhttp_metrics\x18\x03 \x01(\v2#.controlplane.config.v1.Server.HTTPR\vhttpMetrics\x1a\x8c\x01\n" + + "\fhttp_metrics\x18\x03 \x01(\v2#.controlplane.config.v1.Server.HTTPR\vhttpMetrics\x125\n" + + "\x14legacy_grpc_listener\x18\x04 \x01(\bH\x00R\x12legacyGrpcListener\x88\x01\x01\x1a\xcf\x01\n" + "\x04HTTP\x12\x18\n" + "\anetwork\x18\x01 \x01(\tR\anetwork\x12\x12\n" + "\x04addr\x18\x02 \x01(\tR\x04addr\x12!\n" + "\fexternal_url\x18\x04 \x01(\tR\vexternalUrl\x123\n" + - "\atimeout\x18\x05 \x01(\v2\x19.google.protobuf.DurationR\atimeout\x1aH\n" + + "\atimeout\x18\x05 \x01(\v2\x19.google.protobuf.DurationR\atimeout\x12A\n" + + "\n" + + "tls_config\x18\x06 \x01(\v2\".controlplane.config.v1.Server.TLSR\ttlsConfig\x1aH\n" + "\x03TLS\x12 \n" + "\vcertificate\x18\x01 \x01(\tR\vcertificate\x12\x1f\n" + "\vprivate_key\x18\x02 \x01(\tR\n" + @@ -1961,7 +1990,8 @@ const file_controlplane_config_v1_conf_proto_rawDesc = "" + "\atimeout\x18\x03 \x01(\v2\x19.google.protobuf.DurationR\atimeout\x12A\n" + "\n" + "tls_config\x18\x04 \x01(\v2\".controlplane.config.v1.Server.TLSR\ttlsConfig\x12)\n" + - "\x11max_recv_msg_size\x18\x05 \x01(\x05R\x0emaxRecvMsgSize\"\x9a\x02\n" + + "\x11max_recv_msg_size\x18\x05 \x01(\x05R\x0emaxRecvMsgSizeB\x17\n" + + "\x15_legacy_grpc_listener\"\x9a\x02\n" + "\x04Data\x12A\n" + "\bdatabase\x18\x01 \x01(\v2%.controlplane.config.v1.Data.DatabaseR\bdatabase\x1a\xce\x01\n" + "\bDatabase\x12\x16\n" + @@ -2085,14 +2115,15 @@ var file_controlplane_config_v1_conf_proto_depIdxs = []int32{ 16, // 28: controlplane.config.v1.Bootstrap.Observability.tracing:type_name -> controlplane.config.v1.Bootstrap.Observability.Tracing 19, // 29: controlplane.config.v1.Bootstrap.CASServer.grpc:type_name -> controlplane.config.v1.Server.GRPC 26, // 30: controlplane.config.v1.Server.HTTP.timeout:type_name -> google.protobuf.Duration - 26, // 31: controlplane.config.v1.Server.GRPC.timeout:type_name -> google.protobuf.Duration - 18, // 32: controlplane.config.v1.Server.GRPC.tls_config:type_name -> controlplane.config.v1.Server.TLS - 26, // 33: controlplane.config.v1.Data.Database.max_conn_idle_time:type_name -> google.protobuf.Duration - 34, // [34:34] is the sub-list for method output_type - 34, // [34:34] is the sub-list for method input_type - 34, // [34:34] is the sub-list for extension type_name - 34, // [34:34] is the sub-list for extension extendee - 0, // [0:34] is the sub-list for field type_name + 18, // 31: controlplane.config.v1.Server.HTTP.tls_config:type_name -> controlplane.config.v1.Server.TLS + 26, // 32: controlplane.config.v1.Server.GRPC.timeout:type_name -> google.protobuf.Duration + 18, // 33: controlplane.config.v1.Server.GRPC.tls_config:type_name -> controlplane.config.v1.Server.TLS + 26, // 34: controlplane.config.v1.Data.Database.max_conn_idle_time:type_name -> google.protobuf.Duration + 35, // [35:35] is the sub-list for method output_type + 35, // [35:35] is the sub-list for method input_type + 35, // [35:35] is the sub-list for extension type_name + 35, // [35:35] is the sub-list for extension extendee + 0, // [0:35] is the sub-list for field type_name } func init() { file_controlplane_config_v1_conf_proto_init() } @@ -2101,6 +2132,7 @@ func file_controlplane_config_v1_conf_proto_init() { return } file_controlplane_config_v1_conf_proto_msgTypes[2].OneofWrappers = []any{} + file_controlplane_config_v1_conf_proto_msgTypes[6].OneofWrappers = []any{} file_controlplane_config_v1_conf_proto_msgTypes[8].OneofWrappers = []any{} file_controlplane_config_v1_conf_proto_msgTypes[10].OneofWrappers = []any{ (*CA_FileCa)(nil), diff --git a/app/controlplane/internal/conf/controlplane/config/v1/conf.proto b/app/controlplane/internal/conf/controlplane/config/v1/conf.proto index 36140d8a3..40e36c69e 100644 --- a/app/controlplane/internal/conf/controlplane/config/v1/conf.proto +++ b/app/controlplane/internal/conf/controlplane/config/v1/conf.proto @@ -226,6 +226,12 @@ message Server { // Optional string external_url = 4; google.protobuf.Duration timeout = 5; + // TLS for the single port, which serves gRPC, gRPC-Web and HTTP: the + // listener negotiates HTTP/2 and HTTP/1.1 through ALPN. A partial + // configuration (only one of certificate/private_key) is a startup + // error. Note: server.http_metrics reuses this message type but does + // not support TLS. + TLS tls_config = 6; } message TLS { @@ -247,6 +253,12 @@ message Server { GRPC grpc = 2; // HTTPMetrics defines the HTTP server that exposes prometheus metrics HTTP http_metrics = 3; + // Legacy gRPC-only listener that serves native gRPC on its own port, next + // to the single port that already serves gRPC, gRPC-Web and HTTP. It exists + // for the deprecation period of docs/specs/issue-3628-single-endpoint.md. + // Defaults to true when unset. Set it to false to listen on the single port + // only, plus the internal metrics and profiler ports. + optional bool legacy_grpc_listener = 4; } message Data { diff --git a/app/controlplane/internal/server/apimux.go b/app/controlplane/internal/server/apimux.go new file mode 100644 index 000000000..1783cae8a --- /dev/null +++ b/app/controlplane/internal/server/apimux.go @@ -0,0 +1,111 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "crypto/tls" + "fmt" + "net" + + conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1" + "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" + "github.com/go-kratos/kratos/v2/log" +) + +// NewAPIMux creates the connection multiplexer that serves gRPC, gRPC-Web and +// HTTP from the single HTTP port, as specified in +// docs/specs/issue-3628-single-endpoint.md. The mux owns the listener on the +// current HTTP address, so the HTTP-facing clients and hostnames keep working +// unchanged. When the operator configures TLS, the listener terminates it +// before the classifier and offers HTTP/2 and HTTP/1.1 through ALPN, so one +// certificate covers all protocols. +// +// The returned cleanup closes the mux and its listener; wire runs it when a +// later provider fails or on shutdown. +func NewAPIMux(c *conf.Server, logger log.Logger) (*httpgrpcmux.Mux, func(), error) { + httpCfg := c.GetHttp() + if httpCfg.GetAddr() == "" { + // An empty address would silently bind an ephemeral port, and the + // shared API would not be where operators expect it. + return nil, nil, fmt.Errorf("server.http.addr must be set: the single port cannot bind an empty address") + } + network := httpCfg.GetNetwork() + if network == "" { + network = "tcp" + } + l, err := net.Listen(network, httpCfg.GetAddr()) + if err != nil { + return nil, nil, fmt.Errorf("listening on the single port %q: %w", httpCfg.GetAddr(), err) + } + + var raw = l + tlsEnabled := false + if tlsConf, err := loadServerTLSConfig(httpCfg.GetTlsConfig()); err != nil { + _ = l.Close() + return nil, nil, err + } else if tlsConf != nil { + raw = tls.NewListener(l, tlsConf) + tlsEnabled = true + } + + mux := httpgrpcmux.New(raw, &httpgrpcmux.Options{Logger: logger}) + helper := log.NewHelper(logger) + helper.Infow("msg", "serving gRPC, gRPC-Web and HTTP from the single port", "addr", httpCfg.GetAddr(), "tls", tlsEnabled) + if legacyGRPCListenerEnabled(c) { + helper.Warnw("msg", fmt.Sprintf("the gRPC-only listener on %q is deprecated; it will be disabled by default in a future release and removed afterwards. Set server.legacy_grpc_listener to false to listen on the single port only", c.GetGrpc().GetAddr())) + } + + return mux, mux.Close, nil +} + +// loadServerTLSConfig builds a tls.Config from the certificate and private key +// paths of the server configuration. It returns nil when no TLS is configured. +// For the single port the config negotiates HTTP/2 and HTTP/1.1 through ALPN. +func loadServerTLSConfig(tlsConf *conf.Server_TLS) (*tls.Config, error) { + if tlsConf == nil { + return nil, nil + } + cert := tlsConf.GetCertificate() + privKey := tlsConf.GetPrivateKey() + switch { + case cert == "" && privKey == "": + return nil, nil + case cert == "" || privKey == "": + // A partial configuration is a typo: refuse it instead of silently + // exposing the endpoint over plaintext. + return nil, fmt.Errorf("incomplete TLS configuration: certificate and private key must be set together") + } + certPair, err := tls.LoadX509KeyPair(cert, privKey) + if err != nil { + return nil, fmt.Errorf("loading server TLS certificate: %w", err) + } + return &tls.Config{ + Certificates: []tls.Certificate{certPair}, + MinVersion: tls.VersionTLS12, + // Serve gRPC (HTTP/2) and HTTP/1.1 from the same certificate. + NextProtos: []string{"h2", "http/1.1"}, + }, nil +} + +// legacyGRPCListenerEnabled reports whether the deprecated gRPC-only listener +// must stay on. It defaults to true when unset, so that existing +// installations and the Helm chart keep working during the deprecation period. +func legacyGRPCListenerEnabled(c *conf.Server) bool { + if c.LegacyGrpcListener != nil { + return *c.LegacyGrpcListener + } + return true +} diff --git a/app/controlplane/internal/server/apimux_test.go b/app/controlplane/internal/server/apimux_test.go new file mode 100644 index 000000000..988aaffc1 --- /dev/null +++ b/app/controlplane/internal/server/apimux_test.go @@ -0,0 +1,393 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "io" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "buf.build/go/protovalidate" + conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1" + "github.com/chainloop-dev/chainloop/app/controlplane/internal/service" + "github.com/go-kratos/kratos/v2/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + grpcLib "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +// Shared literals across the server tests. +const ( + testAnyAddr = "127.0.0.1:0" + testLoginURL = "http://localhost" +) + +// startSinglePortApp boots the real gRPC and HTTP servers over an API mux on +// 127.0.0.1:0 and returns the single port address, the deprecated legacy gRPC +// port (empty when the legacy listener is off) and the number of gRPC servers +// in use. +func startSinglePortApp(t *testing.T, legacyListener *bool) (string, string, int) { + t.Helper() + + validator, err := protovalidate.New() + require.NoError(t, err) + + serverConfig := &conf.Server{ + Grpc: &conf.Server_GRPC{Addr: testAnyAddr}, + Http: &conf.Server_HTTP{Addr: testAnyAddr}, + LegacyGrpcListener: legacyListener, + } + + mux, cleanup, err := NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) + require.NoError(t, err) + t.Cleanup(cleanup) + + opts := &Opts{ + ServerConfig: serverConfig, + AuthConfig: &conf.Auth{}, + Logger: log.NewStdLogger(io.Discard), + Validator: validator, + AuthSvc: &service.AuthService{AuthURLs: &service.AuthURLs{Login: testLoginURL}}, + APIMux: mux, + } + + gs, err := NewGRPCServer(opts) + require.NoError(t, err) + httpServers, err := NewHTTPServer(opts, gs) + require.NoError(t, err) + + legacyPort := "" + if len(gs) > 1 { + // Resolve the legacy gRPC port before the servers start; kratos + // resolves the advertise host on Endpoint(), so only the port is + // reused here and dialed on the loopback bind address. + ep, err := gs[1].Endpoint() + require.NoError(t, err) + _, port, err := net.SplitHostPort(ep.Host) + require.NoError(t, err) + legacyPort = port + } + + require.NoError(t, mux.Start(context.Background())) + t.Cleanup(func() { _ = mux.Stop(context.Background()) }) + + for _, g := range gs { + gsrv := g + go func() { _ = gsrv.Start(context.Background()) }() + t.Cleanup(func() { _ = gsrv.Stop(context.Background()) }) + } + for _, s := range httpServers { + srv := s + go func() { _ = srv.Start(context.Background()) }() + t.Cleanup(func() { _ = srv.Stop(context.Background()) }) + } + + return mux.GRPC().Addr().String(), legacyPort, len(gs) +} + +func TestSinglePortServesAllProtocols(t *testing.T) { + addr, _, _ := startSinglePortApp(t, nil) + + t.Run("native gRPC", func(t *testing.T) { + conn, err := grpcLib.NewClient(addr, grpcLib.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + // The RPC must reach the native gRPC server, whose middleware chain + // rejects unauthenticated calls exactly as it does today on the + // dedicated gRPC port (R-002: same behavior as today). + require.Error(t, err) + assert.Equal(t, codes.Unauthenticated, status.Code(err)) + }) + + t.Run("plain HTTP/1.1", func(t *testing.T) { + resp, err := http.Get("http://" + addr + "/statusz") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + + t.Run("plain HTTP/2 without TLS", func(t *testing.T) { + tr := &http.Transport{} + tr.Protocols = new(http.Protocols) + tr.Protocols.SetUnencryptedHTTP2(true) + client := &http.Client{Transport: tr} + resp, err := client.Get("http://" + addr + "/statusz") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + + t.Run("gRPC-Web", func(t *testing.T) { + // An empty HealthCheckRequest serialized as a grpc-web frame: + // compression flag 0 + 4 zero length bytes. + req, err := http.NewRequest(http.MethodPost, "http://"+addr+"/grpc.health.v1.Health/Check", bytes.NewReader([]byte{0, 0, 0, 0, 0})) + require.NoError(t, err) + req.Header.Set("content-type", "application/grpc-web+proto") + req.Header.Set("x-grpc-web", "1") + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + + t.Run("native gRPC on an HTTP-owned HTTP/2 connection", func(t *testing.T) { + // One HTTP/2 client connection that first speaks plain HTTP and then + // a native gRPC call: the spec promises this direction works, via the + // HTTP server handing the request to the gRPC server. + tr := &http.Transport{} + tr.Protocols = new(http.Protocols) + tr.Protocols.SetUnencryptedHTTP2(true) + client := &http.Client{Transport: tr} + + resp, err := client.Get("http://" + addr + "/statusz") + require.NoError(t, err) + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // A native gRPC call (content-type application/grpc) with an empty + // HealthCheckRequest frame on the same connection. + req, err := http.NewRequest(http.MethodPost, "http://"+addr+"/grpc.health.v1.Health/Check", bytes.NewReader([]byte{0, 0, 0, 0, 0})) + require.NoError(t, err) + req.Header.Set("content-type", "application/grpc") + req.Header.Set("te", "trailers") + + resp, err = client.Do(req) + require.NoError(t, err) + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + // The gRPC server answered through the same middleware chain as on + // the dedicated gRPC port: the unauthenticated call is rejected. + assert.Equal(t, "16", resp.Trailer.Get("Grpc-Status"), "expected an Unauthenticated gRPC status") + }) +} + +func TestLegacyGRPCListenerFlag(t *testing.T) { + t.Run("on by default", func(t *testing.T) { + _, legacyPort, gsCount := startSinglePortApp(t, nil) + require.NotEmpty(t, legacyPort) + assert.Equal(t, 2, gsCount) + + // The legacy listener serves native gRPC: the call reaches the server + // and is rejected by the same middleware chain as on the single port. + conn, err := grpcLib.NewClient("127.0.0.1:"+legacyPort, grpcLib.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + require.Error(t, err) + assert.Equal(t, codes.Unauthenticated, status.Code(err)) + + // It does NOT serve the HTTP API: grpc-go rejects non-HTTP/2 requests + // outright. + _, herr := http.Get("http://127.0.0.1:" + legacyPort + "/statusz") + require.Error(t, herr, "the legacy gRPC listener must not serve the HTTP API") + }) + + t.Run("explicitly on", func(t *testing.T) { + on := true + _, legacyPort, gsCount := startSinglePortApp(t, &on) + assert.Equal(t, 2, gsCount) + assert.NotEmpty(t, legacyPort) + }) + + t.Run("off listens on the single port only", func(t *testing.T) { + off := false + _, legacyPort, gsCount := startSinglePortApp(t, &off) + assert.Equal(t, 1, gsCount) + assert.Empty(t, legacyPort) + }) +} + +// TestSinglePortTLS proves R-004: one TLS configuration on the single port +// serves gRPC (HTTP/2), gRPC-Web and plain HTTP, negotiated through ALPN. +// TestPartialTLSConfigRejected proves that a partial tls_config is a hard +// startup error instead of silently disabling TLS on the shared endpoint. +func TestPartialTLSConfigRejected(t *testing.T) { + certPath, keyPath := writeSelfSignedCert(t) + + serverConfig := &conf.Server{ + Grpc: &conf.Server_GRPC{ + Addr: testAnyAddr, + }, + Http: &conf.Server_HTTP{ + Addr: testAnyAddr, + TlsConfig: &conf.Server_TLS{Certificate: certPath}, // private key missing + }, + } + + _, _, err := NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) + require.Error(t, err) + assert.Contains(t, err.Error(), "incomplete TLS configuration") + + serverConfig.Http.TlsConfig = &conf.Server_TLS{PrivateKey: keyPath} // certificate missing + _, _, err = NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) + require.Error(t, err) +} + +func TestSinglePortTLS(t *testing.T) { + certPath, keyPath := writeSelfSignedCert(t) + + validator, err := protovalidate.New() + require.NoError(t, err) + + serverConfig := &conf.Server{ + Grpc: &conf.Server_GRPC{ + Addr: testAnyAddr, + }, + Http: &conf.Server_HTTP{ + Addr: testAnyAddr, + TlsConfig: &conf.Server_TLS{ + Certificate: certPath, + PrivateKey: keyPath, + }, + }, + } + + mux, cleanup, err := NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) + require.NoError(t, err) + t.Cleanup(cleanup) + + opts := &Opts{ + ServerConfig: serverConfig, + AuthConfig: &conf.Auth{}, + Logger: log.NewStdLogger(io.Discard), + Validator: validator, + AuthSvc: &service.AuthService{AuthURLs: &service.AuthURLs{Login: testLoginURL}}, + APIMux: mux, + } + + gs, err := NewGRPCServer(opts) + require.NoError(t, err) + httpServers, err := NewHTTPServer(opts, gs) + require.NoError(t, err) + require.Len(t, httpServers, 1) + + require.NoError(t, mux.Start(context.Background())) + t.Cleanup(func() { _ = mux.Stop(context.Background()) }) + for _, g := range gs { + gsrv := g + go func() { _ = gsrv.Start(context.Background()) }() + t.Cleanup(func() { _ = gsrv.Stop(context.Background()) }) + } + for _, s := range httpServers { + srv := s + go func() { _ = srv.Start(context.Background()) }() + t.Cleanup(func() { _ = srv.Stop(context.Background()) }) + } + addr := mux.GRPC().Addr().String() + + // TLS client without ALPN restriction: HTTP/1.1 over TLS must work. + client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} //nolint:gosec // test server with a self-signed certificate + resp, err := client.Get("https://" + addr + "/statusz") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // Native gRPC over TLS: handed to the gRPC server through the mux. + gconn, err := grpcLib.NewClient(addr, grpcLib.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}))) //nolint:gosec // test server with a self-signed certificate + require.NoError(t, err) + defer gconn.Close() + gctx, gcancel := context.WithTimeout(context.Background(), 10*time.Second) + defer gcancel() + gresp, gerr := grpc_health_v1.NewHealthClient(gconn).Check(gctx, &grpc_health_v1.HealthCheckRequest{}) + require.Error(t, gerr) + assert.Nil(t, gresp) + assert.Equal(t, codes.Unauthenticated, status.Code(gerr)) + + // gRPC-Web over TLS: served by the HTTP server's wrapper. + wreq, werr := http.NewRequest(http.MethodPost, "https://"+addr+"/grpc.health.v1.Health/Check", strings.NewReader("\x00\x00\x00\x00\x00")) + require.NoError(t, werr) + wreq.Header.Set("content-type", "application/grpc-web+proto") + wreq.Header.Set("x-grpc-web", "1") + wresp, werr := client.Do(wreq) + require.NoError(t, werr) + defer wresp.Body.Close() + assert.Equal(t, http.StatusOK, wresp.StatusCode) + + // TLS client forcing HTTP/2: ALPN must negotiate h2 against the same + // certificate and port. + h2Client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // test server with a self-signed certificate + ForceAttemptHTTP2: true, + }} + resp, err = h2Client.Get("https://" + addr + "/statusz") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "HTTP/2.0", resp.Proto) +} + +func writeSelfSignedCert(t *testing.T) (string, string) { + t.Helper() + + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + tmpl := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + require.NoError(t, err) + + certPath := filepath.Join(t.TempDir(), "tls.crt") + keyPath := filepath.Join(t.TempDir(), "tls.key") + require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)) + require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), 0o600)) + return certPath, keyPath +} + +// TestEmptyHTTPAddrRejected proves the single port never binds an ephemeral +// address: a missing server.http.addr is a startup error, not a silent :0. +func TestEmptyHTTPAddrRejected(t *testing.T) { + serverConfig := &conf.Server{ + Grpc: &conf.Server_GRPC{Addr: testAnyAddr}, + Http: &conf.Server_HTTP{}, // addr omitted + } + _, _, err := NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) + require.Error(t, err) + assert.Contains(t, err.Error(), "server.http.addr must be set") +} diff --git a/app/controlplane/internal/server/grpc.go b/app/controlplane/internal/server/grpc.go index f781d1481..55b6ce000 100644 --- a/app/controlplane/internal/server/grpc.go +++ b/app/controlplane/internal/server/grpc.go @@ -35,6 +35,7 @@ import ( "github.com/chainloop-dev/chainloop/app/controlplane/internal/service" "github.com/chainloop-dev/chainloop/app/controlplane/internal/usercontext" "github.com/chainloop-dev/chainloop/pkg/credentials" + "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" "github.com/chainloop-dev/chainloop/pkg/servicelogger" "github.com/golang-jwt/jwt/v5" @@ -90,8 +91,12 @@ type Opts struct { GroupSvc *service.GroupService ProjectSvc *service.ProjectService // Utils - Logger log.Logger - ServerConfig *conf.Server + Logger log.Logger + ServerConfig *conf.Server + // APIMux serves gRPC, gRPC-Web and HTTP from the single HTTP port. When + // set, both the gRPC and the HTTP server consume connections from it + // instead of opening their own listeners (issue-3628 single endpoint). + APIMux *httpgrpcmux.Mux AuthConfig *conf.Auth FederatedConfig *conf.FederatedAuthentication OperationAuthConfig *conf.OperationAuthorizationProvider @@ -109,29 +114,68 @@ var ( allButOrganizationOperationsSkipRegexp = regexp.MustCompile("/controlplane.v1.OrganizationService/Create|/controlplane.v1.UserService/ListMemberships|/controlplane.v1.ContextService/Current|/controlplane.v1.AuthService/DeleteAccount") ) -// NewGRPCServer new a gRPC server. -func NewGRPCServer(opts *Opts) (*grpc.Server, error) { +// NewGRPCServer builds the gRPC server(s) that serve native gRPC. +// +// In single-port mode (API mux present) the first server consumes the +// connections that the classifier marks as native gRPC; while the legacy gRPC +// listener is on (default) a second, deprecated server also binds the +// gRPC-only port with its configured TLS — the behavior the dedicated gRPC +// port had before the single port existed. Without the mux, a single server +// binds the gRPC-only port, as before. +func NewGRPCServer(opts *Opts) ([]*grpc.Server, error) { // Opt-in histogram metrics for the interceptor grpc_prometheus.EnableHandlingTimeHistogram() - // NOTE: kratos middlewares will always be run before plain grpc unary interceptors - // If you want to change the behavior, please refer to wrappers.go and the Prometheus wrapper as example - var serverOpts = []grpc.ServerOption{ - grpc.Middleware(craftMiddleware(opts)...), - grpc.UnaryInterceptor( - protovalidateMiddleware.UnaryServerInterceptor(opts.Validator), - ), + if opts.APIMux != nil { + // Single port mode: the mux owns the listener on the HTTP address + // (and its TLS); this server only consumes the connections that the + // classifier marks as native gRPC. + single, err := newGRPCServer(opts, grpc.Listener(opts.APIMux.GRPC())) + if err != nil { + return nil, err + } + + servers := []*grpc.Server{single} + if legacyGRPCListenerEnabled(opts.ServerConfig) { + // The legacy TLS options are loaded only when the deprecated + // listener actually starts: an operator turning it off must not + // be blocked by a dangling cert mount. + legacyOpts, err := legacyGRPCServerOptions(opts) + if err != nil { + return nil, err + } + legacy, err := newGRPCServer(opts, legacyOpts...) + if err != nil { + return nil, err + } + servers = append(servers, legacy) + } + return servers, nil + } + + legacyOpts, err := legacyGRPCServerOptions(opts) + if err != nil { + return nil, err + } + srv, err := newGRPCServer(opts, legacyOpts...) + if err != nil { + return nil, err } + return []*grpc.Server{srv}, nil +} +// legacyGRPCServerOptions binds the deprecated gRPC-only listener: its +// configured network and address, with its TLS settings when present — the +// behavior the dedicated gRPC port had before the single port existed. A +// partial TLS configuration keeps this listener plaintext, as it always did. +func legacyGRPCServerOptions(opts *Opts) ([]grpc.ServerOption, error) { + var serverOpts []grpc.ServerOption if v := opts.ServerConfig.Grpc.Network; v != "" { serverOpts = append(serverOpts, grpc.Network(v)) } if v := opts.ServerConfig.Grpc.Addr; v != "" { serverOpts = append(serverOpts, grpc.Address(v)) } - if v := opts.ServerConfig.Grpc.Timeout; v != nil { - serverOpts = append(serverOpts, grpc.Timeout(v.AsDuration())) - } if tlsConf := opts.ServerConfig.Grpc.GetTlsConfig(); tlsConf != nil { cert := tlsConf.GetCertificate() privKey := tlsConf.GetPrivateKey() @@ -146,6 +190,23 @@ func NewGRPCServer(opts *Opts) (*grpc.Server, error) { })) } } + return serverOpts, nil +} + +// newGRPCServer builds one gRPC server with the shared middleware chain, +// interceptors and service registrations, bound through the given extra +// options. +func newGRPCServer(opts *Opts, extraOpts ...grpc.ServerOption) (*grpc.Server, error) { + var serverOpts = append([]grpc.ServerOption{ + grpc.Middleware(craftMiddleware(opts)...), + grpc.UnaryInterceptor( + protovalidateMiddleware.UnaryServerInterceptor(opts.Validator), + ), + }, extraOpts...) + + if v := opts.ServerConfig.Grpc.Timeout; v != nil { + serverOpts = append(serverOpts, grpc.Timeout(v.AsDuration())) + } // Raw grpc-go server options must be passed in a single grpc.Options call: // kratos' grpc.Options assigns s.grpcOpts instead of appending, so a second @@ -288,13 +349,13 @@ func craftMiddleware(opts *Opts) []middleware.Middleware { // If we should load the user func requireCurrentUserMatcher() selector.MatchFunc { - return func(ctx context.Context, operation string) bool { + return func(_ context.Context, operation string) bool { return !currentUserSkipRegexp.MatchString(operation) } } func requireFullyConfiguredOrgMatcher() selector.MatchFunc { - return func(ctx context.Context, operation string) bool { + return func(_ context.Context, operation string) bool { return !fullyConfiguredOrgSkipRegexp.MatchString(operation) } } @@ -306,20 +367,20 @@ func requireFullyConfiguredCASBackendMatcher() selector.MatchFunc { } func allowListEnabled() selector.MatchFunc { - return func(ctx context.Context, operation string) bool { + return func(_ context.Context, operation string) bool { return !allowListSkipRegexp.MatchString(operation) } } func requireRobotAccountMatcher() selector.MatchFunc { - return func(ctx context.Context, operation string) bool { + return func(_ context.Context, operation string) bool { return robotAccountRequireRegexp.MatchString(operation) } } // Matches all operations that require to have a current organization func requireAllButOrganizationOperationsMatcher() selector.MatchFunc { - return func(ctx context.Context, operation string) bool { + return func(_ context.Context, operation string) bool { return !allButOrganizationOperationsSkipRegexp.MatchString(operation) } } diff --git a/app/controlplane/internal/server/grpc_test.go b/app/controlplane/internal/server/grpc_test.go index 9c25ffcac..ead63aa42 100644 --- a/app/controlplane/internal/server/grpc_test.go +++ b/app/controlplane/internal/server/grpc_test.go @@ -22,13 +22,19 @@ import ( "github.com/stretchr/testify/assert" ) +// Operations used across the matcher tests. +const ( + opWorkflowServiceList = "/controlplane.v1.WorkflowService/List" + opWorkflowRunServiceList = "/controlplane.v1.WorkflowRunService/List" +) + func TestRequireCurrentUserMatcher(t *testing.T) { testCases := []struct { operation string matches bool }{ - {"/controlplane.v1.WorkflowService/List", true}, - {"/controlplane.v1.WorkflowRunService/List", true}, + {opWorkflowServiceList, true}, + {opWorkflowRunServiceList, true}, {"/controlplane.v1.StatusService/Infoz", false}, {"/controlplane.v1.StatusService/Statusz", false}, {"/controlplane.v1.AttestationService/Init", false}, @@ -71,11 +77,11 @@ func TestRequireRobotAccountMatcher(t *testing.T) { operation string matches bool }{ - {"/controlplane.v1.WorkflowService/List", false}, + {opWorkflowServiceList, false}, {"/controlplane.v1.StatusService/Infoz", false}, {"/controlplane.v1.AttestationService/Init", true}, {"/controlplane.v1.AttestationService/Store", true}, - {"/controlplane.v1.WorkflowRunService/List", false}, + {opWorkflowRunServiceList, false}, } matchFunc := requireRobotAccountMatcher() @@ -89,8 +95,8 @@ func TestRequireAllButOrganizationOperationsMatcher(t *testing.T) { operation string matches bool }{ - {"/controlplane.v1.WorkflowService/List", true}, - {"/controlplane.v1.WorkflowRunService/List", true}, + {opWorkflowServiceList, true}, + {opWorkflowRunServiceList, true}, {"/controlplane.v1.OrganizationService/Create", false}, {"/controlplane.v1.CASBackendService/List", true}, {"/controlplane.v1.CASBackendService/Add", true}, diff --git a/app/controlplane/internal/server/http.go b/app/controlplane/internal/server/http.go index 80665f9e5..6b0c16edb 100644 --- a/app/controlplane/internal/server/http.go +++ b/app/controlplane/internal/server/http.go @@ -18,6 +18,7 @@ package server import ( "context" "errors" + "time" "github.com/chainloop-dev/chainloop/app/controlplane/pkg/jwt/apitoken" middlewares_http "github.com/chainloop-dev/chainloop/pkg/middlewares/http" @@ -26,6 +27,7 @@ import ( "buf.build/go/protovalidate" v1 "github.com/chainloop-dev/chainloop/app/controlplane/api/controlplane/v1" "github.com/chainloop-dev/chainloop/app/controlplane/internal/service" + "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" "github.com/go-kratos/kratos/v2/middleware" "github.com/go-kratos/kratos/v2/middleware/tracing" "github.com/improbable-eng/grpc-web/go/grpcweb" @@ -40,11 +42,37 @@ import ( "github.com/go-kratos/kratos/v2/transport/http" ) -// NewHTTPServer new an HTTP server. -func NewHTTPServer(opts *Opts, grpcSrv *grpc.Server) (*http.Server, error) { - var serverOpts = []http.ServerOption{ - http.Middleware(httpMiddlewares(opts)...), +// NewHTTPServer builds the HTTP server that serves plain HTTP and gRPC-Web. +// +// In single-port mode (API mux present, see +// docs/specs/issue-3628-single-endpoint.md) it consumes the connections that +// the classifier did not mark as native gRPC, so gRPC-Web and HTTP are served +// from the single port. The gRPC-Web wrapper and the native-gRPC fallback wrap +// the mux-fed gRPC server (grpcServers[0]); the legacy gRPC server, when +// present, serves its own deprecated port. Without the mux, the server binds +// the HTTP-only port, as before. +func NewHTTPServer(opts *Opts, grpcServers []*grpc.Server) ([]*http.Server, error) { + if opts.APIMux != nil { + single, err := newHTTPServer(opts, grpcServers[0], http.Listener(opts.APIMux.HTTP())) + if err != nil { + return nil, err + } + return []*http.Server{single}, nil } + + srv, err := newHTTPServer(opts, grpcServers[0], http.Address(opts.ServerConfig.GetHttp().GetAddr())) + if err != nil { + return nil, err + } + return []*http.Server{srv}, nil +} + +// newHTTPServer builds one HTTP server that serves the routes, the gRPC-Web +// wrapper and the transcoded handlers, bound through the given extra options. +func newHTTPServer(opts *Opts, grpcSrv *grpc.Server, extraOpts ...http.ServerOption) (*http.Server, error) { + var serverOpts = append([]http.ServerOption{ + http.Middleware(httpMiddlewares(opts)...), + }, extraOpts...) // Prevent unmatched routes from falling through to http.DefaultServeMux, // which would otherwise expose net/http/pprof's /debug/pprof/* endpoints // unauthenticated on the public API listener. @@ -53,9 +81,6 @@ func NewHTTPServer(opts *Opts, grpcSrv *grpc.Server) (*http.Server, error) { if v := opts.ServerConfig.Http.Network; v != "" { serverOpts = append(serverOpts, http.Network(v)) } - if v := opts.ServerConfig.Http.Addr; v != "" { - serverOpts = append(serverOpts, http.Address(v)) - } if v := opts.ServerConfig.Http.Timeout; v != nil { serverOpts = append(serverOpts, http.Timeout(v.AsDuration())) } @@ -97,8 +122,28 @@ func NewHTTPServer(opts *Opts, grpcSrv *grpc.Server) (*http.Server, error) { // Include the OpenAPI spec handler r.GET("/openapi.yaml", statusSvc.HandleOpenAPISpec) - // Handle grpc-web requests or fallback + // Handle grpc-web requests or fallback. Unencrypted HTTP/2 (h2c) is enabled + // natively so that connections that are not native gRPC still work: browsers + // doing gRPC-Web over HTTP/2 and h2c-aware proxies. + wrappedServer.Protocols = new(h.Protocols) + wrappedServer.Protocols.SetHTTP1(true) + wrappedServer.Protocols.SetHTTP2(true) + wrappedServer.Protocols.SetUnencryptedHTTP2(true) + // Bound how long a client may take to deliver its request headers over + // HTTP/1.x, on either listener (slowloris protection). Response streaming + // is not affected. The bound does not protect HTTP/2 header completion: + // Go's HTTP/2 path clears the connection deadline, so that case is + // tracked separately. + wrappedServer.ReadHeaderTimeout = 60 * time.Second + wrappedServer.Handler = h.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + // Native gRPC requests that arrive on a connection the classifier + // gave to the HTTP server: hand them to the gRPC server, which runs + // the same middleware chain as on the dedicated gRPC port. + if httpgrpcmux.IsNativeGRPCContentType(req.Header.Get("Content-Type")) { + grpcSrv.ServeHTTP(res, req) + return + } if wrappedGrpc.IsGrpcWebRequest(req) || wrappedGrpc.IsAcceptableGrpcCorsRequest(req) { wrappedGrpc.ServeHTTP(res, req) return diff --git a/app/controlplane/internal/server/otel_test.go b/app/controlplane/internal/server/otel_test.go index cf0e66fd6..9d3233106 100644 --- a/app/controlplane/internal/server/otel_test.go +++ b/app/controlplane/internal/server/otel_test.go @@ -68,25 +68,25 @@ func TestGRPCServerSpanWithMaxRecvMsgSize(t *testing.T) { validator, err := protovalidate.New() require.NoError(t, err) - srv, err := NewGRPCServer(&Opts{ + srvs, err := NewGRPCServer(&Opts{ ServerConfig: &conf.Server{ Grpc: &conf.Server_GRPC{ - Addr: "127.0.0.1:0", + Addr: testAnyAddr, MaxRecvMsgSize: 8 * 1024 * 1024, }, }, AuthConfig: &conf.Auth{}, Logger: log.NewStdLogger(io.Discard), Validator: validator, - AuthSvc: &service.AuthService{AuthURLs: &service.AuthURLs{Login: "http://localhost"}}, + AuthSvc: &service.AuthService{AuthURLs: &service.AuthURLs{Login: testLoginURL}}, }) require.NoError(t, err) // Endpoint() binds the listener so the address is known before Start. - endpoint, err := srv.Endpoint() + endpoint, err := srvs[0].Endpoint() require.NoError(t, err) - go func() { _ = srv.Start(context.Background()) }() - defer func() { _ = srv.Stop(context.Background()) }() + go func() { _ = srvs[0].Start(context.Background()) }() + defer func() { _ = srvs[0].Stop(context.Background()) }() conn, err := grpcLib.NewClient(endpoint.Host, grpcLib.WithTransportCredentials(insecure.NewCredentials())) require.NoError(t, err) @@ -118,7 +118,7 @@ func TestHTTPServerSpanFromTracingMiddleware(t *testing.T) { require.NoError(t, err) srv := kratoshttp.NewServer( - kratoshttp.Address("127.0.0.1:0"), + kratoshttp.Address(testAnyAddr), kratoshttp.Middleware(httpMiddlewares(&Opts{ AuthConfig: &conf.Auth{}, Logger: log.NewStdLogger(io.Discard), diff --git a/app/controlplane/internal/server/server.go b/app/controlplane/internal/server/server.go index 07827a3e9..b18caa895 100644 --- a/app/controlplane/internal/server/server.go +++ b/app/controlplane/internal/server/server.go @@ -24,6 +24,7 @@ import ( // ProviderSet is server providers. var ProviderSet = wire.NewSet( + NewAPIMux, NewGRPCServer, NewHTTPServer, NewHTTPMetricsServer, diff --git a/go.mod b/go.mod index 56eb3fd5b..9d53da20b 100644 --- a/go.mod +++ b/go.mod @@ -114,6 +114,7 @@ require ( go.uber.org/zap v1.28.0 golang.org/x/crypto v0.57.0 golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba + golang.org/x/net v0.60.0 golang.org/x/oauth2 v0.37.0 golang.org/x/sync v0.24.0 golang.org/x/term v0.46.0 @@ -459,7 +460,6 @@ require ( go4.org v0.0.0-20260112195520-a5071408f32f // indirect goa.design/goa/v3 v3.32.0 // indirect golang.org/x/mod v0.41.0 // indirect - golang.org/x/net v0.60.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.16.0 // indirect golang.org/x/tools v0.51.0 // indirect diff --git a/pkg/httpgrpcmux/mux.go b/pkg/httpgrpcmux/mux.go new file mode 100644 index 000000000..f9a06c0d7 --- /dev/null +++ b/pkg/httpgrpcmux/mux.go @@ -0,0 +1,989 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package httpgrpcmux multiplexes a single TCP listener between a native gRPC +// server and an HTTP server, so one port can serve gRPC, gRPC-Web and plain +// HTTP at the same time. +// +// It implements the connection-level split described in +// docs/specs/issue-3628-single-endpoint.md: the mux looks at the start of each +// new connection without consuming it. An HTTP/2 connection whose first request +// carries a gRPC content type is handed to the gRPC listener; every other +// connection (HTTP/1.x, HTTP/2 without gRPC, gRPC-Web) goes to the HTTP +// listener. A connection classified as gRPC carries only gRPC; a native gRPC +// request that arrives on a connection owned by the HTTP server is forwarded +// to the gRPC server by the HTTP handler. +// +// The mux implements kratos' transport.Server interface (Start/Stop with +// standard signatures) so it can join the application server lifecycle. +package httpgrpcmux + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/go-kratos/kratos/v2/log" + "golang.org/x/net/http2/hpack" +) + +// clientConnPreface is the connection preface that every HTTP/2 client sends +// right after the connection is established. +const clientConnPreface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + +// frameHeaderLen is the fixed size of an HTTP/2 frame header. +const frameHeaderLen = 9 + +// defaultClassifyTimeout bounds how long the classifier waits for a new +// connection's first request before dropping it. It applies to the +// classification step only, and is cleared before the connection is handed +// over, so long-lived streams are not affected. +const defaultClassifyTimeout = 10 * time.Second + +// defaultSettingsGrace is how long the classifier waits for the client's first +// HTTP/2 frame before assuming the client is waiting for the server to speak +// first, and sending an empty SETTINGS frame to unblock it. +const defaultSettingsGrace = time.Second + +// maxClassifiedBytes bounds how much of a connection the classifier may read +// while looking for the first request headers. Anything beyond that is +// considered malformed and the connection is dropped. +const maxClassifiedBytes = 512 << 10 + +// maxHeaderBlockSize bounds a single HTTP/2 header block (plus its +// CONTINUATION frames). Real gRPC requests stay far below it. +const maxHeaderBlockSize = 1 << 20 + +// Decoded-header-list limits: a small compressed block can otherwise expand +// into a very large header list (HPACK amplification). +const ( + maxDecodedFields = 1024 + maxDecodedBytes = 256 << 10 + maxHpackString = 64 << 10 +) + +// maxClassifiedFrames bounds how many HTTP/2 frames the classifier processes +// before the first request. +const maxClassifiedFrames = 4096 + +// defaultMaxInflightClassifications caps how many connections may be in +// classification at the same time; beyond it, connections are shed. +const defaultMaxInflightClassifications = 8192 + +// maxTotalClassifiedBytes bounds the aggregate bytes recorded across all +// connections in classification at the same time, so the per-connection budget +// times the connection cap cannot multiply into unbounded memory. +const maxTotalClassifiedBytes = 256 << 20 + +// errTotalBudgetExceeded is returned when the aggregate classification budget +// is exhausted; the connection is dropped. +var errTotalBudgetExceeded = errors.New("aggregate classification budget exceeded") + +// maxAcceptBackoff caps the retry delay after a recoverable accept error. +const maxAcceptBackoff = time.Second + +// grpcContentTypePrefix matches every native gRPC content type, including +// application/grpc+proto and content types with parameters. +const grpcContentTypePrefix = "application/grpc" + +// grpcWebContentTypePrefix is NOT part of the gRPC family above: gRPC-Web +// requests must reach the HTTP server, which owns the gRPC-Web wrapper. +const grpcWebContentTypePrefix = "application/grpc-web" + +// HTTP/2 frame types used by the classifier. +const ( + http2FrameHeaders = 0x1 + http2FramePushPromise = 0x5 + http2FrameSettings = 0x4 + http2FrameContinuation = 0x9 +) + +// HTTP/2 frame flags used by the classifier. +const ( + http2FlagEndHeaders = 0x4 + http2FlagPadded = 0x8 + http2FlagPriority = 0x20 + // http2FlagSettingsAck marks a SETTINGS frame that acknowledges the peer's + // settings. + http2FlagSettingsAck = 0x1 +) + +// http2ServerEmptySettings is an empty SETTINGS frame. When a client stalls +// waiting for the server to speak first, the classifier sends it to unblock +// the client so it can read the first request. +var http2ServerEmptySettings = []byte{0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00} + +// acceptBacklog is the size of each virtual listener's delivery queue. +const acceptBacklog = 16 + +// Options configures a Mux. +type Options struct { + // Logger receives classification debug information. Optional. + Logger log.Logger + // ClassifyTimeout bounds how long the classifier waits for a new + // connection's first request. Defaults to 10s. + ClassifyTimeout time.Duration + // SettingsGrace is how long the classifier waits for the client's first + // HTTP/2 frame before assuming it waits for the server to speak first, + // and sending an empty SETTINGS frame to unblock it. Defaults to 1s. + SettingsGrace time.Duration + // MaxInflightClassifications caps how many connections may be in + // classification at the same time; beyond it, connections are shed. + // Defaults to 8192. + MaxInflightClassifications int + // MaxTotalClassifiedBytes bounds the aggregate bytes recorded across all + // connections in classification at the same time. Defaults to 256 MiB. + MaxTotalClassifiedBytes int64 +} + +// Mux splits one listener between a gRPC server and an HTTP server. +type Mux struct { + raw net.Listener + + grpcL *virtualListener + httpL *virtualListener + + classifyTimeout time.Duration + settingsGrace time.Duration + logger *log.Helper + // admit caps how many connections may be in classification at once. + admit chan struct{} + // stopping is set before Close tears the listeners down, so servers can + // distinguish an orderly shutdown from a fatal accept-loop failure. + stopping atomic.Bool + // inflight tracks the connections that are being classified, so Close + // can release the ones still parked on the classification deadline. + mu sync.Mutex + inflight map[net.Conn]struct{} + // budget tracks the aggregate bytes charged across in-flight + // classifications: the bytes recorded so far, plus the payload + // reservations held while a frame payload is being read; budgetLimit is + // its cap. + budget *atomic.Int64 + budgetLimit int64 + + wg sync.WaitGroup + closeOnce sync.Once +} + +// New creates a mux over l. Both virtual listeners are usable immediately; +// connections start flowing once Start is called. +func New(l net.Listener, opts *Options) *Mux { + if opts == nil { + opts = &Options{} + } + logger := opts.Logger + if logger == nil { + logger = log.DefaultLogger + } + timeout := opts.ClassifyTimeout + if timeout <= 0 { + timeout = defaultClassifyTimeout + } + grace := opts.SettingsGrace + if grace <= 0 { + grace = defaultSettingsGrace + } + maxInflight := opts.MaxInflightClassifications + if maxInflight <= 0 { + maxInflight = defaultMaxInflightClassifications + } + totalBytes := opts.MaxTotalClassifiedBytes + if totalBytes <= 0 { + totalBytes = maxTotalClassifiedBytes + } + return &Mux{ + raw: l, + grpcL: newVirtualListener(l.Addr()), + httpL: newVirtualListener(l.Addr()), + classifyTimeout: timeout, + settingsGrace: grace, + admit: make(chan struct{}, maxInflight), + inflight: make(map[net.Conn]struct{}), + budget: &atomic.Int64{}, + budgetLimit: totalBytes, + logger: log.NewHelper(logger), + } +} + +// GRPC returns the listener that receives connections classified as native +// gRPC: HTTP/2 connections whose first request carries a gRPC content type. +func (m *Mux) GRPC() net.Listener { return m.grpcL } + +// HTTP returns the listener that receives every other connection: HTTP/1.x, +// HTTP/2 without gRPC, and gRPC-Web. +func (m *Mux) HTTP() net.Listener { return m.httpL } + +// Start runs the accept loop until Stop is called. +func (m *Mux) Start(_ context.Context) error { + m.wg.Add(1) + go m.acceptLoop() + return nil +} + +// Stop closes the underlying listener and both virtual listeners, unblocking +// any Accept call still in progress. +func (m *Mux) Stop(_ context.Context) error { + m.Close() + return nil +} + +// Stopping reports whether Close has been called: servers fed by the mux use +// it to tell an orderly shutdown apart from a fatal accept-loop failure. +func (m *Mux) Stopping() bool { return m.stopping.Load() } + +// Close stops the mux and waits for the accept loop to drain. +func (m *Mux) Close() { + m.closeOnce.Do(func() { + m.stopping.Store(true) + _ = m.raw.Close() + // Close the virtual listeners even when Start never ran: the accept + // loop would otherwise be the only place doing it, and their + // dispatchers would keep running with queued connections inside. + _ = m.grpcL.Close() + _ = m.httpL.Close() + // Release the connections still parked in classification: closing + // them unblocks their goroutines, so the wait below is fast instead + // of riding out the classification deadline. + m.mu.Lock() + for conn := range m.inflight { + _ = conn.Close() + } + m.mu.Unlock() + m.wg.Wait() + }) +} + +func (m *Mux) acceptLoop() { + defer m.wg.Done() + var backoff time.Duration + for { + conn, err := m.raw.Accept() + if err != nil { + // Descriptor exhaustion and timeouts are recoverable: retry with + // backoff instead of killing the accept loop. + if isAcceptRetryable(err) { + if backoff == 0 { + backoff = 5 * time.Millisecond + } else if backoff < maxAcceptBackoff { + backoff *= 2 + } + time.Sleep(backoff) + continue + } + // Listener closed or fatal error: the accept loop is gone. Log it loudly — + // the servers notice through the closed virtual listeners, and the process + // must be restarted (a fatal accept failure is not recoverable). + m.logger.Errorf("mux accept loop terminated: %v", err) + _ = m.grpcL.Close() + _ = m.httpL.Close() + return + } + backoff = 0 + // Shed connections beyond the admission limit: the peer retries. + select { + case m.admit <- struct{}{}: + default: + m.logger.Debugf("shedding connection: classification admission limit reached") + _ = conn.Close() + continue + } + // Track the connection BEFORE the goroutine launches, so Close cannot + // walk past it: registering inside classify would leave a window where + // Close misses the accepted connection and rides out its full + // classification deadline. Ownership transfers to the server at + // handover, after which the entry is removed. + m.mu.Lock() + m.inflight[conn] = struct{}{} + m.mu.Unlock() + m.wg.Add(1) + go func() { + defer func() { + m.mu.Lock() + delete(m.inflight, conn) + m.mu.Unlock() + <-m.admit + }() + // classify owns the WaitGroup accounting for this goroutine. + m.classify(conn) + }() + } +} + +// isAcceptRetryable reports whether an accept error is recoverable: read +// timeouts, and descriptor exhaustion, which is temporary on Linux. +func isAcceptRetryable(err error) bool { + return isTimeout(err) || errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE) +} + +// classify looks at the start of a new connection and hands it to the right +// server. It never consumes more than what it replays to the server. The +// caller registered the connection in the inflight set before launching it +// here; it is removed when this goroutine ends. +func (m *Mux) classify(conn net.Conn) { + defer m.wg.Done() + + bc := &replayConn{Conn: conn, recording: true, budget: m.budget, budgetLimit: m.budgetLimit} + // One absolute deadline bounds the whole classification step: reads and + // writes (the TLS handshake and the injected SETTINGS frame included). + // It is cleared before the connection is handed over, so long-lived + // streams are not affected. + classifyEnd := time.Now().Add(m.classifyTimeout) + _ = bc.SetDeadline(classifyEnd) + // Release this connection's share of the aggregate budget when the + // classification ends, whichever way it ends. + defer func() { + if bc.appended > 0 { + bc.budget.Add(-bc.appended) + } + }() + + if tc, ok := conn.(*tls.Conn); ok { + if err := tc.Handshake(); err != nil { + m.logger.Debugf("dropping connection: TLS handshake failed: %v", err) + _ = conn.Close() + return + } + switch tc.ConnectionState().NegotiatedProtocol { + case "", "http/1.1": + // HTTP/1.1 was negotiated: no HTTP/2 frames to parse. + m.handOver(bc, false) + return + } + // "h2" or another protocol: sniff the first request below. + } + + isGRPC, err := m.sniff(bc, classifyEnd) + if err != nil { + m.logger.Debugf("dropping connection: could not classify: %v", err) + _ = conn.Close() + return + } + m.handOver(bc, isGRPC) +} + +// handOver clears the classification deadline and delivers the connection to +// the right virtual listener, replaying everything the classifier consumed. +func (m *Mux) handOver(bc *replayConn, isGRPC bool) { + _ = bc.SetDeadline(time.Time{}) + if isGRPC { + m.grpcL.deliver(bc.handoff()) + } else { + m.httpL.deliver(bc.handoff()) + } +} + +// sniff decides whether the connection's first request is native gRPC. +func (m *Mux) sniff(bc *replayConn, classifyEnd time.Time) (bool, error) { + isH2, err := readClientPreface(bc) + if err != nil { + return false, err + } + if !isH2 { + // HTTP/1.x or anything else. The HTTP server produces a proper + // error response for malformed traffic. + return false, nil + } + + // A client that waits for the server to speak first stalls in the frame + // loop below; sniffFirstRequest unblocks it after a short grace period. + return m.sniffFirstRequest(bc, classifyEnd) +} + +// readClientPreface reports whether the connection starts with the HTTP/2 +// client preface. It reads byte ranges without over-consuming: everything it +// reads stays buffered in bc and is replayed to the server. +func readClientPreface(bc *replayConn) (bool, error) { + preface := []byte(clientConnPreface) + buf := make([]byte, len(preface)) + n := 0 + for n < len(preface) { + read, err := bc.Read(buf[n:]) + if err != nil { + return false, err + } + n += read + if !bytes.HasPrefix(buf[:n], preface[:n]) { + return false, nil + } + } + return true, nil +} + +// sniffFirstRequest walks HTTP/2 frames until the first HEADERS block is +// complete, then reports whether its content type is a native gRPC one. +// +// A client that waits for the server to speak first before sending anything +// stalls here. After a short grace period the classifier sends an empty +// SETTINGS frame to unblock it, and removes the SETTINGS-ACK that the client +// then sends for it from the replay, so the downstream server only ever sees +// ACKs for its own SETTINGS. The ACK may also arrive after the first request +// — a client whose frame-read loop races its request-write goroutine can send +// HEADERS first — in which case the classifier keeps consuming frames until +// it shows up (maybeWaitForSettingsAck). All reads stay bounded by +// classifyEnd, the absolute end of the classification window. +// http2FrameHeader is the decoded fixed part of an HTTP/2 frame. +type http2FrameHeader struct { + length int + frameType byte + flags byte + streamID uint32 +} + +// readFrameHeader reads and decodes the fixed part of the next HTTP/2 frame. +func readFrameHeader(bc *replayConn, injected *bool, classifyEnd time.Time) (http2FrameHeader, error) { + var hdr [frameHeaderLen]byte + if err := readInto(bc, hdr[:], injected, classifyEnd); err != nil { + return http2FrameHeader{}, err + } + return http2FrameHeader{ + length: int(hdr[0])<<16 | int(hdr[1])<<8 | int(hdr[2]), + frameType: hdr[3], + flags: hdr[4], + streamID: uint32(hdr[5])<<24 | uint32(hdr[6])<<16 | uint32(hdr[7])<<8 | uint32(hdr[8]), + }, nil +} + +// consumeSettingsAck validates a SETTINGS-ACK (zero payload, zero stream) and +// removes its bytes from the replay: the downstream server never expects the +// ACK of the injected SETTINGS. +func consumeSettingsAck(bc *replayConn, hdr http2FrameHeader) error { + if hdr.length != 0 || hdr.streamID != 0 { + return errors.New("malformed SETTINGS ACK") + } + bc.dropLast(frameHeaderLen) + return nil +} + +// checkClassificationBounds enforces the frame-count and byte caps shared by +// both classification loops. +func checkClassificationBounds(bc *replayConn, frames int) error { + if frames > maxClassifiedFrames { + return errors.New("too many frames before the first request") + } + if bc.recorded() > maxClassifiedBytes { + return errors.New("classification buffer limit exceeded") + } + return nil +} + +func (m *Mux) sniffFirstRequest(bc *replayConn, classifyEnd time.Time) (bool, error) { + // Only the content-type decision is retained, under decoded-list caps: a + // small compressed block can otherwise expand into a huge header list. + counters := &headerCounters{} + dec := hpack.NewDecoder(4096, counters.observe) + dec.SetMaxStringLength(maxHpackString) + + var block []byte + var inBlock bool + // The client gets a short window to send its first frame on its own. The + // classification deadline stays in force as the upper bound. + graceEnd := time.Now().Add(m.settingsGrace) + if graceEnd.After(classifyEnd) { + graceEnd = classifyEnd + } + _ = bc.SetDeadline(graceEnd) + injected := false + // ackConsumed tracks whether the ACK of the injected SETTINGS was already + // consumed. It only matters once injected is true: until then any ACK is + // rejected below as unexpected traffic. + ackConsumed := false + frames := 0 + for { + frames++ + if err := checkClassificationBounds(bc, frames); err != nil { + return false, err + } + + hdr, err := readFrameHeader(bc, &injected, classifyEnd) + if err != nil { + return false, err + } + + if hdr.frameType == http2FrameSettings && hdr.flags&http2FlagSettingsAck != 0 { + // A SETTINGS-ACK can only acknowledge the injected SETTINGS: no + // server SETTINGS were sent before the handover. Anything else, + // including an ACK with a payload or on a stream, is invalid and + // would desynchronize the frame stream if replayed. + if !injected { + return false, errors.New("unexpected SETTINGS ACK before any injected SETTINGS") + } + if err := consumeSettingsAck(bc, hdr); err != nil { + return false, err + } + ackConsumed = true + continue + } + + // Every payload read must fit in the classification budget. + if bc.recorded()+hdr.length > maxClassifiedBytes { + return false, errors.New("frame payload exceeds classification buffer limit") + } + + switch hdr.frameType { + case http2FrameHeaders: + if inBlock { + return false, errors.New("unexpected HEADERS inside a header block") + } + if hdr.length > maxHeaderBlockSize { + return false, errors.New("HEADERS frame too large") + } + payload, err := readFramePayload(bc, hdr.length, &injected, classifyEnd) + if err != nil { + return false, err + } + frag, err := headerBlockFragment(payload, hdr.flags) + if err != nil { + return false, err + } + block = append(block, frag...) + if hdr.flags&http2FlagEndHeaders == 0 { + inBlock = true + continue + } + return finishHeaderBlock(bc, dec, counters, block, injected, ackConsumed, classifyEnd) + case http2FrameContinuation: + if !inBlock { + return false, errors.New("unexpected CONTINUATION frame") + } + if len(block)+hdr.length > maxHeaderBlockSize { + return false, errors.New("header block too large") + } + payload, err := readFramePayload(bc, hdr.length, &injected, classifyEnd) + if err != nil { + return false, err + } + block = append(block, payload...) + if hdr.flags&http2FlagEndHeaders == 0 { + continue + } + return finishHeaderBlock(bc, dec, counters, block, injected, ackConsumed, classifyEnd) + case http2FramePushPromise: + // Only servers may push; a client PUSH_PROMISE is a protocol error. + return false, errors.New("unexpected PUSH_PROMISE frame") + default: + // SETTINGS, WINDOW_UPDATE, PING, GOAWAY, ...: read the payload + // (it must be replayed to the server) within the budget checked + // above. + if err := skipFramePayload(bc, hdr.length, &injected, classifyEnd); err != nil { + return false, err + } + } + } +} + +// finishHeaderBlock decodes a completed header block, decides gRPC-ness and, +// for HTTP-classified connections with an outstanding injected SETTINGS, waits +// for its ACK before handover. +func finishHeaderBlock(bc *replayConn, dec *hpack.Decoder, counters *headerCounters, block []byte, injected, ackConsumed bool, classifyEnd time.Time) (bool, error) { + isGRPC, err := contentTypeFromBlock(dec, counters, block) + if err != nil { + return false, err + } + if err := maybeWaitForSettingsAck(bc, isGRPC, injected, ackConsumed, classifyEnd); err != nil { + return false, err + } + return isGRPC, nil +} + +// maxSettingsAckWait bounds how long the classifier waits for the ACK of the +// injected SETTINGS when it arrives after the first request. Well below the +// classification window: a healthy client acknowledges immediately. +const maxSettingsAckWait = 2 * time.Second + +// maybeWaitForSettingsAck removes the client's ACK of the injected SETTINGS +// from the replay even when it arrives after the first request: a client +// whose frame-read loop races its request-write goroutine can send HEADERS +// first and the ACK afterwards. If the late ACK were replayed, the downstream +// server would see two ACKs for one SETTINGS and fail with PROTOCOL_ERROR. +// +// The wait only applies to HTTP-classified connections: net/http breaks on +// the extra ACK, while the gRPC server ignores it, and waiting on a +// gRPC-classified connection would record large attestation DATA frames into +// the replay prefix, hitting the per-connection cap. +// +// While waiting, every frame read stays recorded in the replay prefix, so +// whatever the client sends in the meantime reaches the downstream server. If +// the ACK does not arrive in time the connection is dropped: the HTTP/2 +// connection would break on the extra ACK anyway. +func maybeWaitForSettingsAck(bc *replayConn, isGRPC, injected, ackConsumed bool, classifyEnd time.Time) error { + if isGRPC || !injected || ackConsumed { + return nil + } + + // Bound the wait by the classification window when the wait bound does + // not fit inside it. + waitEnd := time.Now().Add(maxSettingsAckWait) + if waitEnd.After(classifyEnd) { + waitEnd = classifyEnd + } + _ = bc.SetDeadline(waitEnd) + // The SETTINGS were already injected: a timeout past waitEnd must drop + // the connection, not inject a second time. + injected = true + frames := 0 + for { + frames++ + if err := checkClassificationBounds(bc, frames); err != nil { + return err + } + + hdr, err := readFrameHeader(bc, &injected, classifyEnd) + if err != nil { + return fmt.Errorf("waiting for the SETTINGS ACK of the injected SETTINGS: %w", err) + } + + if hdr.frameType == http2FrameSettings && hdr.flags&http2FlagSettingsAck != 0 { + if err := consumeSettingsAck(bc, hdr); err != nil { + return err + } + return nil + } + if bc.recorded()+hdr.length > maxClassifiedBytes { + return errors.New("frame payload exceeds classification buffer limit") + } + // Any other frame (HEADERS, DATA, WINDOW_UPDATE, ...): skip it; its + // bytes stay recorded and are replayed to the server. + if err := skipFramePayload(bc, hdr.length, &injected, classifyEnd); err != nil { + return err + } + } +} + +// readInto fills buf from bc, keeping the bytes read before an error. A +// timeout while the client is still waiting for the server to speak first (no +// SETTINGS injected yet) unblocks it: the deadline is raised to classifyEnd, +// an empty SETTINGS frame is written, injected is marked accordingly, and the +// read continues with the partial progress intact. Any other error is +// returned as is. +func readInto(bc *replayConn, buf []byte, injected *bool, classifyEnd time.Time) error { + n := 0 + for n < len(buf) { + read, err := bc.Read(buf[n:]) + n += read + if err == nil { + continue + } + if isTimeout(err) && !*injected { + // The client is waiting for the server to speak first. Raise the + // deadline to the classification end BEFORE writing, so the write + // itself is bounded by it. + _ = bc.SetDeadline(classifyEnd) + if _, err := bc.Write(http2ServerEmptySettings); err != nil { + return err + } + *injected = true + continue + } + // A Read may legally return data together with an error (io.Reader + // contract): account the data above, and if the buffer is complete + // the error is irrelevant — a trailing EOF after a full payload, for + // instance. A budget failure returns (0, err) and still propagates. + if n == len(buf) { + return nil + } + return err + } + return nil +} + +// skipFramePayloadLen is the size of the scratch buffer used to skip frame +// payloads that need no parsing. +const skipFramePayloadLen = 1024 + +// skipFramePayload reads and discards a frame payload of the given length. It +// reads into a small scratch buffer instead of io.CopyN, which discards the +// read error whenever the full count arrives — including an aggregate-budget +// failure delivered together with the last bytes of the payload. +func skipFramePayload(bc *replayConn, length int, injected *bool, classifyEnd time.Time) error { + var scratch [skipFramePayloadLen]byte + for remaining := length; remaining > 0; { + chunk := scratch[:min(remaining, len(scratch))] + if err := readInto(bc, chunk, injected, classifyEnd); err != nil { + return err + } + remaining -= len(chunk) + } + return nil +} + +// readFramePayload reads a frame payload of the given length. The allocation +// is charged against the aggregate budget before it happens: the recorded +// bytes alone would otherwise let every in-flight classification allocate up +// to maxClassifiedBytes without ever contributing to the budget. The charge +// is released when the read completes, whatever its outcome: by then Read has +// recorded — and charged — the payload bytes themselves, so only they remain. +func readFramePayload(bc *replayConn, length int, injected *bool, classifyEnd time.Time) ([]byte, error) { + if bc.budget != nil { + if bc.budget.Add(int64(length)) > bc.budgetLimit { + bc.budget.Add(-int64(length)) + return nil, errTotalBudgetExceeded + } + // Release the reservation on every path, including the error returns + // below: the recorded bytes are charged by Read itself. + defer bc.budget.Add(-int64(length)) + } + payload := make([]byte, length) + // readInto, unlike io.ReadFull, never swallows a read error that arrives + // together with the last bytes of the payload. + if err := readInto(bc, payload, injected, classifyEnd); err != nil { + return nil, err + } + return payload, nil +} + +// headerBlockFragment strips the padding and the priority fields from a +// HEADERS frame payload. +func headerBlockFragment(payload []byte, flags byte) ([]byte, error) { + if flags&http2FlagPadded != 0 { + if len(payload) == 0 { + return nil, errors.New("truncated HEADERS frame") + } + pad := int(payload[0]) + payload = payload[1:] + if pad > len(payload) { + return nil, errors.New("invalid HEADERS padding") + } + payload = payload[:len(payload)-pad] + } + if flags&http2FlagPriority != 0 { + if len(payload) < 5 { + return nil, errors.New("truncated HEADERS frame") + } + payload = payload[5:] + } + return payload, nil +} + +// headerCounters tracks the decoded header list under its caps; only the +// content-type decision is retained. +type headerCounters struct { + fields int + decodedSize int + contentType string +} + +func (c *headerCounters) observe(f hpack.HeaderField) { + c.fields++ + c.decodedSize += len(f.Name) + len(f.Value) + if strings.EqualFold(f.Name, "content-type") { + c.contentType = f.Value + } +} + +// contentTypeFromBlock decodes an HTTP/2 header block and reports whether its +// content type belongs to the native gRPC family. +func contentTypeFromBlock(dec *hpack.Decoder, counters *headerCounters, block []byte) (bool, error) { + if _, err := dec.Write(block); err != nil { + return false, err + } + if err := dec.Close(); err != nil { + return false, err + } + if counters.fields > maxDecodedFields || counters.decodedSize > maxDecodedBytes { + return false, errors.New("decoded header list exceeds limits") + } + return isGRPCContentType(counters.contentType), nil +} + +// isGRPCContentType reports whether the content type belongs to the native +// gRPC family. gRPC-Web content types start with the same prefix but belong +// to the HTTP server instead. +func isGRPCContentType(contentType string) bool { + ct := strings.ToLower(contentType) + return strings.HasPrefix(ct, grpcContentTypePrefix) && + !strings.HasPrefix(ct, grpcWebContentTypePrefix) +} + +// IsNativeGRPCContentType reports whether the given content type identifies a +// native gRPC request, as opposed to a gRPC-Web or plain HTTP one. The HTTP +// server uses it to forward native gRPC requests that arrive on a connection +// the classifier handed to the HTTP side. +func IsNativeGRPCContentType(contentType string) bool { + return isGRPCContentType(contentType) +} + +// replayConn is used in two modes. During classification (replay=false) it +// reads from the live connection and records every byte it consumes. The +// handoff conn produced by handoff() (replay=true) first replays those bytes +// to the server, then streams the rest of the connection. +type replayConn struct { + net.Conn + + prefix []byte + recording bool + replay bool + pos int + // budget counts this connection's share of the mux's aggregate + // classification budget; appended is its total recorded bytes so the + // share can be released exactly. + budget *atomic.Int64 + budgetLimit int64 + appended int64 +} + +func (c *replayConn) Read(p []byte) (int, error) { + if c.replay && c.pos < len(c.prefix) { + n := copy(p, c.prefix[c.pos:]) + c.pos += n + if c.pos == len(c.prefix) { + // The replay is done: release the buffer for the rest of the + // connection's lifetime. + c.prefix = nil + } + return n, nil + } + n, err := c.Conn.Read(p) + if c.recording && n > 0 { + c.prefix = append(c.prefix, p[:n]...) + c.appended += int64(n) + if c.budget != nil { + if c.budget.Add(int64(n)) > c.budgetLimit { + // The aggregate budget is exhausted: give the bytes back (the + // classifier releases the rest) and fail the read, which + // drops the connection. The bytes stay recorded in the prefix, + // but the connection is dropped either way. The error must be + // returned with n==0: io.CopyN and io.ReadFull swallow a + // trailing error when the requested bytes were all delivered. + c.budget.Add(-int64(n)) + c.appended -= int64(n) + return 0, errTotalBudgetExceeded + } + } + } + return n, err +} + +func (c *replayConn) recorded() int { return len(c.prefix) } + +// dropLast removes the n most recently recorded bytes from the replay, for +// frames the classifier consumed on behalf of the downstream server, such as +// the ACK of the injected SETTINGS. +func (c *replayConn) dropLast(n int) { + if n > len(c.prefix) { + return + } + c.prefix = c.prefix[:len(c.prefix)-n] +} + +// handoff returns the connection handed over to the server: the recorded +// prefix is replayed first, then the connection continues live. +func (c *replayConn) handoff() *replayConn { + return &replayConn{Conn: c.Conn, prefix: c.prefix, replay: true} +} + +// virtualListener is a net.Listener fed by the classifier. A dedicated +// dispatcher goroutine owns the queue: it forwards connections to Accept and, +// on Close, closes everything still owned, so no connection is ever enqueued +// after shutdown or leaked between delivery and close. +type virtualListener struct { + addr net.Addr + in chan net.Conn // deliverers; unbuffered so a closed listener never retains anything + out chan net.Conn // the server's Accept queue + done chan struct{} + dispatcherDone chan struct{} + once sync.Once +} + +func newVirtualListener(addr net.Addr) *virtualListener { + l := &virtualListener{ + addr: addr, + in: make(chan net.Conn), + out: make(chan net.Conn, acceptBacklog), + done: make(chan struct{}), + dispatcherDone: make(chan struct{}), + } + go l.dispatch() + return l +} + +func (l *virtualListener) dispatch() { + defer close(l.dispatcherDone) + for { + select { + case c := <-l.in: + select { + case l.out <- c: + case <-l.done: + _ = c.Close() + l.drain() + return + } + case <-l.done: + l.drain() + return + } + } +} + +// drain closes every connection still owned by the dispatcher. A connection +// that raced into out and was taken by the server's Accept is not owned +// anymore; a connection still queued is closed exactly once here. +func (l *virtualListener) drain() { + for { + select { + case c := <-l.in: + _ = c.Close() + case c := <-l.out: + _ = c.Close() + default: + return + } + } +} + +func (l *virtualListener) Accept() (net.Conn, error) { + select { + case c := <-l.out: + return c, nil + case <-l.done: + // Closed. Everything the dispatcher still owned was closed; a + // connection that raced out before the close is served by the + // server's graceful stop, everything else reports closure. + return nil, net.ErrClosed + } +} + +func (l *virtualListener) Close() error { + l.once.Do(func() { + close(l.done) + <-l.dispatcherDone + }) + return nil +} + +func (l *virtualListener) Addr() net.Addr { return l.addr } + +func (l *virtualListener) deliver(c net.Conn) { + select { + case l.in <- c: + case <-l.done: + _ = c.Close() + } +} + +// isTimeout reports whether the error means a read deadline was exceeded. +func isTimeout(err error) bool { + var ne net.Error + return errors.As(err, &ne) && ne.Timeout() +} diff --git a/pkg/httpgrpcmux/mux_test.go b/pkg/httpgrpcmux/mux_test.go new file mode 100644 index 000000000..74155ce66 --- /dev/null +++ b/pkg/httpgrpcmux/mux_test.go @@ -0,0 +1,938 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpgrpcmux + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "io" + "math/big" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/http2/hpack" +) + +// Header names and content types used across the tests. +const ( + hdrMethod = ":method" + hdrContentType = "content-type" + hdrMethodPost = "POST" + ctGRPC = "application/grpc" + ctGRPCProto = "application/grpc+proto" +) + +// startMux creates a real TCP listener, wraps it in a mux and starts it. +// It returns the mux and the client-side dial function. +func startMux(t *testing.T, opts *Options) (*Mux, func() (net.Conn, error)) { + t.Helper() + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = l.Close() }) + + m := New(l, opts) + require.NoError(t, m.Start(context.Background())) + t.Cleanup(func() { m.Close() }) + + dial := func() (net.Conn, error) { return net.Dial("tcp", l.Addr().String()) } + return m, dial +} + +// acceptWithTimeout waits for a connection on the given listener. +func acceptWithTimeout(t *testing.T, l net.Listener) net.Conn { + t.Helper() + type result struct { + c net.Conn + err error + } + ch := make(chan result, 1) + go func() { + c, err := l.Accept() + ch <- result{c, err} + }() + select { + case r := <-ch: + require.NoError(t, r.err) + return r.c + case <-time.After(5 * time.Second): + require.Fail(t, "timed out waiting for the connection to be classified") + return nil + } +} + +// writeClientStream sends an HTTP/2 client preface, a client SETTINGS frame +// and one HEADERS frame carrying the given headers. +func writeClientStream(t *testing.T, c net.Conn, contentType string, extraHeaders ...hpack.HeaderField) { + t.Helper() + + _, err := io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + var buf bytes.Buffer + enc := hpack.NewEncoder(&buf) + for _, hf := range append([]hpack.HeaderField{ + {Name: hdrMethod, Value: hdrMethodPost}, + {Name: ":scheme", Value: "http"}, + {Name: ":path", Value: "/controlplane.v1.FooService/Bar"}, + {Name: hdrContentType, Value: contentType}, + }, extraHeaders...) { + require.NoError(t, enc.WriteField(hf)) + } + writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) +} + +func writeFrame(t *testing.T, c net.Conn, frameType, flags byte, streamID uint32, payload []byte) { + t.Helper() + + hdr := []byte{ + byte(len(payload) >> 16), byte(len(payload) >> 8), byte(len(payload)), + frameType, flags, + byte(streamID >> 24), byte(streamID >> 16), byte(streamID >> 8), byte(streamID), + } + _, err := c.Write(hdr) + require.NoError(t, err) + if len(payload) > 0 { + _, err = c.Write(payload) + require.NoError(t, err) + } +} + +func TestClassifyHTTP1(t *testing.T) { + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + request := "GET /statusz HTTP/1.1\r\nHost: example.com\r\n\r\n" + _, err = io.WriteString(c, request) + require.NoError(t, err) + + got := acceptWithTimeout(t, m.HTTP()) + defer got.Close() + + require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) + data := make([]byte, len(request)) + _, err = io.ReadFull(got, data) + require.NoError(t, err) + assert.Equal(t, request, string(data), "the HTTP server must receive the request intact") +} + +func TestClassifyShortHTTP10Request(t *testing.T) { + // A request shorter than the HTTP/2 preface must not deadlock the + // classifier waiting for 24 bytes. + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + request := "GET / HTTP/1.0\r\n\r\n" // 18 bytes + _, err = io.WriteString(c, request) + require.NoError(t, err) + + got := acceptWithTimeout(t, m.HTTP()) + defer got.Close() + + require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) + data := make([]byte, len(request)) + _, err = io.ReadFull(got, data) + require.NoError(t, err) + assert.Equal(t, request, string(data), "short request not replayed intact") +} + +func TestClassifyNativeGRPC(t *testing.T) { + for _, ct := range []string{ + ctGRPC, + ctGRPCProto, + "application/grpc+json", + ctGRPC + "; charset=utf-8", // content type with parameters + } { + t.Run(ct, func(t *testing.T) { + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + writeClientStream(t, c, ct) + + got := acceptWithTimeout(t, m.GRPC()) + defer got.Close() + + // The gRPC server must see the preface again. + require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) + prefix := make([]byte, len(clientConnPreface)) + _, err = io.ReadFull(got, prefix) + require.NoError(t, err) + assert.Equal(t, clientConnPreface, string(prefix), "preface not replayed intact") + }) + } +} + +func TestClassifyGRPCWebGoesToHTTP(t *testing.T) { + for _, ct := range []string{ + "application/grpc-web", + "application/grpc-web+proto", + "application/grpc-web+json", + } { + t.Run(ct, func(t *testing.T) { + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + writeClientStream(t, c, ct) + + got := acceptWithTimeout(t, m.HTTP()) + got.Close() + }) + } +} + +func TestClassifyPlainH2GoesToHTTP(t *testing.T) { + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + writeClientStream(t, c, "application/json") + + got := acceptWithTimeout(t, m.HTTP()) + got.Close() +} + +func TestClassifySkipsIntermediateFrames(t *testing.T) { + // WINDOW_UPDATE frames between the preface and the first HEADERS must + // not disturb the classification. + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + writeFrame(t, c, 0x8 /* WINDOW_UPDATE */, 0, 0, []byte{0, 0, 0, 1}) + writeFrame(t, c, 0x8 /* WINDOW_UPDATE */, 0, 1, []byte{0, 0, 0, 2}) + + var buf bytes.Buffer + enc := hpack.NewEncoder(&buf) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) + writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) + + got := acceptWithTimeout(t, m.GRPC()) + got.Close() +} + +func TestClassifyPaddedAndPriorityHeaders(t *testing.T) { + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + var buf bytes.Buffer + enc := hpack.NewEncoder(&buf) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) + + // PADDED (0x8) + PRIORITY (0x20) + END_HEADERS (0x4) + // Payload: [pad length][stream dependency][weight][fragment][padding] + frag := buf.Bytes() + payload := make([]byte, 0, 6+len(frag)) + payload = append(payload, 2) // pad length = 2 + payload = append(payload, 0, 0, 0, 0, 16) // stream dep + weight + payload = append(payload, frag...) // header block fragment + payload = append(payload, 0, 0) // padding + writeFrame(t, c, http2FrameHeaders, 0x8|0x20|0x4, 1, payload) + + got := acceptWithTimeout(t, m.GRPC()) + got.Close() +} + +// TestSettingsAckNotReplayed proves that the ACK of the injected SETTINGS is +// removed from the replay: the downstream gRPC server must never see it, or it +// would count two SETTINGS acknowledgements and fail with PROTOCOL_ERROR. +func TestSettingsAckNotReplayed(t *testing.T) { + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + // The classifier injects its empty SETTINGS after the grace period. + require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) + hdr := make([]byte, 9) + _, err = io.ReadFull(c, hdr) + require.NoError(t, err) + + // The client acknowledges the injected SETTINGS, then sends its request. + writeFrame(t, c, http2FrameSettings, http2FlagSettingsAck, 0, nil) + writeClientStreamAfterPreface(t, c) + + got := acceptWithTimeout(t, m.GRPC()) + defer got.Close() + + // Parse the frames the server receives: SETTINGS (client) then HEADERS, + // and no SETTINGS-ACK. + require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) + reader := bufio.NewReader(got) + preface := make([]byte, len(clientConnPreface)) + _, err = io.ReadFull(reader, preface) + require.NoError(t, err) + assert.Equal(t, clientConnPreface, string(preface)) + + type frame struct { + frameType byte + flags byte + } + var frames []frame + for { + var fh [9]byte + if _, err := io.ReadFull(reader, fh[:]); err != nil { + break // idle connection: everything available was parsed + } + length := int(fh[0])<<16 | int(fh[1])<<8 | int(fh[2]) + if length > 0 { + if _, err := io.CopyN(io.Discard, reader, int64(length)); err != nil { + break + } + } + frames = append(frames, frame{fh[3], fh[4]}) + } + + require.NotEmpty(t, frames) + assert.Equal(t, byte(http2FrameSettings), frames[0].frameType, "first replayed frame must be the client SETTINGS") + assert.Equal(t, byte(0), frames[0].flags&http2FlagSettingsAck, "first replayed frame must not be an ACK") + foundAck := false + for _, f := range frames { + if f.frameType == http2FrameSettings && f.flags&http2FlagSettingsAck != 0 { + foundAck = true + } + } + assert.False(t, foundAck, "the ACK of the injected SETTINGS must not reach the server") +} + +// TestLateSettingsAckNotReplayed proves that the ACK of the injected SETTINGS +// is removed from the replay even when it arrives after the first request: a +// client whose frame-read loop races its request-write goroutine can send +// HEADERS first and the ACK afterwards. The classifier keeps consuming frames +// until the ACK shows up, drops it, and hands the connection to the HTTP +// server without it. +func TestLateSettingsAckNotReplayed(t *testing.T) { + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + // The classifier injects its empty SETTINGS after the grace period. + require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = io.ReadFull(c, make([]byte, 9)) + require.NoError(t, err) + + // The client sends its (non-gRPC) request BEFORE acknowledging the + // injected SETTINGS, and the ACK only afterwards. + var buf bytes.Buffer + enc := hpack.NewEncoder(&buf) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: "application/json"})) + writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) + writeFrame(t, c, http2FrameSettings, http2FlagSettingsAck, 0, nil) + + // The connection must still reach the HTTP listener: the late ACK does + // not change the classification, it only delays the handover. + got := acceptWithTimeout(t, m.HTTP()) + defer got.Close() + + // Parse the frames the server receives: SETTINGS (client) then HEADERS, + // and no SETTINGS-ACK. + require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) + reader := bufio.NewReader(got) + preface := make([]byte, len(clientConnPreface)) + _, err = io.ReadFull(reader, preface) + require.NoError(t, err) + assert.Equal(t, clientConnPreface, string(preface)) + + type frame struct { + frameType byte + flags byte + } + var frames []frame + for { + var fh [9]byte + if _, err := io.ReadFull(reader, fh[:]); err != nil { + break // idle connection: everything available was parsed + } + length := int(fh[0])<<16 | int(fh[1])<<8 | int(fh[2]) + if length > 0 { + if _, err := io.CopyN(io.Discard, reader, int64(length)); err != nil { + break + } + } + frames = append(frames, frame{fh[3], fh[4]}) + } + + require.NotEmpty(t, frames) + assert.Equal(t, byte(http2FrameSettings), frames[0].frameType, "first replayed frame must be the client SETTINGS") + foundAck := false + sawHeaders := false + for _, f := range frames { + if f.frameType == http2FrameSettings && f.flags&http2FlagSettingsAck != 0 { + foundAck = true + } + if f.frameType == http2FrameHeaders { + sawHeaders = true + } + } + assert.True(t, sawHeaders, "the client's HEADERS must reach the server") + assert.False(t, foundAck, "the ACK of the injected SETTINGS must not reach the server") +} + +// TestLateAckWaitDropsConnection proves the bound on the wait for a late +// SETTINGS ACK: a client that sends its request but never acknowledges the +// injected SETTINGS gets the connection dropped shortly after — the HTTP/2 +// connection would break on the extra ACK anyway — and never handed to a +// server. +func TestLateAckWaitDropsConnection(t *testing.T) { + // A short grace keeps the whole classification far from the end of the + // default 10s classification window, so the drop proves the dedicated + // ACK-wait bound, not the classification timeout. + m, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond}) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + // Consume the injected SETTINGS the classifier sends after the grace. + require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = io.ReadFull(c, make([]byte, 9)) + require.NoError(t, err) + + // The client sends a non-gRPC request and never ACKs. + var buf bytes.Buffer + enc := hpack.NewEncoder(&buf) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: "application/json"})) + writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) + + // The classifier must close the connection when the wait for the ACK + // expires (bounded at 2s), well before the classification window (10s) + // would. The client's own 5s read deadline proves it: a connection held + // past it would surface here as a deadline error instead. + require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) + start := time.Now() + _, err = c.Read(make([]byte, 9)) + require.Error(t, err, "expected the connection to be dropped when the ACK never arrives") + require.NotErrorIs(t, err, os.ErrDeadlineExceeded, "the connection was closed by the classifier, not by the client's deadline") + require.Less(t, time.Since(start), 5*time.Second, "the classifier held the connection past the ACK wait bound") + + // Nothing must have been delivered to either server. + time.Sleep(200 * time.Millisecond) + type result struct { + c net.Conn + err error + } + for _, l := range []net.Listener{m.GRPC(), m.HTTP()} { + ch := make(chan result, 1) + go func(l net.Listener) { + c, err := l.Accept() + ch <- result{c, err} + }(l) + select { + case r := <-ch: + if r.err == nil { + r.c.Close() + assert.Fail(t, "the connection was delivered to a server despite the missing ACK") + } + case <-time.After(100 * time.Millisecond): + // Nothing delivered: correct. + } + } +} + +func TestOversizedFrameDropped(t *testing.T) { + // A frame payload that would exceed the classification budget must get + // the connection dropped, not buffered. + _, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + // A WINDOW_UPDATE whose length field claims almost 16 MiB: the classifier + // must reject it instead of recording it. + huge := []byte{0xFF, 0xFF, 0xFF, 0x8, 0x00, 0x00, 0x00, 0x00, 0x00} + _, err = c.Write(huge) + require.NoError(t, err) + + require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + _, err = c.Read(make([]byte, 9)) + require.Error(t, err, "expected the connection to be dropped for the oversized frame") +} + +func TestCloseClosesQueuedConnections(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + m := New(l, nil) + + // Queue a classified connection without any server accepting it. + vl := m.GRPC().(*virtualListener) + serverEnd, clientEnd := net.Pipe() + vl.deliver(serverEnd) + defer clientEnd.Close() + + m.Close() + + // The queued connection must have been closed by Close, not leaked. + // SetReadDeadline may already fail because the pipe end was closed, + // which is exactly what we are proving; the Read below distinguishes + // closure from a timeout. + _ = clientEnd.SetReadDeadline(time.Now().Add(time.Second)) + _, err = clientEnd.Read(make([]byte, 1)) + require.Error(t, err) + // A read timeout would mean the connection is still open: Close must + // actually have closed it. + require.NotErrorIs(t, err, os.ErrDeadlineExceeded, "the queued connection was not closed when the mux stopped") +} + +// TestAggregateBudgetShedsConnections proves the aggregate byte budget: once +// the total recorded bytes across in-flight classifications exceed it, new +// recording fails and the connection is dropped. +func TestAggregateBudgetShedsConnections(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + m := New(l, &Options{MaxTotalClassifiedBytes: 4096, ClassifyTimeout: 5 * time.Second}) + require.NoError(t, m.Start(context.Background())) + t.Cleanup(m.Close) + + dial := func() (net.Conn, error) { return net.Dial("tcp", l.Addr().String()) } + + // First connection records ~3 KiB and stays in classification. + c1, err := dial() + require.NoError(t, err) + defer c1.Close() + _, err = io.WriteString(c1, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c1, http2FrameSettings, 0, 0, nil) + writeFrame(t, c1, 0x8 /* WINDOW_UPDATE */, 0, 0, bytes.Repeat([]byte{0}, 3072)) + // Wait until the first connection's bytes are all accounted for — exactly: + // nothing else has been sent, so this state is stable — so the second + // connection deterministically trips the aggregate budget. + require.Eventually(t, func() bool { return m.budget.Load() == 3114 }, + 2*time.Second, 10*time.Millisecond, "first connection's bytes were not recorded") + + // Second connection pushes the aggregate over the budget: its recording + // must fail and its connection be dropped. + c2, err := dial() + require.NoError(t, err) + defer c2.Close() + _, err = io.WriteString(c2, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c2, http2FrameSettings, 0, 0, nil) + writeFrame(t, c2, 0x8 /* WINDOW_UPDATE */, 0, 0, bytes.Repeat([]byte{0}, 2048)) + + require.NoError(t, c2.SetReadDeadline(time.Now().Add(2*time.Second))) + _, err = c2.Read(make([]byte, 9)) + require.Error(t, err, "expected the connection to be dropped when the aggregate budget is exhausted") + require.NotErrorIs(t, err, os.ErrDeadlineExceeded, "connection was not dropped; the read merely timed out") +} + +// TestPayloadAllocationChargedToBudget proves that a frame payload's +// allocation is charged against the aggregate budget before it happens: a +// connection that declares a large payload while the aggregate budget is +// nearly exhausted must be dropped instead of silently allocating the bytes +// and contributing nothing to the budget. The refused reservation must also +// be released, keeping the accounting exact. +func TestPayloadAllocationChargedToBudget(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + m := New(l, &Options{MaxTotalClassifiedBytes: 4096, ClassifyTimeout: 5 * time.Second}) + require.NoError(t, m.Start(context.Background())) + t.Cleanup(m.Close) + + dial := func() (net.Conn, error) { return net.Dial("tcp", l.Addr().String()) } + + // First connection records ~3 KiB and stays in classification. + c1, err := dial() + require.NoError(t, err) + defer c1.Close() + _, err = io.WriteString(c1, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c1, http2FrameSettings, 0, 0, nil) + writeFrame(t, c1, 0x8 /* WINDOW_UPDATE */, 0, 0, bytes.Repeat([]byte{0}, 3072)) + require.Eventually(t, func() bool { return m.budget.Load() == 3114 }, + 2*time.Second, 10*time.Millisecond, "first connection's bytes were not recorded") + + // Second connection declares a 2 KiB HEADERS payload without sending it: + // its recorded bytes (3156) plus the 2048 reservation exceed the 4096 + // budget, so the allocation must be refused and the connection dropped at + // once, instead of allocating the bytes and only noticing when the + // payload trickles in. + c2, err := dial() + require.NoError(t, err) + defer c2.Close() + _, err = io.WriteString(c2, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c2, http2FrameSettings, 0, 0, nil) + // HEADERS frame header claiming 2048 bytes of payload, never sent. + _, err = c2.Write([]byte{0x00, 0x08, 0x00, http2FrameHeaders, 0x00, 0, 0, 0, 1}) + require.NoError(t, err) + + require.NoError(t, c2.SetReadDeadline(time.Now().Add(2*time.Second))) + _, err = c2.Read(make([]byte, 9)) + require.Error(t, err, "expected the connection to be dropped when the payload allocation exceeds the budget") + require.NotErrorIs(t, err, os.ErrDeadlineExceeded, "connection was not dropped; the read merely timed out") + + // The refused reservation must leave the accounting exact: only the first + // connection's share stays charged. + require.Eventually(t, func() bool { return m.budget.Load() == 3114 }, + 2*time.Second, 10*time.Millisecond, "the refused reservation was not released") +} + +// TestHPACKAmplificationRejected proves the decoded-header-list caps: a tiny +// compressed block that expands into thousands of header fields must get the +// connection dropped instead of being amplified in memory. +func TestHPACKAmplificationRejected(t *testing.T) { + _, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + var buf bytes.Buffer + enc := hpack.NewEncoder(&buf) + big := hpack.HeaderField{Name: "x-bomb", Value: strings.Repeat("a", 1024)} + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) + require.NoError(t, enc.WriteField(big)) + // One-byte indexed references to the big field: the compressed block stays + // small while the decoded list grows past the cap. + for i := 0; i < 2000; i++ { + require.NoError(t, enc.WriteField(big)) + } + writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) + + require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + _, err = c.Read(make([]byte, 9)) + require.Error(t, err, "expected the connection to be dropped for the amplified header list") +} + +// TestAckWithoutInjectionDropped proves that a SETTINGS-ACK for which the +// classifier never injected any SETTINGS is rejected: no server SETTINGS were +// sent before the handover, so such an ACK is malformed traffic. +func TestAckWithoutInjectionDropped(t *testing.T) { + _, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + writeFrame(t, c, http2FrameSettings, http2FlagSettingsAck, 0, nil) + + require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + _, err = c.Read(make([]byte, 9)) + require.Error(t, err, "expected the connection to be dropped for the unexpected ACK") +} + +// TestMalformedAckDropped proves that a SETTINGS-ACK with a payload is +// rejected: replaying its bytes as frame headers would rewrite the client's +// frame stream. +func TestMalformedAckDropped(t *testing.T) { + _, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + // Trigger the injected SETTINGS, acknowledge it with a bogus payload. + require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = io.ReadFull(c, make([]byte, 9)) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, http2FlagSettingsAck, 0, []byte{0, 0, 0, 1}) + + require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + _, err = c.Read(make([]byte, 9)) + require.Error(t, err, "expected the connection to be dropped for the malformed ACK") +} + +// eofAtEndConn returns the whole payload together with a trailing io.EOF, as +// a TLS connection may when the peer half-closes right after its last record. +type eofAtEndConn struct { + net.Conn + data []byte + sent bool +} + +func (c *eofAtEndConn) Read(p []byte) (int, error) { + if c.sent { + return 0, io.EOF + } + c.sent = true + return copy(p, c.data), io.EOF +} + +// TestReadIntoToleratesTrailingEOF proves readInto honors the io.Reader +// contract: a Read that returns the completing bytes together with an error +// must not fail the read. +func TestReadIntoToleratesTrailingEOF(t *testing.T) { + bc := &replayConn{Conn: &eofAtEndConn{data: []byte("0123456789")}, recording: true} + injected := false + buf := make([]byte, 10) + require.NoError(t, readInto(bc, buf, &injected, time.Now().Add(time.Second))) + assert.Equal(t, "0123456789", string(buf)) +} + +func TestServerSpeaksFirst(t *testing.T) { + // A gRPC client that waits for the server's SETTINGS before sending its + // first request must still be classified. + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, clientConnPreface) + require.NoError(t, err) + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + + // The classifier must send its empty SETTINGS frame after the grace + // period, unblocking the waiting client. + require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) + hdr := make([]byte, 9) + _, err = io.ReadFull(c, hdr) + require.NoError(t, err) + assert.Equal(t, byte(http2FrameSettings), hdr[3], "expected a SETTINGS frame from the classifier") + assert.Zero(t, hdr[4]&http2FlagSettingsAck, "expected a SETTINGS frame, not an ACK") + + writeClientStreamAfterPreface(t, c) + + got := acceptWithTimeout(t, m.GRPC()) + got.Close() +} + +// writeClientStreamAfterPreface sends SETTINGS + HEADERS on a connection that +// already sent its preface. +func writeClientStreamAfterPreface(t *testing.T, c net.Conn) { + t.Helper() + + writeFrame(t, c, http2FrameSettings, 0, 0, nil) + var buf bytes.Buffer + enc := hpack.NewEncoder(&buf) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) + writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) +} + +func TestIdleConnectionDropped(t *testing.T) { + m, dial := startMux(t, &Options{ClassifyTimeout: 100 * time.Millisecond}) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + // The client sends nothing: the classifier must give up on the idle + // connection after the deadline, not hold it. + require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + _, err = c.Read(make([]byte, 9)) + require.Error(t, err, "expected the connection to be closed after the classification timeout") + + // Give the mux a moment to drop it, then make sure nothing was delivered. + time.Sleep(200 * time.Millisecond) + type result struct { + c net.Conn + err error + } + for _, l := range []net.Listener{m.GRPC(), m.HTTP()} { + ch := make(chan result, 1) + go func(l net.Listener) { + c, err := l.Accept() + ch <- result{c, err} + }(l) + select { + case r := <-ch: + if r.err == nil { + r.c.Close() + assert.Fail(t, "idle connection was delivered to a server") + } + case <-time.After(100 * time.Millisecond): + // Nothing delivered: correct. + } + } +} + +func TestCloseUnblocksAccept(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + m := New(l, nil) + require.NoError(t, m.Start(context.Background())) + + errCh := make(chan error, 2) + for _, lis := range []net.Listener{m.GRPC(), m.HTTP()} { + go func(lis net.Listener) { + _, err := lis.Accept() + errCh <- err + }(lis) + } + + time.Sleep(50 * time.Millisecond) + m.Close() + + for i := 0; i < 2; i++ { + select { + case err := <-errCh: + require.ErrorIs(t, err, net.ErrClosed) + case <-time.After(2 * time.Second): + require.Fail(t, "Accept was not unblocked by Close") + } + } +} + +func TestTLSALPNClassification(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer l.Close() + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{selfSignedCert(t)}, + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + } + m := New(tls.NewListener(l, tlsConfig), nil) + require.NoError(t, m.Start(context.Background())) + t.Cleanup(m.Close) + + dialTLS := func(t *testing.T, alpn string) net.Conn { + t.Helper() + conn, err := tls.Dial("tcp", l.Addr().String(), &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // test server with a self-signed certificate + NextProtos: []string{alpn}, + MinVersion: tls.VersionTLS12, + }) + require.NoError(t, err) + return conn + } + + // ALPN h2 with a gRPC first request -> gRPC server. + c := dialTLS(t, "h2") + writeClientStream(t, c, ctGRPCProto) + got := acceptWithTimeout(t, m.GRPC()) + got.Close() + c.Close() + + // ALPN http/1.1 -> HTTP server without any frame parsing. + c = dialTLS(t, "http/1.1") + _, err = io.WriteString(c, "GET /statusz HTTP/1.1\r\nHost: x\r\n\r\n") + require.NoError(t, err) + got = acceptWithTimeout(t, m.HTTP()) + got.Close() + c.Close() +} + +func selfSignedCert(t *testing.T) tls.Certificate { + t.Helper() + + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + tmpl := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + require.NoError(t, err) + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +// TestCloseCoversConnectionAcceptedBeforeClassification proves the tracking +// has no window: a connection accepted but not yet inside its classify +// goroutine is still closed by Close, so shutdown does not ride out the +// classification deadline. +func TestCloseCoversConnectionAcceptedBeforeClassification(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + m := New(l, nil) + require.NoError(t, m.Start(context.Background())) + + c, err := net.Dial("tcp", l.Addr().String()) + require.NoError(t, err) + defer c.Close() + + // Wait until a connection is registered in the inflight set (keyed by the + // server-side connection, not the client side), which is what Close walks + // — this is exactly the state the race used to miss. + require.Eventually(t, func() bool { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.inflight) > 0 + }, 2*time.Second, 5*time.Millisecond, "the accepted connection was not registered in the inflight set") + + start := time.Now() + m.Close() + // The idle connection's deadline (10s) must not gate the shutdown. + require.Less(t, time.Since(start), 2*time.Second) +} From 8d55976cb7fa0381e873f08ea9d0858d203858a4 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 15:51:02 +0200 Subject: [PATCH 02/10] docs(specs): issue-3628 supersede D-001, D-005, D-010 with D-013, D-014, D-015 Record the cmux direction agreed in the implementation PR: cmux replaces the custom connection classifier (D-013), the separate-upstream-connection contract and the one-way fallback are explicit and the mixed-connection open question is answered (D-014), and the in-server solution keeps tested nginx and Traefik configurations instead of any proxy (D-015). The proposal, diagram and risk rows name the classifier and the speak-first settings frame. Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: ses_eabb93119ffegqWXti0mS05TIM --- docs/specs/issue-3628-single-endpoint.md | 284 +++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 docs/specs/issue-3628-single-endpoint.md diff --git a/docs/specs/issue-3628-single-endpoint.md b/docs/specs/issue-3628-single-endpoint.md new file mode 100644 index 000000000..c8eeea2a5 --- /dev/null +++ b/docs/specs/issue-3628-single-endpoint.md @@ -0,0 +1,284 @@ +--- +owner: migmartri +ticket: https://github.com/chainloop-dev/chainloop/issues/3628 +prd: +--- + + + +# Spec issue-3628: Serve gRPC and HTTP from one endpoint per service + +## Summary + +The control plane and the Artifact CAS will each serve gRPC and HTTP on one port. The control plane also serves gRPC-Web on it, as it does today. Today each service has a gRPC port and an HTTP port. An installation thus needs two Services, two ingresses, two hostnames, and two TLS settings per service. The service will accept all connections on its current HTTP port, the one that serves the human-facing hostname today. It will look at each new connection and give it to the gRPC server or to the HTTP server that runs today. The middleware, the authentication, and the native gRPC transport for large CAS streams do not change. The legacy gRPC port stays on by default for a deprecation period, behind a config flag. The Helm chart moves to one Service and one hostname per service. + +## Problem + +- Each service listens on two ports: gRPC on 9000 and HTTP on 8000 (9001 and 8001 for the CAS in local development). The Helm chart creates two Services, two ingresses, and two hostnames for each service. That is four ingresses and four hostnames for one installation. +- Operators must know which hostname carries which traffic: + - The CLI and the control plane use gRPC to reach the CAS. + - Browsers use HTTP for the OIDC login, the status endpoints, the referrer discovery, and the OpenAPI document. + - The web UI uses gRPC-Web, which the control plane serves on its HTTP port. + - CAS downloads in a browser use HTTP. +- TLS can only be set on the gRPC listener. The HTTP listener is always plain text, so operators must terminate TLS for HTTP somewhere else. +- An earlier attempt (#2642) solved this at the chart level and depended on Gateway API. The team closed it as stale. The problem stays. + +## Goals and Non-Goals + +- Goal: one port and one hostname per service, for all gRPC, gRPC-Web, and HTTP traffic. +- Goal: existing clients and configuration keep working without change during the deprecation period. +- Goal: large CAS uploads and downloads keep their current streaming behavior and throughput. +- Non-goal: move the services to another RPC framework (for example Connect). The single port does not need it (see D-001). +- Non-goal: merge the Prometheus metrics port or the profiler port into the single port. These ports are internal and must not be on the public hostname (see D-006). +- Non-goal: add Gateway API resources to the Helm chart. It is a separate chart feature, and the single port works without it (see D-004). +- Non-goal: change the CLI's default endpoints or its configuration format. The CLI holds hostnames on the ingress port, so it needs no change (see R-005). +- Non-goal: replace the gRPC-Web library of the control plane or change its origin policy. The single port keeps the current gRPC-Web wrapper with no change. Follow-up: a separate ticket covers the library replacement and a security review of its origin policy. + +## Requirements + +### R-001: One port serves all protocols +On its single port, each service MUST serve gRPC (HTTP/2) and plain HTTP (HTTP/1.1 and HTTP/2). The control plane MUST also serve gRPC-Web. The CAS has no gRPC-Web today and does not get it (see D-011). This includes every endpoint that the gRPC port and the HTTP port serve today. +- Done when: these clients work against only the single port: + - the CLI and the web UI + - the OIDC login and the status probes + - the referrer discovery and the OpenAPI document + - CAS browser downloads +- Done when: the same checks pass through nginx ingress and through Traefik ingress. + +### R-005: Existing configuration keeps working +An installation that does not change its configuration MUST keep working during the deprecation period. The legacy gRPC port keeps serving gRPC at its current address during that period. CLI configurations that use ingress hostnames keep working when the ingress backend moves to the single port. The control plane connection to the CAS, and any other client that uses the legacy gRPC address, MUST move to the single HTTP address before the legacy gRPC port is disabled. +- Done when: the release notes name this migration step for the CAS connection and for direct gRPC clients. + +### R-002: Same behavior as today +A request on the single port MUST get the same result as the same request gets today on the separate ports. This covers authentication, authorization, validation, logging, tracing, and metrics. +- Done when: server-level tests send the same calls through the single port and get the same result as today. The calls cover: + - no token, a user token, an API token, and a robot account or attestation token + - CAS uploader and downloader tokens on streams and on the download endpoint + - a validation error and a suspended organization + - a gRPC-Web call to the control plane + + These tests merge before the listener change, so that they first record the current behavior. After the change, the gRPC calls also run against the legacy gRPC port. + +### R-003: CAS streaming parity +Through the single port, CAS uploads and downloads MUST stream with bounded memory. Their throughput MUST stay within 10% of the current release for the same artifact. +- Done when: a benchmark uploads and downloads a 1 GB artifact through the single port and through the current release. The implementation PR records the results. + +### R-004: One TLS setting per service +When the operator configures TLS for a service, the single port MUST serve gRPC, gRPC-Web, and HTTP over that TLS configuration. It MUST negotiate HTTP/2 and HTTP/1.1. + +### R-006: Legacy gRPC port behind a flag +A configuration flag MUST control the legacy gRPC port. During the deprecation period the flag MUST default to on. When the flag is off, the service MUST listen only on the single port, plus the internal metrics and profiler ports. When the legacy gRPC port is on, the service SHOULD log a deprecation warning at startup. + +### R-007: Health checks on the single port +The HTTP status endpoint (`/statusz`, including readiness) and the standard gRPC health service MUST be available on the single port. + +### R-008: One Service and one hostname in the Helm chart +The Helm chart MUST create one Service and one external hostname per service. Probes MUST target the single port. The install notes and the CAS download URL MUST use the single hostname. While the legacy flag is on, the chart MAY keep the legacy gRPC port on the Service. +- Done when: a default install with nginx ingress serves the CLI, the web UI, and CAS downloads through one hostname per service. + +## Constraints + +- Public, open-source repository: the design must work with common ingress controllers and with no chart-level proxy. +- Backward compatibility for one deprecation period: no client, CLI configuration, or chart value may break on upgrade. +- The CAS spills uploads to disk and sends staged downloads with the kernel copy path. The design must keep both. + +## Proposal + +### What the operator sees + +After the upgrade, nothing changes for the operator. The HTTP port now also answers native gRPC, and the legacy gRPC port still works. The operator points one hostname at the HTTP port of each service. Before the operator sets the legacy flag to off, the operator moves the control plane connection to the CAS, and any other client that uses the gRPC address directly, to the single HTTP address. Then the operator sets the flag to off, and the gRPC port closes. In a later release the flag defaults to off. A release after that deletes the legacy gRPC port. + +### How the single port works + +Each service opens one listener on its current HTTP address. The cmux connection multiplexer reads the start of each new connection, without consuming it. A service-owned matcher decides from the first request: + +- An HTTP/2 connection whose first request has a content type that starts with `application/grpc` goes to the native gRPC server. This includes `application/grpc+proto` and content types with parameters. gRPC-Web (`application/grpc-web`) is not in this family and stays on the HTTP server. This server is the same one that runs today, with its middleware, interceptors, message size limits, and streaming transport. +- All other connections go to the HTTP server that runs today. These are HTTP/1.1, HTTP/2 without gRPC, and gRPC-Web. The HTTP server keeps its routes and its middleware. In the control plane it also keeps the gRPC-Web wrapper. In the CAS it keeps the CORS rules for browser downloads. The CAS has no gRPC-Web wrapper, and this spec does not add one. + +Both servers keep their own lifecycle. They read from the classifier instead of from their own port. When the operator configures TLS, the listener terminates TLS before the classifier, and it offers HTTP/2 and HTTP/1.1 through ALPN. One certificate then covers all protocols. + +The classifier speaks only when a connection stays silent. gRPC clients wait for the server's HTTP/2 settings frame before they send their first request, so after a short grace the classifier sends one. From that moment the connection belongs to the gRPC server: the client's answer to that frame would break the HTTP server's HTTP/2 framing. The classifier closes the connection when its first request is not native gRPC. Clients that send their first request without waiting never see a write. + +The classifier decides once per connection, from its first request. A connection that starts with gRPC carries only gRPC: the native gRPC server cannot serve a later plain HTTP request on it. The other direction works. When a gRPC request arrives on a connection that the HTTP server owns, the HTTP server gives it to the gRPC server. The gRPC-Web wrapper already works in this way. The request gets the right result through the HTTP adapter of the gRPC server. That adapter is a separate transport. Its performance and features may differ from the native one (D-013). + +Thus the single port supports clients and proxies that keep gRPC and plain HTTP on separate HTTP/2 connections. These include direct clients such as the CLI, nginx ingress, and Traefik with HTTP/1.1 for the HTTP path. A proxy that sends both on one shared upstream connection is not supported (see D-010 and Open Questions). + +CAS streams on the single port use the same native gRPC transport as today, so uploads keep their disk spill and their throughput (R-003). Browser downloads stay on HTTP/1.1 through the ingress, so the kernel copy path stays. + +```mermaid +flowchart LR + CLI[CLI / control plane
gRPC] --> L + UI[Web UI
gRPC-Web] --> L + B[Browser
OIDC, status, downloads] --> L + L["Single port
(optional TLS + ALPN)"] --> C{"cmux:
classify new connection
by first request"} + C -- "HTTP/2 + application/grpc" --> T["Native gRPC transport
(grpc-go Serve)
today's middleware"] + C -- "everything else" --> H["HTTP server
routes, CORS
gRPC-Web: control plane only"] + T --> S["gRPC services"] + H -. "native gRPC request on an HTTP-owned
connection (grpc-go ServeHTTP)" .-> S + LG["Legacy gRPC port
flag, default on"] --> T +``` + +### Configuration + +The current HTTP address becomes the single port. It gets its own TLS settings, which apply to all protocols. The existing gRPC TLS settings stay with the legacy gRPC listener. A new flag in the server configuration controls the legacy gRPC listener, which keeps its current address. The Helm chart exposes the same flag as a value for each service. + +### Helm chart + +- One Service per service. It exposes the single port, plus the legacy gRPC port while the flag is on. +- One hostname per service. Controllers that can speak HTTP/2 without TLS to the backend use one Ingress. nginx sets the backend protocol per Ingress object. For nginx, the chart renders a second Ingress on the same hostname. It routes the gRPC service path prefixes with the gRPC backend protocol. The body size settings that the CAS needs today move with it. +- These items use the single port and hostname: + - the probes and the install notes + - the CLI `config save` hint + - the control plane connection to the CAS + - the CAS download URL +- The current split values (the second Service and the second ingress) keep working during the deprecation period. The chart documentation marks them as deprecated. + +## Decision Record + +### ~~D-001: How to serve both protocols on one port~~ +Superseded by D-013. + +~~The service splits by connection in front of the current gRPC and HTTP servers.~~ +- ~~Why: it keeps the whole middleware chain and the native gRPC transport for CAS streams, and it needs the smallest change.~~ +- ~~Alternatives considered:~~ + - ~~One HTTP handler that routes each request: gRPC calls then use the HTTP adapter of the gRPC server. The gRPC library documents that path as slower and with fewer features. That puts R-003 at risk.~~ + - ~~A move to Connect: it replaces every middleware and route, which is a large migration for the same result.~~ +- ~~Source: drafting~~ + +### D-003: Deprecation default +The legacy gRPC port is on by default, then off by default, then deleted. +- Why: no upgrade breaks, and operators move when they are ready. +- Alternatives considered: + - Legacy gRPC port opt-in from the first release: it breaks clients that connect to that port directly. +- Source: drafting + +### D-004: Ingress shape +The chart uses one Service and one hostname per service. Only where the controller needs it, the chart splits paths into two Ingress objects on the same host. +- Why: nginx sets the backend protocol per Ingress object, so one object cannot carry both protocols. +- Alternatives considered: + - One Ingress only: it excludes nginx, the most common controller. + - Add Gateway API in this work: it is a separate feature. +- Source: drafting + +### ~~D-005: Solve in the server or in the chart~~ +Superseded by D-015. + +~~The services solve it in the server.~~ +- ~~Why: it works with any ingress, any proxy, and outside Kubernetes.~~ +- ~~Alternatives considered:~~ + - ~~A chart-level proxy or Gateway API routes (#2642): they need extra infrastructure and leave two ports in the service.~~ +- ~~Source: drafting~~ + +### D-006: Metrics and profiler ports +The metrics and profiler ports stay separate. +- Why: they are internal and must not be on the public hostname. +- Source: drafting + +### D-007: Length of the deprecation period +The legacy gRPC port defaults to off three months after the single port ships. A release three months after that deletes it. The release notes announce each step. +- Why: operators get two release windows to move their ingress and probes. +- Alternatives considered: + - Wait for a major release: the two-port cost stays for too long. +- Source: drafting + +### D-008: TLS on the single port +When the operator configures TLS, all protocols on the single port use it. An HTTP ingress must use an HTTPS backend. The legacy gRPC port keeps its own TLS settings during the deprecation period. +- Why: one certificate and one setting per service is the goal of this work. +- Alternatives considered: + - Accept plain text and TLS on the same port: it adds a second classification step and weakens the TLS setting. +- Source: drafting + +### D-009: Helm chart in the same spec +The Helm chart change stays in this spec, as milestone 2. +- Why: the goal is one port and one hostname per service. The chart gives operators the single hostname. +- Alternatives considered: + - A separate spec for the chart: the server change alone does not delete the second hostname. +- Source: drafting + +### ~~D-010: Proxies that share one connection for gRPC and HTTP~~ +Superseded by D-014. + +~~The single port routes each connection once. It supports only clients and proxies that keep gRPC and plain HTTP on separate HTTP/2 connections.~~ +- ~~Why: a connection that the native gRPC server owns cannot pass single requests to the HTTP server. Native gRPC transport for CAS streams (R-003) needs the connection split of D-001.~~ +- ~~Alternatives considered:~~ + - ~~Route each request on one HTTP/2 server: it supports every proxy. But all gRPC calls then use the slower HTTP adapter of the gRPC server, which puts R-003 at risk.~~ +- ~~Source: [cubic review](https://github.com/chainloop-dev/chainloop/pull/3629#discussion_r4286475927)~~ + +### D-011: gRPC-Web only on the control plane +The single port serves gRPC-Web on the control plane only. The CAS serves gRPC and HTTP. +- Why: the CAS has no gRPC-Web today, and no CAS client needs it. A CAS adapter would be a new feature with its own streaming, authentication, and CORS design, and this spec is about one port per service. +- Alternatives considered: + - Add a gRPC-Web adapter to the CAS: it adds scope that can ship on its own. A separate spec can cover it if a client needs it. +- Source: [cubic review](https://github.com/chainloop-dev/chainloop/pull/3629#discussion_r4307471522) + +### D-012: The current HTTP port is the single endpoint +Drops D-002. The current HTTP port becomes the single port, not the gRPC port. The legacy listener is the gRPC port. +- Why: the human-facing hostname, with the status, download, and login endpoints, is already on the HTTP port. It becomes the one endpoint, with one URL, one certificate, and one TLS setting. +- Why: clients that use ingress hostnames do not break in either direction. During the deprecation period the legacy gRPC port keeps serving. After it closes, the gRPC ingress changes only its backend port. Direct clients, such as the control plane connection to the CAS, move to the single HTTP address (see R-005). +- Why: the ingress and chart work is the same size in both variants, so it does not decide the question. +- Cost: TLS settings move to the HTTP port, and the deprecation window applies to the gRPC port. +- Alternatives considered: + - The gRPC port (the dropped D-002): it keeps TLS settings in place, but the human-facing hostname stays on a second port. + - A new port: every client changes. +- Source: [PR comment](https://github.com/chainloop-dev/chainloop/pull/3629#issuecomment-6253559302) + +### D-013: How to serve both protocols on one port +Supersedes D-001. The service splits by connection in front of the current gRPC and HTTP servers, with the cmux library as the connection classifier. The library owns the connection buffering and the listener plumbing. The service owns only a small matcher and the HTTP request dispatcher. The matcher sends the server's HTTP/2 settings frame to clients that wait for it. It reads the first request's content type through the standard HTTP/2 library. The matcher closes the connection when its first request is not native gRPC. The dispatcher carries the gRPC-Web wrapper and the native-gRPC fallback. +- Why: it keeps the whole middleware chain and the native gRPC transport for CAS streams. It also takes HTTP/2 parsing out of the code this project maintains. Argo CD and etcd run the same library split in production. +- Why: the gRPC library documents that its HTTP adapter may differ in performance and features from the native transport. It does not document the adapter as slower, and this project has not measured a difference. The native transport stays the normal path for gRPC connections, so R-003 does not rest on the adapter. +- Alternatives considered: + - The custom connection parser of the first implementation: it works, but the project then maintains its own HTTP/2 frame parsing, replay, and resource bounds. + - One HTTP handler that routes each request: every gRPC call, on every connection, then uses the HTTP adapter of the gRPC server. + - A move to Connect: it requires migrating the generated server interfaces and the interceptors. Ordinary HTTP routes can stay, but the migration is larger than the classifier swap for the same single port. +- Source: [PR comment](https://github.com/chainloop-dev/chainloop/pull/3647#issuecomment-6266145446) + +### D-014: Separate upstream connections, one-way fallback +Supersedes D-010. The single port routes each connection once. It supports clients and proxies that keep native gRPC and plain HTTP on separate upstream HTTP/2 connections. Supported ingress configurations use separate upstream pools for the two kinds of traffic. The chart documents and tests them for nginx and Traefik. A connection that the HTTP server owns may carry a native gRPC request later. The HTTP server then gives the request to the gRPC server through the documented fallback, on the adapter transport. The reverse is not supported: a connection that the native gRPC server owns cannot carry a plain HTTP request. Arbitrary mixed native-gRPC and plain-HTTP requests on one shared upstream connection are not required. +- Why: direct clients keep native gRPC and plain HTTP on separate transports, and no measured traffic for this product shows mixed upstream connections. The fallback keeps the HTTP-first direction working, and the separate-pool contract keeps the native transport on the normal gRPC path. +- Alternatives considered: + - Route each request on one HTTP/2 server: it supports every proxy. But every gRPC call then uses the HTTP adapter of the gRPC server, whose performance and features may differ (D-013). +- Source: [PR comment](https://github.com/chainloop-dev/chainloop/pull/3647#issuecomment-6266145446) + +### D-015: Solve in the server, with tested proxy configurations +Supersedes D-005. The services solve the multiplexing in the server process. The single port runs directly on a VM, in Docker Compose, and in Kubernetes, with no proxy or sidecar. Behind an L7 proxy, the supported configurations keep native gRPC and plain HTTP on separate upstream connections (D-014). The chart tests and documents those configurations for nginx and Traefik. +- Why: no additional process, configuration, or lifecycle in any deployment. The earlier "any ingress, any proxy" claim was broader than any test covered. The separate-connection contract is the tested and documented support boundary. +- Alternatives considered: + - A chart-level proxy or Gateway API routes (#2642): they need extra infrastructure and leave two ports in the service. + - A mandatory sidecar or embedded proxy: it works behind more proxy configurations, but it adds a process and its lifecycle to every deployment. +- Source: [PR comment](https://github.com/chainloop-dev/chainloop/pull/3647#issuecomment-6266145446) + +## Open Questions + +- [x] **Must the single port support proxies that share one upstream connection for gRPC and plain HTTP?** Examples: Envoy-based proxies, service meshes. Answered by D-014: not required. Setups that share one upstream pool can route gRPC to the legacy gRPC port during the deprecation period. The chart documents the supported nginx and Traefik configurations, which keep the two kinds of traffic on separate upstream connections. Source: [cubic review](https://github.com/chainloop-dev/chainloop/pull/3629#discussion_r4286475927). + +D-007 and D-008 record the answers to the questions from drafting. + +## Milestones + +1. **Single port in the servers.** Both services serve all protocols on the HTTP port. The legacy gRPC flag exists and defaults to on. The CAS benchmark (R-003) passes. +2. **Helm chart.** One Service and one hostname per service. The split values are deprecated. +3. **Deprecation.** The flag defaults to off. A later release deletes the legacy gRPC port. + +## Risks + +| Risk | Mitigation | +|------|------------| +| A proxy shares one upstream connection between gRPC and HTTP requests, for example an Envoy-based proxy or a service mesh. A plain HTTP request then lands on a connection that the classifier gave to gRPC, and fails. | The chart configures nginx and Traefik to use separate upstream connections. The documentation lists the supported setups. The legacy gRPC port stays available during the deprecation period. | +| A gRPC client waits for the server's HTTP/2 settings frame before it sends its first request, so the matcher cannot read it. | After a short silence the matcher sends an empty settings frame. The connection then stays on the gRPC side, because the client's answer to that frame would break the HTTP server's HTTP/2 framing. The matcher closes the connection when its first request is not native gRPC. Clients that send their first request without waiting never see a write. | +| A slow or idle client holds a connection before the classifier decides. | Put a short read deadline on classification only, and clear it when the classifier hands over the connection. The matcher reads at most a small header budget per connection. The service caps how many connections may be in classification. Long CAS streams are not affected. | +| A middleware behaves differently on the single port, for example an authentication step that does not run. | The server-level tests in R-002 merge first and record the current behavior. The same tests then run against the single port. | +| CAS throughput drops on the single port. | The benchmark in R-003 runs before release. gRPC streams use the native transport, so the expected result is no change. | From 6d499c2a67f8878dda8006f742cc02e988a42e38 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 15:51:06 +0200 Subject: [PATCH 03/10] feat(controlplane): replace the custom single-port classifier with cmux The single port keeps serving gRPC, gRPC-Web and HTTP from the HTTP address, but the connection split now runs on the cmux library instead of the custom HTTP/2 frame parser: the library owns buffering, replay and listener plumbing, and frame parsing and HPACK decoding are golang.org/x/net's. The service owns one matcher that decides native gRPC from the first request's content type, and the HTTP dispatcher and its gRPC-Web wrapper and native-gRPC fallback are unchanged. The matcher speaks first, but only to silent connections: the gRPC Go client waits for the server's HTTP/2 settings frame before its first request, so after a 250ms silence the matcher sends one. A spoken-to connection is committed to the gRPC side, because the client's acknowledgement of that frame would break net/http's framing; a committed connection whose first request is not native gRPC is closed. Clients that send their first request without waiting never see a write. Classification stays bounded: one read deadline per connection, a 512KiB per-connection read cap, a 64KiB HPACK string bound, and admission control beyond 8192 in-flight classifications. Shutdown is bounded to one second of drain; the aggregate classification budget of the first implementation is dropped in favor of the per-connection cap times the admission cap. Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: ses_eabb93119ffegqWXti0mS05TIM --- app/controlplane/cmd/main.go | 11 +- go.mod | 1 + go.sum | 2 + pkg/httpgrpcmux/mux.go | 1155 ++++++++++------------------------ pkg/httpgrpcmux/mux_test.go | 849 +++++++++++-------------- 5 files changed, 703 insertions(+), 1315 deletions(-) diff --git a/app/controlplane/cmd/main.go b/app/controlplane/cmd/main.go index 344f35bd7..1a7ac6777 100644 --- a/app/controlplane/cmd/main.go +++ b/app/controlplane/cmd/main.go @@ -17,9 +17,7 @@ package main import ( "context" - "errors" "math/rand" - "net" "os" "time" @@ -65,9 +63,10 @@ func init() { } // cleanShutdownServer tolerates the shutdown race with the mux: kratos starts -// and stops servers concurrently, so the mux may close its virtual listeners -// before the gRPC/HTTP servers flag their own shutdown. Their Start then -// returns net.ErrClosed even though the shutdown is orderly — kratos would +// and stops servers concurrently, and the mux's sub-listeners close together +// with the root listener, so the gRPC/HTTP servers may see their listener +// close before they flag their own shutdown. Their Start then returns the +// mux's shutdown error even though the shutdown is orderly — kratos would // surface it and main would panic on every SIGTERM the mux wins. The race is // only tolerated when the mux is actually stopping: a fatal accept-loop // failure surfaces as an error, so the process restarts instead of serving a @@ -79,7 +78,7 @@ type cleanShutdownServer struct { func (s *cleanShutdownServer) Start(ctx context.Context) error { err := s.Server.Start(ctx) - if errors.Is(err, net.ErrClosed) && s.mux.Stopping() { + if s.mux.IsOrderlyShutdownErr(err) { return nil } return err diff --git a/go.mod b/go.mod index 9d53da20b..8969fad10 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,7 @@ require ( github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.11.0 github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.11.0 github.com/sigstore/timestamp-authority/v2 v2.1.3 + github.com/soheilhy/cmux v0.1.6-0.20260608033842-7b740c1fa6e0 github.com/spdx/tools-golang v0.5.7 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 diff --git a/go.sum b/go.sum index b92eaf63a..2cc76a706 100644 --- a/go.sum +++ b/go.sum @@ -1189,6 +1189,8 @@ github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.6-0.20260608033842-7b740c1fa6e0 h1:zGDSxPIxTarL5n+Co/R20SbWphTP6htuq+yMxPVS2uE= +github.com/soheilhy/cmux v0.1.6-0.20260608033842-7b740c1fa6e0/go.mod h1:JagfQkcwKpXOzyc2ygh7yP8+5BgvkfOMN8HInUthA5o= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik= github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU= diff --git a/pkg/httpgrpcmux/mux.go b/pkg/httpgrpcmux/mux.go index f9a06c0d7..cb38f8185 100644 --- a/pkg/httpgrpcmux/mux.go +++ b/pkg/httpgrpcmux/mux.go @@ -18,13 +18,22 @@ // HTTP at the same time. // // It implements the connection-level split described in -// docs/specs/issue-3628-single-endpoint.md: the mux looks at the start of each -// new connection without consuming it. An HTTP/2 connection whose first request -// carries a gRPC content type is handed to the gRPC listener; every other -// connection (HTTP/1.x, HTTP/2 without gRPC, gRPC-Web) goes to the HTTP -// listener. A connection classified as gRPC carries only gRPC; a native gRPC -// request that arrives on a connection owned by the HTTP server is forwarded -// to the gRPC server by the HTTP handler. +// docs/specs/issue-3628-single-endpoint.md, with the cmux library as the +// classifier (D-013). cmux reads the start of each new connection without +// consuming it and hands the connection to the gRPC or the HTTP listener. A +// service-owned matcher decides native gRPC from the first request's content +// type; every other connection (HTTP/1.x, HTTP/2 without gRPC, gRPC-Web) goes +// to the HTTP listener. A connection classified as gRPC carries only gRPC; a +// native gRPC request that arrives on a connection owned by the HTTP server is +// forwarded to the gRPC server by the HTTP handler. +// +// The matcher speaks only when a connection stays silent for the settings +// grace: gRPC clients (the CLI, Java) wait for the server's HTTP/2 settings +// frame before their first request. A connection the matcher has spoken to is +// committed to the gRPC side, because the client's acknowledgement of the +// injected frame reaches whichever server owns the connection, and net/http +// fails on an acknowledgement of settings it never sent. Every other client +// sends its first request without waiting and classifies unwritten-to. // // The mux implements kratos' transport.Server interface (Start/Stop with // standard signatures) so it can join the application server lifecycle. @@ -33,9 +42,8 @@ package httpgrpcmux import ( "bytes" "context" - "crypto/tls" "errors" - "fmt" + "io" "net" "strings" "sync" @@ -44,63 +52,50 @@ import ( "time" "github.com/go-kratos/kratos/v2/log" + "github.com/soheilhy/cmux" + "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" ) -// clientConnPreface is the connection preface that every HTTP/2 client sends -// right after the connection is established. -const clientConnPreface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" +// defaultClassifyTimeout bounds how long the matcher waits for a new +// connection's first request before the connection is closed or handed to the +// HTTP server. It applies to the classification step only: cmux clears it +// when the connection is handed over, so long-lived streams are not affected. +const defaultClassifyTimeout = 10 * time.Second -// frameHeaderLen is the fixed size of an HTTP/2 frame header. -const frameHeaderLen = 9 +// defaultSettingsGrace is how long the matcher waits for the first request +// after the connection preface before it assumes the client waits for the +// server to speak first. gRPC clients (the CLI) wait for the server's +// HTTP/2 settings frame before they send their first request, so the matcher +// must send one. Every other client (browsers, plain HTTP/2) sends its first +// request without waiting and stays unspoken-to. +const defaultSettingsGrace = 250 * time.Millisecond -// defaultClassifyTimeout bounds how long the classifier waits for a new -// connection's first request before dropping it. It applies to the -// classification step only, and is cleared before the connection is handed -// over, so long-lived streams are not affected. -const defaultClassifyTimeout = 10 * time.Second +// defaultMaxClassifiedBytes bounds how much of a connection the matcher may +// read while looking for the first request headers. A connection that needs +// more goes to the HTTP server, whose own header limits dispose of it. +const defaultMaxClassifiedBytes = 512 << 10 -// defaultSettingsGrace is how long the classifier waits for the client's first -// HTTP/2 frame before assuming the client is waiting for the server to speak -// first, and sending an empty SETTINGS frame to unblock it. -const defaultSettingsGrace = time.Second - -// maxClassifiedBytes bounds how much of a connection the classifier may read -// while looking for the first request headers. Anything beyond that is -// considered malformed and the connection is dropped. -const maxClassifiedBytes = 512 << 10 - -// maxHeaderBlockSize bounds a single HTTP/2 header block (plus its -// CONTINUATION frames). Real gRPC requests stay far below it. -const maxHeaderBlockSize = 1 << 20 - -// Decoded-header-list limits: a small compressed block can otherwise expand -// into a very large header list (HPACK amplification). -const ( - maxDecodedFields = 1024 - maxDecodedBytes = 256 << 10 - maxHpackString = 64 << 10 -) +// maxHpackString bounds a single decoded HPACK string in the matcher, so a +// small compressed block cannot expand into one huge header value. +const maxHpackString = 64 << 10 -// maxClassifiedFrames bounds how many HTTP/2 frames the classifier processes -// before the first request. -const maxClassifiedFrames = 4096 +// hpackDynamicTableSize is the HPACK dynamic table size the matcher offers, +// the HTTP/2 spec default. +const hpackDynamicTableSize = 4096 // defaultMaxInflightClassifications caps how many connections may be in // classification at the same time; beyond it, connections are shed. const defaultMaxInflightClassifications = 8192 -// maxTotalClassifiedBytes bounds the aggregate bytes recorded across all -// connections in classification at the same time, so the per-connection budget -// times the connection cap cannot multiply into unbounded memory. -const maxTotalClassifiedBytes = 256 << 20 +// acceptRetryDelay paces the retry loop after a recoverable accept error such +// as descriptor exhaustion. +const acceptRetryDelay = 5 * time.Millisecond -// errTotalBudgetExceeded is returned when the aggregate classification budget -// is exhausted; the connection is dropped. -var errTotalBudgetExceeded = errors.New("aggregate classification budget exceeded") - -// maxAcceptBackoff caps the retry delay after a recoverable accept error. -const maxAcceptBackoff = time.Second +// closeDrainBound caps how long Close waits for the cmux serve loop to drain. +// Connections still in classification ride out their own window in the +// background instead of holding up shutdown. +const closeDrainBound = time.Second // grpcContentTypePrefix matches every native gRPC content type, including // application/grpc+proto and content types with parameters. @@ -110,83 +105,83 @@ const grpcContentTypePrefix = "application/grpc" // requests must reach the HTTP server, which owns the gRPC-Web wrapper. const grpcWebContentTypePrefix = "application/grpc-web" -// HTTP/2 frame types used by the classifier. -const ( - http2FrameHeaders = 0x1 - http2FramePushPromise = 0x5 - http2FrameSettings = 0x4 - http2FrameContinuation = 0x9 -) - -// HTTP/2 frame flags used by the classifier. -const ( - http2FlagEndHeaders = 0x4 - http2FlagPadded = 0x8 - http2FlagPriority = 0x20 - // http2FlagSettingsAck marks a SETTINGS frame that acknowledges the peer's - // settings. - http2FlagSettingsAck = 0x1 -) - -// http2ServerEmptySettings is an empty SETTINGS frame. When a client stalls -// waiting for the server to speak first, the classifier sends it to unblock -// the client so it can read the first request. -var http2ServerEmptySettings = []byte{0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00} +// headerContentType is the request header that carries the content type. +const headerContentType = "content-type" -// acceptBacklog is the size of each virtual listener's delivery queue. -const acceptBacklog = 16 +// clientPreface is the HTTP/2 client preface, preallocated because the +// matcher compares it on every classified connection. +var clientPreface = []byte(http2.ClientPreface) // Options configures a Mux. type Options struct { // Logger receives classification debug information. Optional. Logger log.Logger - // ClassifyTimeout bounds how long the classifier waits for a new + // ClassifyTimeout bounds how long the matcher waits for a new // connection's first request. Defaults to 10s. ClassifyTimeout time.Duration - // SettingsGrace is how long the classifier waits for the client's first - // HTTP/2 frame before assuming it waits for the server to speak first, - // and sending an empty SETTINGS frame to unblock it. Defaults to 1s. + // SettingsGrace is how long the matcher waits for the first request + // before it sends the server's HTTP/2 settings frame to waiting + // clients. Defaults to 250ms. SettingsGrace time.Duration // MaxInflightClassifications caps how many connections may be in // classification at the same time; beyond it, connections are shed. // Defaults to 8192. MaxInflightClassifications int - // MaxTotalClassifiedBytes bounds the aggregate bytes recorded across all - // connections in classification at the same time. Defaults to 256 MiB. - MaxTotalClassifiedBytes int64 + // MaxClassifiedBytes bounds how much of a connection the matcher may + // read. Defaults to 512 KiB. + MaxClassifiedBytes int } // Mux splits one listener between a gRPC server and an HTTP server. type Mux struct { - raw net.Listener - - grpcL *virtualListener - httpL *virtualListener - - classifyTimeout time.Duration - settingsGrace time.Duration - logger *log.Helper - // admit caps how many connections may be in classification at once. + // root wraps the raw listener with the stopping marker. cmux sub-listeners + // embed the root listener, so whichever server stops first closes it for + // both, and the marker lets every server tell that shutdown apart from a + // fatal accept failure. + root *stoppingListener + cm cmux.CMux + + grpcL net.Listener + httpL net.Listener + + classifyTimeout time.Duration + settingsGrace time.Duration + maxClassifiedBytes int + logger *log.Helper + // admit caps how many connections may be in classification at once. A + // matcher holds a slot while it runs; a connection that cannot get one + // matches no listener, and cmux closes it. admit chan struct{} - // stopping is set before Close tears the listeners down, so servers can - // distinguish an orderly shutdown from a fatal accept-loop failure. + // poisoned holds the connections the gRPC matcher has sent the server's + // HTTP/2 settings frame to, but whose first request turned out not to be + // native gRPC. Their acknowledgement of that frame is already recorded + // in cmux's replay buffer, and net/http fails a connection that + // acknowledges settings it never sent, so the HTTP matcher declines + // them and cmux closes them. The gRPC matcher sets the entry before it + // declines; the HTTP matcher, which cmux always runs next, clears it. + poisoned sync.Map + stopping atomic.Bool - // inflight tracks the connections that are being classified, so Close - // can release the ones still parked on the classification deadline. - mu sync.Mutex - inflight map[net.Conn]struct{} - // budget tracks the aggregate bytes charged across in-flight - // classifications: the bytes recorded so far, plus the payload - // reservations held while a frame payload is being read; budgetLimit is - // its cap. - budget *atomic.Int64 - budgetLimit int64 - - wg sync.WaitGroup + serveWG sync.WaitGroup + // serveOnce and closeOnce keep Start and Close idempotent: kratos may + // call them more than once, and wire cleanup runs Close even when Start + // never ran. + serveOnce sync.Once closeOnce sync.Once } -// New creates a mux over l. Both virtual listeners are usable immediately; +// stoppingListener marks its Mux as stopping before the listener closes. +type stoppingListener struct { + net.Listener + stopping *atomic.Bool +} + +func (l *stoppingListener) Close() error { + l.stopping.Store(true) + return l.Listener.Close() +} + +// New creates a mux over l. Both listeners are usable immediately; // connections start flowing once Start is called. func New(l net.Listener, opts *Options) *Mux { if opts == nil { @@ -200,30 +195,66 @@ func New(l net.Listener, opts *Options) *Mux { if timeout <= 0 { timeout = defaultClassifyTimeout } - grace := opts.SettingsGrace - if grace <= 0 { - grace = defaultSettingsGrace - } maxInflight := opts.MaxInflightClassifications if maxInflight <= 0 { maxInflight = defaultMaxInflightClassifications } - totalBytes := opts.MaxTotalClassifiedBytes - if totalBytes <= 0 { - totalBytes = maxTotalClassifiedBytes + maxBytes := opts.MaxClassifiedBytes + if maxBytes <= 0 { + maxBytes = defaultMaxClassifiedBytes } - return &Mux{ - raw: l, - grpcL: newVirtualListener(l.Addr()), - httpL: newVirtualListener(l.Addr()), - classifyTimeout: timeout, - settingsGrace: grace, - admit: make(chan struct{}, maxInflight), - inflight: make(map[net.Conn]struct{}), - budget: &atomic.Int64{}, - budgetLimit: totalBytes, - logger: log.NewHelper(logger), + + grace := opts.SettingsGrace + if grace <= 0 { + grace = defaultSettingsGrace } + + m := &Mux{ + classifyTimeout: timeout, + settingsGrace: grace, + maxClassifiedBytes: maxBytes, + logger: log.NewHelper(logger), + admit: make(chan struct{}, maxInflight), + } + m.root = &stoppingListener{Listener: l, stopping: &m.stopping} + + m.cm = cmux.New(m.root) + // One read deadline bounds the whole classification step of a connection, + // including the TLS handshake, which runs on the first read. + m.cm.SetReadTimeout(timeout) + + // Registration order is match priority: the gRPC matcher runs first, so + // an HTTP/2 connection with a native gRPC first request goes to the gRPC + // listener, and everything else falls through to the HTTP listener. Both + // are MatchWriters so they can reach the connection: the gRPC matcher + // writes the server's settings frame to clients that wait for it, and + // the HTTP matcher needs the connection to look up poisoned ones. + m.grpcL = m.cm.MatchWithWriters(m.admitMatchWriter(m.matchNativeGRPC)) + m.httpL = m.cm.MatchWithWriters(m.admitMatchWriter(m.httpMatcher)) + + m.cm.HandleError(func(err error) bool { + var notMatched cmux.ErrNotMatched + if errors.As(err, ¬Matched) { + // A connection that matched no listener was closed; routine. + return true + } + if isAcceptRetryable(err) { + m.logger.Debugf("mux accept error, retrying: %v", err) + time.Sleep(acceptRetryDelay) + return true + } + if errors.Is(err, net.ErrClosed) { + // The root listener was closed: an orderly shutdown, whichever + // path closed it. + return false + } + // A fatal accept failure is not recoverable: log it loudly, the + // process must be restarted. + m.logger.Errorf("mux accept loop terminated: %v", err) + return false + }) + + return m } // GRPC returns the listener that receives connections classified as native @@ -234,99 +265,80 @@ func (m *Mux) GRPC() net.Listener { return m.grpcL } // HTTP/2 without gRPC, and gRPC-Web. func (m *Mux) HTTP() net.Listener { return m.httpL } -// Start runs the accept loop until Stop is called. +// Start runs the cmux serve loop until Stop is called. func (m *Mux) Start(_ context.Context) error { - m.wg.Add(1) - go m.acceptLoop() + m.serveOnce.Do(func() { + m.serveWG.Add(1) + go func() { + defer m.serveWG.Done() + // Serve returns when the root listener closes: an orderly + // shutdown, or a fatal accept failure that the error handler + // already logged. It then tears down both sub-listeners and + // closes every connection that no server has accepted yet. + if err := m.cm.Serve(); err != nil && !m.stopping.Load() { + m.logger.Debugf("mux serve loop returned: %v", err) + } + }() + }) return nil } -// Stop closes the underlying listener and both virtual listeners, unblocking -// any Accept call still in progress. +// Stop closes the underlying listener and both sub-listeners, unblocking any +// Accept call still in progress. func (m *Mux) Stop(_ context.Context) error { m.Close() return nil } -// Stopping reports whether Close has been called: servers fed by the mux use -// it to tell an orderly shutdown apart from a fatal accept-loop failure. +// Stopping reports whether Close has been called: a server fed by the mux can +// use it to tell an orderly shutdown apart from a fatal accept-loop failure. func (m *Mux) Stopping() bool { return m.stopping.Load() } -// Close stops the mux and waits for the accept loop to drain. +// Close stops the mux and waits for the cmux serve loop to drain, bounded by +// closeDrainBound: connections still in classification ride out their own +// classification window in the background, and die with the process. func (m *Mux) Close() { m.closeOnce.Do(func() { - m.stopping.Store(true) - _ = m.raw.Close() - // Close the virtual listeners even when Start never ran: the accept - // loop would otherwise be the only place doing it, and their - // dispatchers would keep running with queued connections inside. - _ = m.grpcL.Close() - _ = m.httpL.Close() - // Release the connections still parked in classification: closing - // them unblocks their goroutines, so the wait below is fast instead - // of riding out the classification deadline. - m.mu.Lock() - for conn := range m.inflight { - _ = conn.Close() + // Close the root listener first: Serve unblocks and tears down both + // sub-listeners. cmux.Close then releases the connections that + // already matched but were not accepted by a server. + _ = m.root.Close() + m.cm.Close() + drained := make(chan struct{}) + go func() { + m.serveWG.Wait() + close(drained) + }() + select { + case <-drained: + case <-time.After(closeDrainBound): } - m.mu.Unlock() - m.wg.Wait() }) } -func (m *Mux) acceptLoop() { - defer m.wg.Done() - var backoff time.Duration - for { - conn, err := m.raw.Accept() - if err != nil { - // Descriptor exhaustion and timeouts are recoverable: retry with - // backoff instead of killing the accept loop. - if isAcceptRetryable(err) { - if backoff == 0 { - backoff = 5 * time.Millisecond - } else if backoff < maxAcceptBackoff { - backoff *= 2 - } - time.Sleep(backoff) - continue - } - // Listener closed or fatal error: the accept loop is gone. Log it loudly — - // the servers notice through the closed virtual listeners, and the process - // must be restarted (a fatal accept failure is not recoverable). - m.logger.Errorf("mux accept loop terminated: %v", err) - _ = m.grpcL.Close() - _ = m.httpL.Close() - return - } - backoff = 0 - // Shed connections beyond the admission limit: the peer retries. +// IsOrderlyShutdownErr reports whether err is the listener error of this +// mux's orderly shutdown rather than a fatal accept failure. Servers fed by +// the mux use it to keep a lost shutdown race from failing app startup: on a +// fatal accept failure the mux never stops, so the error still surfaces and +// the process restarts instead of serving a dead single port. +func (m *Mux) IsOrderlyShutdownErr(err error) bool { + return m.stopping.Load() && (errors.Is(err, net.ErrClosed) || + errors.Is(err, cmux.ErrServerClosed) || + errors.Is(err, cmux.ErrListenerClosed)) +} + +// admitMatchWriter wraps a MatchWriter with classification admission control: +// it holds a slot while the matcher runs, and reports no match when the +// admission cap is reached, so cmux closes the connection. +func (m *Mux) admitMatchWriter(match cmux.MatchWriter) cmux.MatchWriter { + return func(w io.Writer, r io.Reader) bool { select { case m.admit <- struct{}{}: default: - m.logger.Debugf("shedding connection: classification admission limit reached") - _ = conn.Close() - continue + return false } - // Track the connection BEFORE the goroutine launches, so Close cannot - // walk past it: registering inside classify would leave a window where - // Close misses the accepted connection and rides out its full - // classification deadline. Ownership transfers to the server at - // handover, after which the entry is removed. - m.mu.Lock() - m.inflight[conn] = struct{}{} - m.mu.Unlock() - m.wg.Add(1) - go func() { - defer func() { - m.mu.Lock() - delete(m.inflight, conn) - m.mu.Unlock() - <-m.admit - }() - // classify owns the WaitGroup accounting for this goroutine. - m.classify(conn) - }() + defer func() { <-m.admit }() + return match(w, r) } } @@ -336,483 +348,158 @@ func isAcceptRetryable(err error) bool { return isTimeout(err) || errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE) } -// classify looks at the start of a new connection and hands it to the right -// server. It never consumes more than what it replays to the server. The -// caller registered the connection in the inflight set before launching it -// here; it is removed when this goroutine ends. -func (m *Mux) classify(conn net.Conn) { - defer m.wg.Done() - - bc := &replayConn{Conn: conn, recording: true, budget: m.budget, budgetLimit: m.budgetLimit} - // One absolute deadline bounds the whole classification step: reads and - // writes (the TLS handshake and the injected SETTINGS frame included). - // It is cleared before the connection is handed over, so long-lived - // streams are not affected. - classifyEnd := time.Now().Add(m.classifyTimeout) - _ = bc.SetDeadline(classifyEnd) - // Release this connection's share of the aggregate budget when the - // classification ends, whichever way it ends. - defer func() { - if bc.appended > 0 { - bc.budget.Add(-bc.appended) - } - }() - - if tc, ok := conn.(*tls.Conn); ok { - if err := tc.Handshake(); err != nil { - m.logger.Debugf("dropping connection: TLS handshake failed: %v", err) - _ = conn.Close() - return - } - switch tc.ConnectionState().NegotiatedProtocol { - case "", "http/1.1": - // HTTP/1.1 was negotiated: no HTTP/2 frames to parse. - m.handOver(bc, false) - return - } - // "h2" or another protocol: sniff the first request below. - } - - isGRPC, err := m.sniff(bc, classifyEnd) - if err != nil { - m.logger.Debugf("dropping connection: could not classify: %v", err) - _ = conn.Close() - return - } - m.handOver(bc, isGRPC) -} - -// handOver clears the classification deadline and delivers the connection to -// the right virtual listener, replaying everything the classifier consumed. -func (m *Mux) handOver(bc *replayConn, isGRPC bool) { - _ = bc.SetDeadline(time.Time{}) - if isGRPC { - m.grpcL.deliver(bc.handoff()) - } else { - m.httpL.deliver(bc.handoff()) - } -} - -// sniff decides whether the connection's first request is native gRPC. -func (m *Mux) sniff(bc *replayConn, classifyEnd time.Time) (bool, error) { - isH2, err := readClientPreface(bc) - if err != nil { - return false, err - } - if !isH2 { - // HTTP/1.x or anything else. The HTTP server produces a proper - // error response for malformed traffic. - return false, nil - } - - // A client that waits for the server to speak first stalls in the frame - // loop below; sniffFirstRequest unblocks it after a short grace period. - return m.sniffFirstRequest(bc, classifyEnd) +func isTimeout(err error) bool { + var ne net.Error + return errors.As(err, &ne) && ne.Timeout() } -// readClientPreface reports whether the connection starts with the HTTP/2 -// client preface. It reads byte ranges without over-consuming: everything it -// reads stays buffered in bc and is replayed to the server. -func readClientPreface(bc *replayConn) (bool, error) { - preface := []byte(clientConnPreface) - buf := make([]byte, len(preface)) - n := 0 - for n < len(preface) { - read, err := bc.Read(buf[n:]) - if err != nil { - return false, err +// httpMatcher accepts every remaining connection that produced at least one +// byte and that the gRPC matcher did not speak to. A connection that produced +// none, or whose read failed, matches no listener and is closed by cmux. +func (m *Mux) httpMatcher(w io.Writer, r io.Reader) bool { + // cmux passes the raw connection as the writer, and the gRPC matcher + // keys poisoned connections by it. A connection that is not identifiable + // cannot have been spoken to: the gRPC matcher declines before speaking + // unless it can reach the connection too. + if conn, ok := w.(net.Conn); ok { + if _, poisoned := m.poisoned.LoadAndDelete(conn); poisoned { + m.logger.Warnw("msg", "closing connection: the classifier sent its settings frame, but the first request was not native gRPC") + return false } - n += read - if !bytes.HasPrefix(buf[:n], preface[:n]) { - return false, nil - } - } - return true, nil -} - -// sniffFirstRequest walks HTTP/2 frames until the first HEADERS block is -// complete, then reports whether its content type is a native gRPC one. -// -// A client that waits for the server to speak first before sending anything -// stalls here. After a short grace period the classifier sends an empty -// SETTINGS frame to unblock it, and removes the SETTINGS-ACK that the client -// then sends for it from the replay, so the downstream server only ever sees -// ACKs for its own SETTINGS. The ACK may also arrive after the first request -// — a client whose frame-read loop races its request-write goroutine can send -// HEADERS first — in which case the classifier keeps consuming frames until -// it shows up (maybeWaitForSettingsAck). All reads stay bounded by -// classifyEnd, the absolute end of the classification window. -// http2FrameHeader is the decoded fixed part of an HTTP/2 frame. -type http2FrameHeader struct { - length int - frameType byte - flags byte - streamID uint32 -} - -// readFrameHeader reads and decodes the fixed part of the next HTTP/2 frame. -func readFrameHeader(bc *replayConn, injected *bool, classifyEnd time.Time) (http2FrameHeader, error) { - var hdr [frameHeaderLen]byte - if err := readInto(bc, hdr[:], injected, classifyEnd); err != nil { - return http2FrameHeader{}, err } - return http2FrameHeader{ - length: int(hdr[0])<<16 | int(hdr[1])<<8 | int(hdr[2]), - frameType: hdr[3], - flags: hdr[4], - streamID: uint32(hdr[5])<<24 | uint32(hdr[6])<<16 | uint32(hdr[7])<<8 | uint32(hdr[8]), - }, nil -} - -// consumeSettingsAck validates a SETTINGS-ACK (zero payload, zero stream) and -// removes its bytes from the replay: the downstream server never expects the -// ACK of the injected SETTINGS. -func consumeSettingsAck(bc *replayConn, hdr http2FrameHeader) error { - if hdr.length != 0 || hdr.streamID != 0 { - return errors.New("malformed SETTINGS ACK") - } - bc.dropLast(frameHeaderLen) - return nil -} - -// checkClassificationBounds enforces the frame-count and byte caps shared by -// both classification loops. -func checkClassificationBounds(bc *replayConn, frames int) error { - if frames > maxClassifiedFrames { - return errors.New("too many frames before the first request") - } - if bc.recorded() > maxClassifiedBytes { - return errors.New("classification buffer limit exceeded") - } - return nil -} - -func (m *Mux) sniffFirstRequest(bc *replayConn, classifyEnd time.Time) (bool, error) { - // Only the content-type decision is retained, under decoded-list caps: a - // small compressed block can otherwise expand into a huge header list. - counters := &headerCounters{} - dec := hpack.NewDecoder(4096, counters.observe) - dec.SetMaxStringLength(maxHpackString) - - var block []byte - var inBlock bool - // The client gets a short window to send its first frame on its own. The - // classification deadline stays in force as the upper bound. - graceEnd := time.Now().Add(m.settingsGrace) - if graceEnd.After(classifyEnd) { - graceEnd = classifyEnd - } - _ = bc.SetDeadline(graceEnd) - injected := false - // ackConsumed tracks whether the ACK of the injected SETTINGS was already - // consumed. It only matters once injected is true: until then any ACK is - // rejected below as unexpected traffic. - ackConsumed := false - frames := 0 + // One byte proves the connection carries traffic. The read goes through + // the sniffing reader, so cmux records it and replays it to the HTTP + // server. + var b [1]byte for { - frames++ - if err := checkClassificationBounds(bc, frames); err != nil { - return false, err - } - - hdr, err := readFrameHeader(bc, &injected, classifyEnd) - if err != nil { - return false, err - } - - if hdr.frameType == http2FrameSettings && hdr.flags&http2FlagSettingsAck != 0 { - // A SETTINGS-ACK can only acknowledge the injected SETTINGS: no - // server SETTINGS were sent before the handover. Anything else, - // including an ACK with a payload or on a stream, is invalid and - // would desynchronize the frame stream if replayed. - if !injected { - return false, errors.New("unexpected SETTINGS ACK before any injected SETTINGS") - } - if err := consumeSettingsAck(bc, hdr); err != nil { - return false, err - } - ackConsumed = true - continue + n, err := r.Read(b[:]) + if err != nil || n > 0 { + return n > 0 } - - // Every payload read must fit in the classification budget. - if bc.recorded()+hdr.length > maxClassifiedBytes { - return false, errors.New("frame payload exceeds classification buffer limit") - } - - switch hdr.frameType { - case http2FrameHeaders: - if inBlock { - return false, errors.New("unexpected HEADERS inside a header block") - } - if hdr.length > maxHeaderBlockSize { - return false, errors.New("HEADERS frame too large") - } - payload, err := readFramePayload(bc, hdr.length, &injected, classifyEnd) - if err != nil { - return false, err - } - frag, err := headerBlockFragment(payload, hdr.flags) - if err != nil { - return false, err - } - block = append(block, frag...) - if hdr.flags&http2FlagEndHeaders == 0 { - inBlock = true - continue - } - return finishHeaderBlock(bc, dec, counters, block, injected, ackConsumed, classifyEnd) - case http2FrameContinuation: - if !inBlock { - return false, errors.New("unexpected CONTINUATION frame") - } - if len(block)+hdr.length > maxHeaderBlockSize { - return false, errors.New("header block too large") - } - payload, err := readFramePayload(bc, hdr.length, &injected, classifyEnd) - if err != nil { - return false, err - } - block = append(block, payload...) - if hdr.flags&http2FlagEndHeaders == 0 { - continue - } - return finishHeaderBlock(bc, dec, counters, block, injected, ackConsumed, classifyEnd) - case http2FramePushPromise: - // Only servers may push; a client PUSH_PROMISE is a protocol error. - return false, errors.New("unexpected PUSH_PROMISE frame") - default: - // SETTINGS, WINDOW_UPDATE, PING, GOAWAY, ...: read the payload - // (it must be replayed to the server) within the budget checked - // above. - if err := skipFramePayload(bc, hdr.length, &injected, classifyEnd); err != nil { - return false, err - } - } - } -} - -// finishHeaderBlock decodes a completed header block, decides gRPC-ness and, -// for HTTP-classified connections with an outstanding injected SETTINGS, waits -// for its ACK before handover. -func finishHeaderBlock(bc *replayConn, dec *hpack.Decoder, counters *headerCounters, block []byte, injected, ackConsumed bool, classifyEnd time.Time) (bool, error) { - isGRPC, err := contentTypeFromBlock(dec, counters, block) - if err != nil { - return false, err } - if err := maybeWaitForSettingsAck(bc, isGRPC, injected, ackConsumed, classifyEnd); err != nil { - return false, err - } - return isGRPC, nil } -// maxSettingsAckWait bounds how long the classifier waits for the ACK of the -// injected SETTINGS when it arrives after the first request. Well below the -// classification window: a healthy client acknowledges immediately. -const maxSettingsAckWait = 2 * time.Second - -// maybeWaitForSettingsAck removes the client's ACK of the injected SETTINGS -// from the replay even when it arrives after the first request: a client -// whose frame-read loop races its request-write goroutine can send HEADERS -// first and the ACK afterwards. If the late ACK were replayed, the downstream -// server would see two ACKs for one SETTINGS and fail with PROTOCOL_ERROR. +// matchNativeGRPC reports whether the connection's first request carries a +// native gRPC content type. // -// The wait only applies to HTTP-classified connections: net/http breaks on -// the extra ACK, while the gRPC server ignores it, and waiting on a -// gRPC-classified connection would record large attestation DATA frames into -// the replay prefix, hitting the per-connection cap. +// Clients that send their first request without waiting (browsers, plain +// HTTP/2) classify without a single byte written to them. A client that stays +// silent for the settings grace is a gRPC client waiting for the server's +// HTTP/2 settings frame — the gRPC Go client does this, and so do Java +// clients — so the matcher sends one. From that moment the connection is +// committed: the client's acknowledgement of the injected frame is already +// recorded in cmux's replay buffer, net/http fails a connection that +// acknowledges settings it never sent, and the gRPC server ignores extra +// acknowledgements. A committed connection whose first request is not native +// gRPC, or that never sends its request, is therefore declined and poisoned: +// the HTTP matcher declines it too, and cmux closes it. // -// While waiting, every frame read stays recorded in the replay prefix, so -// whatever the client sends in the meantime reaches the downstream server. If -// the ACK does not arrive in time the connection is dropped: the HTTP/2 -// connection would break on the extra ACK anyway. -func maybeWaitForSettingsAck(bc *replayConn, isGRPC, injected, ackConsumed bool, classifyEnd time.Time) error { - if isGRPC || !injected || ackConsumed { - return nil +// Frame parsing and HPACK decoding are the standard HTTP/2 library's; +// everything the matcher reads stays buffered in cmux and is replayed to the +// server that wins the connection. The matcher bounds how much of the +// connection it may read. +func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { + // The extra byte makes any read past the cap fail instead of stopping + // silently at it: a connection that needs more than the cap to classify + // goes to the HTTP server, whose own limits dispose of it. + lr := &io.LimitedReader{R: r, N: int64(m.maxClassifiedBytes) + 1} + if !hasHTTP2Preface(lr) { + // HTTP/1.x or anything else: the HTTP server produces a proper error + // response for malformed traffic. Nothing was written. + return false + } + + conn, ok := w.(net.Conn) + if !ok { + return false + } + + framer := http2.NewFramer(w, lr) + var contentType string + hdec := hpack.NewDecoder(hpackDynamicTableSize, func(hf hpack.HeaderField) { + if strings.EqualFold(hf.Name, headerContentType) { + contentType = hf.Value + } + }) + hdec.SetMaxStringLength(maxHpackString) + + // Narrow the read deadline cmux set to the classification window down to + // the settings grace: how long to wait for a first request before + // speaking. decline reports no match, and poisons the connection when the + // matcher has already spoken, so it never reaches net/http. + spoken := false + _ = conn.SetReadDeadline(time.Now().Add(m.settingsGrace)) + decline := func() bool { + if spoken { + m.poisoned.Store(conn, struct{}{}) + } + return false + } + speak := func() bool { + // Bound the write, then give the committed classification a fresh + // window. + _ = conn.SetDeadline(time.Now().Add(m.classifyTimeout)) + if err := framer.WriteSettings(); err != nil { + return false + } + spoken = true + _ = conn.SetReadDeadline(time.Now().Add(m.classifyTimeout)) + return true } - // Bound the wait by the classification window when the wait bound does - // not fit inside it. - waitEnd := time.Now().Add(maxSettingsAckWait) - if waitEnd.After(classifyEnd) { - waitEnd = classifyEnd - } - _ = bc.SetDeadline(waitEnd) - // The SETTINGS were already injected: a timeout past waitEnd must drop - // the connection, not inject a second time. - injected = true - frames := 0 for { - frames++ - if err := checkClassificationBounds(bc, frames); err != nil { - return err - } - - hdr, err := readFrameHeader(bc, &injected, classifyEnd) + f, err := framer.ReadFrame() if err != nil { - return fmt.Errorf("waiting for the SETTINGS ACK of the injected SETTINGS: %w", err) - } - - if hdr.frameType == http2FrameSettings && hdr.flags&http2FlagSettingsAck != 0 { - if err := consumeSettingsAck(bc, hdr); err != nil { - return err + if !spoken && isTimeout(err) { + // The client is waiting for the server to speak first. + if !speak() { + return decline() + } + continue } - return nil - } - if bc.recorded()+hdr.length > maxClassifiedBytes { - return errors.New("frame payload exceeds classification buffer limit") + // Read failure, budget exhausted, or deadline: a committed + // connection cannot go to the HTTP server. + return decline() } - // Any other frame (HEADERS, DATA, WINDOW_UPDATE, ...): skip it; its - // bytes stay recorded and are replayed to the server. - if err := skipFramePayload(bc, hdr.length, &injected, classifyEnd); err != nil { - return err + // HEADERS and CONTINUATION are the only frames that carry header + // block fragments; SETTINGS (including the acknowledgement of the + // injected frame), PING, WINDOW_UPDATE, ... are skipped and stay in + // the replay: the gRPC server ignores unsolicited acknowledgements. + if hb, ok := f.(interface{ HeaderBlockFragment() []byte }); ok { + if _, err := hdec.Write(hb.HeaderBlockFragment()); err != nil { + return decline() + } + if f.Header().Flags&http2.FlagHeadersEndHeaders != 0 { + if err := hdec.Close(); err != nil { + return decline() + } + if !IsNativeGRPCContentType(contentType) { + return decline() + } + return true + } } } } -// readInto fills buf from bc, keeping the bytes read before an error. A -// timeout while the client is still waiting for the server to speak first (no -// SETTINGS injected yet) unblocks it: the deadline is raised to classifyEnd, -// an empty SETTINGS frame is written, injected is marked accordingly, and the -// read continues with the partial progress intact. Any other error is -// returned as is. -func readInto(bc *replayConn, buf []byte, injected *bool, classifyEnd time.Time) error { +// hasHTTP2Preface reports whether the connection starts with the HTTP/2 client +// preface. The preface is exact bytes, unlike header names and values. It +// reads byte ranges without over-consuming: everything it reads stays +// buffered in cmux and is replayed to the server. +func hasHTTP2Preface(r io.Reader) bool { + buf := make([]byte, len(clientPreface)) n := 0 - for n < len(buf) { - read, err := bc.Read(buf[n:]) + for n < len(clientPreface) { + read, err := r.Read(buf[n:]) n += read - if err == nil { - continue - } - if isTimeout(err) && !*injected { - // The client is waiting for the server to speak first. Raise the - // deadline to the classification end BEFORE writing, so the write - // itself is bounded by it. - _ = bc.SetDeadline(classifyEnd) - if _, err := bc.Write(http2ServerEmptySettings); err != nil { - return err - } - *injected = true - continue - } - // A Read may legally return data together with an error (io.Reader - // contract): account the data above, and if the buffer is complete - // the error is irrelevant — a trailing EOF after a full payload, for - // instance. A budget failure returns (0, err) and still propagates. - if n == len(buf) { - return nil - } - return err - } - return nil -} - -// skipFramePayloadLen is the size of the scratch buffer used to skip frame -// payloads that need no parsing. -const skipFramePayloadLen = 1024 - -// skipFramePayload reads and discards a frame payload of the given length. It -// reads into a small scratch buffer instead of io.CopyN, which discards the -// read error whenever the full count arrives — including an aggregate-budget -// failure delivered together with the last bytes of the payload. -func skipFramePayload(bc *replayConn, length int, injected *bool, classifyEnd time.Time) error { - var scratch [skipFramePayloadLen]byte - for remaining := length; remaining > 0; { - chunk := scratch[:min(remaining, len(scratch))] - if err := readInto(bc, chunk, injected, classifyEnd); err != nil { - return err - } - remaining -= len(chunk) - } - return nil -} - -// readFramePayload reads a frame payload of the given length. The allocation -// is charged against the aggregate budget before it happens: the recorded -// bytes alone would otherwise let every in-flight classification allocate up -// to maxClassifiedBytes without ever contributing to the budget. The charge -// is released when the read completes, whatever its outcome: by then Read has -// recorded — and charged — the payload bytes themselves, so only they remain. -func readFramePayload(bc *replayConn, length int, injected *bool, classifyEnd time.Time) ([]byte, error) { - if bc.budget != nil { - if bc.budget.Add(int64(length)) > bc.budgetLimit { - bc.budget.Add(-int64(length)) - return nil, errTotalBudgetExceeded - } - // Release the reservation on every path, including the error returns - // below: the recorded bytes are charged by Read itself. - defer bc.budget.Add(-int64(length)) - } - payload := make([]byte, length) - // readInto, unlike io.ReadFull, never swallows a read error that arrives - // together with the last bytes of the payload. - if err := readInto(bc, payload, injected, classifyEnd); err != nil { - return nil, err - } - return payload, nil -} - -// headerBlockFragment strips the padding and the priority fields from a -// HEADERS frame payload. -func headerBlockFragment(payload []byte, flags byte) ([]byte, error) { - if flags&http2FlagPadded != 0 { - if len(payload) == 0 { - return nil, errors.New("truncated HEADERS frame") - } - pad := int(payload[0]) - payload = payload[1:] - if pad > len(payload) { - return nil, errors.New("invalid HEADERS padding") + if err != nil { + return false } - payload = payload[:len(payload)-pad] - } - if flags&http2FlagPriority != 0 { - if len(payload) < 5 { - return nil, errors.New("truncated HEADERS frame") + if !bytes.HasPrefix(buf[:n], clientPreface[:n]) { + return false } - payload = payload[5:] - } - return payload, nil -} - -// headerCounters tracks the decoded header list under its caps; only the -// content-type decision is retained. -type headerCounters struct { - fields int - decodedSize int - contentType string -} - -func (c *headerCounters) observe(f hpack.HeaderField) { - c.fields++ - c.decodedSize += len(f.Name) + len(f.Value) - if strings.EqualFold(f.Name, "content-type") { - c.contentType = f.Value - } -} - -// contentTypeFromBlock decodes an HTTP/2 header block and reports whether its -// content type belongs to the native gRPC family. -func contentTypeFromBlock(dec *hpack.Decoder, counters *headerCounters, block []byte) (bool, error) { - if _, err := dec.Write(block); err != nil { - return false, err - } - if err := dec.Close(); err != nil { - return false, err } - if counters.fields > maxDecodedFields || counters.decodedSize > maxDecodedBytes { - return false, errors.New("decoded header list exceeds limits") - } - return isGRPCContentType(counters.contentType), nil -} - -// isGRPCContentType reports whether the content type belongs to the native -// gRPC family. gRPC-Web content types start with the same prefix but belong -// to the HTTP server instead. -func isGRPCContentType(contentType string) bool { - ct := strings.ToLower(contentType) - return strings.HasPrefix(ct, grpcContentTypePrefix) && - !strings.HasPrefix(ct, grpcWebContentTypePrefix) + return true } // IsNativeGRPCContentType reports whether the given content type identifies a @@ -820,170 +507,12 @@ func isGRPCContentType(contentType string) bool { // server uses it to forward native gRPC requests that arrive on a connection // the classifier handed to the HTTP side. func IsNativeGRPCContentType(contentType string) bool { - return isGRPCContentType(contentType) -} - -// replayConn is used in two modes. During classification (replay=false) it -// reads from the live connection and records every byte it consumes. The -// handoff conn produced by handoff() (replay=true) first replays those bytes -// to the server, then streams the rest of the connection. -type replayConn struct { - net.Conn - - prefix []byte - recording bool - replay bool - pos int - // budget counts this connection's share of the mux's aggregate - // classification budget; appended is its total recorded bytes so the - // share can be released exactly. - budget *atomic.Int64 - budgetLimit int64 - appended int64 -} - -func (c *replayConn) Read(p []byte) (int, error) { - if c.replay && c.pos < len(c.prefix) { - n := copy(p, c.prefix[c.pos:]) - c.pos += n - if c.pos == len(c.prefix) { - // The replay is done: release the buffer for the rest of the - // connection's lifetime. - c.prefix = nil - } - return n, nil - } - n, err := c.Conn.Read(p) - if c.recording && n > 0 { - c.prefix = append(c.prefix, p[:n]...) - c.appended += int64(n) - if c.budget != nil { - if c.budget.Add(int64(n)) > c.budgetLimit { - // The aggregate budget is exhausted: give the bytes back (the - // classifier releases the rest) and fail the read, which - // drops the connection. The bytes stay recorded in the prefix, - // but the connection is dropped either way. The error must be - // returned with n==0: io.CopyN and io.ReadFull swallow a - // trailing error when the requested bytes were all delivered. - c.budget.Add(-int64(n)) - c.appended -= int64(n) - return 0, errTotalBudgetExceeded - } - } - } - return n, err + return hasFoldPrefix(contentType, grpcContentTypePrefix) && + !hasFoldPrefix(contentType, grpcWebContentTypePrefix) } -func (c *replayConn) recorded() int { return len(c.prefix) } - -// dropLast removes the n most recently recorded bytes from the replay, for -// frames the classifier consumed on behalf of the downstream server, such as -// the ACK of the injected SETTINGS. -func (c *replayConn) dropLast(n int) { - if n > len(c.prefix) { - return - } - c.prefix = c.prefix[:len(c.prefix)-n] -} - -// handoff returns the connection handed over to the server: the recorded -// prefix is replayed first, then the connection continues live. -func (c *replayConn) handoff() *replayConn { - return &replayConn{Conn: c.Conn, prefix: c.prefix, replay: true} -} - -// virtualListener is a net.Listener fed by the classifier. A dedicated -// dispatcher goroutine owns the queue: it forwards connections to Accept and, -// on Close, closes everything still owned, so no connection is ever enqueued -// after shutdown or leaked between delivery and close. -type virtualListener struct { - addr net.Addr - in chan net.Conn // deliverers; unbuffered so a closed listener never retains anything - out chan net.Conn // the server's Accept queue - done chan struct{} - dispatcherDone chan struct{} - once sync.Once -} - -func newVirtualListener(addr net.Addr) *virtualListener { - l := &virtualListener{ - addr: addr, - in: make(chan net.Conn), - out: make(chan net.Conn, acceptBacklog), - done: make(chan struct{}), - dispatcherDone: make(chan struct{}), - } - go l.dispatch() - return l -} - -func (l *virtualListener) dispatch() { - defer close(l.dispatcherDone) - for { - select { - case c := <-l.in: - select { - case l.out <- c: - case <-l.done: - _ = c.Close() - l.drain() - return - } - case <-l.done: - l.drain() - return - } - } -} - -// drain closes every connection still owned by the dispatcher. A connection -// that raced into out and was taken by the server's Accept is not owned -// anymore; a connection still queued is closed exactly once here. -func (l *virtualListener) drain() { - for { - select { - case c := <-l.in: - _ = c.Close() - case c := <-l.out: - _ = c.Close() - default: - return - } - } -} - -func (l *virtualListener) Accept() (net.Conn, error) { - select { - case c := <-l.out: - return c, nil - case <-l.done: - // Closed. Everything the dispatcher still owned was closed; a - // connection that raced out before the close is served by the - // server's graceful stop, everything else reports closure. - return nil, net.ErrClosed - } -} - -func (l *virtualListener) Close() error { - l.once.Do(func() { - close(l.done) - <-l.dispatcherDone - }) - return nil -} - -func (l *virtualListener) Addr() net.Addr { return l.addr } - -func (l *virtualListener) deliver(c net.Conn) { - select { - case l.in <- c: - case <-l.done: - _ = c.Close() - } -} - -// isTimeout reports whether the error means a read deadline was exceeded. -func isTimeout(err error) bool { - var ne net.Error - return errors.As(err, &ne) && ne.Timeout() +// hasFoldPrefix reports whether s starts with prefix, ignoring case, without +// allocating a copy of s. +func hasFoldPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && strings.EqualFold(s[:len(prefix)], prefix) } diff --git a/pkg/httpgrpcmux/mux_test.go b/pkg/httpgrpcmux/mux_test.go index 74155ce66..3e38b3edf 100644 --- a/pkg/httpgrpcmux/mux_test.go +++ b/pkg/httpgrpcmux/mux_test.go @@ -16,7 +16,6 @@ package httpgrpcmux import ( - "bufio" "bytes" "context" "crypto/rand" @@ -27,14 +26,19 @@ import ( "io" "math/big" "net" - "os" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" ) // Header names and content types used across the tests. @@ -44,6 +48,7 @@ const ( hdrMethodPost = "POST" ctGRPC = "application/grpc" ctGRPCProto = "application/grpc+proto" + ctGRPCWebProto = "application/grpc-web+proto" ) // startMux creates a real TCP listener, wraps it in a mux and starts it. @@ -90,9 +95,18 @@ func acceptWithTimeout(t *testing.T, l net.Listener) net.Conn { func writeClientStream(t *testing.T, c net.Conn, contentType string, extraHeaders ...hpack.HeaderField) { t.Helper() - _, err := io.WriteString(c, clientConnPreface) + _, err := io.WriteString(c, http2.ClientPreface) require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) + writeRequestFrames(t, c, contentType, extraHeaders...) +} + +// writeRequestFrames sends a client SETTINGS frame and one HEADERS frame +// carrying the given headers: the request flight after the connection +// preface. +func writeRequestFrames(t *testing.T, c net.Conn, contentType string, extraHeaders ...hpack.HeaderField) { + t.Helper() + + writeFrame(t, c, http2.FrameSettings, 0, 0, nil) var buf bytes.Buffer enc := hpack.NewEncoder(&buf) @@ -104,15 +118,15 @@ func writeClientStream(t *testing.T, c net.Conn, contentType string, extraHeader }, extraHeaders...) { require.NoError(t, enc.WriteField(hf)) } - writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) + writeFrame(t, c, http2.FrameHeaders, http2.FlagHeadersEndHeaders, 1, buf.Bytes()) } -func writeFrame(t *testing.T, c net.Conn, frameType, flags byte, streamID uint32, payload []byte) { +func writeFrame(t *testing.T, c net.Conn, frameType http2.FrameType, flags http2.Flags, streamID uint32, payload []byte) { t.Helper() hdr := []byte{ byte(len(payload) >> 16), byte(len(payload) >> 8), byte(len(payload)), - frameType, flags, + byte(frameType), byte(flags), byte(streamID >> 24), byte(streamID >> 16), byte(streamID >> 8), byte(streamID), } _, err := c.Write(hdr) @@ -123,6 +137,15 @@ func writeFrame(t *testing.T, c net.Conn, frameType, flags byte, streamID uint32 } } +// readEOF asserts that the peer closed the connection. +func readEOF(t *testing.T, c net.Conn, within time.Duration) { + t.Helper() + require.NoError(t, c.SetReadDeadline(time.Now().Add(within))) + _, err := c.Read(make([]byte, 1)) + require.Error(t, err) + assert.False(t, isTimeout(err), "the connection must be closed, not merely silent") +} + func TestClassifyHTTP1(t *testing.T) { m, dial := startMux(t, nil) @@ -146,7 +169,7 @@ func TestClassifyHTTP1(t *testing.T) { func TestClassifyShortHTTP10Request(t *testing.T) { // A request shorter than the HTTP/2 preface must not deadlock the - // classifier waiting for 24 bytes. + // matcher waiting for 24 bytes. m, dial := startMux(t, nil) c, err := dial() @@ -186,12 +209,12 @@ func TestClassifyNativeGRPC(t *testing.T) { got := acceptWithTimeout(t, m.GRPC()) defer got.Close() - // The gRPC server must see the preface again. + // The gRPC server must see the preface and the request again. require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) - prefix := make([]byte, len(clientConnPreface)) - _, err = io.ReadFull(got, prefix) + replayed := make([]byte, len(http2.ClientPreface)) + _, err = io.ReadFull(got, replayed) require.NoError(t, err) - assert.Equal(t, clientConnPreface, string(prefix), "preface not replayed intact") + assert.Equal(t, http2.ClientPreface, string(replayed), "preface not replayed intact") }) } } @@ -199,7 +222,7 @@ func TestClassifyNativeGRPC(t *testing.T) { func TestClassifyGRPCWebGoesToHTTP(t *testing.T) { for _, ct := range []string{ "application/grpc-web", - "application/grpc-web+proto", + ctGRPCWebProto, "application/grpc-web+json", } { t.Run(ct, func(t *testing.T) { @@ -239,17 +262,17 @@ func TestClassifySkipsIntermediateFrames(t *testing.T) { require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, clientConnPreface) + _, err = io.WriteString(c, http2.ClientPreface) require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) - writeFrame(t, c, 0x8 /* WINDOW_UPDATE */, 0, 0, []byte{0, 0, 0, 1}) - writeFrame(t, c, 0x8 /* WINDOW_UPDATE */, 0, 1, []byte{0, 0, 0, 2}) + writeFrame(t, c, http2.FrameSettings, 0, 0, nil) + writeFrame(t, c, http2.FrameWindowUpdate, 0, 0, []byte{0, 0, 0, 1}) + writeFrame(t, c, http2.FrameWindowUpdate, 0, 1, []byte{0, 0, 0, 2}) var buf bytes.Buffer enc := hpack.NewEncoder(&buf) require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) - writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) + writeFrame(t, c, http2.FrameHeaders, http2.FlagHeadersEndHeaders, 1, buf.Bytes()) got := acceptWithTimeout(t, m.GRPC()) got.Close() @@ -262,9 +285,9 @@ func TestClassifyPaddedAndPriorityHeaders(t *testing.T) { require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, clientConnPreface) + _, err = io.WriteString(c, http2.ClientPreface) require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) + writeFrame(t, c, http2.FrameSettings, 0, 0, nil) var buf bytes.Buffer enc := hpack.NewEncoder(&buf) @@ -279,547 +302,280 @@ func TestClassifyPaddedAndPriorityHeaders(t *testing.T) { payload = append(payload, 0, 0, 0, 0, 16) // stream dep + weight payload = append(payload, frag...) // header block fragment payload = append(payload, 0, 0) // padding - writeFrame(t, c, http2FrameHeaders, 0x8|0x20|0x4, 1, payload) + writeFrame(t, c, http2.FrameHeaders, http2.FlagHeadersPadded|http2.FlagHeadersPriority|http2.FlagHeadersEndHeaders, 1, payload) got := acceptWithTimeout(t, m.GRPC()) got.Close() } -// TestSettingsAckNotReplayed proves that the ACK of the injected SETTINGS is -// removed from the replay: the downstream gRPC server must never see it, or it -// would count two SETTINGS acknowledgements and fail with PROTOCOL_ERROR. -func TestSettingsAckNotReplayed(t *testing.T) { - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - _, err = io.WriteString(c, clientConnPreface) - require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) - - // The classifier injects its empty SETTINGS after the grace period. - require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) - hdr := make([]byte, 9) - _, err = io.ReadFull(c, hdr) - require.NoError(t, err) +// TestMatcherDoesNotSpeakFirstForImmediateClients proves the classification +// writes nothing to a connection whose first request arrives without waiting. +// Bytes written while classifying are answered by the client, and the answer +// reaches whichever server ends up owning the connection. +func TestMatcherDoesNotSpeakFirstForImmediateClients(t *testing.T) { + for _, tc := range []struct { + name string + ct string + grpc bool + }{ + {"native gRPC", ctGRPCProto, true}, + {"gRPC-Web", ctGRPCWebProto, false}, + {"plain HTTP/2", "application/json", false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + // A long grace keeps the matcher silent for immediate requests. + m, dial := startMux(t, &Options{SettingsGrace: time.Second}) - // The client acknowledges the injected SETTINGS, then sends its request. - writeFrame(t, c, http2FrameSettings, http2FlagSettingsAck, 0, nil) - writeClientStreamAfterPreface(t, c) + c, err := dial() + require.NoError(t, err) + defer c.Close() - got := acceptWithTimeout(t, m.GRPC()) - defer got.Close() + writeClientStream(t, c, tc.ct) - // Parse the frames the server receives: SETTINGS (client) then HEADERS, - // and no SETTINGS-ACK. - require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) - reader := bufio.NewReader(got) - preface := make([]byte, len(clientConnPreface)) - _, err = io.ReadFull(reader, preface) - require.NoError(t, err) - assert.Equal(t, clientConnPreface, string(preface)) + // The mux must stay silent while it classifies. + require.NoError(t, c.SetReadDeadline(time.Now().Add(300*time.Millisecond))) + _, err = c.Read(make([]byte, 1)) + require.Error(t, err, "the mux must not write during classification") + assert.True(t, isTimeout(err), "the read must time out, not fail: %v", err) - type frame struct { - frameType byte - flags byte - } - var frames []frame - for { - var fh [9]byte - if _, err := io.ReadFull(reader, fh[:]); err != nil { - break // idle connection: everything available was parsed - } - length := int(fh[0])<<16 | int(fh[1])<<8 | int(fh[2]) - if length > 0 { - if _, err := io.CopyN(io.Discard, reader, int64(length)); err != nil { - break + if tc.grpc { + got := acceptWithTimeout(t, m.GRPC()) + got.Close() + } else { + got := acceptWithTimeout(t, m.HTTP()) + got.Close() } - } - frames = append(frames, frame{fh[3], fh[4]}) - } - - require.NotEmpty(t, frames) - assert.Equal(t, byte(http2FrameSettings), frames[0].frameType, "first replayed frame must be the client SETTINGS") - assert.Equal(t, byte(0), frames[0].flags&http2FlagSettingsAck, "first replayed frame must not be an ACK") - foundAck := false - for _, f := range frames { - if f.frameType == http2FrameSettings && f.flags&http2FlagSettingsAck != 0 { - foundAck = true - } + }) } - assert.False(t, foundAck, "the ACK of the injected SETTINGS must not reach the server") } -// TestLateSettingsAckNotReplayed proves that the ACK of the injected SETTINGS -// is removed from the replay even when it arrives after the first request: a -// client whose frame-read loop races its request-write goroutine can send -// HEADERS first and the ACK afterwards. The classifier keeps consuming frames -// until the ACK shows up, drops it, and hands the connection to the HTTP -// server without it. -func TestLateSettingsAckNotReplayed(t *testing.T) { - m, dial := startMux(t, nil) +// TestSettingsWaitingClientClassifiesNative covers a client that waits for the +// server's HTTP/2 settings frame before it sends its first request: the gRPC +// Go client works this way. The matcher sends an empty settings frame after +// the grace, the client acknowledges it and sends its request, and the +// connection goes to the gRPC side with the acknowledgement in the replay: +// the gRPC server ignores acknowledgements of settings it never sent. +func TestSettingsWaitingClientClassifiesNative(t *testing.T) { + m, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: time.Second}) c, err := dial() require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, clientConnPreface) + _, err = io.WriteString(c, http2.ClientPreface) require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) + writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - // The classifier injects its empty SETTINGS after the grace period. - require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) - _, err = io.ReadFull(c, make([]byte, 9)) + // After the grace, the matcher sends its empty settings frame. + require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + injected := make([]byte, 9) + _, err = io.ReadFull(c, injected) require.NoError(t, err) + assert.Equal(t, []byte{0, 0, 0, byte(http2.FrameSettings), 0, 0, 0, 0, 0}, injected, + "the matcher must send an empty settings frame to a waiting client") - // The client sends its (non-gRPC) request BEFORE acknowledging the - // injected SETTINGS, and the ACK only afterwards. - var buf bytes.Buffer - enc := hpack.NewEncoder(&buf) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: "application/json"})) - writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) - writeFrame(t, c, http2FrameSettings, http2FlagSettingsAck, 0, nil) + // The client acknowledges the injected settings and sends its request: + // the preface was already sent, only the request flight follows. + writeFrame(t, c, http2.FrameSettings, http2.FlagSettingsAck, 0, nil) + writeRequestFrames(t, c, ctGRPCProto) - // The connection must still reach the HTTP listener: the late ACK does - // not change the classification, it only delays the handover. - got := acceptWithTimeout(t, m.HTTP()) + got := acceptWithTimeout(t, m.GRPC()) defer got.Close() - // Parse the frames the server receives: SETTINGS (client) then HEADERS, - // and no SETTINGS-ACK. + // The gRPC server sees the whole stream: the preface, the client + // settings, the acknowledgement of the injected frame, and the request. require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) - reader := bufio.NewReader(got) - preface := make([]byte, len(clientConnPreface)) - _, err = io.ReadFull(reader, preface) + replay := make([]byte, len(http2.ClientPreface)) + _, err = io.ReadFull(got, replay) require.NoError(t, err) - assert.Equal(t, clientConnPreface, string(preface)) - - type frame struct { - frameType byte - flags byte - } - var frames []frame - for { - var fh [9]byte - if _, err := io.ReadFull(reader, fh[:]); err != nil { - break // idle connection: everything available was parsed - } - length := int(fh[0])<<16 | int(fh[1])<<8 | int(fh[2]) - if length > 0 { - if _, err := io.CopyN(io.Discard, reader, int64(length)); err != nil { - break - } - } - frames = append(frames, frame{fh[3], fh[4]}) - } - - require.NotEmpty(t, frames) - assert.Equal(t, byte(http2FrameSettings), frames[0].frameType, "first replayed frame must be the client SETTINGS") - foundAck := false - sawHeaders := false - for _, f := range frames { - if f.frameType == http2FrameSettings && f.flags&http2FlagSettingsAck != 0 { - foundAck = true - } - if f.frameType == http2FrameHeaders { - sawHeaders = true - } - } - assert.True(t, sawHeaders, "the client's HEADERS must reach the server") - assert.False(t, foundAck, "the ACK of the injected SETTINGS must not reach the server") + assert.Equal(t, http2.ClientPreface, string(replay), "preface not replayed intact") } -// TestLateAckWaitDropsConnection proves the bound on the wait for a late -// SETTINGS ACK: a client that sends its request but never acknowledges the -// injected SETTINGS gets the connection dropped shortly after — the HTTP/2 -// connection would break on the extra ACK anyway — and never handed to a -// server. -func TestLateAckWaitDropsConnection(t *testing.T) { - // A short grace keeps the whole classification far from the end of the - // default 10s classification window, so the drop proves the dedicated - // ACK-wait bound, not the classification timeout. - m, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond}) +// TestCommittedNonGRPCDropped proves a spoken-to connection whose first +// request is not native gRPC is closed, not handed to the HTTP server: its +// acknowledgement of the injected settings frame is already recorded, and +// net/http fails a connection that acknowledges settings it never sent. +func TestCommittedNonGRPCDropped(t *testing.T) { + m, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: time.Second}) c, err := dial() require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, clientConnPreface) + _, err = io.WriteString(c, http2.ClientPreface) require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) + writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - // Consume the injected SETTINGS the classifier sends after the grace. - require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) + // Receive the injected settings frame, acknowledge it, and send a + // gRPC-Web request flight. + require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) _, err = io.ReadFull(c, make([]byte, 9)) require.NoError(t, err) + writeFrame(t, c, http2.FrameSettings, http2.FlagSettingsAck, 0, nil) + writeRequestFrames(t, c, ctGRPCWebProto) - // The client sends a non-gRPC request and never ACKs. - var buf bytes.Buffer - enc := hpack.NewEncoder(&buf) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: "application/json"})) - writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) - - // The classifier must close the connection when the wait for the ACK - // expires (bounded at 2s), well before the classification window (10s) - // would. The client's own 5s read deadline proves it: a connection held - // past it would surface here as a deadline error instead. - require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) - start := time.Now() - _, err = c.Read(make([]byte, 9)) - require.Error(t, err, "expected the connection to be dropped when the ACK never arrives") - require.NotErrorIs(t, err, os.ErrDeadlineExceeded, "the connection was closed by the classifier, not by the client's deadline") - require.Less(t, time.Since(start), 5*time.Second, "the classifier held the connection past the ACK wait bound") - - // Nothing must have been delivered to either server. - time.Sleep(200 * time.Millisecond) - type result struct { - c net.Conn - err error - } - for _, l := range []net.Listener{m.GRPC(), m.HTTP()} { - ch := make(chan result, 1) - go func(l net.Listener) { - c, err := l.Accept() - ch <- result{c, err} - }(l) - select { - case r := <-ch: - if r.err == nil { - r.c.Close() - assert.Fail(t, "the connection was delivered to a server despite the missing ACK") - } - case <-time.After(100 * time.Millisecond): - // Nothing delivered: correct. - } - } + // The connection matches no listener and cmux closes it. + readEOF(t, c, 2*time.Second) + _ = m // keep the mux referenced for the duration of the test } -func TestOversizedFrameDropped(t *testing.T) { - // A frame payload that would exceed the classification budget must get - // the connection dropped, not buffered. - _, dial := startMux(t, nil) +// TestOversizedHeaderBlockDeclined proves a connection whose first request +// needs more than the classification byte budget goes to the HTTP server, +// whose own header limits dispose of it. +func TestOversizedHeaderBlockDeclined(t *testing.T) { + m, dial := startMux(t, &Options{MaxClassifiedBytes: 1024}) c, err := dial() require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, clientConnPreface) - require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) - - // A WINDOW_UPDATE whose length field claims almost 16 MiB: the classifier - // must reject it instead of recording it. - huge := []byte{0xFF, 0xFF, 0xFF, 0x8, 0x00, 0x00, 0x00, 0x00, 0x00} - _, err = c.Write(huge) - require.NoError(t, err) - - require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = c.Read(make([]byte, 9)) - require.Error(t, err, "expected the connection to be dropped for the oversized frame") -} - -func TestCloseClosesQueuedConnections(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - m := New(l, nil) - - // Queue a classified connection without any server accepting it. - vl := m.GRPC().(*virtualListener) - serverEnd, clientEnd := net.Pipe() - vl.deliver(serverEnd) - defer clientEnd.Close() - - m.Close() - - // The queued connection must have been closed by Close, not leaked. - // SetReadDeadline may already fail because the pipe end was closed, - // which is exactly what we are proving; the Read below distinguishes - // closure from a timeout. - _ = clientEnd.SetReadDeadline(time.Now().Add(time.Second)) - _, err = clientEnd.Read(make([]byte, 1)) - require.Error(t, err) - // A read timeout would mean the connection is still open: Close must - // actually have closed it. - require.NotErrorIs(t, err, os.ErrDeadlineExceeded, "the queued connection was not closed when the mux stopped") -} - -// TestAggregateBudgetShedsConnections proves the aggregate byte budget: once -// the total recorded bytes across in-flight classifications exceed it, new -// recording fails and the connection is dropped. -func TestAggregateBudgetShedsConnections(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - m := New(l, &Options{MaxTotalClassifiedBytes: 4096, ClassifyTimeout: 5 * time.Second}) - require.NoError(t, m.Start(context.Background())) - t.Cleanup(m.Close) - - dial := func() (net.Conn, error) { return net.Dial("tcp", l.Addr().String()) } - - // First connection records ~3 KiB and stays in classification. - c1, err := dial() - require.NoError(t, err) - defer c1.Close() - _, err = io.WriteString(c1, clientConnPreface) - require.NoError(t, err) - writeFrame(t, c1, http2FrameSettings, 0, 0, nil) - writeFrame(t, c1, 0x8 /* WINDOW_UPDATE */, 0, 0, bytes.Repeat([]byte{0}, 3072)) - // Wait until the first connection's bytes are all accounted for — exactly: - // nothing else has been sent, so this state is stable — so the second - // connection deterministically trips the aggregate budget. - require.Eventually(t, func() bool { return m.budget.Load() == 3114 }, - 2*time.Second, 10*time.Millisecond, "first connection's bytes were not recorded") - - // Second connection pushes the aggregate over the budget: its recording - // must fail and its connection be dropped. - c2, err := dial() - require.NoError(t, err) - defer c2.Close() - _, err = io.WriteString(c2, clientConnPreface) - require.NoError(t, err) - writeFrame(t, c2, http2FrameSettings, 0, 0, nil) - writeFrame(t, c2, 0x8 /* WINDOW_UPDATE */, 0, 0, bytes.Repeat([]byte{0}, 2048)) - - require.NoError(t, c2.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = c2.Read(make([]byte, 9)) - require.Error(t, err, "expected the connection to be dropped when the aggregate budget is exhausted") - require.NotErrorIs(t, err, os.ErrDeadlineExceeded, "connection was not dropped; the read merely timed out") -} - -// TestPayloadAllocationChargedToBudget proves that a frame payload's -// allocation is charged against the aggregate budget before it happens: a -// connection that declares a large payload while the aggregate budget is -// nearly exhausted must be dropped instead of silently allocating the bytes -// and contributing nothing to the budget. The refused reservation must also -// be released, keeping the accounting exact. -func TestPayloadAllocationChargedToBudget(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - m := New(l, &Options{MaxTotalClassifiedBytes: 4096, ClassifyTimeout: 5 * time.Second}) - require.NoError(t, m.Start(context.Background())) - t.Cleanup(m.Close) - - dial := func() (net.Conn, error) { return net.Dial("tcp", l.Addr().String()) } - - // First connection records ~3 KiB and stays in classification. - c1, err := dial() - require.NoError(t, err) - defer c1.Close() - _, err = io.WriteString(c1, clientConnPreface) - require.NoError(t, err) - writeFrame(t, c1, http2FrameSettings, 0, 0, nil) - writeFrame(t, c1, 0x8 /* WINDOW_UPDATE */, 0, 0, bytes.Repeat([]byte{0}, 3072)) - require.Eventually(t, func() bool { return m.budget.Load() == 3114 }, - 2*time.Second, 10*time.Millisecond, "first connection's bytes were not recorded") - - // Second connection declares a 2 KiB HEADERS payload without sending it: - // its recorded bytes (3156) plus the 2048 reservation exceed the 4096 - // budget, so the allocation must be refused and the connection dropped at - // once, instead of allocating the bytes and only noticing when the - // payload trickles in. - c2, err := dial() - require.NoError(t, err) - defer c2.Close() - _, err = io.WriteString(c2, clientConnPreface) - require.NoError(t, err) - writeFrame(t, c2, http2FrameSettings, 0, 0, nil) - // HEADERS frame header claiming 2048 bytes of payload, never sent. - _, err = c2.Write([]byte{0x00, 0x08, 0x00, http2FrameHeaders, 0x00, 0, 0, 0, 1}) - require.NoError(t, err) + big := strings.Repeat("a", 2048) + writeClientStream(t, c, ctGRPCProto, hpack.HeaderField{Name: "x-big", Value: big}) - require.NoError(t, c2.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = c2.Read(make([]byte, 9)) - require.Error(t, err, "expected the connection to be dropped when the payload allocation exceeds the budget") - require.NotErrorIs(t, err, os.ErrDeadlineExceeded, "connection was not dropped; the read merely timed out") - - // The refused reservation must leave the accounting exact: only the first - // connection's share stays charged. - require.Eventually(t, func() bool { return m.budget.Load() == 3114 }, - 2*time.Second, 10*time.Millisecond, "the refused reservation was not released") + got := acceptWithTimeout(t, m.HTTP()) + got.Close() } -// TestHPACKAmplificationRejected proves the decoded-header-list caps: a tiny -// compressed block that expands into thousands of header fields must get the -// connection dropped instead of being amplified in memory. -func TestHPACKAmplificationRejected(t *testing.T) { - _, dial := startMux(t, nil) +// TestHugeHPACKStringDeclined proves a single HPACK string beyond the decoder +// bound makes the matcher decline, even when the whole block fits the byte +// budget: a small compressed block must not expand into a huge header value. +func TestHugeHPACKStringDeclined(t *testing.T) { + m, dial := startMux(t, nil) c, err := dial() require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, clientConnPreface) + _, err = io.WriteString(c, http2.ClientPreface) require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) + writeFrame(t, c, http2.FrameSettings, 0, 0, nil) var buf bytes.Buffer enc := hpack.NewEncoder(&buf) - big := hpack.HeaderField{Name: "x-bomb", Value: strings.Repeat("a", 1024)} - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) - require.NoError(t, enc.WriteField(big)) - // One-byte indexed references to the big field: the compressed block stays - // small while the decoded list grows past the cap. - for i := 0; i < 2000; i++ { - require.NoError(t, enc.WriteField(big)) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) + require.NoError(t, enc.WriteField(hpack.HeaderField{Name: "x-big", Value: strings.Repeat("a", 70<<10)})) + + // The block spans the HEADERS frame and CONTINUATION frames: no single + // frame may exceed the HTTP/2 frame size limit. + block := buf.Bytes() + const maxFragment = 1 << 14 + first := block + if len(first) > maxFragment { + first = block[:maxFragment] + } + writeFrame(t, c, http2.FrameHeaders, 0, 1, first) + for rest := block[len(first):]; len(rest) > 0; { + frag := rest + if len(frag) > maxFragment { + frag = rest[:maxFragment] + } + rest = rest[len(frag):] + flags := http2.Flags(0) + if len(rest) == 0 { + flags = http2.FlagContinuationEndHeaders + } + writeFrame(t, c, http2.FrameContinuation, flags, 1, frag) } - writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) - require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = c.Read(make([]byte, 9)) - require.Error(t, err, "expected the connection to be dropped for the amplified header list") + got := acceptWithTimeout(t, m.HTTP()) + got.Close() } -// TestAckWithoutInjectionDropped proves that a SETTINGS-ACK for which the -// classifier never injected any SETTINGS is rejected: no server SETTINGS were -// sent before the handover, so such an ACK is malformed traffic. -func TestAckWithoutInjectionDropped(t *testing.T) { - _, dial := startMux(t, nil) +// TestAdmissionControlSheds proves a connection is shed when too many +// connections are in classification: it matches no listener and cmux closes +// it, while the connection that holds the slot stays open. +func TestAdmissionControlSheds(t *testing.T) { + m, dial := startMux(t, &Options{MaxInflightClassifications: 1, ClassifyTimeout: time.Second}) - c, err := dial() + // The first connection parks in classification: it sent a preface but no + // request, so the matcher holds the only slot waiting for frames. + parked, err := dial() require.NoError(t, err) - defer c.Close() + defer parked.Close() + _, err = io.WriteString(parked, http2.ClientPreface) + require.NoError(t, err) + require.Eventually(t, func() bool { + select { + case m.admit <- struct{}{}: + // The slot was free: put the token back for the matchers. + <-m.admit + return false + default: + return true + } + }, 2*time.Second, 5*time.Millisecond, "the first connection never parked in classification") - _, err = io.WriteString(c, clientConnPreface) + // The second connection cannot get a slot in either matcher, so cmux + // closes it without reading anything from it. + shed, err := dial() require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) - writeFrame(t, c, http2FrameSettings, http2FlagSettingsAck, 0, nil) + readEOF(t, shed, time.Second) - require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = c.Read(make([]byte, 9)) - require.Error(t, err, "expected the connection to be dropped for the unexpected ACK") + // The parked connection is still open past its grace: it received the + // injected settings frame, and no request follows. + require.NoError(t, parked.SetReadDeadline(time.Now().Add(2*time.Second))) + injected := make([]byte, 9) + _, err = io.ReadFull(parked, injected) + require.NoError(t, err) + assert.Equal(t, []byte{0, 0, 0, byte(http2.FrameSettings), 0, 0, 0, 0, 0}, injected, + "the matcher must send an empty settings frame to the parked connection") + require.NoError(t, parked.SetReadDeadline(time.Now().Add(300*time.Millisecond))) + _, err = parked.Read(make([]byte, 1)) + require.Error(t, err) + assert.True(t, isTimeout(err), "the parked connection must stay open: %v", err) } -// TestMalformedAckDropped proves that a SETTINGS-ACK with a payload is -// rejected: replaying its bytes as frame headers would rewrite the client's -// frame stream. -func TestMalformedAckDropped(t *testing.T) { - _, dial := startMux(t, nil) +// TestIdleConnectionDropped proves a connection that never sends anything is +// closed at the end of the classification window: both matchers decline it. +func TestIdleConnectionDropped(t *testing.T) { + _, dial := startMux(t, &Options{ClassifyTimeout: 300 * time.Millisecond}) c, err := dial() require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, clientConnPreface) - require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) - - // Trigger the injected SETTINGS, acknowledge it with a bogus payload. - require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) - _, err = io.ReadFull(c, make([]byte, 9)) - require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, http2FlagSettingsAck, 0, []byte{0, 0, 0, 1}) - - require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = c.Read(make([]byte, 9)) - require.Error(t, err, "expected the connection to be dropped for the malformed ACK") -} - -// eofAtEndConn returns the whole payload together with a trailing io.EOF, as -// a TLS connection may when the peer half-closes right after its last record. -type eofAtEndConn struct { - net.Conn - data []byte - sent bool -} - -func (c *eofAtEndConn) Read(p []byte) (int, error) { - if c.sent { - return 0, io.EOF - } - c.sent = true - return copy(p, c.data), io.EOF + readEOF(t, c, 2*time.Second) } -// TestReadIntoToleratesTrailingEOF proves readInto honors the io.Reader -// contract: a Read that returns the completing bytes together with an error -// must not fail the read. -func TestReadIntoToleratesTrailingEOF(t *testing.T) { - bc := &replayConn{Conn: &eofAtEndConn{data: []byte("0123456789")}, recording: true} - injected := false - buf := make([]byte, 10) - require.NoError(t, readInto(bc, buf, &injected, time.Now().Add(time.Second))) - assert.Equal(t, "0123456789", string(buf)) -} - -func TestServerSpeaksFirst(t *testing.T) { - // A gRPC client that waits for the server's SETTINGS before sending its - // first request must still be classified. - m, dial := startMux(t, nil) +// TestStalledAfterPrefaceDropped proves a connection that sends a preface and +// then stalls is closed at the end of its classification window: the matcher +// speaks to it after the grace, and a spoken-to connection never reaches the +// HTTP server. +func TestStalledAfterPrefaceDropped(t *testing.T) { + _, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: 300 * time.Millisecond}) c, err := dial() require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, clientConnPreface) + _, err = io.WriteString(c, http2.ClientPreface) require.NoError(t, err) - writeFrame(t, c, http2FrameSettings, 0, 0, nil) - // The classifier must send its empty SETTINGS frame after the grace - // period, unblocking the waiting client. - require.NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) - hdr := make([]byte, 9) - _, err = io.ReadFull(c, hdr) + // The matcher speaks after the grace; the connection never sends its + // request and is dropped at the classification deadline. + require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + _, err = io.ReadFull(c, make([]byte, 9)) require.NoError(t, err) - assert.Equal(t, byte(http2FrameSettings), hdr[3], "expected a SETTINGS frame from the classifier") - assert.Zero(t, hdr[4]&http2FlagSettingsAck, "expected a SETTINGS frame, not an ACK") - - writeClientStreamAfterPreface(t, c) - - got := acceptWithTimeout(t, m.GRPC()) - got.Close() -} - -// writeClientStreamAfterPreface sends SETTINGS + HEADERS on a connection that -// already sent its preface. -func writeClientStreamAfterPreface(t *testing.T, c net.Conn) { - t.Helper() - - writeFrame(t, c, http2FrameSettings, 0, 0, nil) - var buf bytes.Buffer - enc := hpack.NewEncoder(&buf) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) - writeFrame(t, c, http2FrameHeaders, http2FlagEndHeaders, 1, buf.Bytes()) + readEOF(t, c, 2*time.Second) } -func TestIdleConnectionDropped(t *testing.T) { - m, dial := startMux(t, &Options{ClassifyTimeout: 100 * time.Millisecond}) +func TestCloseClosesQueuedConnections(t *testing.T) { + m, dial := startMux(t, &Options{ClassifyTimeout: 300 * time.Millisecond}) c, err := dial() require.NoError(t, err) defer c.Close() - // The client sends nothing: the classifier must give up on the idle - // connection after the deadline, not hold it. - require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = c.Read(make([]byte, 9)) - require.Error(t, err, "expected the connection to be closed after the classification timeout") + writeClientStream(t, c, ctGRPCProto) - // Give the mux a moment to drop it, then make sure nothing was delivered. - time.Sleep(200 * time.Millisecond) - type result struct { - c net.Conn - err error - } - for _, l := range []net.Listener{m.GRPC(), m.HTTP()} { - ch := make(chan result, 1) - go func(l net.Listener) { - c, err := l.Accept() - ch <- result{c, err} - }(l) - select { - case r := <-ch: - if r.err == nil { - r.c.Close() - assert.Fail(t, "idle connection was delivered to a server") - } - case <-time.After(100 * time.Millisecond): - // Nothing delivered: correct. - } - } + // Leave the classified connection queued: no server accepts it. + // Classification completes in microseconds locally; 100ms is a wide + // margin. + time.Sleep(100 * time.Millisecond) + m.Close() + + readEOF(t, c, 2*time.Second) } func TestCloseUnblocksAccept(t *testing.T) { @@ -836,19 +592,74 @@ func TestCloseUnblocksAccept(t *testing.T) { }(lis) } - time.Sleep(50 * time.Millisecond) m.Close() for i := 0; i < 2; i++ { select { case err := <-errCh: - require.ErrorIs(t, err, net.ErrClosed) + require.True(t, m.IsOrderlyShutdownErr(err), "unexpected accept error: %v", err) case <-time.After(2 * time.Second): require.Fail(t, "Accept was not unblocked by Close") } } } +// TestSubListenerCloseMarksStopping proves the shutdown race tolerance from +// the other side: a server that stops first closes the root listener through +// its sub-listener, and the mux reports an orderly shutdown so the remaining +// server can tell it apart from a fatal accept failure. +func TestSubListenerCloseMarksStopping(t *testing.T) { + m, dial := startMux(t, nil) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + writeClientStream(t, c, ctGRPCProto) + + require.NoError(t, m.GRPC().Close()) + assert.True(t, m.Stopping(), "closing a sub-listener must mark the mux as stopping") + + errCh := make(chan error, 1) + go func() { + _, err := m.HTTP().Accept() + errCh <- err + }() + select { + case err := <-errCh: + require.True(t, m.IsOrderlyShutdownErr(err), "unexpected accept error: %v", err) + case <-time.After(2 * time.Second): + require.Fail(t, "the HTTP listener did not close with the mux") + } +} + +// TestCloseReturnsWhileConnParked proves Close does not ride out an idle +// classification beyond the classification window. +func TestCloseReturnsWhileConnParked(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + m := New(l, &Options{ClassifyTimeout: 300 * time.Millisecond, MaxInflightClassifications: 1}) + require.NoError(t, m.Start(context.Background())) + + c, err := net.Dial("tcp", l.Addr().String()) + require.NoError(t, err) + defer c.Close() + + // The connection parks in classification when it holds the admission + // slot: exactly the state Close must not ride out. + require.Eventually(t, func() bool { + select { + case m.admit <- struct{}{}: + <-m.admit + return false + default: + return true + } + }, 2*time.Second, 5*time.Millisecond, "the connection never parked in classification") + start := time.Now() + m.Close() + require.Less(t, time.Since(start), 2*time.Second, "Close rode out the classification window") +} + func TestTLSALPNClassification(t *testing.T) { l, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) @@ -881,7 +692,7 @@ func TestTLSALPNClassification(t *testing.T) { got.Close() c.Close() - // ALPN http/1.1 -> HTTP server without any frame parsing. + // ALPN http/1.1 -> HTTP server. c = dialTLS(t, "http/1.1") _, err = io.WriteString(c, "GET /statusz HTTP/1.1\r\nHost: x\r\n\r\n") require.NoError(t, err) @@ -908,31 +719,77 @@ func selfSignedCert(t *testing.T) tls.Certificate { return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} } -// TestCloseCoversConnectionAcceptedBeforeClassification proves the tracking -// has no window: a connection accepted but not yet inside its classify -// goroutine is still closed by Close, so shutdown does not ride out the -// classification deadline. -func TestCloseCoversConnectionAcceptedBeforeClassification(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - m := New(l, nil) - require.NoError(t, m.Start(context.Background())) +// TestGRPCServeOverMux runs a real grpc-go server and client through the mux, +// over plaintext and over TLS: the exact wiring the control plane uses. It +// guards the handoff of matched connections to grpc.Server.Serve. +func TestGRPCServeOverMux(t *testing.T) { + for _, tc := range []struct { + name string + tls bool + }{ + {"plaintext", false}, + {"TLS", true}, + } { + t.Run(tc.name, func(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) - c, err := net.Dial("tcp", l.Addr().String()) - require.NoError(t, err) - defer c.Close() + root := l + if tc.tls { + root = tls.NewListener(l, &tls.Config{ + Certificates: []tls.Certificate{selfSignedCert(t)}, + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + }) + } + m := New(root, nil) + require.NoError(t, m.Start(context.Background())) + t.Cleanup(m.Close) + + srv := grpc.NewServer() + grpc_health_v1.RegisterHealthServer(srv, health.NewServer()) + go func() { _ = srv.Serve(m.GRPC()) }() + t.Cleanup(srv.Stop) + + creds := insecure.NewCredentials() + if tc.tls { + creds = credentials.NewTLS(&tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // test server with a self-signed certificate + MinVersion: tls.VersionTLS12, + }) + } + conn, err := grpc.NewClient(l.Addr().String(), grpc.WithTransportCredentials(creds)) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) - // Wait until a connection is registered in the inflight set (keyed by the - // server-side connection, not the client side), which is what Close walks - // — this is exactly the state the race used to miss. - require.Eventually(t, func() bool { - m.mu.Lock() - defer m.mu.Unlock() - return len(m.inflight) > 0 - }, 2*time.Second, 5*time.Millisecond, "the accepted connection was not registered in the inflight set") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + require.NoError(t, err) + assert.Equal(t, grpc_health_v1.HealthCheckResponse_SERVING, resp.GetStatus()) + }) + } +} - start := time.Now() - m.Close() - // The idle connection's deadline (10s) must not gate the shutdown. - require.Less(t, time.Since(start), 2*time.Second) +func TestIsNativeGRPCContentType(t *testing.T) { + for _, tc := range []struct { + ct string + want bool + }{ + {"application/grpc", true}, + {"application/grpc+proto", true}, + {"application/grpc+json", true}, + {"application/grpc; charset=utf-8", true}, + {"Application/GRPC", true}, + {"application/grpc-web", false}, + {"application/grpc-web+proto", false}, + {"application/grpc-web+json", false}, + {"application/json", false}, + {"text/plain", false}, + {"", false}, + } { + t.Run(tc.ct, func(t *testing.T) { + assert.Equal(t, tc.want, IsNativeGRPCContentType(tc.ct)) + }) + } } From 5ca931173545359ff35732e3ad318b9a01d1706c Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 16:15:41 +0200 Subject: [PATCH 04/10] fix(controlplane): speak only at frame boundaries and charge admission on data The settings grace now fires only between frames: a mid-frame timeout means a slow sender, not a settings-waiting client, so the matcher declines without speaking and the intact replay lets the HTTP server serve the request through the documented fallback, instead of feeding the framer the rest of the interrupted frame. Found by the cubic review of 6d499c2a. Admission slots are now charged only when a connection prefaced (gRPC side) or produced a byte (HTTP side): a silent connection buffers nothing and costs only a file descriptor, as before this package existed, and HTTP/1.x connections never touch admission at all. The spec's live references to superseded decisions now point at D-013 and D-014. Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: ses_eabb93119ffegqWXti0mS05TIM --- docs/specs/issue-3628-single-endpoint.md | 4 +- pkg/httpgrpcmux/mux.go | 111 +++++++++++++++++------ pkg/httpgrpcmux/mux_test.go | 69 +++++++++++++- 3 files changed, 151 insertions(+), 33 deletions(-) diff --git a/docs/specs/issue-3628-single-endpoint.md b/docs/specs/issue-3628-single-endpoint.md index c8eeea2a5..c562193ae 100644 --- a/docs/specs/issue-3628-single-endpoint.md +++ b/docs/specs/issue-3628-single-endpoint.md @@ -42,7 +42,7 @@ The control plane and the Artifact CAS will each serve gRPC and HTTP on one port - Goal: one port and one hostname per service, for all gRPC, gRPC-Web, and HTTP traffic. - Goal: existing clients and configuration keep working without change during the deprecation period. - Goal: large CAS uploads and downloads keep their current streaming behavior and throughput. -- Non-goal: move the services to another RPC framework (for example Connect). The single port does not need it (see D-001). +- Non-goal: move the services to another RPC framework (for example Connect). The single port does not need it (see D-013). - Non-goal: merge the Prometheus metrics port or the profiler port into the single port. These ports are internal and must not be on the public hostname (see D-006). - Non-goal: add Gateway API resources to the Helm chart. It is a separate chart feature, and the single port works without it (see D-004). - Non-goal: change the CLI's default endpoints or its configuration format. The CLI holds hostnames on the ingress port, so it needs no change (see R-005). @@ -115,7 +115,7 @@ The classifier speaks only when a connection stays silent. gRPC clients wait for The classifier decides once per connection, from its first request. A connection that starts with gRPC carries only gRPC: the native gRPC server cannot serve a later plain HTTP request on it. The other direction works. When a gRPC request arrives on a connection that the HTTP server owns, the HTTP server gives it to the gRPC server. The gRPC-Web wrapper already works in this way. The request gets the right result through the HTTP adapter of the gRPC server. That adapter is a separate transport. Its performance and features may differ from the native one (D-013). -Thus the single port supports clients and proxies that keep gRPC and plain HTTP on separate HTTP/2 connections. These include direct clients such as the CLI, nginx ingress, and Traefik with HTTP/1.1 for the HTTP path. A proxy that sends both on one shared upstream connection is not supported (see D-010 and Open Questions). +Thus the single port supports clients and proxies that keep gRPC and plain HTTP on separate HTTP/2 connections. These include direct clients such as the CLI, nginx ingress, and Traefik with HTTP/1.1 for the HTTP path. A proxy that sends both on one shared upstream connection is not supported (see D-014). CAS streams on the single port use the same native gRPC transport as today, so uploads keep their disk spill and their throughput (R-003). Browser downloads stay on HTTP/1.1 through the ingress, so the kernel copy path stays. diff --git a/pkg/httpgrpcmux/mux.go b/pkg/httpgrpcmux/mux.go index cb38f8185..8b491bdaa 100644 --- a/pkg/httpgrpcmux/mux.go +++ b/pkg/httpgrpcmux/mux.go @@ -228,8 +228,10 @@ func New(l net.Listener, opts *Options) *Mux { // listener, and everything else falls through to the HTTP listener. Both // are MatchWriters so they can reach the connection: the gRPC matcher // writes the server's settings frame to clients that wait for it, and - // the HTTP matcher needs the connection to look up poisoned ones. - m.grpcL = m.cm.MatchWithWriters(m.admitMatchWriter(m.matchNativeGRPC)) + // takes its admission slot itself, once the connection prefaced; the + // HTTP matcher's wrapper takes its slot when the connection produced a + // byte. + m.grpcL = m.cm.MatchWithWriters(m.matchNativeGRPC) m.httpL = m.cm.MatchWithWriters(m.admitMatchWriter(m.httpMatcher)) m.cm.HandleError(func(err error) bool { @@ -327,11 +329,28 @@ func (m *Mux) IsOrderlyShutdownErr(err error) bool { errors.Is(err, cmux.ErrListenerClosed)) } -// admitMatchWriter wraps a MatchWriter with classification admission control: -// it holds a slot while the matcher runs, and reports no match when the +// admitMatchWriter wraps a MatchWriter with classification admission control. +// It first waits for the connection to produce a byte: a silent connection +// buffers nothing, so it must not consume an admission slot — the slots bound +// buffered bytes, and the file descriptor limit bounds idle connections, as +// before this package existed. When the connection produces a byte the wrapper +// holds a slot while the matcher runs, and reports no match when the // admission cap is reached, so cmux closes the connection. func (m *Mux) admitMatchWriter(match cmux.MatchWriter) cmux.MatchWriter { return func(w io.Writer, r io.Reader) bool { + // The byte read is safe: cmux re-serves sniffed bytes to the next + // matcher, and the HTTP matcher itself never re-reads within its own + // sniffing session. + var b [1]byte + for { + n, err := r.Read(b[:]) + if n > 0 { + break + } + if err != nil { + return false + } + } select { case m.admit <- struct{}{}: default: @@ -353,30 +372,23 @@ func isTimeout(err error) bool { return errors.As(err, &ne) && ne.Timeout() } -// httpMatcher accepts every remaining connection that produced at least one -// byte and that the gRPC matcher did not speak to. A connection that produced -// none, or whose read failed, matches no listener and is closed by cmux. -func (m *Mux) httpMatcher(w io.Writer, r io.Reader) bool { - // cmux passes the raw connection as the writer, and the gRPC matcher - // keys poisoned connections by it. A connection that is not identifiable - // cannot have been spoken to: the gRPC matcher declines before speaking - // unless it can reach the connection too. +// httpMatcher accepts every remaining connection the gRPC matcher did not +// speak to. Its admission wrapper has already proved the connection carries +// traffic. A poisoned connection — one the gRPC matcher sent its settings +// frame to — is declined so cmux closes it: the client's acknowledgement of +// that frame is in the replay, and net/http fails a connection that +// acknowledges settings it never sent. +func (m *Mux) httpMatcher(w io.Writer, _ io.Reader) bool { if conn, ok := w.(net.Conn); ok { if _, poisoned := m.poisoned.LoadAndDelete(conn); poisoned { m.logger.Warnw("msg", "closing connection: the classifier sent its settings frame, but the first request was not native gRPC") return false } } - // One byte proves the connection carries traffic. The read goes through - // the sniffing reader, so cmux records it and replays it to the HTTP - // server. - var b [1]byte - for { - n, err := r.Read(b[:]) - if err != nil || n > 0 { - return n > 0 - } - } + // A connection the gRPC matcher cannot reach cannot have been spoken + // to: that matcher declines before speaking unless it can reach the + // connection too. + return true } // matchNativeGRPC reports whether the connection's first request carries a @@ -399,16 +411,33 @@ func (m *Mux) httpMatcher(w io.Writer, r io.Reader) bool { // server that wins the connection. The matcher bounds how much of the // connection it may read. func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { - // The extra byte makes any read past the cap fail instead of stopping - // silently at it: a connection that needs more than the cap to classify - // goes to the HTTP server, whose own limits dispose of it. - lr := &io.LimitedReader{R: r, N: int64(m.maxClassifiedBytes) + 1} + // The preface is read before any admission slot is taken: a silent + // connection buffers nothing and must not consume one — the slots bound + // buffered bytes, and the file descriptor limit bounds idle connections. + // A non-prefaced connection (HTTP/1.x, garbage) declines here without + // ever touching admission. + // + // The extra byte in the cap makes any read past it fail instead of + // stopping silently at it: a connection that needs more than the cap to + // classify goes to the HTTP server, whose own limits dispose of it. + cr := &countingReader{r: r} + lr := &io.LimitedReader{R: cr, N: int64(m.maxClassifiedBytes) + 1} if !hasHTTP2Preface(lr) { // HTTP/1.x or anything else: the HTTP server produces a proper error // response for malformed traffic. Nothing was written. return false } + // The connection prefaced, so frames will buffer: take an admission + // slot. A connection that cannot get one matches nothing, and cmux + // closes it. + select { + case m.admit <- struct{}{}: + default: + return false + } + defer func() { <-m.admit }() + conn, ok := w.(net.Conn) if !ok { return false @@ -426,8 +455,11 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { // Narrow the read deadline cmux set to the classification window down to // the settings grace: how long to wait for a first request before // speaking. decline reports no match, and poisons the connection when the - // matcher has already spoken, so it never reaches net/http. + // matcher has already spoken, so it never reaches net/http. boundary is + // the byte count at the last completed frame: a timeout with the count + // past it fired mid-frame. spoken := false + boundary := cr.n _ = conn.SetReadDeadline(time.Now().Add(m.settingsGrace)) decline := func() bool { if spoken { @@ -451,6 +483,16 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { f, err := framer.ReadFrame() if err != nil { if !spoken && isTimeout(err) { + if cr.n > boundary { + // The grace fired mid-frame: a slow sender, not a client + // waiting for the server's settings. The framer cannot + // resume a half-read frame, and speaking now would feed + // it the rest of the frame as garbage. Decline without + // speaking: the replay holds everything the client sent, + // and the HTTP server serves the request through the + // documented fallback (D-014). + return false + } // The client is waiting for the server to speak first. if !speak() { return decline() @@ -461,6 +503,7 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { // connection cannot go to the HTTP server. return decline() } + boundary = cr.n // HEADERS and CONTINUATION are the only frames that carry header // block fragments; SETTINGS (including the acknowledgement of the // injected frame), PING, WINDOW_UPDATE, ... are skipped and stay in @@ -482,6 +525,20 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { } } +// countingReader counts the bytes its caller has consumed from the connection, +// so the matcher can tell a frame boundary from a half-read frame when the +// settings grace fires. +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + // hasHTTP2Preface reports whether the connection starts with the HTTP/2 client // preface. The preface is exact bytes, unlike header names and values. It // reads byte ranges without over-consuming: everything it reads stays diff --git a/pkg/httpgrpcmux/mux_test.go b/pkg/httpgrpcmux/mux_test.go index 3e38b3edf..d2f066391 100644 --- a/pkg/httpgrpcmux/mux_test.go +++ b/pkg/httpgrpcmux/mux_test.go @@ -506,10 +506,12 @@ func TestAdmissionControlSheds(t *testing.T) { } }, 2*time.Second, 5*time.Millisecond, "the first connection never parked in classification") - // The second connection cannot get a slot in either matcher, so cmux - // closes it without reading anything from it. + // The second connection prefaced but cannot get a slot in either + // matcher, so cmux closes it without parsing anything from it. shed, err := dial() require.NoError(t, err) + _, err = io.WriteString(shed, http2.ClientPreface) + require.NoError(t, err) readEOF(t, shed, time.Second) // The parked connection is still open past its grace: it received the @@ -560,6 +562,62 @@ func TestStalledAfterPrefaceDropped(t *testing.T) { readEOF(t, c, 2*time.Second) } +// TestSlowMidFrameClientTakesFallback proves a client whose first request +// straddles the settings grace is served, not dropped: the grace fires +// mid-frame, the matcher declines without speaking, and the intact replay +// lets the HTTP server parse the completed request through the documented +// fallback (D-014). The framer cannot resume a half-read frame, so speaking +// there would desynchronize it. +func TestSlowMidFrameClientTakesFallback(t *testing.T) { + m, dial := startMux(t, &Options{SettingsGrace: 150 * time.Millisecond, ClassifyTimeout: 2 * time.Second}) + + c, err := dial() + require.NoError(t, err) + defer c.Close() + + _, err = io.WriteString(c, http2.ClientPreface) + require.NoError(t, err) + writeFrame(t, c, http2.FrameSettings, 0, 0, nil) + + // Send the HEADERS frame header and half of its payload, then stall past + // the grace. + var buf bytes.Buffer + enc := hpack.NewEncoder(&buf) + for _, hf := range []hpack.HeaderField{ + {Name: hdrMethod, Value: hdrMethodPost}, + {Name: ":scheme", Value: "http"}, + {Name: ":path", Value: "/controlplane.v1.FooService/Bar"}, + {Name: hdrContentType, Value: ctGRPCProto}, + } { + require.NoError(t, enc.WriteField(hf)) + } + block := buf.Bytes() + hdr := []byte{ + byte(len(block) >> 16), byte(len(block) >> 8), byte(len(block)), + byte(http2.FrameHeaders), byte(http2.FlagHeadersEndHeaders), + 0, 0, 0, 1, + } + _, err = c.Write(hdr) + require.NoError(t, err) + _, err = c.Write(block[:len(block)/2]) + require.NoError(t, err) + time.Sleep(400 * time.Millisecond) + + // The rest of the request arrives after the grace. + _, err = c.Write(block[len(block)/2:]) + require.NoError(t, err) + + // The connection is not dropped and not poisoned: the HTTP server gets + // it with the whole stream replayed. + got := acceptWithTimeout(t, m.HTTP()) + defer got.Close() + require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) + replay := make([]byte, len(http2.ClientPreface)) + _, err = io.ReadFull(got, replay) + require.NoError(t, err) + assert.Equal(t, http2.ClientPreface, string(replay), "preface not replayed intact") +} + func TestCloseClosesQueuedConnections(t *testing.T) { m, dial := startMux(t, &Options{ClassifyTimeout: 300 * time.Millisecond}) @@ -644,8 +702,11 @@ func TestCloseReturnsWhileConnParked(t *testing.T) { require.NoError(t, err) defer c.Close() - // The connection parks in classification when it holds the admission - // slot: exactly the state Close must not ride out. + // A connection that produced a byte parks in classification while it + // holds the admission slot: exactly the state Close must not ride out. A + // silent connection holds no slot at all. + _, err = io.WriteString(c, http2.ClientPreface) + require.NoError(t, err) require.Eventually(t, func() bool { select { case m.admit <- struct{}{}: From 045fe2af7f145d9fd8dfb16044be163d407e6aca Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 16:26:51 +0200 Subject: [PATCH 05/10] refactor(controlplane): bound the settings write and harden the poison contract Review follow-ups on the speak-first matcher: - The settings write is bounded by its own write deadline, which the matcher clears afterwards: cmux clears only the read deadline at handover, and a lingering write deadline would fail every later write on long-lived streams. A settings-waiting connection now carries a regression test that writes from the server past the classification window. - The matcher commits the connection before it writes, so a partially delivered settings frame still poisons the connection and cmux closes it instead of handing net/http a desynchronized stream. - The HTTP chain drops poisoned connections before it reads or takes an admission slot, so a full admission cap cannot strand the marker and pin the connection object in the map forever. - The frame-boundary rule reads the remaining budget straight from the LimitedReader (countingReader removed), and the HTTP/2 preface check is cmux's own matcher. Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: ses_eabb93119ffegqWXti0mS05TIM --- pkg/httpgrpcmux/mux.go | 183 +++++++++++++++--------------------- pkg/httpgrpcmux/mux_test.go | 89 ++++++++++-------- 2 files changed, 126 insertions(+), 146 deletions(-) diff --git a/pkg/httpgrpcmux/mux.go b/pkg/httpgrpcmux/mux.go index 8b491bdaa..a43968c4e 100644 --- a/pkg/httpgrpcmux/mux.go +++ b/pkg/httpgrpcmux/mux.go @@ -40,7 +40,6 @@ package httpgrpcmux import ( - "bytes" "context" "errors" "io" @@ -108,10 +107,6 @@ const grpcWebContentTypePrefix = "application/grpc-web" // headerContentType is the request header that carries the content type. const headerContentType = "content-type" -// clientPreface is the HTTP/2 client preface, preallocated because the -// matcher compares it on every classified connection. -var clientPreface = []byte(http2.ClientPreface) - // Options configures a Mux. type Options struct { // Logger receives classification debug information. Optional. @@ -156,9 +151,10 @@ type Mux struct { // HTTP/2 settings frame to, but whose first request turned out not to be // native gRPC. Their acknowledgement of that frame is already recorded // in cmux's replay buffer, and net/http fails a connection that - // acknowledges settings it never sent, so the HTTP matcher declines - // them and cmux closes them. The gRPC matcher sets the entry before it - // declines; the HTTP matcher, which cmux always runs next, clears it. + // acknowledges settings it never sent, so they must not reach the HTTP + // side. The gRPC matcher sets the entry before it declines; the HTTP + // listener's dropPoisoned wrapper, which cmux always runs next, clears + // it before reading, so a full admission cap cannot strand the marker. poisoned sync.Map stopping atomic.Bool @@ -228,11 +224,11 @@ func New(l net.Listener, opts *Options) *Mux { // listener, and everything else falls through to the HTTP listener. Both // are MatchWriters so they can reach the connection: the gRPC matcher // writes the server's settings frame to clients that wait for it, and - // takes its admission slot itself, once the connection prefaced; the - // HTTP matcher's wrapper takes its slot when the connection produced a - // byte. + // takes its admission slot itself, once the connection prefaced. The + // HTTP chain drops spoken-to connections before it reads anything, so a + // poison marker never survives a full admission cap. m.grpcL = m.cm.MatchWithWriters(m.matchNativeGRPC) - m.httpL = m.cm.MatchWithWriters(m.admitMatchWriter(m.httpMatcher)) + m.httpL = m.cm.MatchWithWriters(m.dropPoisoned(m.requireTraffic)) m.cm.HandleError(func(err error) bool { var notMatched cmux.ErrNotMatched @@ -329,35 +325,53 @@ func (m *Mux) IsOrderlyShutdownErr(err error) bool { errors.Is(err, cmux.ErrListenerClosed)) } -// admitMatchWriter wraps a MatchWriter with classification admission control. -// It first waits for the connection to produce a byte: a silent connection -// buffers nothing, so it must not consume an admission slot — the slots bound -// buffered bytes, and the file descriptor limit bounds idle connections, as -// before this package existed. When the connection produces a byte the wrapper -// holds a slot while the matcher runs, and reports no match when the -// admission cap is reached, so cmux closes the connection. -func (m *Mux) admitMatchWriter(match cmux.MatchWriter) cmux.MatchWriter { +// dropPoisoned declines the connections the gRPC matcher has spoken to. +// It runs as the outermost wrapper of the HTTP chain, before any read and +// before admission: a spoken-to connection cannot be handed to net/http, and +// its marker must clear even when the admission cap is full, or the entry +// would pin the connection object forever. +func (m *Mux) dropPoisoned(next cmux.MatchWriter) cmux.MatchWriter { return func(w io.Writer, r io.Reader) bool { - // The byte read is safe: cmux re-serves sniffed bytes to the next - // matcher, and the HTTP matcher itself never re-reads within its own - // sniffing session. - var b [1]byte - for { - n, err := r.Read(b[:]) - if n > 0 { - break - } - if err != nil { + if conn, ok := w.(net.Conn); ok { + if _, poisoned := m.poisoned.LoadAndDelete(conn); poisoned { + m.logger.Warnw("msg", "closing connection: the classifier sent its settings frame, but the first request was not native gRPC") return false } } - select { - case m.admit <- struct{}{}: - default: - return false - } - defer func() { <-m.admit }() - return match(w, r) + return next(w, r) + } +} + +// requireTraffic is the terminal matcher of the HTTP chain: it accepts every +// remaining connection that produced at least one byte. A connection that +// produced none, or whose read failed, matches no listener and is closed by +// cmux. The one-byte read waits for the connection to carry traffic before +// the admission slot is taken: a silent connection buffers nothing and must +// not consume a slot — the slots bound buffered bytes, and the file descriptor +// limit bounds idle connections, as before this package existed. +func (m *Mux) requireTraffic(_ io.Writer, r io.Reader) bool { + // cmux re-serves sniffed bytes between matcher sessions, never within + // one, so no matcher may re-read what it already consumed: this is the + // only read of the HTTP chain. + var b [1]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return false + } + if !m.tryAcquire() { + return false + } + defer func() { <-m.admit }() + return true +} + +// tryAcquire takes an admission slot when one is free, so cmux closes the +// connection when the classification cap is reached. +func (m *Mux) tryAcquire() bool { + select { + case m.admit <- struct{}{}: + return true + default: + return false } } @@ -372,25 +386,6 @@ func isTimeout(err error) bool { return errors.As(err, &ne) && ne.Timeout() } -// httpMatcher accepts every remaining connection the gRPC matcher did not -// speak to. Its admission wrapper has already proved the connection carries -// traffic. A poisoned connection — one the gRPC matcher sent its settings -// frame to — is declined so cmux closes it: the client's acknowledgement of -// that frame is in the replay, and net/http fails a connection that -// acknowledges settings it never sent. -func (m *Mux) httpMatcher(w io.Writer, _ io.Reader) bool { - if conn, ok := w.(net.Conn); ok { - if _, poisoned := m.poisoned.LoadAndDelete(conn); poisoned { - m.logger.Warnw("msg", "closing connection: the classifier sent its settings frame, but the first request was not native gRPC") - return false - } - } - // A connection the gRPC matcher cannot reach cannot have been spoken - // to: that matcher declines before speaking unless it can reach the - // connection too. - return true -} - // matchNativeGRPC reports whether the connection's first request carries a // native gRPC content type. // @@ -420,9 +415,8 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { // The extra byte in the cap makes any read past it fail instead of // stopping silently at it: a connection that needs more than the cap to // classify goes to the HTTP server, whose own limits dispose of it. - cr := &countingReader{r: r} - lr := &io.LimitedReader{R: cr, N: int64(m.maxClassifiedBytes) + 1} - if !hasHTTP2Preface(lr) { + lr := &io.LimitedReader{R: r, N: int64(m.maxClassifiedBytes) + 1} + if !matchHTTP2Preface(lr) { // HTTP/1.x or anything else: the HTTP server produces a proper error // response for malformed traffic. Nothing was written. return false @@ -431,9 +425,7 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { // The connection prefaced, so frames will buffer: take an admission // slot. A connection that cannot get one matches nothing, and cmux // closes it. - select { - case m.admit <- struct{}{}: - default: + if !m.tryAcquire() { return false } defer func() { <-m.admit }() @@ -455,11 +447,13 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { // Narrow the read deadline cmux set to the classification window down to // the settings grace: how long to wait for a first request before // speaking. decline reports no match, and poisons the connection when the - // matcher has already spoken, so it never reaches net/http. boundary is - // the byte count at the last completed frame: a timeout with the count - // past it fired mid-frame. + // matcher has spoken, so it never reaches net/http. boundary is the + // remaining read budget at the last completed frame: a timeout with a + // smaller budget fired mid-frame. The rule is correct only while the + // framer reads exactly frame-header plus payload with no read-ahead; + // inserting a buffering reader in this chain breaks it. spoken := false - boundary := cr.n + boundary := lr.N _ = conn.SetReadDeadline(time.Now().Add(m.settingsGrace)) decline := func() bool { if spoken { @@ -468,13 +462,18 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { return false } speak := func() bool { - // Bound the write, then give the committed classification a fresh - // window. - _ = conn.SetDeadline(time.Now().Add(m.classifyTimeout)) - if err := framer.WriteSettings(); err != nil { + // The write is bounded on its own: cmux clears only the read + // deadline when the connection is handed over, so a lingering write + // deadline would kill long-lived streams downstream. + _ = conn.SetWriteDeadline(time.Now().Add(m.classifyTimeout)) + // The connection is committed from the first written byte, even if + // the frame is only partially delivered: decline must poison it. + spoken = true + err := framer.WriteSettings() + _ = conn.SetWriteDeadline(time.Time{}) + if err != nil { return false } - spoken = true _ = conn.SetReadDeadline(time.Now().Add(m.classifyTimeout)) return true } @@ -483,7 +482,7 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { f, err := framer.ReadFrame() if err != nil { if !spoken && isTimeout(err) { - if cr.n > boundary { + if lr.N < boundary { // The grace fired mid-frame: a slow sender, not a client // waiting for the server's settings. The framer cannot // resume a half-read frame, and speaking now would feed @@ -503,7 +502,7 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { // connection cannot go to the HTTP server. return decline() } - boundary = cr.n + boundary = lr.N // HEADERS and CONTINUATION are the only frames that carry header // block fragments; SETTINGS (including the acknowledgement of the // injected frame), PING, WINDOW_UPDATE, ... are skipped and stay in @@ -525,39 +524,11 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { } } -// countingReader counts the bytes its caller has consumed from the connection, -// so the matcher can tell a frame boundary from a half-read frame when the -// settings grace fires. -type countingReader struct { - r io.Reader - n int64 -} - -func (c *countingReader) Read(p []byte) (int, error) { - n, err := c.r.Read(p) - c.n += int64(n) - return n, err -} - -// hasHTTP2Preface reports whether the connection starts with the HTTP/2 client -// preface. The preface is exact bytes, unlike header names and values. It -// reads byte ranges without over-consuming: everything it reads stays -// buffered in cmux and is replayed to the server. -func hasHTTP2Preface(r io.Reader) bool { - buf := make([]byte, len(clientPreface)) - n := 0 - for n < len(clientPreface) { - read, err := r.Read(buf[n:]) - n += read - if err != nil { - return false - } - if !bytes.HasPrefix(buf[:n], clientPreface[:n]) { - return false - } - } - return true -} +// matchHTTP2Preface is cmux's HTTP/2 preface matcher: it reports whether the +// connection starts with the exact HTTP/2 client preface bytes, without +// over-consuming. Everything it reads stays buffered in cmux and is replayed +// to the server. +var matchHTTP2Preface = cmux.HTTP2() // IsNativeGRPCContentType reports whether the given content type identifies a // native gRPC request, as opposed to a gRPC-Web or plain HTTP one. The HTTP diff --git a/pkg/httpgrpcmux/mux_test.go b/pkg/httpgrpcmux/mux_test.go index d2f066391..9d6ab2c01 100644 --- a/pkg/httpgrpcmux/mux_test.go +++ b/pkg/httpgrpcmux/mux_test.go @@ -100,14 +100,11 @@ func writeClientStream(t *testing.T, c net.Conn, contentType string, extraHeader writeRequestFrames(t, c, contentType, extraHeaders...) } -// writeRequestFrames sends a client SETTINGS frame and one HEADERS frame -// carrying the given headers: the request flight after the connection -// preface. -func writeRequestFrames(t *testing.T, c net.Conn, contentType string, extraHeaders ...hpack.HeaderField) { +// encodeRequestHeaders returns the HPACK-encoded header block of a request +// with the given content type. +func encodeRequestHeaders(t *testing.T, contentType string, extraHeaders ...hpack.HeaderField) []byte { t.Helper() - writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - var buf bytes.Buffer enc := hpack.NewEncoder(&buf) for _, hf := range append([]hpack.HeaderField{ @@ -118,7 +115,18 @@ func writeRequestFrames(t *testing.T, c net.Conn, contentType string, extraHeade }, extraHeaders...) { require.NoError(t, enc.WriteField(hf)) } - writeFrame(t, c, http2.FrameHeaders, http2.FlagHeadersEndHeaders, 1, buf.Bytes()) + return buf.Bytes() +} + +// writeRequestFrames sends a client SETTINGS frame and one HEADERS frame +// carrying the given headers: the request flight after the connection +// preface. +func writeRequestFrames(t *testing.T, c net.Conn, contentType string, extraHeaders ...hpack.HeaderField) { + t.Helper() + + writeFrame(t, c, http2.FrameSettings, 0, 0, nil) + writeFrame(t, c, http2.FrameHeaders, http2.FlagHeadersEndHeaders, 1, + encodeRequestHeaders(t, contentType, extraHeaders...)) } func writeFrame(t *testing.T, c net.Conn, frameType http2.FrameType, flags http2.Flags, streamID uint32, payload []byte) { @@ -146,6 +154,21 @@ func readEOF(t *testing.T, c net.Conn, within time.Duration) { assert.False(t, isTimeout(err), "the connection must be closed, not merely silent") } +// requireAdmissionParked waits until a connection holds an admission slot. +func requireAdmissionParked(t *testing.T, m *Mux, msg string) { + t.Helper() + require.Eventually(t, func() bool { + select { + case m.admit <- struct{}{}: + // The slot was free: put the token back for the matchers. + <-m.admit + return false + default: + return true + } + }, 2*time.Second, 5*time.Millisecond, msg) +} + func TestClassifyHTTP1(t *testing.T) { m, dial := startMux(t, nil) @@ -357,7 +380,7 @@ func TestMatcherDoesNotSpeakFirstForImmediateClients(t *testing.T) { // connection goes to the gRPC side with the acknowledgement in the replay: // the gRPC server ignores acknowledgements of settings it never sent. func TestSettingsWaitingClientClassifiesNative(t *testing.T) { - m, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: time.Second}) + m, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: 300 * time.Millisecond}) c, err := dial() require.NoError(t, err) @@ -390,6 +413,20 @@ func TestSettingsWaitingClientClassifiesNative(t *testing.T) { _, err = io.ReadFull(got, replay) require.NoError(t, err) assert.Equal(t, http2.ClientPreface, string(replay), "preface not replayed intact") + + // The matcher must clear the write deadline its settings frame was + // bounded by: cmux clears only the read deadline at handover, and a + // lingering write deadline would fail every later write on the + // connection. Write from the server past the classification window and + // expect the client to receive it. + time.Sleep(450 * time.Millisecond) + _, err = got.Write([]byte("still-writable")) + require.NoError(t, err, "a write deadline survived the handover") + require.NoError(t, c.SetReadDeadline(time.Now().Add(time.Second))) + tail := make([]byte, len("still-writable")) + _, err = io.ReadFull(c, tail) + require.NoError(t, err) + assert.Equal(t, "still-writable", string(tail)) } // TestCommittedNonGRPCDropped proves a spoken-to connection whose first @@ -397,7 +434,7 @@ func TestSettingsWaitingClientClassifiesNative(t *testing.T) { // acknowledgement of the injected settings frame is already recorded, and // net/http fails a connection that acknowledges settings it never sent. func TestCommittedNonGRPCDropped(t *testing.T) { - m, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: time.Second}) + _, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: time.Second}) c, err := dial() require.NoError(t, err) @@ -417,7 +454,6 @@ func TestCommittedNonGRPCDropped(t *testing.T) { // The connection matches no listener and cmux closes it. readEOF(t, c, 2*time.Second) - _ = m // keep the mux referenced for the duration of the test } // TestOversizedHeaderBlockDeclined proves a connection whose first request @@ -495,16 +531,7 @@ func TestAdmissionControlSheds(t *testing.T) { defer parked.Close() _, err = io.WriteString(parked, http2.ClientPreface) require.NoError(t, err) - require.Eventually(t, func() bool { - select { - case m.admit <- struct{}{}: - // The slot was free: put the token back for the matchers. - <-m.admit - return false - default: - return true - } - }, 2*time.Second, 5*time.Millisecond, "the first connection never parked in classification") + requireAdmissionParked(t, m, "the first connection never parked in classification") // The second connection prefaced but cannot get a slot in either // matcher, so cmux closes it without parsing anything from it. @@ -581,17 +608,7 @@ func TestSlowMidFrameClientTakesFallback(t *testing.T) { // Send the HEADERS frame header and half of its payload, then stall past // the grace. - var buf bytes.Buffer - enc := hpack.NewEncoder(&buf) - for _, hf := range []hpack.HeaderField{ - {Name: hdrMethod, Value: hdrMethodPost}, - {Name: ":scheme", Value: "http"}, - {Name: ":path", Value: "/controlplane.v1.FooService/Bar"}, - {Name: hdrContentType, Value: ctGRPCProto}, - } { - require.NoError(t, enc.WriteField(hf)) - } - block := buf.Bytes() + block := encodeRequestHeaders(t, ctGRPCProto) hdr := []byte{ byte(len(block) >> 16), byte(len(block) >> 8), byte(len(block)), byte(http2.FrameHeaders), byte(http2.FlagHeadersEndHeaders), @@ -707,15 +724,7 @@ func TestCloseReturnsWhileConnParked(t *testing.T) { // silent connection holds no slot at all. _, err = io.WriteString(c, http2.ClientPreface) require.NoError(t, err) - require.Eventually(t, func() bool { - select { - case m.admit <- struct{}{}: - <-m.admit - return false - default: - return true - } - }, 2*time.Second, 5*time.Millisecond, "the connection never parked in classification") + requireAdmissionParked(t, m, "the connection never parked in classification") start := time.Now() m.Close() require.Less(t, time.Since(start), 2*time.Second, "Close rode out the classification window") From 1d8d9607f3a422045fc5d1fc8af894f27146c21b Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 17:57:42 +0200 Subject: [PATCH 06/10] fix(controlplane): bound HTTP/2 matching and preserve HTTP handoff Reject oversized frames before allocating their payloads. Track and remove the acknowledgement of classifier-injected SETTINGS before handing HTTP connections to net/http, while keeping native gRPC on its existing transport. Cover waiting HTTP/2 clients over plaintext and TLS, ACK ordering, connection reuse, incomplete handshakes, and oversized frame headers. Record D-016 as the supersession of D-013. Chainloop-Trace-Sessions: ses_eabb93119ffegqWXti0mS05TIM Signed-off-by: Miguel Martinez Trivino --- docs/specs/issue-3628-single-endpoint.md | 29 ++- pkg/httpgrpcmux/mux.go | 173 ++++++++++++------ pkg/httpgrpcmux/mux_test.go | 219 +++++++++++++++++++++-- 3 files changed, 336 insertions(+), 85 deletions(-) diff --git a/docs/specs/issue-3628-single-endpoint.md b/docs/specs/issue-3628-single-endpoint.md index c562193ae..7437de27a 100644 --- a/docs/specs/issue-3628-single-endpoint.md +++ b/docs/specs/issue-3628-single-endpoint.md @@ -236,15 +236,17 @@ Drops D-002. The current HTTP port becomes the single port, not the gRPC port. T - A new port: every client changes. - Source: [PR comment](https://github.com/chainloop-dev/chainloop/pull/3629#issuecomment-6253559302) -### D-013: How to serve both protocols on one port -Supersedes D-001. The service splits by connection in front of the current gRPC and HTTP servers, with the cmux library as the connection classifier. The library owns the connection buffering and the listener plumbing. The service owns only a small matcher and the HTTP request dispatcher. The matcher sends the server's HTTP/2 settings frame to clients that wait for it. It reads the first request's content type through the standard HTTP/2 library. The matcher closes the connection when its first request is not native gRPC. The dispatcher carries the gRPC-Web wrapper and the native-gRPC fallback. -- Why: it keeps the whole middleware chain and the native gRPC transport for CAS streams. It also takes HTTP/2 parsing out of the code this project maintains. Argo CD and etcd run the same library split in production. -- Why: the gRPC library documents that its HTTP adapter may differ in performance and features from the native transport. It does not document the adapter as slower, and this project has not measured a difference. The native transport stays the normal path for gRPC connections, so R-003 does not rest on the adapter. -- Alternatives considered: - - The custom connection parser of the first implementation: it works, but the project then maintains its own HTTP/2 frame parsing, replay, and resource bounds. - - One HTTP handler that routes each request: every gRPC call, on every connection, then uses the HTTP adapter of the gRPC server. - - A move to Connect: it requires migrating the generated server interfaces and the interceptors. Ordinary HTTP routes can stay, but the migration is larger than the classifier swap for the same single port. -- Source: [PR comment](https://github.com/chainloop-dev/chainloop/pull/3647#issuecomment-6266145446) +### ~~D-013: How to serve both protocols on one port~~ +Superseded by D-016. + +~~Supersedes D-001. The service splits by connection in front of the current gRPC and HTTP servers, with the cmux library as the connection classifier. The library owns the connection buffering and the listener plumbing. The service owns only a small matcher and the HTTP request dispatcher. The matcher sends the server's HTTP/2 settings frame to clients that wait for it. It reads the first request's content type through the standard HTTP/2 library. The matcher closes the connection when its first request is not native gRPC. The dispatcher carries the gRPC-Web wrapper and the native-gRPC fallback.~~ +- ~~Why: it keeps the whole middleware chain and the native gRPC transport for CAS streams. It also takes HTTP/2 parsing out of the code this project maintains. Argo CD and etcd run the same library split in production.~~ +- ~~Why: the gRPC library documents that its HTTP adapter may differ in performance and features from the native transport. It does not document the adapter as slower, and this project has not measured a difference. The native transport stays the normal path for gRPC connections, so R-003 does not rest on the adapter.~~ +- ~~Alternatives considered:~~ + - ~~The custom connection parser of the first implementation: it works, but the project then maintains its own HTTP/2 frame parsing, replay, and resource bounds.~~ + - ~~One HTTP handler that routes each request: every gRPC call, on every connection, then uses the HTTP adapter of the gRPC server.~~ + - ~~A move to Connect: it requires migrating the generated server interfaces and the interceptors. Ordinary HTTP routes can stay, but the migration is larger than the classifier swap for the same single port.~~ +- ~~Source: [PR comment](https://github.com/chainloop-dev/chainloop/pull/3647#issuecomment-6266145446)~~ ### D-014: Separate upstream connections, one-way fallback Supersedes D-010. The single port routes each connection once. It supports clients and proxies that keep native gRPC and plain HTTP on separate upstream HTTP/2 connections. Supported ingress configurations use separate upstream pools for the two kinds of traffic. The chart documents and tests them for nginx and Traefik. A connection that the HTTP server owns may carry a native gRPC request later. The HTTP server then gives the request to the gRPC server through the documented fallback, on the adapter transport. The reverse is not supported: a connection that the native gRPC server owns cannot carry a plain HTTP request. Arbitrary mixed native-gRPC and plain-HTTP requests on one shared upstream connection are not required. @@ -261,6 +263,15 @@ Supersedes D-005. The services solve the multiplexing in the server process. The - A mandatory sidecar or embedded proxy: it works behind more proxy configurations, but it adds a process and its lifecycle to every deployment. - Source: [PR comment](https://github.com/chainloop-dev/chainloop/pull/3647#issuecomment-6266145446) +### D-016: Complete the classifier's SETTINGS exchange before HTTP handoff +Supersedes D-013. The service keeps cmux connection classification, the native gRPC server, and the HTTP request dispatcher with its gRPC-Web wrapper and native-gRPC fallback. Both native gRPC and ordinary HTTP/2 clients may wait for server SETTINGS before their first request. Silence does not identify the protocol of that request. +- The matcher uses the standard HTTP/2 framer and HPACK decoder. It limits incoming frames to the initial 16 KiB maximum before reading their payloads, as well as limiting classification bytes, time, and concurrency. +- When the matcher sends an empty SETTINGS frame and the first request is HTTP, it waits within the classification bounds for that SETTINGS acknowledgement. It records the validated ACK's exact offset in cmux's replay. The HTTP listener removes only that nine-byte frame before the HTTP server reads the connection. The HTTP server owns all subsequent SETTINGS exchanges. An incomplete injected handshake is closed. +- Native gRPC connections still go to the existing native server with their original replay. HTTP connections that did not need an injected SETTINGS frame also retain their original replay. HTTP requests and later streams are not parsed again by the handoff adapter. +- Why: rejecting every non-gRPC request after the matcher sends SETTINGS also rejects valid HTTP-only clients. Forwarding the extra ACK unchanged makes net/http reject the connection after the client acknowledges the HTTP server's own SETTINGS. Removing the one ACK preserves its ownership without adding an ongoing frame parser. +- Evidence: focused tests cover HTTP and gRPC-Web content types over plaintext and TLS, ACKs before and after the first request, subsequent requests on the same connection, incomplete handshakes, and an oversized frame header without its payload. These tests do not establish full R-002 parity or R-003 CAS performance. +- Source: [implementation review](https://github.com/chainloop-dev/chainloop/pull/3647#pullrequestreview-5573391966). + ## Open Questions - [x] **Must the single port support proxies that share one upstream connection for gRPC and plain HTTP?** Examples: Envoy-based proxies, service meshes. Answered by D-014: not required. Setups that share one upstream pool can route gRPC to the legacy gRPC port during the deprecation period. The chart documents the supported nginx and Traefik configurations, which keep the two kinds of traffic on separate upstream connections. Source: [cubic review](https://github.com/chainloop-dev/chainloop/pull/3629#discussion_r4286475927). diff --git a/pkg/httpgrpcmux/mux.go b/pkg/httpgrpcmux/mux.go index a43968c4e..7f408be44 100644 --- a/pkg/httpgrpcmux/mux.go +++ b/pkg/httpgrpcmux/mux.go @@ -19,7 +19,7 @@ // // It implements the connection-level split described in // docs/specs/issue-3628-single-endpoint.md, with the cmux library as the -// classifier (D-013). cmux reads the start of each new connection without +// classifier (D-016). cmux reads the start of each new connection without // consuming it and hands the connection to the gRPC or the HTTP listener. A // service-owned matcher decides native gRPC from the first request's content // type; every other connection (HTTP/1.x, HTTP/2 without gRPC, gRPC-Web) goes @@ -29,11 +29,10 @@ // // The matcher speaks only when a connection stays silent for the settings // grace: gRPC clients (the CLI, Java) wait for the server's HTTP/2 settings -// frame before their first request. A connection the matcher has spoken to is -// committed to the gRPC side, because the client's acknowledgement of the -// injected frame reaches whichever server owns the connection, and net/http -// fails on an acknowledgement of settings it never sent. Every other client -// sends its first request without waiting and classifies unwritten-to. +// frame before their first request. Ordinary HTTP/2 clients may also wait for +// this frame. When such a connection goes to HTTP, the matcher consumes its +// SETTINGS acknowledgement and the HTTP listener removes that one frame from +// cmux's replay. The HTTP server owns every subsequent SETTINGS exchange. // // The mux implements kratos' transport.Server interface (Start/Stop with // standard signatures) so it can join the application server lifecycle. @@ -64,12 +63,15 @@ const defaultClassifyTimeout = 10 * time.Second // defaultSettingsGrace is how long the matcher waits for the first request // after the connection preface before it assumes the client waits for the -// server to speak first. gRPC clients (the CLI) wait for the server's -// HTTP/2 settings frame before they send their first request, so the matcher -// must send one. Every other client (browsers, plain HTTP/2) sends its first -// request without waiting and stays unspoken-to. +// server to speak first. Both native gRPC and ordinary HTTP/2 clients may +// wait for the server's SETTINGS before sending their first request. const defaultSettingsGrace = 250 * time.Millisecond +// initialMaxFrameSize is the HTTP/2 frame size limit before the server has +// advertised a larger value. Bound allocation before reading frame payloads; +// a LimitedReader alone only bounds bytes consumed, not allocated buffers. +const initialMaxFrameSize = 1 << 14 + // defaultMaxClassifiedBytes bounds how much of a connection the matcher may // read while looking for the first request headers. A connection that needs // more goes to the HTTP server, whose own header limits dispose of it. @@ -146,17 +148,7 @@ type Mux struct { // admit caps how many connections may be in classification at once. A // matcher holds a slot while it runs; a connection that cannot get one // matches no listener, and cmux closes it. - admit chan struct{} - // poisoned holds the connections the gRPC matcher has sent the server's - // HTTP/2 settings frame to, but whose first request turned out not to be - // native gRPC. Their acknowledgement of that frame is already recorded - // in cmux's replay buffer, and net/http fails a connection that - // acknowledges settings it never sent, so they must not reach the HTTP - // side. The gRPC matcher sets the entry before it declines; the HTTP - // listener's dropPoisoned wrapper, which cmux always runs next, clears - // it before reading, so a full admission cap cannot strand the marker. - poisoned sync.Map - + admit chan struct{} stopping atomic.Bool serveWG sync.WaitGroup // serveOnce and closeOnce keep Start and Close idempotent: kratos may @@ -172,11 +164,74 @@ type stoppingListener struct { stopping *atomic.Bool } +func (l *stoppingListener) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return &classificationConn{Conn: c, settingsACKOffset: -1}, nil +} + func (l *stoppingListener) Close() error { l.stopping.Store(true) return l.Listener.Close() } +// classificationConn carries handshake state through cmux. Matchers update +// it before cmux publishes the connection to a sub-listener. No shared map +// or state remains after an unaccepted connection closes. +type classificationConn struct { + net.Conn + settingsACKOffset int64 + rejectHTTP bool +} + +// httpListener strips only the acknowledgement of the matcher's SETTINGS. +// Its exact offset is known before handoff, so serving HTTP does not require +// a second frame parser or any inspection of later requests or streams. +type httpListener struct{ net.Listener } + +func (l httpListener) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + if mc, ok := c.(*cmux.MuxConn); ok { + if state, ok := mc.Conn.(*classificationConn); ok && state.settingsACKOffset >= 0 { + return &httpConn{Conn: c, beforeACK: state.settingsACKOffset}, nil + } + } + return c, nil +} + +type httpConn struct { + net.Conn + beforeACK int64 +} + +func (c *httpConn) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if c.beforeACK > 0 { + n, err := c.Conn.Read(p[:min(int64(len(p)), c.beforeACK)]) + c.beforeACK -= int64(n) + return n, err + } + if c.beforeACK == 0 { + // The matcher already read and validated this complete, empty ACK. + // It is entirely in cmux's replay, even when the caller reads one + // byte at a time. Leave the next server SETTINGS/ACK pair untouched. + var ack [9]byte + _, err := io.ReadFull(c.Conn, ack[:]) + c.beforeACK = -1 + if err != nil { + return 0, err + } + } + return c.Conn.Read(p) +} + // New creates a mux over l. Both listeners are usable immediately; // connections start flowing once Start is called. func New(l net.Listener, opts *Options) *Mux { @@ -225,10 +280,9 @@ func New(l net.Listener, opts *Options) *Mux { // are MatchWriters so they can reach the connection: the gRPC matcher // writes the server's settings frame to clients that wait for it, and // takes its admission slot itself, once the connection prefaced. The - // HTTP chain drops spoken-to connections before it reads anything, so a - // poison marker never survives a full admission cap. + // HTTP chain rejects incomplete handshakes before it reads anything. m.grpcL = m.cm.MatchWithWriters(m.matchNativeGRPC) - m.httpL = m.cm.MatchWithWriters(m.dropPoisoned(m.requireTraffic)) + m.httpL = httpListener{m.cm.MatchWithWriters(m.rejectIncompleteHandshake(m.requireTraffic))} m.cm.HandleError(func(err error) bool { var notMatched cmux.ErrNotMatched @@ -325,18 +379,13 @@ func (m *Mux) IsOrderlyShutdownErr(err error) bool { errors.Is(err, cmux.ErrListenerClosed)) } -// dropPoisoned declines the connections the gRPC matcher has spoken to. -// It runs as the outermost wrapper of the HTTP chain, before any read and -// before admission: a spoken-to connection cannot be handed to net/http, and -// its marker must clear even when the admission cap is full, or the entry -// would pin the connection object forever. -func (m *Mux) dropPoisoned(next cmux.MatchWriter) cmux.MatchWriter { +// rejectIncompleteHandshake closes connections whose injected SETTINGS +// exchange failed. A completed HTTP handshake is safe to hand off because +// httpListener removes the acknowledgement belonging to the matcher. +func (m *Mux) rejectIncompleteHandshake(next cmux.MatchWriter) cmux.MatchWriter { return func(w io.Writer, r io.Reader) bool { - if conn, ok := w.(net.Conn); ok { - if _, poisoned := m.poisoned.LoadAndDelete(conn); poisoned { - m.logger.Warnw("msg", "closing connection: the classifier sent its settings frame, but the first request was not native gRPC") - return false - } + if conn, ok := w.(*classificationConn); ok && conn.rejectHTTP { + return false } return next(w, r) } @@ -389,17 +438,12 @@ func isTimeout(err error) bool { // matchNativeGRPC reports whether the connection's first request carries a // native gRPC content type. // -// Clients that send their first request without waiting (browsers, plain -// HTTP/2) classify without a single byte written to them. A client that stays -// silent for the settings grace is a gRPC client waiting for the server's -// HTTP/2 settings frame — the gRPC Go client does this, and so do Java -// clients — so the matcher sends one. From that moment the connection is -// committed: the client's acknowledgement of the injected frame is already -// recorded in cmux's replay buffer, net/http fails a connection that -// acknowledges settings it never sent, and the gRPC server ignores extra -// acknowledgements. A committed connection whose first request is not native -// gRPC, or that never sends its request, is therefore declined and poisoned: -// the HTTP matcher declines it too, and cmux closes it. +// Clients that send their first request without waiting classify without a +// single byte written to them. After the settings grace, the matcher sends +// SETTINGS to unblock either kind of waiting HTTP/2 client. Before an HTTP +// handoff it records the ACK's exact replay offset, even if the ACK follows +// the first request. httpListener removes that ACK; native gRPC continues to +// receive the original replay. Incomplete injected handshakes are closed. // // Frame parsing and HPACK decoding are the standard HTTP/2 library's; // everything the matcher reads stays buffered in cmux and is replayed to the @@ -430,12 +474,13 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { } defer func() { <-m.admit }() - conn, ok := w.(net.Conn) + conn, ok := w.(*classificationConn) if !ok { return false } framer := http2.NewFramer(w, lr) + framer.SetMaxReadFrameSize(initialMaxFrameSize) var contentType string hdec := hpack.NewDecoder(hpackDynamicTableSize, func(hf hpack.HeaderField) { if strings.EqualFold(hf.Name, headerContentType) { @@ -446,18 +491,20 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { // Narrow the read deadline cmux set to the classification window down to // the settings grace: how long to wait for a first request before - // speaking. decline reports no match, and poisons the connection when the - // matcher has spoken, so it never reaches net/http. boundary is the + // speaking. decline reports no match, and rejects an incomplete handshake + // when the matcher has spoken, so it never reaches net/http. boundary is the // remaining read budget at the last completed frame: a timeout with a // smaller budget fired mid-frame. The rule is correct only while the // framer reads exactly frame-header plus payload with no read-ahead; // inserting a buffering reader in this chain breaks it. spoken := false + firstRequest := false + nativeGRPC := false boundary := lr.N _ = conn.SetReadDeadline(time.Now().Add(m.settingsGrace)) decline := func() bool { if spoken { - m.poisoned.Store(conn, struct{}{}) + conn.rejectHTTP = true } return false } @@ -466,8 +513,8 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { // deadline when the connection is handed over, so a lingering write // deadline would kill long-lived streams downstream. _ = conn.SetWriteDeadline(time.Now().Add(m.classifyTimeout)) - // The connection is committed from the first written byte, even if - // the frame is only partially delivered: decline must poison it. + // A failed handshake cannot be replayed to HTTP, even if only part + // of the SETTINGS frame was delivered. spoken = true err := framer.WriteSettings() _ = conn.SetWriteDeadline(time.Time{}) @@ -479,6 +526,7 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { } for { + offset := int64(m.maxClassifiedBytes) + 1 - lr.N f, err := framer.ReadFrame() if err != nil { if !spoken && isTimeout(err) { @@ -498,16 +546,19 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { } continue } - // Read failure, budget exhausted, or deadline: a committed - // connection cannot go to the HTTP server. + // Read failure, budget exhausted, or deadline: an incomplete + // injected handshake cannot go to the HTTP server. return decline() } boundary = lr.N + if sf, ok := f.(*http2.SettingsFrame); ok && sf.IsAck() && spoken && conn.settingsACKOffset < 0 { + conn.settingsACKOffset = offset + } // HEADERS and CONTINUATION are the only frames that carry header // block fragments; SETTINGS (including the acknowledgement of the // injected frame), PING, WINDOW_UPDATE, ... are skipped and stay in // the replay: the gRPC server ignores unsolicited acknowledgements. - if hb, ok := f.(interface{ HeaderBlockFragment() []byte }); ok { + if hb, ok := f.(interface{ HeaderBlockFragment() []byte }); ok && !firstRequest { if _, err := hdec.Write(hb.HeaderBlockFragment()); err != nil { return decline() } @@ -515,11 +566,17 @@ func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { if err := hdec.Close(); err != nil { return decline() } - if !IsNativeGRPCContentType(contentType) { - return decline() - } + firstRequest = true + nativeGRPC = IsNativeGRPCContentType(contentType) + } + } + if firstRequest { + if nativeGRPC { return true } + if !spoken || conn.settingsACKOffset >= 0 { + return false + } } } } diff --git a/pkg/httpgrpcmux/mux_test.go b/pkg/httpgrpcmux/mux_test.go index 9d6ab2c01..e38362a61 100644 --- a/pkg/httpgrpcmux/mux_test.go +++ b/pkg/httpgrpcmux/mux_test.go @@ -23,11 +23,14 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "fmt" "io" "math/big" "net" + "net/http" "strings" "testing" + "testing/iotest" "time" "github.com/stretchr/testify/assert" @@ -49,6 +52,7 @@ const ( ctGRPC = "application/grpc" ctGRPCProto = "application/grpc+proto" ctGRPCWebProto = "application/grpc-web+proto" + alpnHTTP1 = "http/1.1" ) // startMux creates a real TCP listener, wraps it in a mux and starts it. @@ -429,33 +433,213 @@ func TestSettingsWaitingClientClassifiesNative(t *testing.T) { assert.Equal(t, "still-writable", string(tail)) } -// TestCommittedNonGRPCDropped proves a spoken-to connection whose first -// request is not native gRPC is closed, not handed to the HTTP server: its -// acknowledgement of the injected settings frame is already recorded, and -// net/http fails a connection that acknowledges settings it never sent. -func TestCommittedNonGRPCDropped(t *testing.T) { - _, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: time.Second}) +// HTTP/2 permits any client to wait for server SETTINGS before its request. +// Exercise real net/http handoff, including the ACK of net/http's subsequent +// SETTINGS and a second request, so an extra replayed ACK cannot go unnoticed. +func TestSettingsWaitingHTTPClient(t *testing.T) { + for _, encrypted := range []bool{false, true} { + for _, contentType := range []string{"application/json", ctGRPCWebProto} { + for _, ackFirst := range []bool{false, true} { + name := fmt.Sprintf("TLS=%t/content-type=%s/ACK-first=%t", encrypted, contentType, ackFirst) + t.Run(name, func(t *testing.T) { + t.Parallel() + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = l.Close() }) + root := l + if encrypted { + root = tls.NewListener(l, &tls.Config{ + Certificates: []tls.Certificate{selfSignedCert(t)}, + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", alpnHTTP1}, + }) + } + m := New(root, &Options{SettingsGrace: 40 * time.Millisecond, ClassifyTimeout: 2 * time.Second}) + require.NoError(t, m.Start(context.Background())) + t.Cleanup(m.Close) + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Request-Content-Type", r.Header.Get(hdrContentType)) + w.WriteHeader(http.StatusOK) + }), + ReadHeaderTimeout: time.Second, + Protocols: new(http.Protocols), + } + srv.Protocols.SetHTTP1(true) + srv.Protocols.SetUnencryptedHTTP2(true) + go func() { _ = srv.Serve(m.HTTP()) }() + t.Cleanup(func() { _ = srv.Close() }) + + var c net.Conn + if encrypted { + c, err = tls.Dial("tcp", l.Addr().String(), &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // self-signed test certificate + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2"}, + }) + } else { + c, err = net.Dial("tcp", l.Addr().String()) + } + require.NoError(t, err) + t.Cleanup(func() { _ = c.Close() }) + require.NoError(t, c.SetDeadline(time.Now().Add(5*time.Second))) + _, err = io.WriteString(c, http2.ClientPreface) + require.NoError(t, err) + fr := http2.NewFramer(c, c) + fr.ReadMetaHeaders = hpack.NewDecoder(4096, nil) + require.NoError(t, fr.WriteSettings()) + frame, err := fr.ReadFrame() + require.NoError(t, err) + settings, ok := frame.(*http2.SettingsFrame) + require.True(t, ok, "expected injected SETTINGS, got %T", frame) + require.False(t, settings.IsAck()) + + var block bytes.Buffer + enc := hpack.NewEncoder(&block) + writeRequest := func(streamID uint32) { + block.Reset() + method, scheme := "GET", "http" + if contentType == ctGRPCWebProto { + method = hdrMethodPost + } + if encrypted { + scheme = "https" + } + for _, hf := range []hpack.HeaderField{ + {Name: hdrMethod, Value: method}, + {Name: ":scheme", Value: scheme}, + {Name: ":authority", Value: l.Addr().String()}, + {Name: ":path", Value: "/statusz"}, + {Name: hdrContentType, Value: contentType}, + } { + require.NoError(t, enc.WriteField(hf)) + } + require.NoError(t, fr.WriteHeaders(http2.HeadersFrameParam{ + StreamID: streamID, BlockFragment: block.Bytes(), EndHeaders: true, EndStream: true, + })) + } + if ackFirst { + require.NoError(t, fr.WriteSettingsAck()) + } + writeRequest(1) + // Include another frame before the ACK to exercise its exact + // offset rather than assuming it immediately follows SETTINGS. + require.NoError(t, fr.WritePing(false, [8]byte{1})) + if !ackFirst { + require.NoError(t, fr.WriteSettingsAck()) + } + + serverSettings := 0 + readResponse := func(streamID uint32) { + for { + frame, err := fr.ReadFrame() + require.NoError(t, err) + switch f := frame.(type) { + case *http2.SettingsFrame: + if !f.IsAck() { + serverSettings++ + require.NoError(t, fr.WriteSettingsAck()) + } + case *http2.MetaHeadersFrame: + if f.StreamID == streamID { + assert.Equal(t, "200", f.PseudoValue("status")) + assert.Contains(t, f.Fields, hpack.HeaderField{Name: "x-request-content-type", Value: contentType}) + return + } + case *http2.GoAwayFrame: + require.FailNow(t, "unexpected GOAWAY", "%v: %s", f.ErrCode, f.DebugData()) + } + } + } + readResponse(1) + writeRequest(3) + readResponse(3) + assert.Equal(t, 1, serverSettings, "the HTTP server must own its SETTINGS/ACK exchange") + }) + } + } + } +} + +func TestHTTPReplayRemovesOnlyMatcherACK(t *testing.T) { + var wire bytes.Buffer + wire.WriteString(http2.ClientPreface) + fr := http2.NewFramer(&wire, nil) + require.NoError(t, fr.WriteSettings()) + prefix := wire.String() + require.NoError(t, fr.WriteSettingsAck()) + require.NoError(t, fr.WritePing(false, [8]byte{1, 2, 3})) + require.NoError(t, fr.WriteSettingsAck()) // belongs to the HTTP server + tail := wire.Bytes()[len(prefix)+9:] + for _, oneByte := range []bool{false, true} { + var reader io.Reader = bytes.NewReader(wire.Bytes()) + if oneByte { + reader = iotest.OneByteReader(reader) + } + c := &httpConn{Conn: &readerConn{reader: reader}, beforeACK: int64(len(prefix))} + got, err := io.ReadAll(c) + require.NoError(t, err) + assert.Equal(t, prefix+string(tail), string(got)) + } +} +func TestSettingsWaitingHTTPWithoutACKDropped(t *testing.T) { + _, dial := startMux(t, &Options{SettingsGrace: 40 * time.Millisecond, ClassifyTimeout: 150 * time.Millisecond}) c, err := dial() require.NoError(t, err) defer c.Close() - _, err = io.WriteString(c, http2.ClientPreface) require.NoError(t, err) writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - - // Receive the injected settings frame, acknowledge it, and send a - // gRPC-Web request flight. require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) _, err = io.ReadFull(c, make([]byte, 9)) require.NoError(t, err) - writeFrame(t, c, http2.FrameSettings, http2.FlagSettingsAck, 0, nil) writeRequestFrames(t, c, ctGRPCWebProto) - - // The connection matches no listener and cmux closes it. + // The first request alone cannot complete handoff: an ACK still pending + // on the wire would belong to the matcher rather than the HTTP server. readEOF(t, c, 2*time.Second) } +type readerConn struct { + net.Conn + reader io.Reader +} + +func (c *readerConn) Read(p []byte) (int, error) { return c.reader.Read(p) } + +// A frame length must be rejected before any read of its advertised payload. +// The sentinel makes this deterministic without process-wide memory or timing +// assertions: the old Framer allocated first, then reached the sentinel. +func TestOversizedFrameRejectedBeforePayloadRead(t *testing.T) { + for _, size := range []uint32{initialMaxFrameSize + 1, (1 << 24) - 1} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + var wire bytes.Buffer + wire.WriteString(http2.ClientPreface) + require.NoError(t, http2.NewFramer(&wire, nil).WriteSettings()) + wire.Write([]byte{byte(size >> 16), byte(size >> 8), byte(size), 0xfe, 0, 0, 0, 0, 0}) + payloadRead := false + reader := io.MultiReader(&wire, testReaderFunc(func([]byte) (int, error) { + payloadRead = true + return 0, io.EOF + })) + server, client := net.Pipe() + defer server.Close() + defer client.Close() + m := &Mux{ + maxClassifiedBytes: defaultMaxClassifiedBytes, + settingsGrace: time.Second, + admit: make(chan struct{}, 1), + } + assert.False(t, m.matchNativeGRPC(&classificationConn{Conn: server, settingsACKOffset: -1}, reader)) + assert.False(t, payloadRead, "oversized frame header must fail before its payload buffer is read") + }) + } +} + +type testReaderFunc func([]byte) (int, error) + +func (f testReaderFunc) Read(p []byte) (int, error) { return f(p) } + // TestOversizedHeaderBlockDeclined proves a connection whose first request // needs more than the classification byte budget goes to the HTTP server, // whose own header limits dispose of it. @@ -569,8 +753,7 @@ func TestIdleConnectionDropped(t *testing.T) { // TestStalledAfterPrefaceDropped proves a connection that sends a preface and // then stalls is closed at the end of its classification window: the matcher -// speaks to it after the grace, and a spoken-to connection never reaches the -// HTTP server. +// speaks to it after the grace, but the handshake never completes. func TestStalledAfterPrefaceDropped(t *testing.T) { _, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: 300 * time.Millisecond}) @@ -624,7 +807,7 @@ func TestSlowMidFrameClientTakesFallback(t *testing.T) { _, err = c.Write(block[len(block)/2:]) require.NoError(t, err) - // The connection is not dropped and not poisoned: the HTTP server gets + // The connection is not dropped: the HTTP server gets // it with the whole stream replayed. got := acceptWithTimeout(t, m.HTTP()) defer got.Close() @@ -738,7 +921,7 @@ func TestTLSALPNClassification(t *testing.T) { tlsConfig := &tls.Config{ Certificates: []tls.Certificate{selfSignedCert(t)}, MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", "http/1.1"}, + NextProtos: []string{"h2", alpnHTTP1}, } m := New(tls.NewListener(l, tlsConfig), nil) require.NoError(t, m.Start(context.Background())) @@ -763,7 +946,7 @@ func TestTLSALPNClassification(t *testing.T) { c.Close() // ALPN http/1.1 -> HTTP server. - c = dialTLS(t, "http/1.1") + c = dialTLS(t, alpnHTTP1) _, err = io.WriteString(c, "GET /statusz HTTP/1.1\r\nHost: x\r\n\r\n") require.NoError(t, err) got = acceptWithTimeout(t, m.HTTP()) From 004900e508b6646fcb1397f967893ffd8eb2c7d4 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 19:22:01 +0200 Subject: [PATCH 07/10] feat(controlplane): serve HTTP and gRPC through one ingress Replace connection classification with HTTP-owned per-request dispatch. Configure the shared Helm ingress for HTTP/2 upstream traffic and retain the legacy native gRPC endpoint. Record the replacement control-plane spec and preserve the CAS streaming qualification requirements. Refs #3628 Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: ses_eabb93119ffegqWXti0mS05TIM --- app/controlplane/cmd/main.go | 41 +- app/controlplane/cmd/main_test.go | 57 - app/controlplane/cmd/wire_gen.go | 46 +- app/controlplane/internal/server/apimux.go | 111 -- app/controlplane/internal/server/grpc.go | 83 +- app/controlplane/internal/server/http.go | 71 +- .../internal/server/http_dispatch_test.go | 243 ++++ .../server/{apimux_test.go => http_test.go} | 184 +-- .../internal/server/ingress_test.go | 146 +++ app/controlplane/internal/server/otel_test.go | 53 +- app/controlplane/internal/server/server.go | 1 - app/controlplane/internal/server/tls.go | 52 + deployment/chainloop/.helmignore | 2 + deployment/chainloop/Chart.yaml | 2 +- deployment/chainloop/README.md | 35 +- deployment/chainloop/chart_test.go | 145 +++ .../templates/controlplane/ingress.yaml | 11 +- deployment/chainloop/values.yaml | 18 +- .../issue-3628-controlplane-http-dispatch.md | 166 +++ docs/specs/issue-3628-single-endpoint.md | 2 + go.mod | 3 +- go.sum | 2 - pkg/httpgrpcmux/mux.go | 603 ---------- pkg/httpgrpcmux/mux_test.go | 1048 ----------------- 24 files changed, 1011 insertions(+), 2114 deletions(-) delete mode 100644 app/controlplane/internal/server/apimux.go create mode 100644 app/controlplane/internal/server/http_dispatch_test.go rename app/controlplane/internal/server/{apimux_test.go => http_test.go} (74%) create mode 100644 app/controlplane/internal/server/ingress_test.go create mode 100644 app/controlplane/internal/server/tls.go create mode 100644 deployment/chainloop/chart_test.go create mode 100644 docs/specs/issue-3628-controlplane-http-dispatch.md delete mode 100644 pkg/httpgrpcmux/mux.go delete mode 100644 pkg/httpgrpcmux/mux_test.go diff --git a/app/controlplane/cmd/main.go b/app/controlplane/cmd/main.go index 1a7ac6777..631c7c488 100644 --- a/app/controlplane/cmd/main.go +++ b/app/controlplane/cmd/main.go @@ -32,7 +32,6 @@ import ( "github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1" "github.com/chainloop-dev/chainloop/pkg/credentials" "github.com/chainloop-dev/chainloop/pkg/credentials/manager" - "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" "github.com/chainloop-dev/chainloop/pkg/natsconn" "github.com/chainloop-dev/chainloop/pkg/servicelogger" @@ -42,8 +41,6 @@ import ( "github.com/go-kratos/kratos/v2/config/file" "github.com/go-kratos/kratos/v2/log" "github.com/go-kratos/kratos/v2/transport" - "github.com/go-kratos/kratos/v2/transport/grpc" - "github.com/go-kratos/kratos/v2/transport/http" ) var ( @@ -62,41 +59,15 @@ func init() { flag.StringVar(&flagconf, "conf", "../configs", "config path, eg: -conf config.yaml") } -// cleanShutdownServer tolerates the shutdown race with the mux: kratos starts -// and stops servers concurrently, and the mux's sub-listeners close together -// with the root listener, so the gRPC/HTTP servers may see their listener -// close before they flag their own shutdown. Their Start then returns the -// mux's shutdown error even though the shutdown is orderly — kratos would -// surface it and main would panic on every SIGTERM the mux wins. The race is -// only tolerated when the mux is actually stopping: a fatal accept-loop -// failure surfaces as an error, so the process restarts instead of serving a -// dead single port. -type cleanShutdownServer struct { - transport.Server - mux *httpgrpcmux.Mux -} - -func (s *cleanShutdownServer) Start(ctx context.Context) error { - err := s.Server.Start(ctx) - if s.mux.IsOrderlyShutdownErr(err) { - return nil - } - return err -} - -func newApp(logger log.Logger, am *httpgrpcmux.Mux, gs []*grpc.Server, hs []*http.Server, ms *server.HTTPMetricsServer, profilerSvc *server.HTTPProfilerServer, +func newApp(logger log.Logger, gs *server.GRPCServers, api *server.APIServer, ms *server.HTTPMetricsServer, profilerSvc *server.HTTPProfilerServer, expirer *biz.WorkflowRunExpirerUseCase, plugins sdk.AvailablePlugins, userAccessSyncer *biz.UserAccessSyncerUseCase, casBackendChecker *biz.CASBackendChecker, apiTokenStaleRevoker *biz.APITokenStaleRevoker, loginHandoffs *biz.LoginHandoffUseCase, cfg *conf.Bootstrap) *app { - servers := make([]transport.Server, 0, len(gs)+len(hs)+3) - // The mux owns the single-port listener; kratos starts servers concurrently, - // so no ordering is required — the virtual listeners work before Start. - servers = append(servers, am, ms) - for _, g := range gs { - servers = append(servers, &cleanShutdownServer{Server: g, mux: am}) - } - for _, h := range hs { - servers = append(servers, &cleanShutdownServer{Server: h, mux: am}) + servers := []transport.Server{api, ms} + // APIServer owns the gRPC handler's lifecycle. Only the legacy server + // starts a native gRPC listener. + if gs.Legacy != nil { + servers = append(servers, gs.Legacy) } if cfg.EnableProfiler { servers = append(servers, profilerSvc) diff --git a/app/controlplane/cmd/main_test.go b/app/controlplane/cmd/main_test.go index 5b4b09ffa..243ca7105 100644 --- a/app/controlplane/cmd/main_test.go +++ b/app/controlplane/cmd/main_test.go @@ -15,20 +15,15 @@ package main import ( - "context" "fmt" - "net" "os" "path/filepath" "testing" "time" conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1" - "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" "github.com/go-kratos/kratos/v2/config" "github.com/go-kratos/kratos/v2/config/file" - kratosgrpc "github.com/go-kratos/kratos/v2/transport/grpc" - kratoshttp "github.com/go-kratos/kratos/v2/transport/http" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/durationpb" @@ -221,55 +216,3 @@ func loadBootstrap(t *testing.T, path string) (*conf.Bootstrap, error) { return &bootstrap, nil } - -// TestServerToleratesMuxShutdown proves the shutdown-race fix: when the mux -// stops first, the servers' Start returns net.ErrClosed — an orderly -// shutdown that must not surface as an error (kratos would propagate it and -// main would panic on SIGTERM). -func TestServerToleratesMuxShutdown(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - - m := httpgrpcmux.New(l, nil) - require.NoError(t, m.Start(context.Background())) - - // A kratos HTTP server on the mux's HTTP side, and a kratos gRPC server - // on its gRPC side: both fed by virtual listeners. - hs := kratoshttp.NewServer(kratoshttp.Listener(m.HTTP())) - gs := kratosgrpc.NewServer(kratosgrpc.Listener(m.GRPC())) - - errs := make(chan error, 2) - go func() { errs <- (&cleanShutdownServer{Server: hs, mux: m}).Start(context.Background()) }() - go func() { errs <- (&cleanShutdownServer{Server: gs, mux: m}).Start(context.Background()) }() - - // Give the servers time to enter their accept loops, then close the mux - // first — exactly the SIGTERM ordering that used to panic. - time.Sleep(100 * time.Millisecond) - m.Close() - - for i := 0; i < 2; i++ { - select { - case err := <-errs: - require.NoError(t, err, "an orderly mux shutdown must not surface as a server error") - case <-time.After(5 * time.Second): - require.Fail(t, "server Start did not return after the mux stopped") - } - } -} - -// TestCleanShutdownServerPropagatesRealErrors proves the decorator only -// tolerates the mux shutdown: a real failure (bind conflict) surfaces as an -// error, so the process still crashes on genuine startup problems. -func TestCleanShutdownServerPropagatesRealErrors(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - defer l.Close() - - decorated := &cleanShutdownServer{ - Server: kratoshttp.NewServer(kratoshttp.Address(l.Addr().String())), - mux: httpgrpcmux.New(l, nil), - } - err = decorated.Start(context.Background()) - require.Error(t, err) - assert.NotErrorIs(t, err, net.ErrClosed) -} diff --git a/app/controlplane/cmd/wire_gen.go b/app/controlplane/cmd/wire_gen.go index bdf1c6a88..3881ab4c1 100644 --- a/app/controlplane/cmd/wire_gen.go +++ b/app/controlplane/cmd/wire_gen.go @@ -35,28 +35,20 @@ import ( // Injectors from wire.go: func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWriter credentials.ReaderWriter, logger log.Logger, availablePlugins sdk.AvailablePlugins) (*app, func(), error) { - confServer := bootstrap.Server - mux, cleanup, err := server.NewAPIMux(confServer, logger) - if err != nil { - return nil, nil, err - } config := authzConfig() casbinEnforcer, err := authz.NewCasbinEnforcer(config) if err != nil { - cleanup() return nil, nil, err } confData := bootstrap.Data data_Database := confData.Database databaseConfig := newDataConf(data_Database) - tracerProvider, cleanup2, err := server.NewTracerProvider(bootstrap, logger) + tracerProvider, cleanup, err := server.NewTracerProvider(bootstrap, logger) if err != nil { - cleanup() return nil, nil, err } - dataData, cleanup3, err := data.NewData(databaseConfig, tracerProvider, logger) + dataData, cleanup2, err := data.NewData(databaseConfig, tracerProvider, logger) if err != nil { - cleanup2() cleanup() return nil, nil, err } @@ -73,16 +65,14 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr casServerDefaultOpts := newCASServerOptions(bootstrap_CASServer) bootstrap_NatsServer := bootstrap.NatsServer natsconnConfig := newNatsConfig(bootstrap_NatsServer) - reloadableConnection, cleanup4, err := natsconn.New(natsconnConfig, logger) + reloadableConnection, cleanup3, err := natsconn.New(natsconnConfig, logger) if err != nil { - cleanup3() cleanup2() cleanup() return nil, nil, err } auditLogPublisher, err := auditor.NewAuditLogPublisher(contextContext, reloadableConnection, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -91,7 +81,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr auditorUseCase := biz.NewAuditorUseCase(auditLogPublisher, logger) casBackendUseCase, err := biz.NewCASBackendUseCase(casBackendRepo, readerWriter, providers, casServerDefaultOpts, auditorUseCase, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -128,7 +117,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr auth := bootstrap.Auth casCredentialsUseCase, err := biz.NewCASCredentialsUseCase(auth) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -139,7 +127,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr referrerRepo := data.NewReferrerRepo(dataData, workflowRepo, logger) referrerUseCase, err := biz.NewReferrerUseCase(referrerRepo, workflowRepo, membershipUseCase, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -148,7 +135,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr apiTokenJWTConfig := newJWTConfig(auth) apiTokenUseCase, err := biz.NewAPITokenUseCase(apiTokenRepo, apiTokenJWTConfig, authzUseCase, organizationUseCase, auditorUseCase, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -160,7 +146,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr v4 := newPolicyProviderConfig(v3) registry, err := policies.NewRegistry(logger, v4...) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -170,7 +155,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr workflowUseCase := biz.NewWorkflowUsecase(workflowRepo, projectsRepo, workflowContractUseCase, auditorUseCase, membershipUseCase, organizationRepo, logger) cache, err := newMembershipsCache(logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -178,7 +162,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } cacheCache, err := newClaimsCache(logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -187,7 +170,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr orgInvitationRepo := data.NewOrgInvitation(dataData, logger) orgInvitationUseCase, err := biz.NewOrgInvitationUseCase(orgInvitationRepo, membershipRepo, userRepo, auditorUseCase, groupRepo, projectsRepo, authzUseCase, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -201,7 +183,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr loginHandoffUseCase := biz.NewLoginHandoffUseCase(loginHandoffRepo, logger) authService, err := service.NewAuthService(userUseCase, organizationUseCase, membershipUseCase, orgInvitationUseCase, loginHandoffUseCase, auth, bootstrap, auditorUseCase, v5...) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -210,7 +191,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr workflowRunRepo := data.NewWorkflowRunRepo(dataData, logger) signingUseCase, err := biz.NewChainloopSigningUseCase(bootstrap, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -218,7 +198,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } attestationbundleCache, err := attestationbundle.New(contextContext, reloadableConnection, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -240,7 +219,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } workflowRunUseCase, err := biz.NewWorkflowRunUseCase(workflowRunUseCaseOpts) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -250,7 +228,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr projectVersionUseCase := biz.NewProjectVersionUseCase(projectVersionRepo, auditorUseCase, logger) policyevalbundleCache, err := policyevalbundle.New(contextContext, reloadableConnection, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -276,7 +253,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr orgMetricsRepo := data.NewOrgMetricsRepo(dataData, logger) orgMetricsUseCase, err := biz.NewOrgMetricsUseCase(orgMetricsRepo, organizationRepo, workflowUseCase, logger) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -315,7 +291,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr casBackendService := service.NewCASBackendService(casBackendUseCase, providers, v5...) casRedirectService, err := service.NewCASRedirectService(casMappingUseCase, casCredentialsUseCase, bootstrap_CASServer, authService, v5...) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -327,7 +302,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr attestationStateRepo := data.NewAttestationStateRepo(dataData, logger) attestationStateUseCase, err := biz.NewAttestationStateUseCase(attestationStateRepo, workflowRunRepo) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -345,11 +319,11 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr prometheusService := service.NewPrometheusService(organizationUseCase, prometheusUseCase, v5...) groupService := service.NewGroupService(groupUseCase, v5...) projectService := service.NewProjectService(v5...) + confServer := bootstrap.Server federatedAuthentication := bootstrap.FederatedAuthentication operationAuthorizationProvider := bootstrap.OperationAuthorizationProvider validator, err := newProtoValidator() if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -392,7 +366,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr ProjectSvc: projectService, Logger: logger, ServerConfig: confServer, - APIMux: mux, AuthConfig: auth, FederatedConfig: federatedAuthentication, OperationAuthConfig: operationAuthorizationProvider, @@ -400,17 +373,15 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr Credentials: readerWriter, Validator: validator, } - v7, err := server.NewGRPCServer(opts) + grpcServers, err := server.NewGRPCServer(opts) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() return nil, nil, err } - v8, err := server.NewHTTPServer(opts, v7) + apiServer, err := server.NewHTTPServer(opts, grpcServers) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -418,7 +389,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } httpMetricsServer, err := server.NewHTTPMetricsServer(opts) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -426,7 +396,6 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr } httpProfilerServer, err := server.NewHTTPProfilerServer(opts) if err != nil { - cleanup4() cleanup3() cleanup2() cleanup() @@ -436,9 +405,8 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr distributedLock := data.NewPostgresLock(dataData, logger) casBackendChecker := biz.NewCASBackendChecker(logger, casBackendRepo, casBackendUseCase, distributedLock) apiTokenStaleRevoker := biz.NewAPITokenStaleRevoker(organizationRepo, apiTokenRepo, apiTokenUseCase, logger) - mainApp := newApp(logger, mux, v7, v8, httpMetricsServer, httpProfilerServer, workflowRunExpirerUseCase, availablePlugins, userAccessSyncerUseCase, casBackendChecker, apiTokenStaleRevoker, loginHandoffUseCase, bootstrap) + mainApp := newApp(logger, grpcServers, apiServer, httpMetricsServer, httpProfilerServer, workflowRunExpirerUseCase, availablePlugins, userAccessSyncerUseCase, casBackendChecker, apiTokenStaleRevoker, loginHandoffUseCase, bootstrap) return mainApp, func() { - cleanup4() cleanup3() cleanup2() cleanup() diff --git a/app/controlplane/internal/server/apimux.go b/app/controlplane/internal/server/apimux.go deleted file mode 100644 index 1783cae8a..000000000 --- a/app/controlplane/internal/server/apimux.go +++ /dev/null @@ -1,111 +0,0 @@ -// -// Copyright 2026 The Chainloop Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package server - -import ( - "crypto/tls" - "fmt" - "net" - - conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1" - "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" - "github.com/go-kratos/kratos/v2/log" -) - -// NewAPIMux creates the connection multiplexer that serves gRPC, gRPC-Web and -// HTTP from the single HTTP port, as specified in -// docs/specs/issue-3628-single-endpoint.md. The mux owns the listener on the -// current HTTP address, so the HTTP-facing clients and hostnames keep working -// unchanged. When the operator configures TLS, the listener terminates it -// before the classifier and offers HTTP/2 and HTTP/1.1 through ALPN, so one -// certificate covers all protocols. -// -// The returned cleanup closes the mux and its listener; wire runs it when a -// later provider fails or on shutdown. -func NewAPIMux(c *conf.Server, logger log.Logger) (*httpgrpcmux.Mux, func(), error) { - httpCfg := c.GetHttp() - if httpCfg.GetAddr() == "" { - // An empty address would silently bind an ephemeral port, and the - // shared API would not be where operators expect it. - return nil, nil, fmt.Errorf("server.http.addr must be set: the single port cannot bind an empty address") - } - network := httpCfg.GetNetwork() - if network == "" { - network = "tcp" - } - l, err := net.Listen(network, httpCfg.GetAddr()) - if err != nil { - return nil, nil, fmt.Errorf("listening on the single port %q: %w", httpCfg.GetAddr(), err) - } - - var raw = l - tlsEnabled := false - if tlsConf, err := loadServerTLSConfig(httpCfg.GetTlsConfig()); err != nil { - _ = l.Close() - return nil, nil, err - } else if tlsConf != nil { - raw = tls.NewListener(l, tlsConf) - tlsEnabled = true - } - - mux := httpgrpcmux.New(raw, &httpgrpcmux.Options{Logger: logger}) - helper := log.NewHelper(logger) - helper.Infow("msg", "serving gRPC, gRPC-Web and HTTP from the single port", "addr", httpCfg.GetAddr(), "tls", tlsEnabled) - if legacyGRPCListenerEnabled(c) { - helper.Warnw("msg", fmt.Sprintf("the gRPC-only listener on %q is deprecated; it will be disabled by default in a future release and removed afterwards. Set server.legacy_grpc_listener to false to listen on the single port only", c.GetGrpc().GetAddr())) - } - - return mux, mux.Close, nil -} - -// loadServerTLSConfig builds a tls.Config from the certificate and private key -// paths of the server configuration. It returns nil when no TLS is configured. -// For the single port the config negotiates HTTP/2 and HTTP/1.1 through ALPN. -func loadServerTLSConfig(tlsConf *conf.Server_TLS) (*tls.Config, error) { - if tlsConf == nil { - return nil, nil - } - cert := tlsConf.GetCertificate() - privKey := tlsConf.GetPrivateKey() - switch { - case cert == "" && privKey == "": - return nil, nil - case cert == "" || privKey == "": - // A partial configuration is a typo: refuse it instead of silently - // exposing the endpoint over plaintext. - return nil, fmt.Errorf("incomplete TLS configuration: certificate and private key must be set together") - } - certPair, err := tls.LoadX509KeyPair(cert, privKey) - if err != nil { - return nil, fmt.Errorf("loading server TLS certificate: %w", err) - } - return &tls.Config{ - Certificates: []tls.Certificate{certPair}, - MinVersion: tls.VersionTLS12, - // Serve gRPC (HTTP/2) and HTTP/1.1 from the same certificate. - NextProtos: []string{"h2", "http/1.1"}, - }, nil -} - -// legacyGRPCListenerEnabled reports whether the deprecated gRPC-only listener -// must stay on. It defaults to true when unset, so that existing -// installations and the Helm chart keep working during the deprecation period. -func legacyGRPCListenerEnabled(c *conf.Server) bool { - if c.LegacyGrpcListener != nil { - return *c.LegacyGrpcListener - } - return true -} diff --git a/app/controlplane/internal/server/grpc.go b/app/controlplane/internal/server/grpc.go index 55b6ce000..1bc3ef318 100644 --- a/app/controlplane/internal/server/grpc.go +++ b/app/controlplane/internal/server/grpc.go @@ -19,6 +19,7 @@ import ( "context" "crypto/tls" "fmt" + "net/url" "regexp" v1 "github.com/chainloop-dev/chainloop/app/controlplane/api/controlplane/v1" @@ -35,7 +36,6 @@ import ( "github.com/chainloop-dev/chainloop/app/controlplane/internal/service" "github.com/chainloop-dev/chainloop/app/controlplane/internal/usercontext" "github.com/chainloop-dev/chainloop/pkg/credentials" - "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" "github.com/chainloop-dev/chainloop/pkg/servicelogger" "github.com/golang-jwt/jwt/v5" @@ -91,12 +91,8 @@ type Opts struct { GroupSvc *service.GroupService ProjectSvc *service.ProjectService // Utils - Logger log.Logger - ServerConfig *conf.Server - // APIMux serves gRPC, gRPC-Web and HTTP from the single HTTP port. When - // set, both the gRPC and the HTTP server consume connections from it - // instead of opening their own listeners (issue-3628 single endpoint). - APIMux *httpgrpcmux.Mux + Logger log.Logger + ServerConfig *conf.Server AuthConfig *conf.Auth FederatedConfig *conf.FederatedAuthentication OperationAuthConfig *conf.OperationAuthorizationProvider @@ -114,54 +110,43 @@ var ( allButOrganizationOperationsSkipRegexp = regexp.MustCompile("/controlplane.v1.OrganizationService/Create|/controlplane.v1.UserService/ListMemberships|/controlplane.v1.ContextService/Current|/controlplane.v1.AuthService/DeleteAccount") ) -// NewGRPCServer builds the gRPC server(s) that serve native gRPC. -// -// In single-port mode (API mux present) the first server consumes the -// connections that the classifier marks as native gRPC; while the legacy gRPC -// listener is on (default) a second, deprecated server also binds the -// gRPC-only port with its configured TLS — the behavior the dedicated gRPC -// port had before the single port existed. Without the mux, a single server -// binds the gRPC-only port, as before. -func NewGRPCServer(opts *Opts) ([]*grpc.Server, error) { +// GRPCServers separates the HTTP handler from the optional native listener. +// Only Legacy must be started as a transport server. +type GRPCServers struct { + Handler *grpc.Server + Legacy *grpc.Server +} + +// NewGRPCServer builds both servers with the same services and middleware. +func NewGRPCServer(opts *Opts) (*GRPCServers, error) { // Opt-in histogram metrics for the interceptor grpc_prometheus.EnableHandlingTimeHistogram() - if opts.APIMux != nil { - // Single port mode: the mux owns the listener on the HTTP address - // (and its TLS); this server only consumes the connections that the - // classifier marks as native gRPC. - single, err := newGRPCServer(opts, grpc.Listener(opts.APIMux.GRPC())) + // Kratos' stream interceptor needs a non-nil endpoint even when the gRPC + // handler never starts a native listener. Supply it explicitly rather than + // calling Endpoint(), which would bind an otherwise unused socket. + scheme := "grpc" + if opts.ServerConfig.GetHttp().GetTlsConfig().GetCertificate() != "" { + scheme = "grpcs" + } + handler := newGRPCServer(opts, grpc.Endpoint(&url.URL{ + Scheme: scheme, + Host: opts.ServerConfig.GetHttp().GetAddr(), + })) + servers := &GRPCServers{Handler: handler} + if legacyGRPCListenerEnabled(opts.ServerConfig) { + // Ignore unused legacy certificates when this listener is disabled. + legacyOpts, err := legacyGRPCServerOptions(opts) if err != nil { return nil, err } - - servers := []*grpc.Server{single} - if legacyGRPCListenerEnabled(opts.ServerConfig) { - // The legacy TLS options are loaded only when the deprecated - // listener actually starts: an operator turning it off must not - // be blocked by a dangling cert mount. - legacyOpts, err := legacyGRPCServerOptions(opts) - if err != nil { - return nil, err - } - legacy, err := newGRPCServer(opts, legacyOpts...) - if err != nil { - return nil, err - } - servers = append(servers, legacy) - } - return servers, nil + servers.Legacy = newGRPCServer(opts, legacyOpts...) } + return servers, nil +} - legacyOpts, err := legacyGRPCServerOptions(opts) - if err != nil { - return nil, err - } - srv, err := newGRPCServer(opts, legacyOpts...) - if err != nil { - return nil, err - } - return []*grpc.Server{srv}, nil +func legacyGRPCListenerEnabled(c *conf.Server) bool { + return c.LegacyGrpcListener == nil || *c.LegacyGrpcListener } // legacyGRPCServerOptions binds the deprecated gRPC-only listener: its @@ -196,7 +181,7 @@ func legacyGRPCServerOptions(opts *Opts) ([]grpc.ServerOption, error) { // newGRPCServer builds one gRPC server with the shared middleware chain, // interceptors and service registrations, bound through the given extra // options. -func newGRPCServer(opts *Opts, extraOpts ...grpc.ServerOption) (*grpc.Server, error) { +func newGRPCServer(opts *Opts, extraOpts ...grpc.ServerOption) *grpc.Server { var serverOpts = append([]grpc.ServerOption{ grpc.Middleware(craftMiddleware(opts)...), grpc.UnaryInterceptor( @@ -243,7 +228,7 @@ func newGRPCServer(opts *Opts, extraOpts ...grpc.ServerOption) (*grpc.Server, er // Register Prometheus metrics grpc_prometheus.Register(srv.Server) - return srv, nil + return srv } func craftMiddleware(opts *Opts) []middleware.Middleware { diff --git a/app/controlplane/internal/server/http.go b/app/controlplane/internal/server/http.go index 6b0c16edb..5270b8e92 100644 --- a/app/controlplane/internal/server/http.go +++ b/app/controlplane/internal/server/http.go @@ -18,6 +18,8 @@ package server import ( "context" "errors" + "fmt" + "strings" "time" "github.com/chainloop-dev/chainloop/app/controlplane/pkg/jwt/apitoken" @@ -27,7 +29,7 @@ import ( "buf.build/go/protovalidate" v1 "github.com/chainloop-dev/chainloop/app/controlplane/api/controlplane/v1" "github.com/chainloop-dev/chainloop/app/controlplane/internal/service" - "github.com/chainloop-dev/chainloop/pkg/httpgrpcmux" + "github.com/go-kratos/kratos/v2/log" "github.com/go-kratos/kratos/v2/middleware" "github.com/go-kratos/kratos/v2/middleware/tracing" "github.com/improbable-eng/grpc-web/go/grpcweb" @@ -42,34 +44,41 @@ import ( "github.com/go-kratos/kratos/v2/transport/http" ) -// NewHTTPServer builds the HTTP server that serves plain HTTP and gRPC-Web. -// -// In single-port mode (API mux present, see -// docs/specs/issue-3628-single-endpoint.md) it consumes the connections that -// the classifier did not mark as native gRPC, so gRPC-Web and HTTP are served -// from the single port. The gRPC-Web wrapper and the native-gRPC fallback wrap -// the mux-fed gRPC server (grpcServers[0]); the legacy gRPC server, when -// present, serves its own deprecated port. Without the mux, the server binds -// the HTTP-only port, as before. -func NewHTTPServer(opts *Opts, grpcServers []*grpc.Server) ([]*http.Server, error) { - if opts.APIMux != nil { - single, err := newHTTPServer(opts, grpcServers[0], http.Listener(opts.APIMux.HTTP())) - if err != nil { - return nil, err - } - return []*http.Server{single}, nil - } +// APIServer owns the public HTTP listener and its gRPC handler. The HTTP +// server must drain before the handler is stopped, so in-flight RPCs finish. +type APIServer struct { + *http.Server + grpcHandler *grpc.Server +} - srv, err := newHTTPServer(opts, grpcServers[0], http.Address(opts.ServerConfig.GetHttp().GetAddr())) +func (s *APIServer) Stop(ctx context.Context) error { + httpErr := s.Server.Stop(ctx) + return errors.Join(httpErr, s.grpcHandler.Stop(ctx)) +} + +// NewHTTPServer serves HTTP, gRPC-Web and native gRPC per request on one +// HTTP/1.1 and HTTP/2 listener, without classifying connections. +func NewHTTPServer(opts *Opts, grpcServers *GRPCServers) (*APIServer, error) { + if opts.ServerConfig.GetHttp().GetAddr() == "" { + return nil, errors.New("server.http.addr must be set: the shared endpoint cannot bind an empty address") + } + srv, err := newHTTPServer(opts, grpcServers.Handler, http.Address(opts.ServerConfig.GetHttp().GetAddr())) if err != nil { return nil, err } - return []*http.Server{srv}, nil + if legacyGRPCListenerEnabled(opts.ServerConfig) { + log.NewHelper(opts.Logger).Warnw("msg", fmt.Sprintf("the gRPC-only listener on %q is deprecated; use the shared HTTP endpoint and set server.legacy_grpc_listener to false when all clients have moved", opts.ServerConfig.GetGrpc().GetAddr())) + } + return &APIServer{Server: srv, grpcHandler: grpcServers.Handler}, nil } // newHTTPServer builds one HTTP server that serves the routes, the gRPC-Web // wrapper and the transcoded handlers, bound through the given extra options. func newHTTPServer(opts *Opts, grpcSrv *grpc.Server, extraOpts ...http.ServerOption) (*http.Server, error) { + tlsConfig, err := loadServerTLSConfig(opts.ServerConfig.GetHttp().GetTlsConfig()) + if err != nil { + return nil, err + } var serverOpts = append([]http.ServerOption{ http.Middleware(httpMiddlewares(opts)...), }, extraOpts...) @@ -84,6 +93,9 @@ func newHTTPServer(opts *Opts, grpcSrv *grpc.Server, extraOpts ...http.ServerOpt if v := opts.ServerConfig.Http.Timeout; v != nil { serverOpts = append(serverOpts, http.Timeout(v.AsDuration())) } + if tlsConfig != nil { + serverOpts = append(serverOpts, http.TLSConfig(tlsConfig)) + } // initialize the underneath http server httpSrv := http.NewServer(serverOpts...) @@ -122,9 +134,8 @@ func newHTTPServer(opts *Opts, grpcSrv *grpc.Server, extraOpts ...http.ServerOpt // Include the OpenAPI spec handler r.GET("/openapi.yaml", statusSvc.HandleOpenAPISpec) - // Handle grpc-web requests or fallback. Unencrypted HTTP/2 (h2c) is enabled - // natively so that connections that are not native gRPC still work: browsers - // doing gRPC-Web over HTTP/2 and h2c-aware proxies. + // The standard HTTP server owns every connection and dispatches each + // stream independently, including mixed HTTP/2 upstream proxy pools. wrappedServer.Protocols = new(h.Protocols) wrappedServer.Protocols.SetHTTP1(true) wrappedServer.Protocols.SetHTTP2(true) @@ -137,10 +148,10 @@ func newHTTPServer(opts *Opts, grpcSrv *grpc.Server, extraOpts ...http.ServerOpt wrappedServer.ReadHeaderTimeout = 60 * time.Second wrappedServer.Handler = h.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - // Native gRPC requests that arrive on a connection the classifier - // gave to the HTTP server: hand them to the gRPC server, which runs - // the same middleware chain as on the dedicated gRPC port. - if httpgrpcmux.IsNativeGRPCContentType(req.Header.Get("Content-Type")) { + // Native gRPC uses grpc-go's HTTP adapter and its existing middleware. + // gRPC-Web must reach its wrapper even though its content type shares + // the application/grpc prefix. Native gRPC requires HTTP/2. + if req.ProtoMajor == 2 && isNativeGRPCContentType(req.Header.Get("Content-Type")) { grpcSrv.ServeHTTP(res, req) return } @@ -154,6 +165,12 @@ func newHTTPServer(opts *Opts, grpcSrv *grpc.Server, extraOpts ...http.ServerOpt return wrappedServer, nil } +func isNativeGRPCContentType(contentType string) bool { + contentType = strings.ToLower(contentType) + return strings.HasPrefix(contentType, "application/grpc") && + !strings.HasPrefix(contentType, "application/grpc-web") +} + // httpMiddlewares returns the middleware chain for the HTTP server. // Transport-level tracing comes first: the gRPC server is instrumented via the // otelgrpc stats handler, but transcoded HTTP requests get no server span diff --git a/app/controlplane/internal/server/http_dispatch_test.go b/app/controlplane/internal/server/http_dispatch_test.go new file mode 100644 index 000000000..513a85af1 --- /dev/null +++ b/app/controlplane/internal/server/http_dispatch_test.go @@ -0,0 +1,243 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "context" + "crypto/tls" + "io" + "net" + "net/http" + "net/url" + "sync" + "sync/atomic" + "testing" + "time" + + v1 "github.com/chainloop-dev/chainloop/app/controlplane/api/controlplane/v1" + conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1" + "github.com/go-kratos/kratos/v2/middleware" + "github.com/go-kratos/kratos/v2/transport" + kratosgrpc "github.com/go-kratos/kratos/v2/transport/grpc" + "github.com/stretchr/testify/require" + grpcLib "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + reflection "google.golang.org/grpc/reflection/grpc_reflection_v1" + "google.golang.org/grpc/status" +) + +// A proxy can send browser traffic after native gRPC on the same upstream +// connection. Verify real HTTP/2 reuse, rather than separate client sockets. +func TestSharedHTTP2ConnectionAfterNativeGRPC(t *testing.T) { + for _, useTLS := range []bool{false, true} { + t.Run(map[bool]string{false: "h2c", true: "TLS"}[useTLS], func(t *testing.T) { + testSharedHTTP2Connection(t, useTLS) + }) + } +} + +func testSharedHTTP2Connection(t *testing.T, useTLS bool) { + t.Helper() + const httpKind, grpcKind, preflightKind = "http", "grpc", "preflight" + off := false + config := &conf.Server{ + Grpc: &conf.Server_GRPC{Addr: testAnyAddr}, + Http: &conf.Server_HTTP{Addr: testAnyAddr}, + LegacyGrpcListener: &off, + } + scheme := httpKind + if useTLS { + cert, key := writeSelfSignedCert(t) + config.Http.TlsConfig = &conf.Server_TLS{Certificate: cert, PrivateKey: key} + scheme = "https" + } + addr, _, _ := startSinglePortAppWithConfig(t, config) + var connections atomic.Int32 + tr := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + connections.Add(1) + return (&net.Dialer{}).DialContext(ctx, network, addr) + }, + Protocols: new(http.Protocols), + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // test certificate + } + tr.Protocols.SetUnencryptedHTTP2(true) + tr.Protocols.SetHTTP2(true) + t.Cleanup(tr.CloseIdleConnections) + client := &http.Client{Transport: tr, Timeout: 5 * time.Second} + + for _, kind := range []string{grpcKind, httpKind, "web", preflightKind, grpcKind, httpKind} { + path, method := "/grpc.health.v1.Health/Check", http.MethodPost + switch kind { + case httpKind: + path, method = "/statusz", http.MethodGet + case preflightKind: + method = http.MethodOptions + } + req, err := http.NewRequest(method, scheme+"://"+addr+path, bytes.NewReader([]byte{0, 0, 0, 0, 0})) + require.NoError(t, err) + switch kind { + case grpcKind: + req.Header.Set("Content-Type", "application/grpc") + req.Header.Set("Te", "trailers") + case "web": + req.Header.Set("Content-Type", "application/grpc-web+proto") + req.Header.Set("X-Grpc-Web", "1") + case preflightKind: + req.Header.Set("Origin", "https://example.test") + req.Header.Set("Access-Control-Request-Method", "POST") + req.Header.Set("Access-Control-Request-Headers", "content-type,x-grpc-web") + } + resp, err := client.Do(req) + require.NoError(t, err, kind) + _, err = io.Copy(io.Discard, resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + expectedStatus := http.StatusOK + if kind == preflightKind { + expectedStatus = http.StatusNoContent + } + require.Equal(t, expectedStatus, resp.StatusCode, kind) + require.Equal(t, 2, resp.ProtoMajor) + if kind == grpcKind { + grpcStatus := resp.Trailer.Get("Grpc-Status") + if grpcStatus == "" { + grpcStatus = resp.Header.Get("Grpc-Status") + } + require.Equal(t, "16", grpcStatus) + } + } + require.EqualValues(t, 1, connections.Load(), "all request types must share one HTTP/2 connection") +} + +// This is a focused parity check. The full token and authorization matrix in +// the spec remains a separate qualification requirement. +func TestSharedAndLegacyGRPCBehavior(t *testing.T) { + addr, legacyPort, _ := startSinglePortApp(t, nil) + for _, endpoint := range []string{addr, "127.0.0.1:" + legacyPort} { + t.Run(endpoint, func(t *testing.T) { + conn, err := grpcLib.NewClient(endpoint, grpcLib.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err = v1.NewStatusServiceClient(conn).Statusz(ctx, &v1.StatuszRequest{}) + require.NoError(t, err) + for _, token := range []string{"", "Bearer invalid.jwt.token"} { + callCtx := metadata.AppendToOutgoingContext(ctx, "authorization", token) + _, err = grpc_health_v1.NewHealthClient(conn).Check(callCtx, &grpc_health_v1.HealthCheckRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + } + // Reflection exercises the Kratos bidirectional-stream interceptor, + // including its endpoint metadata on a handler with no native listener. + stream, err := reflection.NewServerReflectionClient(conn).ServerReflectionInfo(ctx) + require.NoError(t, err) + for range 2 { + require.NoError(t, stream.Send(&reflection.ServerReflectionRequest{ + MessageRequest: &reflection.ServerReflectionRequest_ListServices{ListServices: ""}, + })) + resp, err := stream.Recv() + require.NoError(t, err) + require.NotEmpty(t, resp.GetListServicesResponse().GetService()) + } + require.NoError(t, stream.CloseSend()) + _, err = stream.Recv() + require.ErrorIs(t, err, io.EOF) + }) + } +} + +func TestAPIServerDrainsRPCAndPreservesTLSMetadata(t *testing.T) { + off := false + cert, key := writeSelfSignedCert(t) + opts := testServerOpts(t, &conf.Server{ + Grpc: &conf.Server_GRPC{Addr: testAnyAddr}, + Http: &conf.Server_HTTP{Addr: testAnyAddr, TlsConfig: &conf.Server_TLS{Certificate: cert, PrivateKey: key}}, + LegacyGrpcListener: &off, + }) + release := make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + defer unblock() + entered := make(chan context.Context, 1) + g := newGRPCServer(opts, kratosgrpc.Endpoint(&url.URL{Scheme: "grpcs", Host: testAnyAddr}), + kratosgrpc.Middleware(func(next middleware.Handler) middleware.Handler { + return func(ctx context.Context, req any) (any, error) { + entered <- ctx + select { + case <-release: + return next(ctx, req) + case <-ctx.Done(): + return nil, ctx.Err() + } + } + })) + api, err := NewHTTPServer(opts, &GRPCServers{Handler: g}) + require.NoError(t, err) + endpoint, err := api.Endpoint() + require.NoError(t, err) + serveErr := make(chan error, 1) + go func() { serveErr <- api.Start(context.Background()) }() + t.Cleanup(func() { + unblock() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, api.Stop(ctx)) + }) + conn, err := grpcLib.NewClient(endpoint.Host, grpcLib.WithTransportCredentials(credentials.NewTLS( + &tls.Config{InsecureSkipVerify: true}))) //nolint:gosec // test certificate + require.NoError(t, err) + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + rpcErr := make(chan error, 1) + go func() { + callCtx := metadata.AppendToOutgoingContext(ctx, "x-dispatch-test", "preserved") + _, err := v1.NewStatusServiceClient(conn).Statusz(callCtx, &v1.StatuszRequest{}) + rpcErr <- err + }() + select { + case callCtx := <-entered: + p, ok := peer.FromContext(callCtx) + require.True(t, ok) + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + require.True(t, ok, "the HTTP adapter must retain TLS peer information") + require.Equal(t, "h2", tlsInfo.State.NegotiatedProtocol) + tr, ok := transport.FromServerContext(callCtx) + require.True(t, ok) + require.Equal(t, "/controlplane.v1.StatusService/Statusz", tr.Operation()) + require.Equal(t, "preserved", tr.RequestHeader().Get("x-dispatch-test")) + case <-ctx.Done(): + require.FailNow(t, "RPC did not enter the middleware") + } + shutdownStarted := make(chan struct{}) + api.RegisterOnShutdown(sync.OnceFunc(func() { close(shutdownStarted) })) + stopErr := make(chan error, 1) + go func() { stopErr <- api.Stop(ctx) }() + select { + case <-shutdownStarted: + case <-ctx.Done(): + require.FailNow(t, "HTTP shutdown did not start") + } + unblock() + require.NoError(t, <-rpcErr, "shutdown must allow the in-flight RPC to finish") + require.NoError(t, <-stopErr) + require.NoError(t, <-serveErr) +} diff --git a/app/controlplane/internal/server/apimux_test.go b/app/controlplane/internal/server/http_test.go similarity index 74% rename from app/controlplane/internal/server/apimux_test.go rename to app/controlplane/internal/server/http_test.go index 988aaffc1..e549fb7a2 100644 --- a/app/controlplane/internal/server/apimux_test.go +++ b/app/controlplane/internal/server/http_test.go @@ -54,67 +54,88 @@ const ( testLoginURL = "http://localhost" ) -// startSinglePortApp boots the real gRPC and HTTP servers over an API mux on -// 127.0.0.1:0 and returns the single port address, the deprecated legacy gRPC -// port (empty when the legacy listener is off) and the number of gRPC servers -// in use. +// startSinglePortApp starts the real HTTP-owned endpoint and optional legacy +// native gRPC listener. The HTTP-served gRPC handler never starts a listener. func startSinglePortApp(t *testing.T, legacyListener *bool) (string, string, int) { t.Helper() - - validator, err := protovalidate.New() - require.NoError(t, err) - - serverConfig := &conf.Server{ + return startSinglePortAppWithConfig(t, &conf.Server{ Grpc: &conf.Server_GRPC{Addr: testAnyAddr}, Http: &conf.Server_HTTP{Addr: testAnyAddr}, LegacyGrpcListener: legacyListener, - } - - mux, cleanup, err := NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) - require.NoError(t, err) - t.Cleanup(cleanup) - - opts := &Opts{ - ServerConfig: serverConfig, - AuthConfig: &conf.Auth{}, - Logger: log.NewStdLogger(io.Discard), - Validator: validator, - AuthSvc: &service.AuthService{AuthURLs: &service.AuthURLs{Login: testLoginURL}}, - APIMux: mux, - } + }) +} +func startSinglePortAppWithConfig(t *testing.T, serverConfig *conf.Server) (string, string, int) { + t.Helper() + opts := testServerOpts(t, serverConfig) gs, err := NewGRPCServer(opts) require.NoError(t, err) - httpServers, err := NewHTTPServer(opts, gs) + api, err := NewHTTPServer(opts, gs) + require.NoError(t, err) + endpoint, err := api.Endpoint() + require.NoError(t, err) + _, port, err := net.SplitHostPort(endpoint.Host) require.NoError(t, err) legacyPort := "" - if len(gs) > 1 { - // Resolve the legacy gRPC port before the servers start; kratos - // resolves the advertise host on Endpoint(), so only the port is - // reused here and dialed on the loopback bind address. - ep, err := gs[1].Endpoint() + serverCount := 1 + if gs.Legacy != nil { + serverCount++ + } + errs := make(chan error, serverCount) + if g := gs.Legacy; g != nil { + endpoint, err := g.Endpoint() require.NoError(t, err) - _, port, err := net.SplitHostPort(ep.Host) + _, legacyPort, err = net.SplitHostPort(endpoint.Host) require.NoError(t, err) - legacyPort = port + go func() { errs <- g.Start(context.Background()) }() } - - require.NoError(t, mux.Start(context.Background())) - t.Cleanup(func() { _ = mux.Stop(context.Background()) }) - - for _, g := range gs { - gsrv := g - go func() { _ = gsrv.Start(context.Background()) }() - t.Cleanup(func() { _ = gsrv.Stop(context.Background()) }) - } - for _, s := range httpServers { - srv := s - go func() { _ = srv.Start(context.Background()) }() - t.Cleanup(func() { _ = srv.Stop(context.Background()) }) + go func() { errs <- api.Start(context.Background()) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, api.Stop(ctx)) + if g := gs.Legacy; g != nil { + require.NoError(t, g.Stop(ctx)) + } + for range serverCount { + select { + case err := <-errs: + require.NoError(t, err) + case <-ctx.Done(): + require.FailNow(t, "API servers did not stop") + } + } + }) + // Wait for the legacy server to enter Serve before a test can stop it. + // Endpoint binds its socket, but Start runs in another goroutine. + if legacyPort != "" { + legacyCreds := insecure.NewCredentials() + if cfg := serverConfig.GetGrpc().GetTlsConfig(); cfg.GetCertificate() != "" && cfg.GetPrivateKey() != "" { + legacyCreds = credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) //nolint:gosec // test certificate + } + conn, err := grpcLib.NewClient(net.JoinHostPort("127.0.0.1", legacyPort), grpcLib.WithTransportCredentials(legacyCreds)) + require.NoError(t, err) + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err = grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) } + return net.JoinHostPort("127.0.0.1", port), legacyPort, serverCount +} - return mux.GRPC().Addr().String(), legacyPort, len(gs) +func testServerOpts(t *testing.T, serverConfig *conf.Server) *Opts { + t.Helper() + validator, err := protovalidate.New() + require.NoError(t, err) + return &Opts{ + ServerConfig: serverConfig, + AuthConfig: &conf.Auth{}, + Logger: log.NewStdLogger(io.Discard), + Validator: validator, + AuthSvc: &service.AuthService{AuthURLs: &service.AuthURLs{Login: testLoginURL}}, + } } func TestSinglePortServesAllProtocols(t *testing.T) { @@ -128,7 +149,7 @@ func TestSinglePortServesAllProtocols(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _, err = grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{}) - // The RPC must reach the native gRPC server, whose middleware chain + // The RPC must reach the HTTP-served gRPC handler, whose middleware chain // rejects unauthenticated calls exactly as it does today on the // dedicated gRPC port (R-002: same behavior as today). require.Error(t, err) @@ -255,21 +276,18 @@ func TestPartialTLSConfigRejected(t *testing.T) { }, } - _, _, err := NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) + _, err := newHTTPServer(&Opts{ServerConfig: serverConfig}, nil) require.Error(t, err) assert.Contains(t, err.Error(), "incomplete TLS configuration") serverConfig.Http.TlsConfig = &conf.Server_TLS{PrivateKey: keyPath} // certificate missing - _, _, err = NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) + _, err = newHTTPServer(&Opts{ServerConfig: serverConfig}, nil) require.Error(t, err) } func TestSinglePortTLS(t *testing.T) { certPath, keyPath := writeSelfSignedCert(t) - validator, err := protovalidate.New() - require.NoError(t, err) - serverConfig := &conf.Server{ Grpc: &conf.Server_GRPC{ Addr: testAnyAddr, @@ -283,38 +301,7 @@ func TestSinglePortTLS(t *testing.T) { }, } - mux, cleanup, err := NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) - require.NoError(t, err) - t.Cleanup(cleanup) - - opts := &Opts{ - ServerConfig: serverConfig, - AuthConfig: &conf.Auth{}, - Logger: log.NewStdLogger(io.Discard), - Validator: validator, - AuthSvc: &service.AuthService{AuthURLs: &service.AuthURLs{Login: testLoginURL}}, - APIMux: mux, - } - - gs, err := NewGRPCServer(opts) - require.NoError(t, err) - httpServers, err := NewHTTPServer(opts, gs) - require.NoError(t, err) - require.Len(t, httpServers, 1) - - require.NoError(t, mux.Start(context.Background())) - t.Cleanup(func() { _ = mux.Stop(context.Background()) }) - for _, g := range gs { - gsrv := g - go func() { _ = gsrv.Start(context.Background()) }() - t.Cleanup(func() { _ = gsrv.Stop(context.Background()) }) - } - for _, s := range httpServers { - srv := s - go func() { _ = srv.Start(context.Background()) }() - t.Cleanup(func() { _ = srv.Stop(context.Background()) }) - } - addr := mux.GRPC().Addr().String() + addr, _, _ := startSinglePortAppWithConfig(t, serverConfig) // TLS client without ALPN restriction: HTTP/1.1 over TLS must work. client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} //nolint:gosec // test server with a self-signed certificate @@ -323,7 +310,7 @@ func TestSinglePortTLS(t *testing.T) { defer resp.Body.Close() assert.Equal(t, http.StatusOK, resp.StatusCode) - // Native gRPC over TLS: handed to the gRPC server through the mux. + // Native gRPC over TLS uses the gRPC HTTP adapter. gconn, err := grpcLib.NewClient(addr, grpcLib.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}))) //nolint:gosec // test server with a self-signed certificate require.NoError(t, err) defer gconn.Close() @@ -357,6 +344,33 @@ func TestSinglePortTLS(t *testing.T) { assert.Equal(t, "HTTP/2.0", resp.Proto) } +func TestLegacyTLSIsIndependentFromSharedEndpoint(t *testing.T) { + cert, key := writeSelfSignedCert(t) + for _, sharedTLS := range []bool{false, true} { + t.Run(map[bool]string{false: "plaintext shared port", true: "TLS shared port"}[sharedTLS], func(t *testing.T) { + cfg := &conf.Server{ + Grpc: &conf.Server_GRPC{Addr: testAnyAddr, TlsConfig: &conf.Server_TLS{Certificate: cert, PrivateKey: key}}, + Http: &conf.Server_HTTP{Addr: testAnyAddr}, + } + sharedCreds := insecure.NewCredentials() + if sharedTLS { + cfg.Http.TlsConfig = cfg.Grpc.TlsConfig + sharedCreds = credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) //nolint:gosec // test certificate + } + addr, _, _ := startSinglePortAppWithConfig(t, cfg) + // The helper verifies that the legacy listener requires TLS. This + // call verifies that its credentials do not affect the HTTP handler. + conn, err := grpcLib.NewClient(addr, grpcLib.WithTransportCredentials(sharedCreds)) + require.NoError(t, err) + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err = grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + }) + } +} + func writeSelfSignedCert(t *testing.T) (string, string) { t.Helper() @@ -387,7 +401,7 @@ func TestEmptyHTTPAddrRejected(t *testing.T) { Grpc: &conf.Server_GRPC{Addr: testAnyAddr}, Http: &conf.Server_HTTP{}, // addr omitted } - _, _, err := NewAPIMux(serverConfig, log.NewStdLogger(io.Discard)) + _, err := NewHTTPServer(&Opts{ServerConfig: serverConfig}, nil) require.Error(t, err) assert.Contains(t, err.Error(), "server.http.addr must be set") } diff --git a/app/controlplane/internal/server/ingress_test.go b/app/controlplane/internal/server/ingress_test.go new file mode 100644 index 000000000..9324aff34 --- /dev/null +++ b/app/controlplane/internal/server/ingress_test.go @@ -0,0 +1,146 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + v1 "github.com/chainloop-dev/chainloop/app/controlplane/api/controlplane/v1" + "github.com/stretchr/testify/require" + grpcLib "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// Run on Linux with Docker and an available image: +// NGINX_INTEGRATION_IMAGE=nginx:1.28.0-alpine go test ./app/controlplane/internal/server -run TestNGINXSharedIngress -v +// This checks the grpc_pass path used by the chart, not a Kubernetes controller. +func TestNGINXSharedIngress(t *testing.T) { + image := os.Getenv("NGINX_INTEGRATION_IMAGE") + if image == "" { + t.Skip("set NGINX_INTEGRATION_IMAGE to run the Docker integration test") + } + off := false + upstream, _, _ := startSinglePortApp(t, &off) + cert, key := writeSelfSignedCert(t) + dir := t.TempDir() + for name, src := range map[string]string{"cert.pem": cert, "key.pem": key} { + data, err := os.ReadFile(src) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), data, 0o600)) //nolint:gosec // fixed filenames inside the test's temporary directory + } + lis, err := net.Listen("tcp", testAnyAddr) + require.NoError(t, err) + addr := lis.Addr().String() + require.NoError(t, lis.Close()) + config := fmt.Sprintf(`events { worker_connections 128; } +http { + upstream controlplane { server %s; keepalive 16; } + server { + listen %s ssl http2; + ssl_certificate /probe/cert.pem; + ssl_certificate_key /probe/key.pem; + location / { grpc_pass grpc://controlplane; } + } +} +`, upstream, addr) + require.NoError(t, os.WriteFile(filepath.Join(dir, "nginx.conf"), []byte(config), 0o600)) + name := fmt.Sprintf("chainloop-shared-ingress-%d-%s", os.Getpid(), strings.ReplaceAll(addr, ":", "-")) + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext(ctx, "docker", "run", "--rm", "--name", name, //nolint:gosec // test-selected image passed as an argument, without a shell + "--network", "host", "-v", dir+":/probe:ro", image, + "nginx", "-c", "/probe/nginx.conf", "-g", "daemon off;") + var logs bytes.Buffer + cmd.Stdout, cmd.Stderr = &logs, &logs + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + stopCtx, stopCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer stopCancel() + _ = exec.CommandContext(stopCtx, "docker", "stop", "--timeout", "1", name).Run() + cancel() + _ = cmd.Wait() + t.Log(logs.String()) + }) + require.Eventually(t, func() bool { + conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) + if err != nil { + return false + } + _ = conn.Close() + return true + }, 10*time.Second, 50*time.Millisecond) + + conn, err := grpcLib.NewClient(addr, grpcLib.WithTransportCredentials(credentials.NewTLS( + &tls.Config{InsecureSkipVerify: true}))) //nolint:gosec // test certificate + require.NoError(t, err) + defer conn.Close() + rpcCtx, rpcCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer rpcCancel() + _, err = v1.NewStatusServiceClient(conn).Statusz(rpcCtx, &v1.StatuszRequest{}) + require.NoError(t, err, "native gRPC must succeed before browser requests") + + for _, useHTTP2 := range []bool{false, true} { + t.Run(fmt.Sprintf("client_HTTP2_%t", useHTTP2), func(t *testing.T) { + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, ForceAttemptHTTP2: useHTTP2} //nolint:gosec // test certificate + defer tr.CloseIdleConnections() + client := &http.Client{Transport: tr, Timeout: 5 * time.Second} + for range 3 { + resp, err := client.Get("https://" + addr + "/statusz") + require.NoError(t, err) + _, err = io.Copy(io.Discard, resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, map[bool]int{false: 1, true: 2}[useHTTP2], resp.ProtoMajor) + + url := "https://" + addr + "/controlplane.v1.StatusService/Statusz" + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{0, 0, 0, 0, 0})) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/grpc-web+proto") + req.Header.Set("X-Grpc-Web", "1") + resp, err = client.Do(req) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Contains(t, resp.Header.Get("Content-Type"), "application/grpc-web") + require.Contains(t, string(body), "grpc-status: 0") + + req, err = http.NewRequest(http.MethodOptions, url, nil) + require.NoError(t, err) + req.Header.Set("Origin", "https://example.test") + req.Header.Set("Access-Control-Request-Method", "POST") + req.Header.Set("Access-Control-Request-Headers", "content-type,x-grpc-web") + resp, err = client.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusNoContent, resp.StatusCode) + require.Equal(t, "https://example.test", resp.Header.Get("Access-Control-Allow-Origin")) + } + }) + } +} diff --git a/app/controlplane/internal/server/otel_test.go b/app/controlplane/internal/server/otel_test.go index 9d3233106..37d6d5feb 100644 --- a/app/controlplane/internal/server/otel_test.go +++ b/app/controlplane/internal/server/otel_test.go @@ -24,7 +24,6 @@ import ( "buf.build/go/protovalidate" conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1" - "github.com/chainloop-dev/chainloop/app/controlplane/internal/service" "github.com/go-kratos/kratos/v2/log" kratoshttp "github.com/go-kratos/kratos/v2/transport/http" "github.com/stretchr/testify/assert" @@ -65,47 +64,27 @@ func spanNames(exporter *tracetest.InMemoryExporter) []string { func TestGRPCServerSpanWithMaxRecvMsgSize(t *testing.T) { exporter := withInMemoryTracing(t) - validator, err := protovalidate.New() - require.NoError(t, err) - - srvs, err := NewGRPCServer(&Opts{ - ServerConfig: &conf.Server{ - Grpc: &conf.Server_GRPC{ - Addr: testAnyAddr, - MaxRecvMsgSize: 8 * 1024 * 1024, - }, + addr, legacyPort, _ := startSinglePortAppWithConfig(t, &conf.Server{ + Grpc: &conf.Server_GRPC{ + Addr: testAnyAddr, + MaxRecvMsgSize: 8 * 1024 * 1024, }, - AuthConfig: &conf.Auth{}, - Logger: log.NewStdLogger(io.Discard), - Validator: validator, - AuthSvc: &service.AuthService{AuthURLs: &service.AuthURLs{Login: testLoginURL}}, + Http: &conf.Server_HTTP{Addr: testAnyAddr}, }) - require.NoError(t, err) - - // Endpoint() binds the listener so the address is known before Start. - endpoint, err := srvs[0].Endpoint() - require.NoError(t, err) - go func() { _ = srvs[0].Start(context.Background()) }() - defer func() { _ = srvs[0].Stop(context.Background()) }() - - conn, err := grpcLib.NewClient(endpoint.Host, grpcLib.WithTransportCredentials(insecure.NewCredentials())) - require.NoError(t, err) - defer conn.Close() - - // The request itself is rejected by the auth middleware, but the - // transport-level stats handler records the server span regardless. - deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + for _, endpoint := range []string{addr, "127.0.0.1:" + legacyPort} { + exporter.Reset() + conn, err := grpcLib.NewClient(endpoint, grpcLib.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) _, _ = grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{}) cancel() - if names := spanNames(exporter); len(names) > 0 { - assert.Contains(t, names, "grpc.health.v1.Health/Check") - return - } - time.Sleep(50 * time.Millisecond) + require.NoError(t, conn.Close()) + // The transport records a span even when authentication rejects the RPC. + require.Eventually(t, func() bool { + return len(spanNames(exporter)) > 0 + }, time.Second, time.Millisecond) + assert.Contains(t, spanNames(exporter), "grpc.health.v1.Health/Check", endpoint) } - require.Fail(t, "expected gRPC server span from otelgrpc stats handler") } // The HTTP server must produce a transport-level server span so transcoded diff --git a/app/controlplane/internal/server/server.go b/app/controlplane/internal/server/server.go index b18caa895..07827a3e9 100644 --- a/app/controlplane/internal/server/server.go +++ b/app/controlplane/internal/server/server.go @@ -24,7 +24,6 @@ import ( // ProviderSet is server providers. var ProviderSet = wire.NewSet( - NewAPIMux, NewGRPCServer, NewHTTPServer, NewHTTPMetricsServer, diff --git a/app/controlplane/internal/server/tls.go b/app/controlplane/internal/server/tls.go new file mode 100644 index 000000000..30ce96785 --- /dev/null +++ b/app/controlplane/internal/server/tls.go @@ -0,0 +1,52 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "crypto/tls" + "fmt" + + conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1" +) + +// loadServerTLSConfig builds a tls.Config from the certificate and private key +// paths of the server configuration. It returns nil when no TLS is configured. +// For the single port the config negotiates HTTP/2 and HTTP/1.1 through ALPN. +func loadServerTLSConfig(tlsConf *conf.Server_TLS) (*tls.Config, error) { + if tlsConf == nil { + return nil, nil + } + cert := tlsConf.GetCertificate() + privKey := tlsConf.GetPrivateKey() + switch { + case cert == "" && privKey == "": + return nil, nil + case cert == "" || privKey == "": + // A partial configuration is a typo: refuse it instead of silently + // exposing the endpoint over plaintext. + return nil, fmt.Errorf("incomplete TLS configuration: certificate and private key must be set together") + } + certPair, err := tls.LoadX509KeyPair(cert, privKey) + if err != nil { + return nil, fmt.Errorf("loading server TLS certificate: %w", err) + } + return &tls.Config{ + Certificates: []tls.Certificate{certPair}, + MinVersion: tls.VersionTLS12, + // Serve gRPC (HTTP/2) and HTTP/1.1 from the same certificate. + NextProtos: []string{"h2", "http/1.1"}, + }, nil +} diff --git a/deployment/chainloop/.helmignore b/deployment/chainloop/.helmignore index 0e8a0eb36..bd4da3cac 100644 --- a/deployment/chainloop/.helmignore +++ b/deployment/chainloop/.helmignore @@ -21,3 +21,5 @@ .idea/ *.tmproj .vscode/ +# Go tests run from the source tree and do not belong in the chart package. +*_test.go diff --git a/deployment/chainloop/Chart.yaml b/deployment/chainloop/Chart.yaml index 85b0b28dd..713c92fd9 100644 --- a/deployment/chainloop/Chart.yaml +++ b/deployment/chainloop/Chart.yaml @@ -7,7 +7,7 @@ description: Chainloop is an open source software supply chain control plane, a type: application # Bump the patch (not minor, not major) version on each change in the Chart Source code -version: 1.468.0 +version: 1.468.1 # Do not update appVersion, this is handled automatically by the release process appVersion: v1.125.0 diff --git a/deployment/chainloop/README.md b/deployment/chainloop/README.md index ce50fd1ed..28d51d8d6 100644 --- a/deployment/chainloop/README.md +++ b/deployment/chainloop/README.md @@ -241,7 +241,33 @@ or as shown before, provide them as imperative inputs during Helm Install/Upgrad ### Enable a custom domain with TLS -Chainloop uses three endpoints so we'll need to enable the ingress resource for each one of them. +The control plane ingress serves HTTP, gRPC-Web, and native gRPC on the same hostname. +Its HTTP server routes each request, so these protocols can share an upstream HTTP/2 connection. +The existing gRPC ingress stays available. Existing clients can keep their current endpoints. +To test the shared endpoint, point a gRPC client to the control plane HTTP hostname on port 443. +The Artifact CAS keeps its existing endpoints and native gRPC transport. + +The chart configures HTTP/2 upstream traffic for these controllers: + +- Community ingress-nginx: `controlplane.ingress.annotations` includes `nginx.ingress.kubernetes.io/backend-protocol: GRPC`. + This uses HTTP/2 upstream for all requests, including ordinary HTTP routes. + This setting also applies to `extraPaths`. Place other services that require HTTP/1 upstream in a separate Ingress. +- F5 NGINX Ingress Controller with NGINX or NGINX Plus: the same annotations include `nginx.org/grpc-services` and `nginx.org/http2: "true"`. + For controller versions that require it, also enable `http2` in the controller ConfigMap. + Native gRPC clients normally use TLS on the ingress. Plaintext client HTTP/2 needs controller configuration too. +- Traefik: `controlplane.service.annotations` includes `traefik.ingress.kubernetes.io/service.serversscheme: h2c`. + +These defaults apply to plaintext traffic between the ingress and the pod. Public ingress TLS still works as before. +If you enable TLS on the pod listener, configure the controller for encrypted HTTP/2 upstream traffic instead. +For ingress-nginx, use `GRPCS`. For F5 NGINX, also configure `nginx.org/ssl-services`. +For Traefik, use the `https` service scheme and configure certificate trust as required. +Other controllers need their own upstream protocol settings. + +The NGINX gRPC module path supports the shared endpoint without path or content-type rules. +The regression suite checks that path with NGINX Open Source. It does not certify every controller or NGINX Plus release. +See the [F5 annotation reference](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/) for controller requirements. + +Enable the shared control plane ingress and the existing gRPC ingresses as needed. See below an example of a `values.yaml` override @@ -276,6 +302,7 @@ controlplane: ingressClassName: nginx hostname: cp.chainloop.dev annotations: + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" # This depends on your configured issuer cert-manager.io/cluster-issuer: "letsencrypt-prod" @@ -634,7 +661,7 @@ Once done, you can access with [two predefined users](https://github.com/chainlo | `controlplane.service.loadBalancerIP` | controlplane service Load Balancer IP | `""` | | `controlplane.service.loadBalancerSourceRanges` | controlplane service Load Balancer sources | `[]` | | `controlplane.service.externalTrafficPolicy` | controlplane service external traffic policy | `Cluster` | -| `controlplane.service.annotations` | Additional custom annotations for controlplane service | `{}` | +| `controlplane.service.annotations` | Annotations for the shared HTTP and gRPC service. Traefik uses h2c upstream. | See values.yaml | | `controlplane.service.extraPorts` | Extra ports to expose in controlplane service (normally used with the `sidecars` value) | `[]` | | `controlplane.service.sessionAffinity` | Control where client requests go, to the same pod or round-robin | `None` | | `controlplane.service.sessionAffinityConfig` | Additional settings for the sessionAffinity | `{}` | @@ -651,12 +678,12 @@ Once done, you can access with [two predefined users](https://github.com/chainlo | `controlplane.serviceAPI.extraPorts` | Extra ports to expose in controlplane service (normally used with the `sidecars` value) | `[]` | | `controlplane.serviceAPI.sessionAffinity` | Control where client requests go, to the same pod or round-robin | `None` | | `controlplane.serviceAPI.sessionAffinityConfig` | Additional settings for the sessionAffinity | `{}` | -| `controlplane.ingress.enabled` | Enable ingress record generation for controlplane | `false` | +| `controlplane.ingress.enabled` | Enable the shared HTTP, gRPC-Web and native gRPC ingress for controlplane | `false` | | `controlplane.ingress.pathType` | Ingress path type | `ImplementationSpecific` | | `controlplane.ingress.hostname` | Default host for the ingress record | `cp.dev.local` | | `controlplane.ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `""` | | `controlplane.ingress.path` | Default path for the ingress record | `/` | -| `controlplane.ingress.annotations` | Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations. | `{}` | +| `controlplane.ingress.annotations` | Annotations for the shared ingress. NGINX uses HTTP/2 upstream. Add cert-manager annotations here. | See values.yaml | | `controlplane.ingress.tls` | Enable TLS configuration for the host defined at `controlplane.ingress.hostname` parameter | `false` | | `controlplane.ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | | `controlplane.ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | diff --git a/deployment/chainloop/chart_test.go b/deployment/chainloop/chart_test.go new file mode 100644 index 000000000..149b81484 --- /dev/null +++ b/deployment/chainloop/chart_test.go @@ -0,0 +1,145 @@ +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chart_test + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +// Verify the rendered resources, including templated F5 service names and +// overrides. The legacy gRPC and CAS resources keep their own upstreams. +func TestSharedControlplaneIngress(t *testing.T) { + const ( + setFlag = "--set" + setStringFlag = "--set-string" + grpcBackend = "GRPC" + h2cScheme = "h2c" + chartFullname = "test-chainloop" + ) + helm, err := exec.LookPath("helm") + if err != nil { + t.Skip("helm is required for chart rendering tests") + } + for _, tc := range []struct { + name string + args []string + fullname string + backendProtocol string + traefikScheme string + }{ + {name: "defaults", fullname: chartFullname, backendProtocol: grpcBackend, traefikScheme: h2cScheme}, + {name: "custom name", args: []string{setFlag, "fullnameOverride=shared-api"}, fullname: "shared-api", backendProtocol: grpcBackend, traefikScheme: h2cScheme}, + { + name: "common annotations merge", + args: []string{setStringFlag, `commonAnnotations.nginx\.org/http2=false`}, + fullname: chartFullname, backendProtocol: grpcBackend, traefikScheme: h2cScheme, + }, + { + name: "encrypted upstream overrides", + args: []string{ + setStringFlag, `controlplane.ingress.annotations.nginx\.ingress\.kubernetes\.io/backend-protocol=GRPCS`, + setStringFlag, `controlplane.service.annotations.traefik\.ingress\.kubernetes\.io/service\.serversscheme=https`, + }, + fullname: chartFullname, backendProtocol: "GRPCS", traefikScheme: "https", + }, + } { + t.Run(tc.name, func(t *testing.T) { + serviceName := tc.fullname + "-controlplane" + args := append([]string{"template", "test", ".", setFlag, "development=true", + setFlag, "controlplane.ingress.enabled=true", setFlag, "controlplane.ingress.hostname=shared.example.test", + setFlag, "controlplane.ingressAPI.enabled=true", setFlag, "cas.ingressAPI.enabled=true"}, tc.args...) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, helm, args...) + // Local plugins are unrelated to chart rendering and can print + // startup warnings on stdout before the YAML document. + cmd.Env = append(os.Environ(), "HELM_PLUGINS="+t.TempDir()) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + require.NoError(t, err, "%s", stderr.String()) + docs := yaml.NewDecoder(bytes.NewReader(out)) + resources := make(map[string]chartResource) + for { + var resource chartResource + err := docs.Decode(&resource) + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + resources[resource.Kind+"/"+resource.Metadata.Name] = resource + } + ingress, ok := resources["Ingress/"+serviceName] + require.True(t, ok) + require.Equal(t, tc.backendProtocol, ingress.Metadata.Annotations["nginx.ingress.kubernetes.io/backend-protocol"]) + require.Equal(t, serviceName, ingress.Metadata.Annotations["nginx.org/grpc-services"]) + require.Equal(t, "true", ingress.Metadata.Annotations["nginx.org/http2"]) + require.Len(t, ingress.Spec.Rules, 1) + require.Equal(t, "shared.example.test", ingress.Spec.Rules[0].Host) + require.Len(t, ingress.Spec.Rules[0].HTTP.Paths, 1) + path := ingress.Spec.Rules[0].HTTP.Paths[0] + require.Equal(t, "/", path.Path) + require.Equal(t, serviceName, path.Backend.Service.Name) + require.Equal(t, "http", path.Backend.Service.Port.Name) + svc := resources["Service/"+serviceName] + require.Equal(t, tc.traefikScheme, svc.Metadata.Annotations["traefik.ingress.kubernetes.io/service.serversscheme"]) + legacy := resources["Ingress/"+serviceName+"-api"] + require.Equal(t, grpcBackend, legacy.Metadata.Annotations["nginx.ingress.kubernetes.io/backend-protocol"]) + require.Len(t, legacy.Spec.Rules, 1) + require.Equal(t, serviceName+"-api", legacy.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name) + casName := tc.fullname + "-cas-api" + cas := resources["Ingress/"+casName] + require.Equal(t, grpcBackend, cas.Metadata.Annotations["nginx.ingress.kubernetes.io/backend-protocol"]) + require.Len(t, cas.Spec.Rules, 1) + require.Equal(t, casName, cas.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name) + }) + } +} + +type chartResource struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + Annotations map[string]string `yaml:"annotations"` + } `yaml:"metadata"` + Spec struct { + Rules []struct { + Host string `yaml:"host"` + HTTP struct { + Paths []struct { + Path string `yaml:"path"` + Backend struct { + Service struct { + Name string `yaml:"name"` + Port struct { + Name string `yaml:"name"` + } `yaml:"port"` + } `yaml:"service"` + } `yaml:"backend"` + } `yaml:"paths"` + } `yaml:"http"` + } `yaml:"rules"` + } `yaml:"spec"` +} diff --git a/deployment/chainloop/templates/controlplane/ingress.yaml b/deployment/chainloop/templates/controlplane/ingress.yaml index de35e887b..e5725f5ac 100644 --- a/deployment/chainloop/templates/controlplane/ingress.yaml +++ b/deployment/chainloop/templates/controlplane/ingress.yaml @@ -12,13 +12,8 @@ metadata: namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "chainloop.controlplane.labels" . | nindent 4 }} {{- if or .Values.controlplane.ingress.annotations .Values.commonAnnotations }} - annotations: - {{- if .Values.controlplane.ingress.annotations }} - {{- include "common.tplvalues.render" ( dict "value" .Values.controlplane.ingress.annotations "context" $) | nindent 4 }} - {{- end }} - {{- if .Values.commonAnnotations }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} + {{- $annotations := include "common.tplvalues.merge" (dict "values" (list .Values.controlplane.ingress.annotations .Values.commonAnnotations) "context" .) }} + annotations: {{- include "common.tplvalues.render" (dict "value" $annotations "context" $) | nindent 4 }} {{- end }} spec: {{- if and .Values.controlplane.ingress.ingressClassName (eq "true" (include "common.ingress.supportsIngressClassname" .)) }} @@ -62,4 +57,4 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.controlplane.ingress.extraTls "context" $) | nindent 4 }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/deployment/chainloop/values.yaml b/deployment/chainloop/values.yaml index 03865dc18..4fb5d843a 100644 --- a/deployment/chainloop/values.yaml +++ b/deployment/chainloop/values.yaml @@ -345,9 +345,11 @@ controlplane: ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip ## externalTrafficPolicy: Cluster - ## @param controlplane.service.annotations Additional custom annotations for controlplane service + ## @extra controlplane.service.annotations Annotations for the shared HTTP and gRPC service. Traefik uses h2c for HTTP/2 upstream traffic. ## - annotations: {} + annotations: + ## @skip controlplane.service.annotations.traefik.ingress.kubernetes.io/service.serversscheme + traefik.ingress.kubernetes.io/service.serversscheme: h2c ## @param controlplane.service.extraPorts Extra ports to expose in controlplane service (normally used with the `sidecars` value) ## extraPorts: [] @@ -432,7 +434,7 @@ controlplane: ## ref: http://kubernetes.io/docs/user-guide/ingress/ ingress: - ## @param controlplane.ingress.enabled Enable ingress record generation for controlplane + ## @param controlplane.ingress.enabled Enable the shared HTTP, gRPC-Web and native gRPC ingress for controlplane ## enabled: false ## @param controlplane.ingress.pathType Ingress path type @@ -450,7 +452,7 @@ controlplane: ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers ## path: / - ## @param controlplane.ingress.annotations Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations. + ## @extra controlplane.ingress.annotations Annotations for the shared ingress. NGINX uses HTTP/2 upstream for every request. Add cert-manager annotations here. ## Use this parameter to set the required annotations for cert-manager, see ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations ## e.g: @@ -458,7 +460,13 @@ controlplane: ## kubernetes.io/controlplane.ingress.class: nginx ## cert-manager.io/cluster-issuer: cluster-issuer-name ## - annotations: {} + annotations: + ## @skip controlplane.ingress.annotations.nginx.ingress.kubernetes.io/backend-protocol + nginx.ingress.kubernetes.io/backend-protocol: GRPC + ## @skip controlplane.ingress.annotations.nginx.org/grpc-services + nginx.org/grpc-services: '{{ include "chainloop.controlplane.fullname" . }}' + ## @skip controlplane.ingress.annotations.nginx.org/http2 + nginx.org/http2: "true" ## @param controlplane.ingress.tls Enable TLS configuration for the host defined at `controlplane.ingress.hostname` parameter ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf "%s-tls" .Values.controlplane.ingress.hostname }}` ## You can: diff --git a/docs/specs/issue-3628-controlplane-http-dispatch.md b/docs/specs/issue-3628-controlplane-http-dispatch.md new file mode 100644 index 000000000..868a9b5c0 --- /dev/null +++ b/docs/specs/issue-3628-controlplane-http-dispatch.md @@ -0,0 +1,166 @@ +--- +owner: migmartri +ticket: https://github.com/chainloop-dev/chainloop/issues/3628 +prd: +--- + + + +# Spec issue-3628-controlplane-http-dispatch: One control-plane ingress for HTTP and gRPC + +## Summary + +This spec replaces the control-plane design in [Spec issue-3628-single-endpoint](issue-3628-single-endpoint.md). +The HTTP server owns the shared endpoint and routes every request to the existing HTTP, gRPC-Web, or gRPC handlers. +HTTP and gRPC requests can share one HTTP/2 connection in either order. +The chart enables HTTP/2 upstream traffic on the existing HTTP ingress, while the legacy gRPC endpoint remains available. +CAS transport changes remain separate work under the parent ticket. + +## Problem + +Connection classification assigns an entire connection to one server. +A proxy can later reuse that connection for a different request type. +If the first request selects the native gRPC server, later HTTP or gRPC-Web requests fail. +Path rules cannot separate native gRPC from gRPC-Web because both use the same RPC paths. +Operators need one control-plane hostname that serves existing browser and native gRPC clients. + +## Goals and Non-Goals + +- Goal: HTTP, gRPC-Web, and native gRPC work through one control-plane ingress hostname. +- Goal: the same endpoint works directly on VMs, in Compose, and in Kubernetes. +- Goal: existing clients keep their endpoints during migration. +- Non-goal: migrate the service or clients to Connect, or add the Connect protocol. +- Non-goal: change the CAS listeners, ingress, or ByteStream transport in this change. + CAS still requires bounded memory and the performance evidence from the original R-003. +- Non-goal: retire the legacy gRPC endpoint or change client defaults. + +## Requirements + +### R-001: Shared control-plane endpoint +The control plane MUST serve native gRPC, gRPC-Web, HTTP/1.1, and HTTP/2 on its current HTTP endpoint. +- Done when: each request type works directly with plaintext or TLS, and through the shared NGINX ingress path. +- Done when: one HTTP/2 connection serves native gRPC followed by HTTP, gRPC-Web, and CORS preflight, without a reconnect. + The reverse order also works. + +### R-002: Existing behavior +The shared endpoint MUST retain the existing authentication, authorization, validation, status, tracing, and metrics behavior. +- Done when: the authentication matrix in the original R-002 passes through the shared and legacy gRPC endpoints. +- Focused protocol tests do not establish full alignment with this requirement. + +### R-003: Existing endpoints remain available +The legacy gRPC endpoint MUST remain enabled by default, with its own TLS settings and the existing disable flag. +The CAS endpoints MUST retain their current behavior. +- Done when: an upgrade needs no client endpoint changes, and clients can test the shared control-plane hostname separately. + +### R-004: Shared ingress +The chart MUST support one control-plane ingress path for HTTP, gRPC-Web, and native gRPC, with HTTP/2 upstream traffic. +- Done when: the NGINX and Traefik protocol settings target the HTTP service, while legacy ingress settings remain independent. +- Done when: ordinary HTTP clients still receive their existing responses through that ingress. + +## Constraints + +- Keep the existing generated gRPC services, interceptors, HTTP routes, and gRPC-Web wrapper. +- Keep metrics and profiler listeners separate from the public endpoint. +- A proxy must preserve HTTP/2 upstream for native gRPC, even when a browser uses HTTP/1.1 at the edge. +- Do not claim CAS streaming parity from control-plane tests or a transport microbenchmark. + +## Proposal + +Operators enable the existing HTTP ingress to expose all control-plane protocols on one hostname. +Existing gRPC clients can continue to use the legacy hostname. +A client can test the shared hostname without changing other clients. + +The HTTP server handles connection setup, optional TLS, HTTP/2 settings, and stream ownership. +For every request, it selects the native gRPC handler, the existing gRPC-Web wrapper, or an existing HTTP route. +A native gRPC request requires HTTP/2 and a native gRPC content type. +The native gRPC handler uses grpc-go's HTTP adapter and the existing services and interceptors. +The service contains no connection classifier, frame replay, or injected settings exchange. +On shutdown, the HTTP server drains requests before the gRPC handler stops. + +The ingress sends every request on its root path to the shared HTTP service over HTTP/2. +NGINX uses its gRPC upstream setting for that path. +Traefik uses its unencrypted HTTP/2 upstream setting for the service. +F5 NGINX and NGINX Plus use their gRPC service and HTTP/2 settings. +Controller versions can require additional configuration, so the chart documentation states those prerequisites. +TLS between the ingress and the pod requires the controller's encrypted HTTP/2 settings. + +```mermaid +flowchart TD + clients["Browsers, HTTP clients, and native gRPC clients"] + ingress["Optional shared ingress hostname
One root path, HTTP/2 upstream"] + http["HTTP server owns the shared port
HTTP/1.1, HTTP/2, optional TLS
Dispatch every request"] + grpc["Native gRPC over HTTP/2
grpc-go HTTP adapter"] + web["Existing gRPC-Web wrapper"] + routes["Existing HTTP routes"] + rpc["Existing RPC services and interceptors"] + legacy["Optional legacy gRPC listener
Native transport, enabled by default"] + clients --> ingress --> http + clients -->|"Direct VM or Compose access"| http + http --> grpc --> rpc + http --> web --> rpc + http --> routes + clients -->|"Existing gRPC endpoint"| legacy --> rpc +``` + +## Decision Record + +### D-001: HTTP owns the control-plane connection +This replaces the control-plane parts of the original D-014, D-015, and D-016. +The service routes each request through the HTTP server, including all native gRPC requests on the shared endpoint. +- Why: proxies can reuse upstream connections without depending on the first request's protocol. + The standard HTTP server owns all HTTP/2 framing and settings. +- Alternatives considered: + - cmux with a native gRPC fast path: requires separate upstream pools and retains custom handshake code. + - A proxy sidecar: adds a process that this control-plane dispatch does not need. + - Full Connect migration: changes generated service interfaces and middleware for a larger scope. +- Source: [implementation and design discussion](https://github.com/chainloop-dev/chainloop/pull/3647). + +### D-002: One ingress path with HTTP/2 upstream +This replaces the control-plane ingress design in the original D-004. +The shared ingress sends all requests to the HTTP service, without path or content-type splits. +The chart keeps the separate legacy gRPC ingress and its hostname. +- Why: per-request dispatch supports all request types on the same upstream connection. +- Alternatives considered: + - Split RPC paths: native gRPC and gRPC-Web share those paths. + - Switch existing clients to the shared hostname: optional client testing does not require a coordinated migration. +- Source: [ingress reproduction and research](https://github.com/chainloop-dev/chainloop/pull/3647#issuecomment-6272007730). + +### D-003: Defer CAS transport selection +CAS keeps its current native gRPC transport and separate endpoints. +The original CAS requirements remain open under the parent ticket. +- Why: a paused-reader probe showed that the HTTP adapter can read ahead beyond the application's receive pace. + That result prevents an unqualified claim of bounded memory for ByteStream uploads. +- Alternatives considered: + - Apply the control-plane design to CAS now: lacks the original R-003 benchmark and bounded-memory evidence. +- Source: [implementation and design discussion](https://github.com/chainloop-dev/chainloop/pull/3647). + +## Open Questions + +None for this control-plane change. CAS transport selection remains separate work under the parent ticket. + +## Milestones + +1. **Shared server and ingress.** The control plane dispatches every request and the chart exposes the shared hostname. +2. **Behavior qualification.** The full authentication matrix passes, and deployed controller tests check the documented configurations. + +## Risks + +| Risk | Mitigation | +|------|------------| +| The HTTP adapter differs from the native gRPC transport in performance or features. | Retain the legacy endpoint. Measure relevant workloads before client migration. Keep CAS on its current transport. | +| A controller uses HTTP/1.1 upstream or needs extra HTTP/2 settings. | Supply protocol annotations and document prerequisites. Test deployed controller versions before production use. | +| Focused tests miss authentication or authorization differences. | Keep the full original R-002 matrix as a qualification requirement. | diff --git a/docs/specs/issue-3628-single-endpoint.md b/docs/specs/issue-3628-single-endpoint.md index 7437de27a..727d9ca0e 100644 --- a/docs/specs/issue-3628-single-endpoint.md +++ b/docs/specs/issue-3628-single-endpoint.md @@ -22,6 +22,8 @@ When your code goes against this spec, say so in your PR, and supersede the entr # Spec issue-3628: Serve gRPC and HTTP from one endpoint per service +Control-plane design superseded by [Spec issue-3628-controlplane-http-dispatch](issue-3628-controlplane-http-dispatch.md). CAS requirements remain open under the parent ticket. + ## Summary The control plane and the Artifact CAS will each serve gRPC and HTTP on one port. The control plane also serves gRPC-Web on it, as it does today. Today each service has a gRPC port and an HTTP port. An installation thus needs two Services, two ingresses, two hostnames, and two TLS settings per service. The service will accept all connections on its current HTTP port, the one that serves the human-facing hostname today. It will look at each new connection and give it to the gRPC server or to the HTTP server that runs today. The middleware, the authentication, and the native gRPC transport for large CAS streams do not change. The legacy gRPC port stays on by default for a deprecation period, behind a config flag. The Helm chart moves to one Service and one hostname per service. diff --git a/go.mod b/go.mod index 89edcd718..d91a3f4f2 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,6 @@ require ( github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.11.0 github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.11.0 github.com/sigstore/timestamp-authority/v2 v2.1.4 - github.com/soheilhy/cmux v0.1.6-0.20260608033842-7b740c1fa6e0 github.com/spdx/tools-golang v0.5.7 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 @@ -115,7 +114,6 @@ require ( go.uber.org/zap v1.28.0 golang.org/x/crypto v0.58.0 golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba - golang.org/x/net v0.61.0 golang.org/x/oauth2 v0.37.0 golang.org/x/sync v0.24.0 golang.org/x/term v0.47.0 @@ -461,6 +459,7 @@ require ( go4.org v0.0.0-20260112195520-a5071408f32f // indirect goa.design/goa/v3 v3.32.0 // indirect golang.org/x/mod v0.41.0 // indirect + golang.org/x/net v0.61.0 // indirect golang.org/x/sys v0.49.0 // indirect golang.org/x/time v0.16.0 // indirect golang.org/x/tools v0.51.0 // indirect diff --git a/go.sum b/go.sum index 77b0fc0bc..89f3201b3 100644 --- a/go.sum +++ b/go.sum @@ -1189,8 +1189,6 @@ github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.6-0.20260608033842-7b740c1fa6e0 h1:zGDSxPIxTarL5n+Co/R20SbWphTP6htuq+yMxPVS2uE= -github.com/soheilhy/cmux v0.1.6-0.20260608033842-7b740c1fa6e0/go.mod h1:JagfQkcwKpXOzyc2ygh7yP8+5BgvkfOMN8HInUthA5o= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik= github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU= diff --git a/pkg/httpgrpcmux/mux.go b/pkg/httpgrpcmux/mux.go deleted file mode 100644 index 7f408be44..000000000 --- a/pkg/httpgrpcmux/mux.go +++ /dev/null @@ -1,603 +0,0 @@ -// -// Copyright 2026 The Chainloop Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package httpgrpcmux multiplexes a single TCP listener between a native gRPC -// server and an HTTP server, so one port can serve gRPC, gRPC-Web and plain -// HTTP at the same time. -// -// It implements the connection-level split described in -// docs/specs/issue-3628-single-endpoint.md, with the cmux library as the -// classifier (D-016). cmux reads the start of each new connection without -// consuming it and hands the connection to the gRPC or the HTTP listener. A -// service-owned matcher decides native gRPC from the first request's content -// type; every other connection (HTTP/1.x, HTTP/2 without gRPC, gRPC-Web) goes -// to the HTTP listener. A connection classified as gRPC carries only gRPC; a -// native gRPC request that arrives on a connection owned by the HTTP server is -// forwarded to the gRPC server by the HTTP handler. -// -// The matcher speaks only when a connection stays silent for the settings -// grace: gRPC clients (the CLI, Java) wait for the server's HTTP/2 settings -// frame before their first request. Ordinary HTTP/2 clients may also wait for -// this frame. When such a connection goes to HTTP, the matcher consumes its -// SETTINGS acknowledgement and the HTTP listener removes that one frame from -// cmux's replay. The HTTP server owns every subsequent SETTINGS exchange. -// -// The mux implements kratos' transport.Server interface (Start/Stop with -// standard signatures) so it can join the application server lifecycle. -package httpgrpcmux - -import ( - "context" - "errors" - "io" - "net" - "strings" - "sync" - "sync/atomic" - "syscall" - "time" - - "github.com/go-kratos/kratos/v2/log" - "github.com/soheilhy/cmux" - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" -) - -// defaultClassifyTimeout bounds how long the matcher waits for a new -// connection's first request before the connection is closed or handed to the -// HTTP server. It applies to the classification step only: cmux clears it -// when the connection is handed over, so long-lived streams are not affected. -const defaultClassifyTimeout = 10 * time.Second - -// defaultSettingsGrace is how long the matcher waits for the first request -// after the connection preface before it assumes the client waits for the -// server to speak first. Both native gRPC and ordinary HTTP/2 clients may -// wait for the server's SETTINGS before sending their first request. -const defaultSettingsGrace = 250 * time.Millisecond - -// initialMaxFrameSize is the HTTP/2 frame size limit before the server has -// advertised a larger value. Bound allocation before reading frame payloads; -// a LimitedReader alone only bounds bytes consumed, not allocated buffers. -const initialMaxFrameSize = 1 << 14 - -// defaultMaxClassifiedBytes bounds how much of a connection the matcher may -// read while looking for the first request headers. A connection that needs -// more goes to the HTTP server, whose own header limits dispose of it. -const defaultMaxClassifiedBytes = 512 << 10 - -// maxHpackString bounds a single decoded HPACK string in the matcher, so a -// small compressed block cannot expand into one huge header value. -const maxHpackString = 64 << 10 - -// hpackDynamicTableSize is the HPACK dynamic table size the matcher offers, -// the HTTP/2 spec default. -const hpackDynamicTableSize = 4096 - -// defaultMaxInflightClassifications caps how many connections may be in -// classification at the same time; beyond it, connections are shed. -const defaultMaxInflightClassifications = 8192 - -// acceptRetryDelay paces the retry loop after a recoverable accept error such -// as descriptor exhaustion. -const acceptRetryDelay = 5 * time.Millisecond - -// closeDrainBound caps how long Close waits for the cmux serve loop to drain. -// Connections still in classification ride out their own window in the -// background instead of holding up shutdown. -const closeDrainBound = time.Second - -// grpcContentTypePrefix matches every native gRPC content type, including -// application/grpc+proto and content types with parameters. -const grpcContentTypePrefix = "application/grpc" - -// grpcWebContentTypePrefix is NOT part of the gRPC family above: gRPC-Web -// requests must reach the HTTP server, which owns the gRPC-Web wrapper. -const grpcWebContentTypePrefix = "application/grpc-web" - -// headerContentType is the request header that carries the content type. -const headerContentType = "content-type" - -// Options configures a Mux. -type Options struct { - // Logger receives classification debug information. Optional. - Logger log.Logger - // ClassifyTimeout bounds how long the matcher waits for a new - // connection's first request. Defaults to 10s. - ClassifyTimeout time.Duration - // SettingsGrace is how long the matcher waits for the first request - // before it sends the server's HTTP/2 settings frame to waiting - // clients. Defaults to 250ms. - SettingsGrace time.Duration - // MaxInflightClassifications caps how many connections may be in - // classification at the same time; beyond it, connections are shed. - // Defaults to 8192. - MaxInflightClassifications int - // MaxClassifiedBytes bounds how much of a connection the matcher may - // read. Defaults to 512 KiB. - MaxClassifiedBytes int -} - -// Mux splits one listener between a gRPC server and an HTTP server. -type Mux struct { - // root wraps the raw listener with the stopping marker. cmux sub-listeners - // embed the root listener, so whichever server stops first closes it for - // both, and the marker lets every server tell that shutdown apart from a - // fatal accept failure. - root *stoppingListener - cm cmux.CMux - - grpcL net.Listener - httpL net.Listener - - classifyTimeout time.Duration - settingsGrace time.Duration - maxClassifiedBytes int - logger *log.Helper - // admit caps how many connections may be in classification at once. A - // matcher holds a slot while it runs; a connection that cannot get one - // matches no listener, and cmux closes it. - admit chan struct{} - stopping atomic.Bool - serveWG sync.WaitGroup - // serveOnce and closeOnce keep Start and Close idempotent: kratos may - // call them more than once, and wire cleanup runs Close even when Start - // never ran. - serveOnce sync.Once - closeOnce sync.Once -} - -// stoppingListener marks its Mux as stopping before the listener closes. -type stoppingListener struct { - net.Listener - stopping *atomic.Bool -} - -func (l *stoppingListener) Accept() (net.Conn, error) { - c, err := l.Listener.Accept() - if err != nil { - return nil, err - } - return &classificationConn{Conn: c, settingsACKOffset: -1}, nil -} - -func (l *stoppingListener) Close() error { - l.stopping.Store(true) - return l.Listener.Close() -} - -// classificationConn carries handshake state through cmux. Matchers update -// it before cmux publishes the connection to a sub-listener. No shared map -// or state remains after an unaccepted connection closes. -type classificationConn struct { - net.Conn - settingsACKOffset int64 - rejectHTTP bool -} - -// httpListener strips only the acknowledgement of the matcher's SETTINGS. -// Its exact offset is known before handoff, so serving HTTP does not require -// a second frame parser or any inspection of later requests or streams. -type httpListener struct{ net.Listener } - -func (l httpListener) Accept() (net.Conn, error) { - c, err := l.Listener.Accept() - if err != nil { - return nil, err - } - if mc, ok := c.(*cmux.MuxConn); ok { - if state, ok := mc.Conn.(*classificationConn); ok && state.settingsACKOffset >= 0 { - return &httpConn{Conn: c, beforeACK: state.settingsACKOffset}, nil - } - } - return c, nil -} - -type httpConn struct { - net.Conn - beforeACK int64 -} - -func (c *httpConn) Read(p []byte) (int, error) { - if len(p) == 0 { - return 0, nil - } - if c.beforeACK > 0 { - n, err := c.Conn.Read(p[:min(int64(len(p)), c.beforeACK)]) - c.beforeACK -= int64(n) - return n, err - } - if c.beforeACK == 0 { - // The matcher already read and validated this complete, empty ACK. - // It is entirely in cmux's replay, even when the caller reads one - // byte at a time. Leave the next server SETTINGS/ACK pair untouched. - var ack [9]byte - _, err := io.ReadFull(c.Conn, ack[:]) - c.beforeACK = -1 - if err != nil { - return 0, err - } - } - return c.Conn.Read(p) -} - -// New creates a mux over l. Both listeners are usable immediately; -// connections start flowing once Start is called. -func New(l net.Listener, opts *Options) *Mux { - if opts == nil { - opts = &Options{} - } - logger := opts.Logger - if logger == nil { - logger = log.DefaultLogger - } - timeout := opts.ClassifyTimeout - if timeout <= 0 { - timeout = defaultClassifyTimeout - } - maxInflight := opts.MaxInflightClassifications - if maxInflight <= 0 { - maxInflight = defaultMaxInflightClassifications - } - maxBytes := opts.MaxClassifiedBytes - if maxBytes <= 0 { - maxBytes = defaultMaxClassifiedBytes - } - - grace := opts.SettingsGrace - if grace <= 0 { - grace = defaultSettingsGrace - } - - m := &Mux{ - classifyTimeout: timeout, - settingsGrace: grace, - maxClassifiedBytes: maxBytes, - logger: log.NewHelper(logger), - admit: make(chan struct{}, maxInflight), - } - m.root = &stoppingListener{Listener: l, stopping: &m.stopping} - - m.cm = cmux.New(m.root) - // One read deadline bounds the whole classification step of a connection, - // including the TLS handshake, which runs on the first read. - m.cm.SetReadTimeout(timeout) - - // Registration order is match priority: the gRPC matcher runs first, so - // an HTTP/2 connection with a native gRPC first request goes to the gRPC - // listener, and everything else falls through to the HTTP listener. Both - // are MatchWriters so they can reach the connection: the gRPC matcher - // writes the server's settings frame to clients that wait for it, and - // takes its admission slot itself, once the connection prefaced. The - // HTTP chain rejects incomplete handshakes before it reads anything. - m.grpcL = m.cm.MatchWithWriters(m.matchNativeGRPC) - m.httpL = httpListener{m.cm.MatchWithWriters(m.rejectIncompleteHandshake(m.requireTraffic))} - - m.cm.HandleError(func(err error) bool { - var notMatched cmux.ErrNotMatched - if errors.As(err, ¬Matched) { - // A connection that matched no listener was closed; routine. - return true - } - if isAcceptRetryable(err) { - m.logger.Debugf("mux accept error, retrying: %v", err) - time.Sleep(acceptRetryDelay) - return true - } - if errors.Is(err, net.ErrClosed) { - // The root listener was closed: an orderly shutdown, whichever - // path closed it. - return false - } - // A fatal accept failure is not recoverable: log it loudly, the - // process must be restarted. - m.logger.Errorf("mux accept loop terminated: %v", err) - return false - }) - - return m -} - -// GRPC returns the listener that receives connections classified as native -// gRPC: HTTP/2 connections whose first request carries a gRPC content type. -func (m *Mux) GRPC() net.Listener { return m.grpcL } - -// HTTP returns the listener that receives every other connection: HTTP/1.x, -// HTTP/2 without gRPC, and gRPC-Web. -func (m *Mux) HTTP() net.Listener { return m.httpL } - -// Start runs the cmux serve loop until Stop is called. -func (m *Mux) Start(_ context.Context) error { - m.serveOnce.Do(func() { - m.serveWG.Add(1) - go func() { - defer m.serveWG.Done() - // Serve returns when the root listener closes: an orderly - // shutdown, or a fatal accept failure that the error handler - // already logged. It then tears down both sub-listeners and - // closes every connection that no server has accepted yet. - if err := m.cm.Serve(); err != nil && !m.stopping.Load() { - m.logger.Debugf("mux serve loop returned: %v", err) - } - }() - }) - return nil -} - -// Stop closes the underlying listener and both sub-listeners, unblocking any -// Accept call still in progress. -func (m *Mux) Stop(_ context.Context) error { - m.Close() - return nil -} - -// Stopping reports whether Close has been called: a server fed by the mux can -// use it to tell an orderly shutdown apart from a fatal accept-loop failure. -func (m *Mux) Stopping() bool { return m.stopping.Load() } - -// Close stops the mux and waits for the cmux serve loop to drain, bounded by -// closeDrainBound: connections still in classification ride out their own -// classification window in the background, and die with the process. -func (m *Mux) Close() { - m.closeOnce.Do(func() { - // Close the root listener first: Serve unblocks and tears down both - // sub-listeners. cmux.Close then releases the connections that - // already matched but were not accepted by a server. - _ = m.root.Close() - m.cm.Close() - drained := make(chan struct{}) - go func() { - m.serveWG.Wait() - close(drained) - }() - select { - case <-drained: - case <-time.After(closeDrainBound): - } - }) -} - -// IsOrderlyShutdownErr reports whether err is the listener error of this -// mux's orderly shutdown rather than a fatal accept failure. Servers fed by -// the mux use it to keep a lost shutdown race from failing app startup: on a -// fatal accept failure the mux never stops, so the error still surfaces and -// the process restarts instead of serving a dead single port. -func (m *Mux) IsOrderlyShutdownErr(err error) bool { - return m.stopping.Load() && (errors.Is(err, net.ErrClosed) || - errors.Is(err, cmux.ErrServerClosed) || - errors.Is(err, cmux.ErrListenerClosed)) -} - -// rejectIncompleteHandshake closes connections whose injected SETTINGS -// exchange failed. A completed HTTP handshake is safe to hand off because -// httpListener removes the acknowledgement belonging to the matcher. -func (m *Mux) rejectIncompleteHandshake(next cmux.MatchWriter) cmux.MatchWriter { - return func(w io.Writer, r io.Reader) bool { - if conn, ok := w.(*classificationConn); ok && conn.rejectHTTP { - return false - } - return next(w, r) - } -} - -// requireTraffic is the terminal matcher of the HTTP chain: it accepts every -// remaining connection that produced at least one byte. A connection that -// produced none, or whose read failed, matches no listener and is closed by -// cmux. The one-byte read waits for the connection to carry traffic before -// the admission slot is taken: a silent connection buffers nothing and must -// not consume a slot — the slots bound buffered bytes, and the file descriptor -// limit bounds idle connections, as before this package existed. -func (m *Mux) requireTraffic(_ io.Writer, r io.Reader) bool { - // cmux re-serves sniffed bytes between matcher sessions, never within - // one, so no matcher may re-read what it already consumed: this is the - // only read of the HTTP chain. - var b [1]byte - if _, err := io.ReadFull(r, b[:]); err != nil { - return false - } - if !m.tryAcquire() { - return false - } - defer func() { <-m.admit }() - return true -} - -// tryAcquire takes an admission slot when one is free, so cmux closes the -// connection when the classification cap is reached. -func (m *Mux) tryAcquire() bool { - select { - case m.admit <- struct{}{}: - return true - default: - return false - } -} - -// isAcceptRetryable reports whether an accept error is recoverable: read -// timeouts, and descriptor exhaustion, which is temporary on Linux. -func isAcceptRetryable(err error) bool { - return isTimeout(err) || errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE) -} - -func isTimeout(err error) bool { - var ne net.Error - return errors.As(err, &ne) && ne.Timeout() -} - -// matchNativeGRPC reports whether the connection's first request carries a -// native gRPC content type. -// -// Clients that send their first request without waiting classify without a -// single byte written to them. After the settings grace, the matcher sends -// SETTINGS to unblock either kind of waiting HTTP/2 client. Before an HTTP -// handoff it records the ACK's exact replay offset, even if the ACK follows -// the first request. httpListener removes that ACK; native gRPC continues to -// receive the original replay. Incomplete injected handshakes are closed. -// -// Frame parsing and HPACK decoding are the standard HTTP/2 library's; -// everything the matcher reads stays buffered in cmux and is replayed to the -// server that wins the connection. The matcher bounds how much of the -// connection it may read. -func (m *Mux) matchNativeGRPC(w io.Writer, r io.Reader) bool { - // The preface is read before any admission slot is taken: a silent - // connection buffers nothing and must not consume one — the slots bound - // buffered bytes, and the file descriptor limit bounds idle connections. - // A non-prefaced connection (HTTP/1.x, garbage) declines here without - // ever touching admission. - // - // The extra byte in the cap makes any read past it fail instead of - // stopping silently at it: a connection that needs more than the cap to - // classify goes to the HTTP server, whose own limits dispose of it. - lr := &io.LimitedReader{R: r, N: int64(m.maxClassifiedBytes) + 1} - if !matchHTTP2Preface(lr) { - // HTTP/1.x or anything else: the HTTP server produces a proper error - // response for malformed traffic. Nothing was written. - return false - } - - // The connection prefaced, so frames will buffer: take an admission - // slot. A connection that cannot get one matches nothing, and cmux - // closes it. - if !m.tryAcquire() { - return false - } - defer func() { <-m.admit }() - - conn, ok := w.(*classificationConn) - if !ok { - return false - } - - framer := http2.NewFramer(w, lr) - framer.SetMaxReadFrameSize(initialMaxFrameSize) - var contentType string - hdec := hpack.NewDecoder(hpackDynamicTableSize, func(hf hpack.HeaderField) { - if strings.EqualFold(hf.Name, headerContentType) { - contentType = hf.Value - } - }) - hdec.SetMaxStringLength(maxHpackString) - - // Narrow the read deadline cmux set to the classification window down to - // the settings grace: how long to wait for a first request before - // speaking. decline reports no match, and rejects an incomplete handshake - // when the matcher has spoken, so it never reaches net/http. boundary is the - // remaining read budget at the last completed frame: a timeout with a - // smaller budget fired mid-frame. The rule is correct only while the - // framer reads exactly frame-header plus payload with no read-ahead; - // inserting a buffering reader in this chain breaks it. - spoken := false - firstRequest := false - nativeGRPC := false - boundary := lr.N - _ = conn.SetReadDeadline(time.Now().Add(m.settingsGrace)) - decline := func() bool { - if spoken { - conn.rejectHTTP = true - } - return false - } - speak := func() bool { - // The write is bounded on its own: cmux clears only the read - // deadline when the connection is handed over, so a lingering write - // deadline would kill long-lived streams downstream. - _ = conn.SetWriteDeadline(time.Now().Add(m.classifyTimeout)) - // A failed handshake cannot be replayed to HTTP, even if only part - // of the SETTINGS frame was delivered. - spoken = true - err := framer.WriteSettings() - _ = conn.SetWriteDeadline(time.Time{}) - if err != nil { - return false - } - _ = conn.SetReadDeadline(time.Now().Add(m.classifyTimeout)) - return true - } - - for { - offset := int64(m.maxClassifiedBytes) + 1 - lr.N - f, err := framer.ReadFrame() - if err != nil { - if !spoken && isTimeout(err) { - if lr.N < boundary { - // The grace fired mid-frame: a slow sender, not a client - // waiting for the server's settings. The framer cannot - // resume a half-read frame, and speaking now would feed - // it the rest of the frame as garbage. Decline without - // speaking: the replay holds everything the client sent, - // and the HTTP server serves the request through the - // documented fallback (D-014). - return false - } - // The client is waiting for the server to speak first. - if !speak() { - return decline() - } - continue - } - // Read failure, budget exhausted, or deadline: an incomplete - // injected handshake cannot go to the HTTP server. - return decline() - } - boundary = lr.N - if sf, ok := f.(*http2.SettingsFrame); ok && sf.IsAck() && spoken && conn.settingsACKOffset < 0 { - conn.settingsACKOffset = offset - } - // HEADERS and CONTINUATION are the only frames that carry header - // block fragments; SETTINGS (including the acknowledgement of the - // injected frame), PING, WINDOW_UPDATE, ... are skipped and stay in - // the replay: the gRPC server ignores unsolicited acknowledgements. - if hb, ok := f.(interface{ HeaderBlockFragment() []byte }); ok && !firstRequest { - if _, err := hdec.Write(hb.HeaderBlockFragment()); err != nil { - return decline() - } - if f.Header().Flags&http2.FlagHeadersEndHeaders != 0 { - if err := hdec.Close(); err != nil { - return decline() - } - firstRequest = true - nativeGRPC = IsNativeGRPCContentType(contentType) - } - } - if firstRequest { - if nativeGRPC { - return true - } - if !spoken || conn.settingsACKOffset >= 0 { - return false - } - } - } -} - -// matchHTTP2Preface is cmux's HTTP/2 preface matcher: it reports whether the -// connection starts with the exact HTTP/2 client preface bytes, without -// over-consuming. Everything it reads stays buffered in cmux and is replayed -// to the server. -var matchHTTP2Preface = cmux.HTTP2() - -// IsNativeGRPCContentType reports whether the given content type identifies a -// native gRPC request, as opposed to a gRPC-Web or plain HTTP one. The HTTP -// server uses it to forward native gRPC requests that arrive on a connection -// the classifier handed to the HTTP side. -func IsNativeGRPCContentType(contentType string) bool { - return hasFoldPrefix(contentType, grpcContentTypePrefix) && - !hasFoldPrefix(contentType, grpcWebContentTypePrefix) -} - -// hasFoldPrefix reports whether s starts with prefix, ignoring case, without -// allocating a copy of s. -func hasFoldPrefix(s, prefix string) bool { - return len(s) >= len(prefix) && strings.EqualFold(s[:len(prefix)], prefix) -} diff --git a/pkg/httpgrpcmux/mux_test.go b/pkg/httpgrpcmux/mux_test.go deleted file mode 100644 index e38362a61..000000000 --- a/pkg/httpgrpcmux/mux_test.go +++ /dev/null @@ -1,1048 +0,0 @@ -// -// Copyright 2026 The Chainloop Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package httpgrpcmux - -import ( - "bytes" - "context" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "fmt" - "io" - "math/big" - "net" - "net/http" - "strings" - "testing" - "testing/iotest" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/health" - "google.golang.org/grpc/health/grpc_health_v1" -) - -// Header names and content types used across the tests. -const ( - hdrMethod = ":method" - hdrContentType = "content-type" - hdrMethodPost = "POST" - ctGRPC = "application/grpc" - ctGRPCProto = "application/grpc+proto" - ctGRPCWebProto = "application/grpc-web+proto" - alpnHTTP1 = "http/1.1" -) - -// startMux creates a real TCP listener, wraps it in a mux and starts it. -// It returns the mux and the client-side dial function. -func startMux(t *testing.T, opts *Options) (*Mux, func() (net.Conn, error)) { - t.Helper() - - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - t.Cleanup(func() { _ = l.Close() }) - - m := New(l, opts) - require.NoError(t, m.Start(context.Background())) - t.Cleanup(func() { m.Close() }) - - dial := func() (net.Conn, error) { return net.Dial("tcp", l.Addr().String()) } - return m, dial -} - -// acceptWithTimeout waits for a connection on the given listener. -func acceptWithTimeout(t *testing.T, l net.Listener) net.Conn { - t.Helper() - type result struct { - c net.Conn - err error - } - ch := make(chan result, 1) - go func() { - c, err := l.Accept() - ch <- result{c, err} - }() - select { - case r := <-ch: - require.NoError(t, r.err) - return r.c - case <-time.After(5 * time.Second): - require.Fail(t, "timed out waiting for the connection to be classified") - return nil - } -} - -// writeClientStream sends an HTTP/2 client preface, a client SETTINGS frame -// and one HEADERS frame carrying the given headers. -func writeClientStream(t *testing.T, c net.Conn, contentType string, extraHeaders ...hpack.HeaderField) { - t.Helper() - - _, err := io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - writeRequestFrames(t, c, contentType, extraHeaders...) -} - -// encodeRequestHeaders returns the HPACK-encoded header block of a request -// with the given content type. -func encodeRequestHeaders(t *testing.T, contentType string, extraHeaders ...hpack.HeaderField) []byte { - t.Helper() - - var buf bytes.Buffer - enc := hpack.NewEncoder(&buf) - for _, hf := range append([]hpack.HeaderField{ - {Name: hdrMethod, Value: hdrMethodPost}, - {Name: ":scheme", Value: "http"}, - {Name: ":path", Value: "/controlplane.v1.FooService/Bar"}, - {Name: hdrContentType, Value: contentType}, - }, extraHeaders...) { - require.NoError(t, enc.WriteField(hf)) - } - return buf.Bytes() -} - -// writeRequestFrames sends a client SETTINGS frame and one HEADERS frame -// carrying the given headers: the request flight after the connection -// preface. -func writeRequestFrames(t *testing.T, c net.Conn, contentType string, extraHeaders ...hpack.HeaderField) { - t.Helper() - - writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - writeFrame(t, c, http2.FrameHeaders, http2.FlagHeadersEndHeaders, 1, - encodeRequestHeaders(t, contentType, extraHeaders...)) -} - -func writeFrame(t *testing.T, c net.Conn, frameType http2.FrameType, flags http2.Flags, streamID uint32, payload []byte) { - t.Helper() - - hdr := []byte{ - byte(len(payload) >> 16), byte(len(payload) >> 8), byte(len(payload)), - byte(frameType), byte(flags), - byte(streamID >> 24), byte(streamID >> 16), byte(streamID >> 8), byte(streamID), - } - _, err := c.Write(hdr) - require.NoError(t, err) - if len(payload) > 0 { - _, err = c.Write(payload) - require.NoError(t, err) - } -} - -// readEOF asserts that the peer closed the connection. -func readEOF(t *testing.T, c net.Conn, within time.Duration) { - t.Helper() - require.NoError(t, c.SetReadDeadline(time.Now().Add(within))) - _, err := c.Read(make([]byte, 1)) - require.Error(t, err) - assert.False(t, isTimeout(err), "the connection must be closed, not merely silent") -} - -// requireAdmissionParked waits until a connection holds an admission slot. -func requireAdmissionParked(t *testing.T, m *Mux, msg string) { - t.Helper() - require.Eventually(t, func() bool { - select { - case m.admit <- struct{}{}: - // The slot was free: put the token back for the matchers. - <-m.admit - return false - default: - return true - } - }, 2*time.Second, 5*time.Millisecond, msg) -} - -func TestClassifyHTTP1(t *testing.T) { - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - request := "GET /statusz HTTP/1.1\r\nHost: example.com\r\n\r\n" - _, err = io.WriteString(c, request) - require.NoError(t, err) - - got := acceptWithTimeout(t, m.HTTP()) - defer got.Close() - - require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) - data := make([]byte, len(request)) - _, err = io.ReadFull(got, data) - require.NoError(t, err) - assert.Equal(t, request, string(data), "the HTTP server must receive the request intact") -} - -func TestClassifyShortHTTP10Request(t *testing.T) { - // A request shorter than the HTTP/2 preface must not deadlock the - // matcher waiting for 24 bytes. - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - request := "GET / HTTP/1.0\r\n\r\n" // 18 bytes - _, err = io.WriteString(c, request) - require.NoError(t, err) - - got := acceptWithTimeout(t, m.HTTP()) - defer got.Close() - - require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) - data := make([]byte, len(request)) - _, err = io.ReadFull(got, data) - require.NoError(t, err) - assert.Equal(t, request, string(data), "short request not replayed intact") -} - -func TestClassifyNativeGRPC(t *testing.T) { - for _, ct := range []string{ - ctGRPC, - ctGRPCProto, - "application/grpc+json", - ctGRPC + "; charset=utf-8", // content type with parameters - } { - t.Run(ct, func(t *testing.T) { - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - writeClientStream(t, c, ct) - - got := acceptWithTimeout(t, m.GRPC()) - defer got.Close() - - // The gRPC server must see the preface and the request again. - require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) - replayed := make([]byte, len(http2.ClientPreface)) - _, err = io.ReadFull(got, replayed) - require.NoError(t, err) - assert.Equal(t, http2.ClientPreface, string(replayed), "preface not replayed intact") - }) - } -} - -func TestClassifyGRPCWebGoesToHTTP(t *testing.T) { - for _, ct := range []string{ - "application/grpc-web", - ctGRPCWebProto, - "application/grpc-web+json", - } { - t.Run(ct, func(t *testing.T) { - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - writeClientStream(t, c, ct) - - got := acceptWithTimeout(t, m.HTTP()) - got.Close() - }) - } -} - -func TestClassifyPlainH2GoesToHTTP(t *testing.T) { - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - writeClientStream(t, c, "application/json") - - got := acceptWithTimeout(t, m.HTTP()) - got.Close() -} - -func TestClassifySkipsIntermediateFrames(t *testing.T) { - // WINDOW_UPDATE frames between the preface and the first HEADERS must - // not disturb the classification. - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - writeFrame(t, c, http2.FrameWindowUpdate, 0, 0, []byte{0, 0, 0, 1}) - writeFrame(t, c, http2.FrameWindowUpdate, 0, 1, []byte{0, 0, 0, 2}) - - var buf bytes.Buffer - enc := hpack.NewEncoder(&buf) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) - writeFrame(t, c, http2.FrameHeaders, http2.FlagHeadersEndHeaders, 1, buf.Bytes()) - - got := acceptWithTimeout(t, m.GRPC()) - got.Close() -} - -func TestClassifyPaddedAndPriorityHeaders(t *testing.T) { - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - - var buf bytes.Buffer - enc := hpack.NewEncoder(&buf) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrMethod, Value: hdrMethodPost})) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) - - // PADDED (0x8) + PRIORITY (0x20) + END_HEADERS (0x4) - // Payload: [pad length][stream dependency][weight][fragment][padding] - frag := buf.Bytes() - payload := make([]byte, 0, 6+len(frag)) - payload = append(payload, 2) // pad length = 2 - payload = append(payload, 0, 0, 0, 0, 16) // stream dep + weight - payload = append(payload, frag...) // header block fragment - payload = append(payload, 0, 0) // padding - writeFrame(t, c, http2.FrameHeaders, http2.FlagHeadersPadded|http2.FlagHeadersPriority|http2.FlagHeadersEndHeaders, 1, payload) - - got := acceptWithTimeout(t, m.GRPC()) - got.Close() -} - -// TestMatcherDoesNotSpeakFirstForImmediateClients proves the classification -// writes nothing to a connection whose first request arrives without waiting. -// Bytes written while classifying are answered by the client, and the answer -// reaches whichever server ends up owning the connection. -func TestMatcherDoesNotSpeakFirstForImmediateClients(t *testing.T) { - for _, tc := range []struct { - name string - ct string - grpc bool - }{ - {"native gRPC", ctGRPCProto, true}, - {"gRPC-Web", ctGRPCWebProto, false}, - {"plain HTTP/2", "application/json", false}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - // A long grace keeps the matcher silent for immediate requests. - m, dial := startMux(t, &Options{SettingsGrace: time.Second}) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - writeClientStream(t, c, tc.ct) - - // The mux must stay silent while it classifies. - require.NoError(t, c.SetReadDeadline(time.Now().Add(300*time.Millisecond))) - _, err = c.Read(make([]byte, 1)) - require.Error(t, err, "the mux must not write during classification") - assert.True(t, isTimeout(err), "the read must time out, not fail: %v", err) - - if tc.grpc { - got := acceptWithTimeout(t, m.GRPC()) - got.Close() - } else { - got := acceptWithTimeout(t, m.HTTP()) - got.Close() - } - }) - } -} - -// TestSettingsWaitingClientClassifiesNative covers a client that waits for the -// server's HTTP/2 settings frame before it sends its first request: the gRPC -// Go client works this way. The matcher sends an empty settings frame after -// the grace, the client acknowledges it and sends its request, and the -// connection goes to the gRPC side with the acknowledgement in the replay: -// the gRPC server ignores acknowledgements of settings it never sent. -func TestSettingsWaitingClientClassifiesNative(t *testing.T) { - m, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: 300 * time.Millisecond}) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - - // After the grace, the matcher sends its empty settings frame. - require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) - injected := make([]byte, 9) - _, err = io.ReadFull(c, injected) - require.NoError(t, err) - assert.Equal(t, []byte{0, 0, 0, byte(http2.FrameSettings), 0, 0, 0, 0, 0}, injected, - "the matcher must send an empty settings frame to a waiting client") - - // The client acknowledges the injected settings and sends its request: - // the preface was already sent, only the request flight follows. - writeFrame(t, c, http2.FrameSettings, http2.FlagSettingsAck, 0, nil) - writeRequestFrames(t, c, ctGRPCProto) - - got := acceptWithTimeout(t, m.GRPC()) - defer got.Close() - - // The gRPC server sees the whole stream: the preface, the client - // settings, the acknowledgement of the injected frame, and the request. - require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) - replay := make([]byte, len(http2.ClientPreface)) - _, err = io.ReadFull(got, replay) - require.NoError(t, err) - assert.Equal(t, http2.ClientPreface, string(replay), "preface not replayed intact") - - // The matcher must clear the write deadline its settings frame was - // bounded by: cmux clears only the read deadline at handover, and a - // lingering write deadline would fail every later write on the - // connection. Write from the server past the classification window and - // expect the client to receive it. - time.Sleep(450 * time.Millisecond) - _, err = got.Write([]byte("still-writable")) - require.NoError(t, err, "a write deadline survived the handover") - require.NoError(t, c.SetReadDeadline(time.Now().Add(time.Second))) - tail := make([]byte, len("still-writable")) - _, err = io.ReadFull(c, tail) - require.NoError(t, err) - assert.Equal(t, "still-writable", string(tail)) -} - -// HTTP/2 permits any client to wait for server SETTINGS before its request. -// Exercise real net/http handoff, including the ACK of net/http's subsequent -// SETTINGS and a second request, so an extra replayed ACK cannot go unnoticed. -func TestSettingsWaitingHTTPClient(t *testing.T) { - for _, encrypted := range []bool{false, true} { - for _, contentType := range []string{"application/json", ctGRPCWebProto} { - for _, ackFirst := range []bool{false, true} { - name := fmt.Sprintf("TLS=%t/content-type=%s/ACK-first=%t", encrypted, contentType, ackFirst) - t.Run(name, func(t *testing.T) { - t.Parallel() - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - t.Cleanup(func() { _ = l.Close() }) - root := l - if encrypted { - root = tls.NewListener(l, &tls.Config{ - Certificates: []tls.Certificate{selfSignedCert(t)}, - MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", alpnHTTP1}, - }) - } - m := New(root, &Options{SettingsGrace: 40 * time.Millisecond, ClassifyTimeout: 2 * time.Second}) - require.NoError(t, m.Start(context.Background())) - t.Cleanup(m.Close) - srv := &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-Request-Content-Type", r.Header.Get(hdrContentType)) - w.WriteHeader(http.StatusOK) - }), - ReadHeaderTimeout: time.Second, - Protocols: new(http.Protocols), - } - srv.Protocols.SetHTTP1(true) - srv.Protocols.SetUnencryptedHTTP2(true) - go func() { _ = srv.Serve(m.HTTP()) }() - t.Cleanup(func() { _ = srv.Close() }) - - var c net.Conn - if encrypted { - c, err = tls.Dial("tcp", l.Addr().String(), &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // self-signed test certificate - MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2"}, - }) - } else { - c, err = net.Dial("tcp", l.Addr().String()) - } - require.NoError(t, err) - t.Cleanup(func() { _ = c.Close() }) - require.NoError(t, c.SetDeadline(time.Now().Add(5*time.Second))) - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - fr := http2.NewFramer(c, c) - fr.ReadMetaHeaders = hpack.NewDecoder(4096, nil) - require.NoError(t, fr.WriteSettings()) - frame, err := fr.ReadFrame() - require.NoError(t, err) - settings, ok := frame.(*http2.SettingsFrame) - require.True(t, ok, "expected injected SETTINGS, got %T", frame) - require.False(t, settings.IsAck()) - - var block bytes.Buffer - enc := hpack.NewEncoder(&block) - writeRequest := func(streamID uint32) { - block.Reset() - method, scheme := "GET", "http" - if contentType == ctGRPCWebProto { - method = hdrMethodPost - } - if encrypted { - scheme = "https" - } - for _, hf := range []hpack.HeaderField{ - {Name: hdrMethod, Value: method}, - {Name: ":scheme", Value: scheme}, - {Name: ":authority", Value: l.Addr().String()}, - {Name: ":path", Value: "/statusz"}, - {Name: hdrContentType, Value: contentType}, - } { - require.NoError(t, enc.WriteField(hf)) - } - require.NoError(t, fr.WriteHeaders(http2.HeadersFrameParam{ - StreamID: streamID, BlockFragment: block.Bytes(), EndHeaders: true, EndStream: true, - })) - } - if ackFirst { - require.NoError(t, fr.WriteSettingsAck()) - } - writeRequest(1) - // Include another frame before the ACK to exercise its exact - // offset rather than assuming it immediately follows SETTINGS. - require.NoError(t, fr.WritePing(false, [8]byte{1})) - if !ackFirst { - require.NoError(t, fr.WriteSettingsAck()) - } - - serverSettings := 0 - readResponse := func(streamID uint32) { - for { - frame, err := fr.ReadFrame() - require.NoError(t, err) - switch f := frame.(type) { - case *http2.SettingsFrame: - if !f.IsAck() { - serverSettings++ - require.NoError(t, fr.WriteSettingsAck()) - } - case *http2.MetaHeadersFrame: - if f.StreamID == streamID { - assert.Equal(t, "200", f.PseudoValue("status")) - assert.Contains(t, f.Fields, hpack.HeaderField{Name: "x-request-content-type", Value: contentType}) - return - } - case *http2.GoAwayFrame: - require.FailNow(t, "unexpected GOAWAY", "%v: %s", f.ErrCode, f.DebugData()) - } - } - } - readResponse(1) - writeRequest(3) - readResponse(3) - assert.Equal(t, 1, serverSettings, "the HTTP server must own its SETTINGS/ACK exchange") - }) - } - } - } -} - -func TestHTTPReplayRemovesOnlyMatcherACK(t *testing.T) { - var wire bytes.Buffer - wire.WriteString(http2.ClientPreface) - fr := http2.NewFramer(&wire, nil) - require.NoError(t, fr.WriteSettings()) - prefix := wire.String() - require.NoError(t, fr.WriteSettingsAck()) - require.NoError(t, fr.WritePing(false, [8]byte{1, 2, 3})) - require.NoError(t, fr.WriteSettingsAck()) // belongs to the HTTP server - tail := wire.Bytes()[len(prefix)+9:] - for _, oneByte := range []bool{false, true} { - var reader io.Reader = bytes.NewReader(wire.Bytes()) - if oneByte { - reader = iotest.OneByteReader(reader) - } - c := &httpConn{Conn: &readerConn{reader: reader}, beforeACK: int64(len(prefix))} - got, err := io.ReadAll(c) - require.NoError(t, err) - assert.Equal(t, prefix+string(tail), string(got)) - } -} - -func TestSettingsWaitingHTTPWithoutACKDropped(t *testing.T) { - _, dial := startMux(t, &Options{SettingsGrace: 40 * time.Millisecond, ClassifyTimeout: 150 * time.Millisecond}) - c, err := dial() - require.NoError(t, err) - defer c.Close() - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = io.ReadFull(c, make([]byte, 9)) - require.NoError(t, err) - writeRequestFrames(t, c, ctGRPCWebProto) - // The first request alone cannot complete handoff: an ACK still pending - // on the wire would belong to the matcher rather than the HTTP server. - readEOF(t, c, 2*time.Second) -} - -type readerConn struct { - net.Conn - reader io.Reader -} - -func (c *readerConn) Read(p []byte) (int, error) { return c.reader.Read(p) } - -// A frame length must be rejected before any read of its advertised payload. -// The sentinel makes this deterministic without process-wide memory or timing -// assertions: the old Framer allocated first, then reached the sentinel. -func TestOversizedFrameRejectedBeforePayloadRead(t *testing.T) { - for _, size := range []uint32{initialMaxFrameSize + 1, (1 << 24) - 1} { - t.Run(fmt.Sprint(size), func(t *testing.T) { - var wire bytes.Buffer - wire.WriteString(http2.ClientPreface) - require.NoError(t, http2.NewFramer(&wire, nil).WriteSettings()) - wire.Write([]byte{byte(size >> 16), byte(size >> 8), byte(size), 0xfe, 0, 0, 0, 0, 0}) - payloadRead := false - reader := io.MultiReader(&wire, testReaderFunc(func([]byte) (int, error) { - payloadRead = true - return 0, io.EOF - })) - server, client := net.Pipe() - defer server.Close() - defer client.Close() - m := &Mux{ - maxClassifiedBytes: defaultMaxClassifiedBytes, - settingsGrace: time.Second, - admit: make(chan struct{}, 1), - } - assert.False(t, m.matchNativeGRPC(&classificationConn{Conn: server, settingsACKOffset: -1}, reader)) - assert.False(t, payloadRead, "oversized frame header must fail before its payload buffer is read") - }) - } -} - -type testReaderFunc func([]byte) (int, error) - -func (f testReaderFunc) Read(p []byte) (int, error) { return f(p) } - -// TestOversizedHeaderBlockDeclined proves a connection whose first request -// needs more than the classification byte budget goes to the HTTP server, -// whose own header limits dispose of it. -func TestOversizedHeaderBlockDeclined(t *testing.T) { - m, dial := startMux(t, &Options{MaxClassifiedBytes: 1024}) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - big := strings.Repeat("a", 2048) - writeClientStream(t, c, ctGRPCProto, hpack.HeaderField{Name: "x-big", Value: big}) - - got := acceptWithTimeout(t, m.HTTP()) - got.Close() -} - -// TestHugeHPACKStringDeclined proves a single HPACK string beyond the decoder -// bound makes the matcher decline, even when the whole block fits the byte -// budget: a small compressed block must not expand into a huge header value. -func TestHugeHPACKStringDeclined(t *testing.T) { - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - - var buf bytes.Buffer - enc := hpack.NewEncoder(&buf) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: hdrContentType, Value: ctGRPCProto})) - require.NoError(t, enc.WriteField(hpack.HeaderField{Name: "x-big", Value: strings.Repeat("a", 70<<10)})) - - // The block spans the HEADERS frame and CONTINUATION frames: no single - // frame may exceed the HTTP/2 frame size limit. - block := buf.Bytes() - const maxFragment = 1 << 14 - first := block - if len(first) > maxFragment { - first = block[:maxFragment] - } - writeFrame(t, c, http2.FrameHeaders, 0, 1, first) - for rest := block[len(first):]; len(rest) > 0; { - frag := rest - if len(frag) > maxFragment { - frag = rest[:maxFragment] - } - rest = rest[len(frag):] - flags := http2.Flags(0) - if len(rest) == 0 { - flags = http2.FlagContinuationEndHeaders - } - writeFrame(t, c, http2.FrameContinuation, flags, 1, frag) - } - - got := acceptWithTimeout(t, m.HTTP()) - got.Close() -} - -// TestAdmissionControlSheds proves a connection is shed when too many -// connections are in classification: it matches no listener and cmux closes -// it, while the connection that holds the slot stays open. -func TestAdmissionControlSheds(t *testing.T) { - m, dial := startMux(t, &Options{MaxInflightClassifications: 1, ClassifyTimeout: time.Second}) - - // The first connection parks in classification: it sent a preface but no - // request, so the matcher holds the only slot waiting for frames. - parked, err := dial() - require.NoError(t, err) - defer parked.Close() - _, err = io.WriteString(parked, http2.ClientPreface) - require.NoError(t, err) - requireAdmissionParked(t, m, "the first connection never parked in classification") - - // The second connection prefaced but cannot get a slot in either - // matcher, so cmux closes it without parsing anything from it. - shed, err := dial() - require.NoError(t, err) - _, err = io.WriteString(shed, http2.ClientPreface) - require.NoError(t, err) - readEOF(t, shed, time.Second) - - // The parked connection is still open past its grace: it received the - // injected settings frame, and no request follows. - require.NoError(t, parked.SetReadDeadline(time.Now().Add(2*time.Second))) - injected := make([]byte, 9) - _, err = io.ReadFull(parked, injected) - require.NoError(t, err) - assert.Equal(t, []byte{0, 0, 0, byte(http2.FrameSettings), 0, 0, 0, 0, 0}, injected, - "the matcher must send an empty settings frame to the parked connection") - require.NoError(t, parked.SetReadDeadline(time.Now().Add(300*time.Millisecond))) - _, err = parked.Read(make([]byte, 1)) - require.Error(t, err) - assert.True(t, isTimeout(err), "the parked connection must stay open: %v", err) -} - -// TestIdleConnectionDropped proves a connection that never sends anything is -// closed at the end of the classification window: both matchers decline it. -func TestIdleConnectionDropped(t *testing.T) { - _, dial := startMux(t, &Options{ClassifyTimeout: 300 * time.Millisecond}) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - readEOF(t, c, 2*time.Second) -} - -// TestStalledAfterPrefaceDropped proves a connection that sends a preface and -// then stalls is closed at the end of its classification window: the matcher -// speaks to it after the grace, but the handshake never completes. -func TestStalledAfterPrefaceDropped(t *testing.T) { - _, dial := startMux(t, &Options{SettingsGrace: 100 * time.Millisecond, ClassifyTimeout: 300 * time.Millisecond}) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - - // The matcher speaks after the grace; the connection never sends its - // request and is dropped at the classification deadline. - require.NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) - _, err = io.ReadFull(c, make([]byte, 9)) - require.NoError(t, err) - readEOF(t, c, 2*time.Second) -} - -// TestSlowMidFrameClientTakesFallback proves a client whose first request -// straddles the settings grace is served, not dropped: the grace fires -// mid-frame, the matcher declines without speaking, and the intact replay -// lets the HTTP server parse the completed request through the documented -// fallback (D-014). The framer cannot resume a half-read frame, so speaking -// there would desynchronize it. -func TestSlowMidFrameClientTakesFallback(t *testing.T) { - m, dial := startMux(t, &Options{SettingsGrace: 150 * time.Millisecond, ClassifyTimeout: 2 * time.Second}) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - writeFrame(t, c, http2.FrameSettings, 0, 0, nil) - - // Send the HEADERS frame header and half of its payload, then stall past - // the grace. - block := encodeRequestHeaders(t, ctGRPCProto) - hdr := []byte{ - byte(len(block) >> 16), byte(len(block) >> 8), byte(len(block)), - byte(http2.FrameHeaders), byte(http2.FlagHeadersEndHeaders), - 0, 0, 0, 1, - } - _, err = c.Write(hdr) - require.NoError(t, err) - _, err = c.Write(block[:len(block)/2]) - require.NoError(t, err) - time.Sleep(400 * time.Millisecond) - - // The rest of the request arrives after the grace. - _, err = c.Write(block[len(block)/2:]) - require.NoError(t, err) - - // The connection is not dropped: the HTTP server gets - // it with the whole stream replayed. - got := acceptWithTimeout(t, m.HTTP()) - defer got.Close() - require.NoError(t, got.SetReadDeadline(time.Now().Add(time.Second))) - replay := make([]byte, len(http2.ClientPreface)) - _, err = io.ReadFull(got, replay) - require.NoError(t, err) - assert.Equal(t, http2.ClientPreface, string(replay), "preface not replayed intact") -} - -func TestCloseClosesQueuedConnections(t *testing.T) { - m, dial := startMux(t, &Options{ClassifyTimeout: 300 * time.Millisecond}) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - - writeClientStream(t, c, ctGRPCProto) - - // Leave the classified connection queued: no server accepts it. - // Classification completes in microseconds locally; 100ms is a wide - // margin. - time.Sleep(100 * time.Millisecond) - m.Close() - - readEOF(t, c, 2*time.Second) -} - -func TestCloseUnblocksAccept(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - m := New(l, nil) - require.NoError(t, m.Start(context.Background())) - - errCh := make(chan error, 2) - for _, lis := range []net.Listener{m.GRPC(), m.HTTP()} { - go func(lis net.Listener) { - _, err := lis.Accept() - errCh <- err - }(lis) - } - - m.Close() - - for i := 0; i < 2; i++ { - select { - case err := <-errCh: - require.True(t, m.IsOrderlyShutdownErr(err), "unexpected accept error: %v", err) - case <-time.After(2 * time.Second): - require.Fail(t, "Accept was not unblocked by Close") - } - } -} - -// TestSubListenerCloseMarksStopping proves the shutdown race tolerance from -// the other side: a server that stops first closes the root listener through -// its sub-listener, and the mux reports an orderly shutdown so the remaining -// server can tell it apart from a fatal accept failure. -func TestSubListenerCloseMarksStopping(t *testing.T) { - m, dial := startMux(t, nil) - - c, err := dial() - require.NoError(t, err) - defer c.Close() - writeClientStream(t, c, ctGRPCProto) - - require.NoError(t, m.GRPC().Close()) - assert.True(t, m.Stopping(), "closing a sub-listener must mark the mux as stopping") - - errCh := make(chan error, 1) - go func() { - _, err := m.HTTP().Accept() - errCh <- err - }() - select { - case err := <-errCh: - require.True(t, m.IsOrderlyShutdownErr(err), "unexpected accept error: %v", err) - case <-time.After(2 * time.Second): - require.Fail(t, "the HTTP listener did not close with the mux") - } -} - -// TestCloseReturnsWhileConnParked proves Close does not ride out an idle -// classification beyond the classification window. -func TestCloseReturnsWhileConnParked(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - m := New(l, &Options{ClassifyTimeout: 300 * time.Millisecond, MaxInflightClassifications: 1}) - require.NoError(t, m.Start(context.Background())) - - c, err := net.Dial("tcp", l.Addr().String()) - require.NoError(t, err) - defer c.Close() - - // A connection that produced a byte parks in classification while it - // holds the admission slot: exactly the state Close must not ride out. A - // silent connection holds no slot at all. - _, err = io.WriteString(c, http2.ClientPreface) - require.NoError(t, err) - requireAdmissionParked(t, m, "the connection never parked in classification") - start := time.Now() - m.Close() - require.Less(t, time.Since(start), 2*time.Second, "Close rode out the classification window") -} - -func TestTLSALPNClassification(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - defer l.Close() - - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{selfSignedCert(t)}, - MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", alpnHTTP1}, - } - m := New(tls.NewListener(l, tlsConfig), nil) - require.NoError(t, m.Start(context.Background())) - t.Cleanup(m.Close) - - dialTLS := func(t *testing.T, alpn string) net.Conn { - t.Helper() - conn, err := tls.Dial("tcp", l.Addr().String(), &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // test server with a self-signed certificate - NextProtos: []string{alpn}, - MinVersion: tls.VersionTLS12, - }) - require.NoError(t, err) - return conn - } - - // ALPN h2 with a gRPC first request -> gRPC server. - c := dialTLS(t, "h2") - writeClientStream(t, c, ctGRPCProto) - got := acceptWithTimeout(t, m.GRPC()) - got.Close() - c.Close() - - // ALPN http/1.1 -> HTTP server. - c = dialTLS(t, alpnHTTP1) - _, err = io.WriteString(c, "GET /statusz HTTP/1.1\r\nHost: x\r\n\r\n") - require.NoError(t, err) - got = acceptWithTimeout(t, m.HTTP()) - got.Close() - c.Close() -} - -func selfSignedCert(t *testing.T) tls.Certificate { - t.Helper() - - key, err := rsa.GenerateKey(rand.Reader, 2048) - require.NoError(t, err) - tmpl := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{CommonName: "localhost"}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), - IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, - DNSNames: []string{"localhost"}, - } - der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) - require.NoError(t, err) - return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} -} - -// TestGRPCServeOverMux runs a real grpc-go server and client through the mux, -// over plaintext and over TLS: the exact wiring the control plane uses. It -// guards the handoff of matched connections to grpc.Server.Serve. -func TestGRPCServeOverMux(t *testing.T) { - for _, tc := range []struct { - name string - tls bool - }{ - {"plaintext", false}, - {"TLS", true}, - } { - t.Run(tc.name, func(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - - root := l - if tc.tls { - root = tls.NewListener(l, &tls.Config{ - Certificates: []tls.Certificate{selfSignedCert(t)}, - MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", "http/1.1"}, - }) - } - m := New(root, nil) - require.NoError(t, m.Start(context.Background())) - t.Cleanup(m.Close) - - srv := grpc.NewServer() - grpc_health_v1.RegisterHealthServer(srv, health.NewServer()) - go func() { _ = srv.Serve(m.GRPC()) }() - t.Cleanup(srv.Stop) - - creds := insecure.NewCredentials() - if tc.tls { - creds = credentials.NewTLS(&tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // test server with a self-signed certificate - MinVersion: tls.VersionTLS12, - }) - } - conn, err := grpc.NewClient(l.Addr().String(), grpc.WithTransportCredentials(creds)) - require.NoError(t, err) - t.Cleanup(func() { _ = conn.Close() }) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - resp, err := grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{}) - require.NoError(t, err) - assert.Equal(t, grpc_health_v1.HealthCheckResponse_SERVING, resp.GetStatus()) - }) - } -} - -func TestIsNativeGRPCContentType(t *testing.T) { - for _, tc := range []struct { - ct string - want bool - }{ - {"application/grpc", true}, - {"application/grpc+proto", true}, - {"application/grpc+json", true}, - {"application/grpc; charset=utf-8", true}, - {"Application/GRPC", true}, - {"application/grpc-web", false}, - {"application/grpc-web+proto", false}, - {"application/grpc-web+json", false}, - {"application/json", false}, - {"text/plain", false}, - {"", false}, - } { - t.Run(tc.ct, func(t *testing.T) { - assert.Equal(t, tc.want, IsNativeGRPCContentType(tc.ct)) - }) - } -} From 37f5e5318f1c471774cddef14a7e91c5ae1b843f Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 19:40:05 +0200 Subject: [PATCH 08/10] docs(spec): clarify control-plane supersession and qualification Make the original control-plane design explicitly historical and scope the replacement authentication matrix to control-plane cases. CAS qualification remains separate. Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino --- docs/specs/issue-3628-controlplane-http-dispatch.md | 8 +++++--- docs/specs/issue-3628-single-endpoint.md | 6 +++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/specs/issue-3628-controlplane-http-dispatch.md b/docs/specs/issue-3628-controlplane-http-dispatch.md index 868a9b5c0..c4bd3a9c9 100644 --- a/docs/specs/issue-3628-controlplane-http-dispatch.md +++ b/docs/specs/issue-3628-controlplane-http-dispatch.md @@ -58,7 +58,9 @@ The control plane MUST serve native gRPC, gRPC-Web, HTTP/1.1, and HTTP/2 on its ### R-002: Existing behavior The shared endpoint MUST retain the existing authentication, authorization, validation, status, tracing, and metrics behavior. -- Done when: the authentication matrix in the original R-002 passes through the shared and legacy gRPC endpoints. +- Done when: the control-plane cases in the original R-002 matrix pass through the shared endpoint. + Native gRPC cases also pass through the legacy gRPC endpoint. +- CAS stream and download cases remain part of the separate CAS qualification under the parent ticket. - Focused protocol tests do not establish full alignment with this requirement. ### R-003: Existing endpoints remain available @@ -155,7 +157,7 @@ None for this control-plane change. CAS transport selection remains separate wor ## Milestones 1. **Shared server and ingress.** The control plane dispatches every request and the chart exposes the shared hostname. -2. **Behavior qualification.** The full authentication matrix passes, and deployed controller tests check the documented configurations. +2. **Behavior qualification.** The full control-plane authentication matrix passes, and deployed controller tests check the documented configurations. ## Risks @@ -163,4 +165,4 @@ None for this control-plane change. CAS transport selection remains separate wor |------|------------| | The HTTP adapter differs from the native gRPC transport in performance or features. | Retain the legacy endpoint. Measure relevant workloads before client migration. Keep CAS on its current transport. | | A controller uses HTTP/1.1 upstream or needs extra HTTP/2 settings. | Supply protocol annotations and document prerequisites. Test deployed controller versions before production use. | -| Focused tests miss authentication or authorization differences. | Keep the full original R-002 matrix as a qualification requirement. | +| Focused tests miss authentication or authorization differences. | Keep all control-plane cases from the original R-002 matrix as a qualification requirement. | diff --git a/docs/specs/issue-3628-single-endpoint.md b/docs/specs/issue-3628-single-endpoint.md index 727d9ca0e..b23215c7c 100644 --- a/docs/specs/issue-3628-single-endpoint.md +++ b/docs/specs/issue-3628-single-endpoint.md @@ -22,7 +22,11 @@ When your code goes against this spec, say so in your PR, and supersede the entr # Spec issue-3628: Serve gRPC and HTTP from one endpoint per service -Control-plane design superseded by [Spec issue-3628-controlplane-http-dispatch](issue-3628-controlplane-http-dispatch.md). CAS requirements remain open under the parent ticket. +Control-plane requirements and design superseded by [Spec issue-3628-controlplane-http-dispatch](issue-3628-controlplane-http-dispatch.md). +For the control plane, the requirements, constraints, proposal, diagram, decision records, milestones, and risks below are historical. +Use the replacement spec for all control-plane implementation and qualification. +Only CAS requirements remain in scope here. +CAS transport selection and qualification remain open under the parent ticket. ## Summary From ccce6a4be13e42364e8af346fc67fffe12c652d3 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 19:57:16 +0200 Subject: [PATCH 09/10] refactor(controlplane): retain native gRPC server constructor Keep NewGRPCServer dedicated to the existing native listener configuration. Assemble it with a separate HTTP-served handler through NewGRPCServers, sharing the service and middleware setup. Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino --- app/controlplane/cmd/wire_gen.go | 2 +- app/controlplane/internal/server/grpc.go | 44 ++++++++++--------- app/controlplane/internal/server/http_test.go | 2 +- app/controlplane/internal/server/server.go | 2 +- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/app/controlplane/cmd/wire_gen.go b/app/controlplane/cmd/wire_gen.go index 3881ab4c1..1775e07b6 100644 --- a/app/controlplane/cmd/wire_gen.go +++ b/app/controlplane/cmd/wire_gen.go @@ -373,7 +373,7 @@ func wireApp(contextContext context.Context, bootstrap *conf.Bootstrap, readerWr Credentials: readerWriter, Validator: validator, } - grpcServers, err := server.NewGRPCServer(opts) + grpcServers, err := server.NewGRPCServers(opts) if err != nil { cleanup3() cleanup2() diff --git a/app/controlplane/internal/server/grpc.go b/app/controlplane/internal/server/grpc.go index 1bc3ef318..74bc1231a 100644 --- a/app/controlplane/internal/server/grpc.go +++ b/app/controlplane/internal/server/grpc.go @@ -117,11 +117,22 @@ type GRPCServers struct { Legacy *grpc.Server } -// NewGRPCServer builds both servers with the same services and middleware. -func NewGRPCServer(opts *Opts) (*GRPCServers, error) { - // Opt-in histogram metrics for the interceptor - grpc_prometheus.EnableHandlingTimeHistogram() +// NewGRPCServers assembles the HTTP handler and optional native listener. +func NewGRPCServers(opts *Opts) (*GRPCServers, error) { + servers := &GRPCServers{Handler: newGRPCHandler(opts)} + if legacyGRPCListenerEnabled(opts.ServerConfig) { + // Ignore unused legacy certificates when this listener is disabled. + legacy, err := NewGRPCServer(opts) + if err != nil { + return nil, err + } + servers.Legacy = legacy + } + return servers, nil +} +// newGRPCHandler builds the handler served on the shared HTTP endpoint. +func newGRPCHandler(opts *Opts) *grpc.Server { // Kratos' stream interceptor needs a non-nil endpoint even when the gRPC // handler never starts a native listener. Supply it explicitly rather than // calling Endpoint(), which would bind an otherwise unused socket. @@ -129,31 +140,19 @@ func NewGRPCServer(opts *Opts) (*GRPCServers, error) { if opts.ServerConfig.GetHttp().GetTlsConfig().GetCertificate() != "" { scheme = "grpcs" } - handler := newGRPCServer(opts, grpc.Endpoint(&url.URL{ + return newGRPCServer(opts, grpc.Endpoint(&url.URL{ Scheme: scheme, Host: opts.ServerConfig.GetHttp().GetAddr(), })) - servers := &GRPCServers{Handler: handler} - if legacyGRPCListenerEnabled(opts.ServerConfig) { - // Ignore unused legacy certificates when this listener is disabled. - legacyOpts, err := legacyGRPCServerOptions(opts) - if err != nil { - return nil, err - } - servers.Legacy = newGRPCServer(opts, legacyOpts...) - } - return servers, nil } func legacyGRPCListenerEnabled(c *conf.Server) bool { return c.LegacyGrpcListener == nil || *c.LegacyGrpcListener } -// legacyGRPCServerOptions binds the deprecated gRPC-only listener: its -// configured network and address, with its TLS settings when present — the -// behavior the dedicated gRPC port had before the single port existed. A -// partial TLS configuration keeps this listener plaintext, as it always did. -func legacyGRPCServerOptions(opts *Opts) ([]grpc.ServerOption, error) { +// NewGRPCServer builds the native gRPC listener with its existing configuration. +// A partial TLS configuration keeps this listener plaintext, as it always did. +func NewGRPCServer(opts *Opts) (*grpc.Server, error) { var serverOpts []grpc.ServerOption if v := opts.ServerConfig.Grpc.Network; v != "" { serverOpts = append(serverOpts, grpc.Network(v)) @@ -175,13 +174,16 @@ func legacyGRPCServerOptions(opts *Opts) ([]grpc.ServerOption, error) { })) } } - return serverOpts, nil + return newGRPCServer(opts, serverOpts...), nil } // newGRPCServer builds one gRPC server with the shared middleware chain, // interceptors and service registrations, bound through the given extra // options. func newGRPCServer(opts *Opts, extraOpts ...grpc.ServerOption) *grpc.Server { + // Opt-in histogram metrics for the interceptor. + grpc_prometheus.EnableHandlingTimeHistogram() + var serverOpts = append([]grpc.ServerOption{ grpc.Middleware(craftMiddleware(opts)...), grpc.UnaryInterceptor( diff --git a/app/controlplane/internal/server/http_test.go b/app/controlplane/internal/server/http_test.go index e549fb7a2..f23e92dab 100644 --- a/app/controlplane/internal/server/http_test.go +++ b/app/controlplane/internal/server/http_test.go @@ -68,7 +68,7 @@ func startSinglePortApp(t *testing.T, legacyListener *bool) (string, string, int func startSinglePortAppWithConfig(t *testing.T, serverConfig *conf.Server) (string, string, int) { t.Helper() opts := testServerOpts(t, serverConfig) - gs, err := NewGRPCServer(opts) + gs, err := NewGRPCServers(opts) require.NoError(t, err) api, err := NewHTTPServer(opts, gs) require.NoError(t, err) diff --git a/app/controlplane/internal/server/server.go b/app/controlplane/internal/server/server.go index 07827a3e9..25d0a0865 100644 --- a/app/controlplane/internal/server/server.go +++ b/app/controlplane/internal/server/server.go @@ -24,7 +24,7 @@ import ( // ProviderSet is server providers. var ProviderSet = wire.NewSet( - NewGRPCServer, + NewGRPCServers, NewHTTPServer, NewHTTPMetricsServer, NewHTTPProfilerServer, From 007d88de60f41b4afcd14ce56a2bf6e46daeed39 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Mon, 19 Oct 2026 20:03:17 +0200 Subject: [PATCH 10/10] refactor(controlplane): simplify shared HTTP server setup Read the shared listener address directly in its builder and remove redundant nil handling already provided by protobuf getters. Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino --- app/controlplane/internal/server/http.go | 13 ++++++------- app/controlplane/internal/server/tls.go | 3 --- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/app/controlplane/internal/server/http.go b/app/controlplane/internal/server/http.go index 5270b8e92..4ed4234a2 100644 --- a/app/controlplane/internal/server/http.go +++ b/app/controlplane/internal/server/http.go @@ -62,7 +62,7 @@ func NewHTTPServer(opts *Opts, grpcServers *GRPCServers) (*APIServer, error) { if opts.ServerConfig.GetHttp().GetAddr() == "" { return nil, errors.New("server.http.addr must be set: the shared endpoint cannot bind an empty address") } - srv, err := newHTTPServer(opts, grpcServers.Handler, http.Address(opts.ServerConfig.GetHttp().GetAddr())) + srv, err := newHTTPServer(opts, grpcServers.Handler) if err != nil { return nil, err } @@ -72,16 +72,16 @@ func NewHTTPServer(opts *Opts, grpcServers *GRPCServers) (*APIServer, error) { return &APIServer{Server: srv, grpcHandler: grpcServers.Handler}, nil } -// newHTTPServer builds one HTTP server that serves the routes, the gRPC-Web -// wrapper and the transcoded handlers, bound through the given extra options. -func newHTTPServer(opts *Opts, grpcSrv *grpc.Server, extraOpts ...http.ServerOption) (*http.Server, error) { +// newHTTPServer builds the shared HTTP listener and its request dispatcher. +func newHTTPServer(opts *Opts, grpcSrv *grpc.Server) (*http.Server, error) { tlsConfig, err := loadServerTLSConfig(opts.ServerConfig.GetHttp().GetTlsConfig()) if err != nil { return nil, err } - var serverOpts = append([]http.ServerOption{ + serverOpts := []http.ServerOption{ http.Middleware(httpMiddlewares(opts)...), - }, extraOpts...) + http.Address(opts.ServerConfig.GetHttp().GetAddr()), + } // Prevent unmatched routes from falling through to http.DefaultServeMux, // which would otherwise expose net/http/pprof's /debug/pprof/* endpoints // unauthenticated on the public API listener. @@ -97,7 +97,6 @@ func newHTTPServer(opts *Opts, grpcSrv *grpc.Server, extraOpts ...http.ServerOpt serverOpts = append(serverOpts, http.TLSConfig(tlsConfig)) } - // initialize the underneath http server httpSrv := http.NewServer(serverOpts...) // Relax secure cookie settings in development mode opts.AuthSvc.SetDevMode(Version == "dev") diff --git a/app/controlplane/internal/server/tls.go b/app/controlplane/internal/server/tls.go index 30ce96785..74019baf6 100644 --- a/app/controlplane/internal/server/tls.go +++ b/app/controlplane/internal/server/tls.go @@ -26,9 +26,6 @@ import ( // paths of the server configuration. It returns nil when no TLS is configured. // For the single port the config negotiates HTTP/2 and HTTP/1.1 through ALPN. func loadServerTLSConfig(tlsConf *conf.Server_TLS) (*tls.Config, error) { - if tlsConf == nil { - return nil, nil - } cert := tlsConf.GetCertificate() privKey := tlsConf.GetPrivateKey() switch {